diff options
Diffstat (limited to 'ca')
| -rwxr-xr-x | ca/irbe_cli | 422 | ||||
| -rwxr-xr-x | ca/irdbd | 4 | ||||
| -rwxr-xr-x | ca/pubd | 4 | ||||
| -rwxr-xr-x | ca/rootd | 4 | ||||
| -rwxr-xr-x | ca/rpki-confgen | 455 | ||||
| -rwxr-xr-x | ca/rpki-sql-backup | 14 | ||||
| -rwxr-xr-x | ca/rpki-sql-setup | 378 | ||||
| -rwxr-xr-x | ca/rpki-start-servers | 36 | ||||
| -rwxr-xr-x | ca/rpkic | 4 | ||||
| -rwxr-xr-x | ca/rpkid | 4 | ||||
| -rwxr-xr-x | ca/rpkigui-apache-conf-gen | 672 | ||||
| -rwxr-xr-x | ca/rpkigui-import-routes | 1 | ||||
| -rwxr-xr-x | ca/tests/bgpsec-yaml.py | 56 | ||||
| -rw-r--r-- | ca/tests/myrpki-xml-parse-test.py | 74 | ||||
| -rw-r--r-- | ca/tests/old_irdbd.py | 4 | ||||
| -rw-r--r-- | ca/tests/smoketest.py | 2196 | ||||
| -rw-r--r-- | ca/tests/sql-cleaner.py | 32 | ||||
| -rw-r--r-- | ca/tests/sql-dumper.py | 20 | ||||
| -rwxr-xr-x | ca/tests/test-rrdp.py | 114 | ||||
| -rw-r--r-- | ca/tests/testpoke.py | 118 | ||||
| -rw-r--r-- | ca/tests/xml-parse-test.py | 108 | ||||
| -rw-r--r-- | ca/tests/yamlconf.py | 1426 | ||||
| -rw-r--r-- | ca/tests/yamltest.py | 1606 |
23 files changed, 3878 insertions, 3874 deletions
diff --git a/ca/irbe_cli b/ca/irbe_cli index ebd5d2a5..59039bf9 100755 --- a/ca/irbe_cli +++ b/ca/irbe_cli @@ -51,231 +51,231 @@ import rpki.publication pem_out = None class UsageWrapper(textwrap.TextWrapper): - """ - Call interface around Python textwrap.Textwrapper class. - """ - - def __call__(self, *args): """ - Format arguments, with TextWrapper indentation. + Call interface around Python textwrap.Textwrapper class. """ - return self.fill(textwrap.dedent(" ".join(args))) + + def __call__(self, *args): + """ + Format arguments, with TextWrapper indentation. + """ + return self.fill(textwrap.dedent(" ".join(args))) usage_fill = UsageWrapper(subsequent_indent = " " * 4) class reply_elt_mixin(object): - """ - Protocol mix-in for printout of reply PDUs. - """ - - is_cmd = False - - def client_reply_decode(self): - pass - - def client_reply_show(self): - print self.element_name - for i in self.attributes + self.elements: - if getattr(self, i) is not None: - print " %s: %s" % (i, getattr(self, i)) - -class cmd_elt_mixin(reply_elt_mixin): - """ - Protocol mix-in for command line client element PDUs. - """ - - is_cmd = True - - ## @var excludes - # XML attributes and elements that should not be allowed as command - # line arguments. - excludes = () - - @classmethod - def usage(cls): """ - Generate usage message for this PDU. + Protocol mix-in for printout of reply PDUs. """ - args = " ".join("--" + x + "=" for x in cls.attributes + cls.elements if x not in cls.excludes) - bools = " ".join("--" + x for x in cls.booleans) - if args and bools: - return args + " " + bools - else: - return args or bools - def client_getopt(self, argv): - """ - Parse options for this class. - """ - # pylint: disable=W0621 - opts, argv = getopt.getopt(argv, "", [x + "=" for x in self.attributes + self.elements if x not in self.excludes] + list(self.booleans)) - for o, a in opts: - o = o[2:] - handler = getattr(self, "client_query_" + o, None) - if handler is not None: - handler(a) - elif o in self.booleans: - setattr(self, o, True) - else: - assert o in self.attributes - setattr(self, o, a) - return argv - - def client_query_bpki_cert(self, arg): - """ - Special handler for --bpki_cert option. - """ - self.bpki_cert = rpki.x509.X509(Auto_file = arg) + is_cmd = False - def client_query_glue(self, arg): - """ - Special handler for --bpki_glue option. - """ - self.bpki_glue = rpki.x509.X509(Auto_file = arg) + def client_reply_decode(self): + pass - def client_query_bpki_cms_cert(self, arg): - """ - Special handler for --bpki_cms_cert option. - """ - self.bpki_cms_cert = rpki.x509.X509(Auto_file = arg) + def client_reply_show(self): + print self.element_name + for i in self.attributes + self.elements: + if getattr(self, i) is not None: + print " %s: %s" % (i, getattr(self, i)) - def client_query_cms_glue(self, arg): +class cmd_elt_mixin(reply_elt_mixin): """ - Special handler for --bpki_cms_glue option. + Protocol mix-in for command line client element PDUs. """ - self.bpki_cms_glue = rpki.x509.X509(Auto_file = arg) -class cmd_msg_mixin(object): - """ - Protocol mix-in for command line client message PDUs. - """ + is_cmd = True + + ## @var excludes + # XML attributes and elements that should not be allowed as command + # line arguments. + excludes = () + + @classmethod + def usage(cls): + """ + Generate usage message for this PDU. + """ + args = " ".join("--" + x + "=" for x in cls.attributes + cls.elements if x not in cls.excludes) + bools = " ".join("--" + x for x in cls.booleans) + if args and bools: + return args + " " + bools + else: + return args or bools + + def client_getopt(self, argv): + """ + Parse options for this class. + """ + # pylint: disable=W0621 + opts, argv = getopt.getopt(argv, "", [x + "=" for x in self.attributes + self.elements if x not in self.excludes] + list(self.booleans)) + for o, a in opts: + o = o[2:] + handler = getattr(self, "client_query_" + o, None) + if handler is not None: + handler(a) + elif o in self.booleans: + setattr(self, o, True) + else: + assert o in self.attributes + setattr(self, o, a) + return argv + + def client_query_bpki_cert(self, arg): + """ + Special handler for --bpki_cert option. + """ + self.bpki_cert = rpki.x509.X509(Auto_file = arg) + + def client_query_glue(self, arg): + """ + Special handler for --bpki_glue option. + """ + self.bpki_glue = rpki.x509.X509(Auto_file = arg) + + def client_query_bpki_cms_cert(self, arg): + """ + Special handler for --bpki_cms_cert option. + """ + self.bpki_cms_cert = rpki.x509.X509(Auto_file = arg) + + def client_query_cms_glue(self, arg): + """ + Special handler for --bpki_cms_glue option. + """ + self.bpki_cms_glue = rpki.x509.X509(Auto_file = arg) - @classmethod - def usage(cls): +class cmd_msg_mixin(object): """ - Generate usage message for this PDU. + Protocol mix-in for command line client message PDUs. """ - for k, v in cls.pdus.items(): - if v.is_cmd: - print usage_fill(k, v.usage()) + + @classmethod + def usage(cls): + """ + Generate usage message for this PDU. + """ + for k, v in cls.pdus.items(): + if v.is_cmd: + print usage_fill(k, v.usage()) # left-right protcol class left_right_msg(cmd_msg_mixin, rpki.left_right.msg): - class self_elt(cmd_elt_mixin, rpki.left_right.self_elt): - pass + class self_elt(cmd_elt_mixin, rpki.left_right.self_elt): + pass - class bsc_elt(cmd_elt_mixin, rpki.left_right.bsc_elt): + class bsc_elt(cmd_elt_mixin, rpki.left_right.bsc_elt): - excludes = ("pkcs10_request",) + excludes = ("pkcs10_request",) - def client_query_signing_cert(self, arg): - """--signing_cert option.""" - self.signing_cert = rpki.x509.X509(Auto_file = arg) + def client_query_signing_cert(self, arg): + """--signing_cert option.""" + self.signing_cert = rpki.x509.X509(Auto_file = arg) - def client_query_signing_cert_crl(self, arg): - """--signing_cert_crl option.""" - self.signing_cert_crl = rpki.x509.CRL(Auto_file = arg) + def client_query_signing_cert_crl(self, arg): + """--signing_cert_crl option.""" + self.signing_cert_crl = rpki.x509.CRL(Auto_file = arg) - def client_reply_decode(self): - global pem_out - if pem_out is not None and self.pkcs10_request is not None: - if isinstance(pem_out, str): - pem_out = open(pem_out, "w") - pem_out.write(self.pkcs10_request.get_PEM()) + def client_reply_decode(self): + global pem_out + if pem_out is not None and self.pkcs10_request is not None: + if isinstance(pem_out, str): + pem_out = open(pem_out, "w") + pem_out.write(self.pkcs10_request.get_PEM()) - class parent_elt(cmd_elt_mixin, rpki.left_right.parent_elt): - pass + class parent_elt(cmd_elt_mixin, rpki.left_right.parent_elt): + pass - class child_elt(cmd_elt_mixin, rpki.left_right.child_elt): - pass + class child_elt(cmd_elt_mixin, rpki.left_right.child_elt): + pass - class repository_elt(cmd_elt_mixin, rpki.left_right.repository_elt): - pass + class repository_elt(cmd_elt_mixin, rpki.left_right.repository_elt): + pass - class list_published_objects_elt(cmd_elt_mixin, rpki.left_right.list_published_objects_elt): - excludes = ("uri",) + class list_published_objects_elt(cmd_elt_mixin, rpki.left_right.list_published_objects_elt): + excludes = ("uri",) - class list_received_resources_elt(cmd_elt_mixin, rpki.left_right.list_received_resources_elt): - excludes = ("parent_handle", "notBefore", "notAfter", "uri", "sia_uri", "aia_uri", "asn", "ipv4", "ipv6") + class list_received_resources_elt(cmd_elt_mixin, rpki.left_right.list_received_resources_elt): + excludes = ("parent_handle", "notBefore", "notAfter", "uri", "sia_uri", "aia_uri", "asn", "ipv4", "ipv6") - class report_error_elt(reply_elt_mixin, rpki.left_right.report_error_elt): - pass + class report_error_elt(reply_elt_mixin, rpki.left_right.report_error_elt): + pass - pdus = dict((x.element_name, x) - for x in (self_elt, bsc_elt, parent_elt, child_elt, repository_elt, - list_published_objects_elt, list_received_resources_elt, report_error_elt)) + pdus = dict((x.element_name, x) + for x in (self_elt, bsc_elt, parent_elt, child_elt, repository_elt, + list_published_objects_elt, list_received_resources_elt, report_error_elt)) class left_right_sax_handler(rpki.left_right.sax_handler): # pylint: disable=W0232 - pdu = left_right_msg + pdu = left_right_msg class left_right_cms_msg(rpki.left_right.cms_msg): - saxify = left_right_sax_handler.saxify + saxify = left_right_sax_handler.saxify # Publication protocol class publication_msg(cmd_msg_mixin, rpki.publication.msg): - class config_elt(cmd_elt_mixin, rpki.publication.config_elt): + class config_elt(cmd_elt_mixin, rpki.publication.config_elt): - def client_query_bpki_crl(self, arg): - """ - Special handler for --bpki_crl option. - """ - self.bpki_crl = rpki.x509.CRL(Auto_file = arg) + def client_query_bpki_crl(self, arg): + """ + Special handler for --bpki_crl option. + """ + self.bpki_crl = rpki.x509.CRL(Auto_file = arg) - class client_elt(cmd_elt_mixin, rpki.publication.client_elt): - pass + class client_elt(cmd_elt_mixin, rpki.publication.client_elt): + pass - class certificate_elt(cmd_elt_mixin, rpki.publication.certificate_elt): - pass + class certificate_elt(cmd_elt_mixin, rpki.publication.certificate_elt): + pass - class crl_elt(cmd_elt_mixin, rpki.publication.crl_elt): - pass + class crl_elt(cmd_elt_mixin, rpki.publication.crl_elt): + pass - class manifest_elt(cmd_elt_mixin, rpki.publication.manifest_elt): - pass + class manifest_elt(cmd_elt_mixin, rpki.publication.manifest_elt): + pass - class roa_elt(cmd_elt_mixin, rpki.publication.roa_elt): - pass + class roa_elt(cmd_elt_mixin, rpki.publication.roa_elt): + pass - class report_error_elt(reply_elt_mixin, rpki.publication.report_error_elt): - pass + class report_error_elt(reply_elt_mixin, rpki.publication.report_error_elt): + pass - class ghostbuster_elt(cmd_elt_mixin, rpki.publication.ghostbuster_elt): - pass + class ghostbuster_elt(cmd_elt_mixin, rpki.publication.ghostbuster_elt): + pass - pdus = dict((x.element_name, x) - for x in (config_elt, client_elt, certificate_elt, crl_elt, - manifest_elt, roa_elt, report_error_elt, - ghostbuster_elt)) + pdus = dict((x.element_name, x) + for x in (config_elt, client_elt, certificate_elt, crl_elt, + manifest_elt, roa_elt, report_error_elt, + ghostbuster_elt)) class publication_sax_handler(rpki.publication.sax_handler): # pylint: disable=W0232 - pdu = publication_msg + pdu = publication_msg class publication_cms_msg(rpki.publication.cms_msg): - saxify = publication_sax_handler.saxify + saxify = publication_sax_handler.saxify # Usage top_opts = ["config=", "help", "pem_out=", "quiet", "verbose"] def usage(code = 1): - if __doc__ is not None: - print __doc__.strip() + if __doc__ is not None: + print __doc__.strip() + print + print "Usage:" + print + print "# Top-level options:" + print usage_fill(*["--" + x for x in top_opts]) print - print "Usage:" - print - print "# Top-level options:" - print usage_fill(*["--" + x for x in top_opts]) - print - print "# left-right protocol:" - left_right_msg.usage() - print - print "# publication protocol:" - publication_msg.usage() - sys.exit(code) + print "# left-right protocol:" + left_right_msg.usage() + print + print "# publication protocol:" + publication_msg.usage() + sys.exit(code) # Main program @@ -284,26 +284,26 @@ rpki.log.init("irbe_cli") argv = sys.argv[1:] if not argv: - usage(0) + usage(0) cfg_file = None verbose = True opts, argv = getopt.getopt(argv, "c:hpqv?", top_opts) for o, a in opts: - if o in ("-?", "-h", "--help"): - usage(0) - elif o in ("-c", "--config"): - cfg_file = a - elif o in ("-p", "--pem_out"): - pem_out = a - elif o in ("-q", "--quiet"): - verbose = False - elif o in ("-v", "--verbose"): - verbose = True + if o in ("-?", "-h", "--help"): + usage(0) + elif o in ("-c", "--config"): + cfg_file = a + elif o in ("-p", "--pem_out"): + pem_out = a + elif o in ("-q", "--quiet"): + verbose = False + elif o in ("-v", "--verbose"): + verbose = True if not argv: - usage(1) + usage(1) cfg = rpki.config.parser(set_filename = cfg_file, section = "irbe_cli") @@ -311,16 +311,16 @@ q_msg_left_right = [] q_msg_publication = [] while argv: - if argv[0] in left_right_msg.pdus: - q_pdu = left_right_msg.pdus[argv[0]]() - q_msg = q_msg_left_right - elif argv[0] in publication_msg.pdus: - q_pdu = publication_msg.pdus[argv[0]]() - q_msg = q_msg_publication - else: - usage(1) - argv = q_pdu.client_getopt(argv[1:]) - q_msg.append(q_pdu) + if argv[0] in left_right_msg.pdus: + q_pdu = left_right_msg.pdus[argv[0]]() + q_msg = q_msg_left_right + elif argv[0] in publication_msg.pdus: + q_pdu = publication_msg.pdus[argv[0]]() + q_msg = q_msg_publication + else: + usage(1) + argv = q_pdu.client_getopt(argv[1:]) + q_msg.append(q_pdu) os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb") @@ -331,36 +331,36 @@ irbe = server_ca.ee_certificates.get(purpose = "irbe") if q_msg_left_right: - rpkid = server_ca.ee_certificates.get(purpose = "rpkid") + rpkid = server_ca.ee_certificates.get(purpose = "rpkid") - rpkid_url = "http://%s:%s/left-right/" % ( - cfg.get("server-host", section = "rpkid"), - cfg.get("server-port", section = "rpkid")) + rpkid_url = "http://%s:%s/left-right/" % ( + cfg.get("server-host", section = "rpkid"), + cfg.get("server-port", section = "rpkid")) - rpki.http_simple.client( - proto_cms_msg = left_right_cms_msg, - client_key = irbe.private_key, - client_cert = irbe.certificate, - server_ta = server_ca.certificate, - server_cert = rpkid.certificate, - url = rpkid_url, - debug = verbose, - q_msg = left_right_msg(*q_msg_left_right)) + rpki.http_simple.client( + proto_cms_msg = left_right_cms_msg, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = server_ca.certificate, + server_cert = rpkid.certificate, + url = rpkid_url, + debug = verbose, + q_msg = left_right_msg(*q_msg_left_right)) if q_msg_publication: - pubd = server_ca.ee_certificates.get(purpose = "pubd") - - pubd_url = "http://%s:%s/control/" % ( - cfg.get("server-host", section = "pubd"), - cfg.get("server-port", section = "pubd")) - - rpki.http_simple.client( - proto_cms_msg = publication_cms_msg, - client_key = irbe.private_key, - client_cert = irbe.certificate, - server_ta = server_ca.certificate, - server_cert = pubd.certificate, - url = pubd_url, - debug = verbose, - q_msg = publication_msg(*q_msg_publication)) + pubd = server_ca.ee_certificates.get(purpose = "pubd") + + pubd_url = "http://%s:%s/control/" % ( + cfg.get("server-host", section = "pubd"), + cfg.get("server-port", section = "pubd")) + + rpki.http_simple.client( + proto_cms_msg = publication_cms_msg, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = server_ca.certificate, + server_cert = pubd.certificate, + url = pubd_url, + debug = verbose, + q_msg = publication_msg(*q_msg_publication)) @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.irdbd - rpki.irdbd.main() + import rpki.irdbd + rpki.irdbd.main() @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.pubd - rpki.pubd.main() + import rpki.pubd + rpki.pubd.main() @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.rootd - rpki.rootd.main() + import rpki.rootd + rpki.rootd.main() diff --git a/ca/rpki-confgen b/ca/rpki-confgen index 07c87f0f..beb8506d 100755 --- a/ca/rpki-confgen +++ b/ca/rpki-confgen @@ -38,254 +38,259 @@ xml8_wrapper = textwrap.TextWrapper(initial_indent = space8, subsequent_indent = class Option(object): - def __init__(self, name, value, doc): - self.name = name - self.value = value - self.doc = doc - - @property - def width(self): - return len(self.name) - - def to_xml(self): - x = Element("option", name = self.name) - if self.value is not None: - x.set("value", self.value) - for d in self.doc: - SubElement(x, "doc").text = "\n" + xml8_wrapper.fill(d) + "\n" + space6 - return x - - def to_wiki(self, f): - f.write("\n== %s == #%s\n" % (self.name, self.name)) - for d in self.doc: - f.write("\n%s\n" % wiki_wrapper.fill(d)) - if self.value is None: - f.write("\n%s\n" % wiki_wrapper.fill("No default value.")) - else: - f.write("\n{{{\n#!ini\n%s = %s\n}}}\n" % (self.name, self.value)) - - def to_conf(self, f, width): - for i, d in enumerate(self.doc): - f.write("%s\n%s\n" % ("" if i == 0 else "#", conf_wrapper.fill(d))) - if self.value is None: - f.write("\n#%-*s = ???\n" % (width - 1, self.name)) - else: - f.write("\n%-*s = %s\n" % (width, self.name, self.value)) + def __init__(self, name, value, doc): + self.name = name + self.value = value + self.doc = doc + + @property + def width(self): + return len(self.name) + + def to_xml(self): + x = Element("option", name = self.name) + if self.value is not None: + x.set("value", self.value) + for d in self.doc: + SubElement(x, "doc").text = "\n" + xml8_wrapper.fill(d) + "\n" + space6 + return x + + def to_wiki(self, f): + f.write("\n== %s == #%s\n" % (self.name, self.name)) + for d in self.doc: + f.write("\n%s\n" % wiki_wrapper.fill(d)) + if self.value is None: + f.write("\n%s\n" % wiki_wrapper.fill("No default value.")) + else: + f.write("\n{{{\n#!ini\n%s = %s\n}}}\n" % (self.name, self.value)) + + def to_conf(self, f, width): + for i, d in enumerate(self.doc): + f.write("%s\n%s\n" % ("" if i == 0 else "#", conf_wrapper.fill(d))) + if self.value is None: + f.write("\n#%-*s = ???\n" % (width - 1, self.name)) + else: + f.write("\n%-*s = %s\n" % (width, self.name, self.value)) class Section(object): - def __init__(self, name): - self.name = name - self.doc = [] - self.options = [] - - @property - def width(self): - return max(o.width for o in self.options) - - @classmethod - def from_xml(cls, elt): - self = cls(name = elt.get("name")) - for x in elt.iterchildren("doc"): - self.doc.append(" ".join(x.text.split())) - for x in elt.iterchildren("option"): - self.options.append(Option(name = x.get("name"), value = x.get("value"), - doc = [" ".join(d.text.split()) - for d in x.iterchildren("doc")])) - return self - - def to_xml(self): - x = Element("section", name = self.name) - for d in self.doc: - SubElement(x, "doc").text = "\n" + xml6_wrapper.fill(d) + "\n" + space4 - x.extend(o.to_xml() for o in self.options) - return x - - def to_wiki(self, f): - f.write("\n= [%s] section = #%s\n" % (self.name, self.name)) - for d in self.doc: - f.write("\n%s\n" % wiki_wrapper.fill(d)) - for o in self.options: - o.to_wiki(f) - - def to_conf(self, f, width): - f.write("\n" + "#" * 78 + "\n\n[" + self.name + "]\n") - if self.doc: - f.write("\n##") - for i, d in enumerate(self.doc): - f.write("%s\n%s\n" % ("" if i == 0 else "#", conf_wrapper.fill(d))) - f.write("##\n") - for o in self.options: - o.to_conf(f, width) + def __init__(self, name): + self.name = name + self.doc = [] + self.options = [] + + @property + def width(self): + return max(o.width for o in self.options) + + @classmethod + def from_xml(cls, elt): + self = cls(name = elt.get("name")) + for x in elt.iterchildren("doc"): + self.doc.append(" ".join(x.text.split())) + for x in elt.iterchildren("option"): + self.options.append(Option(name = x.get("name"), value = x.get("value"), + doc = [" ".join(d.text.split()) + for d in x.iterchildren("doc")])) + return self + + def to_xml(self): + x = Element("section", name = self.name) + for d in self.doc: + SubElement(x, "doc").text = "\n" + xml6_wrapper.fill(d) + "\n" + space4 + x.extend(o.to_xml() for o in self.options) + return x + + def to_wiki(self, f): + f.write("\n= [%s] section = #%s\n" % (self.name, self.name)) + for d in self.doc: + f.write("\n%s\n" % wiki_wrapper.fill(d)) + for o in self.options: + o.to_wiki(f) + + def to_conf(self, f, width): + f.write("\n" + "#" * 78 + "\n\n[" + self.name + "]\n") + if self.doc: + f.write("\n##") + for i, d in enumerate(self.doc): + f.write("%s\n%s\n" % ("" if i == 0 else "#", conf_wrapper.fill(d))) + f.write("##\n") + for o in self.options: + o.to_conf(f, width) def wiki_header(f, ident, toc): - f.write("\n".join(( - "{{{", - "#!comment", - "", - star78, - "THIS PAGE WAS GENERATED AUTOMATICALLY, DO NOT EDIT.", - "", - "Generated from " + ident, - " by $Id$", - star78, - "", - "}}}", - ""))) - if toc is not None: - f.write("[[TracNav(%s)]]\n" % toc) - f.write("[[PageOutline]]\n") + f.write("\n".join(( + "{{{", + "#!comment", + "", + star78, + "THIS PAGE WAS GENERATED AUTOMATICALLY, DO NOT EDIT.", + "", + "Generated from " + ident, + " by $Id$", + star78, + "", + "}}}", + ""))) + if toc is not None: + f.write("[[TracNav(%s)]]\n" % toc) + f.write("[[PageOutline]]\n") def conf_header(f, ident): - f.write("\n".join(( - "# Automatically generated. Edit as needed, but be careful of overwriting.", - "#", - "# Generated from " + ident, - "# by $Id$", - ""))) + f.write("\n".join(( + "# Automatically generated. Edit as needed, but be careful of overwriting.", + "#", + "# Generated from " + ident, + "# by $Id$", + ""))) # http://stackoverflow.com/questions/9027028/argparse-argument-order class CustomAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string = None): - if not "ordered_args" in namespace: - namespace.ordered_args = [] - namespace.ordered_args.append((self.dest, values)) + def __call__(self, parser, namespace, values, option_string = None): + if not "ordered_args" in namespace: + namespace.ordered_args = [] + namespace.ordered_args.append((self.dest, values)) class CustomFlagAction(argparse.Action): - def __init__(self, option_strings, dest, default = None, - required = False, help = None): # pylint: disable=W0622 - super(CustomFlagAction, self).__init__( - option_strings = option_strings, dest = dest, nargs = 0, - const = None, default = default, required = required, help = help) - def __call__(self, parser, namespace, values, option_string = None): - if not "ordered_args" in namespace: - namespace.ordered_args = [] - namespace.ordered_args.append((self.dest, None)) + def __init__(self, option_strings, dest, default = None, + required = False, help = None): # pylint: disable=W0622 + super(CustomFlagAction, self).__init__( + option_strings = option_strings, + dest = dest, + nargs = 0, + const = None, + default = default, + required = required, + help = help) + def __call__(self, parser, namespace, values, option_string = None): + if not "ordered_args" in namespace: + namespace.ordered_args = [] + namespace.ordered_args.append((self.dest, None)) class main(object): - def __init__(self): - self.sections = [] - self.section_map = None - self.option_map = None - self.ident = None - self.toc = None - - parser = argparse.ArgumentParser(description = __doc__) - parser.add_argument("--read-xml", metavar = "FILE", action = CustomAction, - required = True, type = argparse.FileType("r"), - help = "XML input file defining sections and options") - parser.add_argument("--write-xml", metavar = "FILE", action = CustomAction, - help = "XML file to write") - parser.add_argument("--write-wiki", metavar = "FILE", action = CustomAction, - help = "TracWiki file to write") - parser.add_argument("--write-conf", metavar = "FILE", action = CustomAction, - help = "rpki.conf configuration file to write") - parser.add_argument("--set", metavar = "VARVAL", action = CustomAction, - help = "variable setting in form \"VAR=VAL\"") - parser.add_argument("--pwgen", metavar = "VAR", action = CustomAction, - help = "set variable to generated password") - parser.add_argument("--toc", metavar = "TRACNAV", action = CustomAction, - help = "set TOC value to use with TracNav plugin") - parser.add_argument("--autoconf", action = CustomFlagAction, - help = "configure [autoconf] section") - args = parser.parse_args() - - for cmd, arg in args.ordered_args: - getattr(self, "do_" + cmd)(arg) - - - def do_read_xml(self, arg): - self.option_map = None - root = ElementTree(file = arg).getroot() - self.ident = root.get("ident") - self.sections.extend(Section.from_xml(x) for x in root) - self.option_map = {} - self.section_map = {} - for section in self.sections: - if section.name in self.section_map: - sys.exit("Duplicate section %s" % section.name) - self.section_map[section.name] = section - for option in section.options: - name = (section.name, option.name) - if name in self.option_map: - sys.exit("Duplicate option %s::%s" % name) - self.option_map[name] = option - - - def do_set(self, arg): - try: - name, value = arg.split("=", 1) - section, option = name.split("::") - except ValueError: - sys.exit("Couldn't parse --set specification \"%s\"" % arg) - name = (section, option) - if name not in self.option_map: - sys.exit("Couldn't find option %s::%s" % name) - self.option_map[name].value = value - - - def do_pwgen(self, arg): - try: - section, option = arg.split("::") - except ValueError: - sys.exit("Couldn't parse --pwgen specification \"%s\"" % arg) - name = (section, option) - if name not in self.option_map: - sys.exit("Couldn't find option %s::%s" % name) - self.option_map[name].value = base64.urlsafe_b64encode(os.urandom(66)) - - - def do_autoconf(self, ignored): - try: - import rpki.autoconf - for option in self.section_map["autoconf"].options: + def __init__(self): + self.sections = [] + self.section_map = None + self.option_map = None + self.ident = None + self.toc = None + + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("--read-xml", metavar = "FILE", action = CustomAction, + required = True, type = argparse.FileType("r"), + help = "XML input file defining sections and options") + parser.add_argument("--write-xml", metavar = "FILE", action = CustomAction, + help = "XML file to write") + parser.add_argument("--write-wiki", metavar = "FILE", action = CustomAction, + help = "TracWiki file to write") + parser.add_argument("--write-conf", metavar = "FILE", action = CustomAction, + help = "rpki.conf configuration file to write") + parser.add_argument("--set", metavar = "VARVAL", action = CustomAction, + help = "variable setting in form \"VAR=VAL\"") + parser.add_argument("--pwgen", metavar = "VAR", action = CustomAction, + help = "set variable to generated password") + parser.add_argument("--toc", metavar = "TRACNAV", action = CustomAction, + help = "set TOC value to use with TracNav plugin") + parser.add_argument("--autoconf", action = CustomFlagAction, + help = "configure [autoconf] section") + args = parser.parse_args() + + for cmd, arg in args.ordered_args: + getattr(self, "do_" + cmd)(arg) + + + def do_read_xml(self, arg): + self.option_map = None + root = ElementTree(file = arg).getroot() + self.ident = root.get("ident") + self.sections.extend(Section.from_xml(x) for x in root) + self.option_map = {} + self.section_map = {} + for section in self.sections: + if section.name in self.section_map: + sys.exit("Duplicate section %s" % section.name) + self.section_map[section.name] = section + for option in section.options: + name = (section.name, option.name) + if name in self.option_map: + sys.exit("Duplicate option %s::%s" % name) + self.option_map[name] = option + + + def do_set(self, arg): try: - option.value = getattr(rpki.autoconf, option.name) - except AttributeError: - pass - except ImportError: - sys.exit("rpki.autoconf module is not available") - except KeyError: - sys.exit("Couldn't find autoconf section") + name, value = arg.split("=", 1) + section, option = name.split("::") + except ValueError: + sys.exit("Couldn't parse --set specification \"%s\"" % arg) + name = (section, option) + if name not in self.option_map: + sys.exit("Couldn't find option %s::%s" % name) + self.option_map[name].value = value - def do_write_xml(self, arg): - x = Element("configuration", ident = self.ident) - x.extend(s.to_xml() for s in self.sections) - ElementTree(x).write(arg, pretty_print = True, encoding = "us-ascii") - - - def do_write_wiki(self, arg): - if "%" in arg: - for section in self.sections: - with open(arg % section.name, "w") as f: - wiki_header(f, self.ident, self.toc) - section.to_wiki(f) - else: - with open(arg, "w") as f: - for i, section in enumerate(self.sections): - if i == 0: - wiki_header(f, self.ident, self.toc) - else: - f.write("\f\n") - section.to_wiki(f) - - - def do_write_conf(self, arg): - with open(arg, "w") as f: - conf_header(f, self.ident) - width = max(s.width for s in self.sections) - for section in self.sections: - section.to_conf(f, width) + def do_pwgen(self, arg): + try: + section, option = arg.split("::") + except ValueError: + sys.exit("Couldn't parse --pwgen specification \"%s\"" % arg) + name = (section, option) + if name not in self.option_map: + sys.exit("Couldn't find option %s::%s" % name) + self.option_map[name].value = base64.urlsafe_b64encode(os.urandom(66)) - def do_toc(self, arg): - self.toc = arg + def do_autoconf(self, ignored): + try: + import rpki.autoconf + for option in self.section_map["autoconf"].options: + try: + option.value = getattr(rpki.autoconf, option.name) + except AttributeError: + pass + except ImportError: + sys.exit("rpki.autoconf module is not available") + except KeyError: + sys.exit("Couldn't find autoconf section") + + + def do_write_xml(self, arg): + x = Element("configuration", ident = self.ident) + x.extend(s.to_xml() for s in self.sections) + ElementTree(x).write(arg, pretty_print = True, encoding = "us-ascii") + + + def do_write_wiki(self, arg): + if "%" in arg: + for section in self.sections: + with open(arg % section.name, "w") as f: + wiki_header(f, self.ident, self.toc) + section.to_wiki(f) + else: + with open(arg, "w") as f: + for i, section in enumerate(self.sections): + if i == 0: + wiki_header(f, self.ident, self.toc) + else: + f.write("\f\n") + section.to_wiki(f) + + + def do_write_conf(self, arg): + with open(arg, "w") as f: + conf_header(f, self.ident) + width = max(s.width for s in self.sections) + for section in self.sections: + section.to_conf(f, width) + + + def do_toc(self, arg): + self.toc = arg if __name__ == "__main__": - main() + main() diff --git a/ca/rpki-sql-backup b/ca/rpki-sql-backup index 02835956..986e10b1 100755 --- a/ca/rpki-sql-backup +++ b/ca/rpki-sql-backup @@ -44,10 +44,10 @@ args = parser.parse_args() cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") for name in ("rpkid", "irdbd", "pubd"): - if cfg.getboolean("start_" + name, False): - subprocess.check_call( - ("mysqldump", "--add-drop-database", - "-u", cfg.get("sql-username", section = name), - "-p" + cfg.get("sql-password", section = name), - "-B", cfg.get("sql-database", section = name)), - stdout = args.output) + if cfg.getboolean("start_" + name, False): + subprocess.check_call( + ("mysqldump", "--add-drop-database", + "-u", cfg.get("sql-username", section = name), + "-p" + cfg.get("sql-password", section = name), + "-B", cfg.get("sql-database", section = name)), + stdout = args.output) diff --git a/ca/rpki-sql-setup b/ca/rpki-sql-setup index b209fec7..c399fcfe 100755 --- a/ca/rpki-sql-setup +++ b/ca/rpki-sql-setup @@ -39,218 +39,218 @@ ER_NO_SUCH_TABLE = 1146 # See mysqld_ername.h class RootDB(object): - """ - Class to wrap MySQL actions that require root-equivalent access so - we can defer such actions until we're sure they're really needed. - Overall goal here is to prompt the user for the root password once - at most, and not at all when not necessary. - """ - - def __init__(self, mysql_defaults = None): - self.initialized = False - self.mysql_defaults = mysql_defaults - - def __getattr__(self, name): - if self.initialized: - raise AttributeError - if self.mysql_defaults is None: - self.db = MySQLdb.connect(db = "mysql", - user = "root", - passwd = getpass.getpass("Please enter your MySQL root password: ")) - else: - mysql_cfg = rpki.config.parser(set_filename = self.mysql_defaults, section = "client") - self.db = MySQLdb.connect(db = "mysql", - user = mysql_cfg.get("user"), - passwd = mysql_cfg.get("password")) - self.cur = self.db.cursor() - self.cur.execute("SHOW DATABASES") - self.databases = set(d[0] for d in self.cur.fetchall()) - self.initialized = True - return getattr(self, name) - - def close(self): - if self.initialized: - self.db.close() + """ + Class to wrap MySQL actions that require root-equivalent access so + we can defer such actions until we're sure they're really needed. + Overall goal here is to prompt the user for the root password once + at most, and not at all when not necessary. + """ + + def __init__(self, mysql_defaults = None): + self.initialized = False + self.mysql_defaults = mysql_defaults + + def __getattr__(self, name): + if self.initialized: + raise AttributeError + if self.mysql_defaults is None: + self.db = MySQLdb.connect(db = "mysql", + user = "root", + passwd = getpass.getpass("Please enter your MySQL root password: ")) + else: + mysql_cfg = rpki.config.parser(set_filename = self.mysql_defaults, section = "client") + self.db = MySQLdb.connect(db = "mysql", + user = mysql_cfg.get("user"), + passwd = mysql_cfg.get("password")) + self.cur = self.db.cursor() + self.cur.execute("SHOW DATABASES") + self.databases = set(d[0] for d in self.cur.fetchall()) + self.initialized = True + return getattr(self, name) + + def close(self): + if self.initialized: + self.db.close() class UserDB(object): - """ - Class to wrap MySQL access parameters for a particular database. - - NB: The SQL definitions for the upgrade_version table is embedded in - this class rather than being declared in any of the .sql files. - This is deliberate: nothing but the upgrade system should ever touch - this table, and it's simpler to keep everything in one place. - - We have to be careful about SQL commits here, because CREATE TABLE - implies an automatic commit. So presence of the magic table per se - isn't significant, only its content (or lack thereof). - """ - - upgrade_version_table_schema = """ - CREATE TABLE upgrade_version ( - version TEXT NOT NULL, - updated DATETIME NOT NULL - ) ENGINE=InnoDB + """ + Class to wrap MySQL access parameters for a particular database. + + NB: The SQL definitions for the upgrade_version table is embedded in + this class rather than being declared in any of the .sql files. + This is deliberate: nothing but the upgrade system should ever touch + this table, and it's simpler to keep everything in one place. + + We have to be careful about SQL commits here, because CREATE TABLE + implies an automatic commit. So presence of the magic table per se + isn't significant, only its content (or lack thereof). """ - def __init__(self, name): - self.name = name - self.database = cfg.get("sql-database", section = name) - self.username = cfg.get("sql-username", section = name) - self.password = cfg.get("sql-password", section = name) - self.db = None - self.cur = None - - def open(self): - self.db = MySQLdb.connect(db = self.database, user = self.username, passwd = self.password) - self.db.autocommit(False) - self.cur = self.db.cursor() - - def close(self): - if self.cur is not None: - self.cur.close() - self.cur = None - if self.db is not None: - self.db.commit() - self.db.close() - self.db = None - - @property - def exists_and_accessible(self): - try: - MySQLdb.connect(db = self.database, user = self.username, passwd = self.password).close() - except: # pylint: disable=W0702 - return False - else: - return True - - @property - def version(self): - try: - self.cur.execute("SELECT version FROM upgrade_version") - v = self.cur.fetchone() - return Version(None if v is None else v[0]) - except _mysql_exceptions.ProgrammingError, e: - if e.args[0] != ER_NO_SUCH_TABLE: - raise - log("Creating upgrade_version table in %s" % self.name) - self.cur.execute(self.upgrade_version_table_schema) - return Version(None) - - @version.setter - def version(self, v): - if v > self.version: - self.cur.execute("DELETE FROM upgrade_version") - self.cur.execute("INSERT upgrade_version (version, updated) VALUES (%s, %s)", (v, datetime.datetime.now())) - self.db.commit() - log("Updated %s to %s" % (self.name, v)) + upgrade_version_table_schema = """ + CREATE TABLE upgrade_version ( + version TEXT NOT NULL, + updated DATETIME NOT NULL + ) ENGINE=InnoDB + """ + + def __init__(self, name): + self.name = name + self.database = cfg.get("sql-database", section = name) + self.username = cfg.get("sql-username", section = name) + self.password = cfg.get("sql-password", section = name) + self.db = None + self.cur = None + + def open(self): + self.db = MySQLdb.connect(db = self.database, user = self.username, passwd = self.password) + self.db.autocommit(False) + self.cur = self.db.cursor() + + def close(self): + if self.cur is not None: + self.cur.close() + self.cur = None + if self.db is not None: + self.db.commit() + self.db.close() + self.db = None + + @property + def exists_and_accessible(self): + try: + MySQLdb.connect(db = self.database, user = self.username, passwd = self.password).close() + except: # pylint: disable=W0702 + return False + else: + return True + + @property + def version(self): + try: + self.cur.execute("SELECT version FROM upgrade_version") + v = self.cur.fetchone() + return Version(None if v is None else v[0]) + except _mysql_exceptions.ProgrammingError, e: + if e.args[0] != ER_NO_SUCH_TABLE: + raise + log("Creating upgrade_version table in %s" % self.name) + self.cur.execute(self.upgrade_version_table_schema) + return Version(None) + + @version.setter + def version(self, v): + if v > self.version: + self.cur.execute("DELETE FROM upgrade_version") + self.cur.execute("INSERT upgrade_version (version, updated) VALUES (%s, %s)", (v, datetime.datetime.now())) + self.db.commit() + log("Updated %s to %s" % (self.name, v)) class Version(object): - """ - A version number. This is a class in its own right to force the - comparision and string I/O behavior we want. - """ + """ + A version number. This is a class in its own right to force the + comparision and string I/O behavior we want. + """ - def __init__(self, v): - if v is None: - v = "0.0" - self.v = tuple(v.lower().split(".")) + def __init__(self, v): + if v is None: + v = "0.0" + self.v = tuple(v.lower().split(".")) - def __str__(self): - return ".".join(self.v) + def __str__(self): + return ".".join(self.v) - def __cmp__(self, other): - return cmp(self.v, other.v) + def __cmp__(self, other): + return cmp(self.v, other.v) class Upgrade(object): - """ - One upgrade script. Really, just its filename and the Version - object we parse from its filename, we don't need to read the script - itself except when applying it, but we do need to sort all the - available upgrade scripts into version order. - """ - - @classmethod - def load_all(cls, name, dn): - g = os.path.join(dn, "upgrade-%s-to-*.py" % name) - for fn in glob.iglob(g): - yield cls(g, fn) - - def __init__(self, g, fn): - head, sep, tail = g.partition("*") # pylint: disable=W0612 - self.fn = fn - self.version = Version(fn[len(head):-len(tail)]) - - def __cmp__(self, other): - return cmp(self.version, other.version) - - def apply(self, db): - # db is an argument here primarily so the script we exec can get at it - log("Applying %s to %s" % (self.fn, db.name)) - with open(self.fn, "r") as f: - exec f # pylint: disable=W0122 + """ + One upgrade script. Really, just its filename and the Version + object we parse from its filename, we don't need to read the script + itself except when applying it, but we do need to sort all the + available upgrade scripts into version order. + """ + + @classmethod + def load_all(cls, name, dn): + g = os.path.join(dn, "upgrade-%s-to-*.py" % name) + for fn in glob.iglob(g): + yield cls(g, fn) + + def __init__(self, g, fn): + head, sep, tail = g.partition("*") # pylint: disable=W0612 + self.fn = fn + self.version = Version(fn[len(head):-len(tail)]) + + def __cmp__(self, other): + return cmp(self.version, other.version) + + def apply(self, db): + # db is an argument here primarily so the script we exec can get at it + log("Applying %s to %s" % (self.fn, db.name)) + with open(self.fn, "r") as f: + exec f # pylint: disable=W0122 def do_drop(name): - db = UserDB(name) - if db.database in root.databases: - log("DROP DATABASE %s" % db.database) - root.cur.execute("DROP DATABASE %s" % db.database) - root.db.commit() + db = UserDB(name) + if db.database in root.databases: + log("DROP DATABASE %s" % db.database) + root.cur.execute("DROP DATABASE %s" % db.database) + root.db.commit() def do_create(name): - db = UserDB(name) - log("CREATE DATABASE %s" % db.database) - root.cur.execute("CREATE DATABASE %s" % db.database) - log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) - root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), - (db.password,)) - root.db.commit() - db.open() - db.version = current_version - db.close() + db = UserDB(name) + log("CREATE DATABASE %s" % db.database) + root.cur.execute("CREATE DATABASE %s" % db.database) + log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) + root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), + (db.password,)) + root.db.commit() + db.open() + db.version = current_version + db.close() def do_script_drop(name): - db = UserDB(name) - print "DROP DATABASE IF EXISTS %s;" % db.database + db = UserDB(name) + print "DROP DATABASE IF EXISTS %s;" % db.database def do_drop_and_create(name): - do_drop(name) - do_create(name) + do_drop(name) + do_create(name) def do_fix_grants(name): - db = UserDB(name) - if not db.exists_and_accessible: - log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) - root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), - (db.password,)) - root.db.commit() + db = UserDB(name) + if not db.exists_and_accessible: + log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) + root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), + (db.password,)) + root.db.commit() def do_create_if_missing(name): - db = UserDB(name) - if not db.exists_and_accessible: - do_create(name) + db = UserDB(name) + if not db.exists_and_accessible: + do_create(name) def do_apply_upgrades(name): - upgrades = sorted(Upgrade.load_all(name, args.upgrade_scripts)) - if upgrades: - db = UserDB(name) - db.open() - log("Current version of %s is %s" % (db.name, db.version)) - for upgrade in upgrades: - if upgrade.version > db.version: - upgrade.apply(db) - db.version = upgrade.version - db.version = current_version - db.close() + upgrades = sorted(Upgrade.load_all(name, args.upgrade_scripts)) + if upgrades: + db = UserDB(name) + db.open() + log("Current version of %s is %s" % (db.name, db.version)) + for upgrade in upgrades: + if upgrade.version > db.version: + upgrade.apply(db) + db.version = upgrade.version + db.version = current_version + db.close() def log(text): - if args.verbose: - print "#", text + if args.verbose: + print "#", text parser = argparse.ArgumentParser(description = """\ Automated setup of all SQL stuff used by the RPKI CA tools. Pulls @@ -291,13 +291,13 @@ parser.set_defaults(dispatch = do_create_if_missing) args = parser.parse_args() try: - cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") - root = RootDB(args.mysql_defaults) - current_version = Version(rpki.version.VERSION) - for program_name in ("irdbd", "rpkid", "pubd"): - if cfg.getboolean("start_" + program_name, False): - args.dispatch(program_name) - root.close() + cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") + root = RootDB(args.mysql_defaults) + current_version = Version(rpki.version.VERSION) + for program_name in ("irdbd", "rpkid", "pubd"): + if cfg.getboolean("start_" + program_name, False): + args.dispatch(program_name) + root.close() except Exception, e: - #raise - sys.exit(str(e)) + #raise + sys.exit(str(e)) diff --git a/ca/rpki-start-servers b/ca/rpki-start-servers index f1f70aa8..e01f4f9b 100755 --- a/ca/rpki-start-servers +++ b/ca/rpki-start-servers @@ -38,9 +38,9 @@ os.environ["TZ"] = "UTC" time.tzset() def non_negative_integer(s): - if int(s) < 0: - raise ValueError - return s + if int(s) < 0: + raise ValueError + return s parser = argparse.ArgumentParser(description = __doc__) parser.add_argument("-c", "--config", @@ -67,21 +67,21 @@ args = parser.parse_args() cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") def run(name, old_flag = None): - if cfg.getboolean("start_" + name, cfg.getboolean("run_" + name if old_flag is None else old_flag, False)): - # pylint: disable=E1103 - log_file = os.path.join(args.log_directory, name + ".log") - cmd = (os.path.join(rpki.autoconf.libexecdir, name), "--log-level", args.log_level) - if args.log_file: - cmd += ("--log-file", log_file) - elif args.log_rotating_file_kbytes: - cmd += ("--log-rotating-file", log_file, args.log_rotating_file_kbytes, args.log_backup_count) - elif args.log_rotating_file_hours: - cmd += ("--log-timed-rotating-file", log_file, args.log_rotating_file_hours, args.log_backup_count) - else: - cmd += ("--log-syslog", args.log_syslog) - proc = subprocess.Popen(cmd) - if proc.wait() != 0: - sys.exit("Problem starting %s, pid %s" % (name, proc.pid)) + if cfg.getboolean("start_" + name, cfg.getboolean("run_" + name if old_flag is None else old_flag, False)): + # pylint: disable=E1103 + log_file = os.path.join(args.log_directory, name + ".log") + cmd = (os.path.join(rpki.autoconf.libexecdir, name), "--log-level", args.log_level) + if args.log_file: + cmd += ("--log-file", log_file) + elif args.log_rotating_file_kbytes: + cmd += ("--log-rotating-file", log_file, args.log_rotating_file_kbytes, args.log_backup_count) + elif args.log_rotating_file_hours: + cmd += ("--log-timed-rotating-file", log_file, args.log_rotating_file_hours, args.log_backup_count) + else: + cmd += ("--log-syslog", args.log_syslog) + proc = subprocess.Popen(cmd) + if proc.wait() != 0: + sys.exit("Problem starting %s, pid %s" % (name, proc.pid)) run("irdbd", "run_rpkid") run("rpkid") @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.rpkic - rpki.rpkic.main() + import rpki.rpkic + rpki.rpkic.main() @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.rpkid - rpki.rpkid.main() + import rpki.rpkid + rpki.rpkid.main() diff --git a/ca/rpkigui-apache-conf-gen b/ca/rpkigui-apache-conf-gen index 8ac9c94a..a6a58c2c 100755 --- a/ca/rpkigui-apache-conf-gen +++ b/ca/rpkigui-apache-conf-gen @@ -188,370 +188,370 @@ NameVirtualHost *:443 ''' def Guess(args): - """ - Guess what platform this is and dispatch to platform constructor. - """ - - system = platform.system() - if system == "FreeBSD": - return FreeBSD(args) - if system == "Darwin": - return Darwin(args) - if system == "Linux": - distro = platform.linux_distribution()[0].lower() - if distro == "debian": - return Debian(args) - if distro == "ubuntu": - return Ubuntu(args) - if distro in ("fedora", "centos"): - return Redhat(args) - raise NotImplementedError("Can't guess what platform this is, sorry") + """ + Guess what platform this is and dispatch to platform constructor. + """ + + system = platform.system() + if system == "FreeBSD": + return FreeBSD(args) + if system == "Darwin": + return Darwin(args) + if system == "Linux": + distro = platform.linux_distribution()[0].lower() + if distro == "debian": + return Debian(args) + if distro == "ubuntu": + return Ubuntu(args) + if distro in ("fedora", "centos"): + return Redhat(args) + raise NotImplementedError("Can't guess what platform this is, sorry") class Platform(object): - """ - Abstract base class representing an operating system platform. - """ - - apache_cer = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.cer") - apache_key = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.key") - - apache_conf = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.conf") - apache_conf_sample = apache_conf + ".sample" - - apache_conf_preface = "" - - def __init__(self, args): - self.args = args - self.log("RPKI Apache configuration: platform \"%s\", action \"%s\"" % ( - self.__class__.__name__, args.action)) - getattr(self, args.action)() - - def log(self, msg): - if self.args.verbose: - print msg - - def run(self, *cmd, **kwargs): - self.log("Running %s" % " ".join(cmd)) - subprocess.check_call(cmd, **kwargs) - - req_cmd = ("openssl", "req", "-new", - "-config", "/dev/stdin", - "-out", "/dev/stdout", - "-keyout", apache_key, - "-newkey", "rsa:2048") - - x509_cmd = ("openssl", "x509", "-req", "-sha256", - "-signkey", apache_key, - "-in", "/dev/stdin", - "-out", apache_cer, - "-days", "3650") - - req_conf = '''\ - [req] - default_bits = 2048 - default_md = sha256 - distinguished_name = req_dn - prompt = no - encrypt_key = no - [req_dn] - CN = %s - ''' % fqdn - - def unlink(self, fn, silent = False): - if os.path.lexists(fn): - if not silent: - self.log("Removing %s" % fn) - os.unlink(fn) - elif not silent: - self.log("Would have removed %s if it existed" % fn) - - def del_certs(self, silent = False): - self.unlink(self.apache_cer, silent) - self.unlink(self.apache_key, silent) - - def add_certs(self): - if os.path.exists(self.apache_cer) and os.path.exists(self.apache_key): - return - self.del_certs() - req = subprocess.Popen(self.req_cmd, - stdin = subprocess.PIPE, - stdout = subprocess.PIPE, - stderr = open("/dev/null", "w")) - x509 = subprocess.Popen(self.x509_cmd, - stdin = req.stdout, - stderr = open("/dev/null", "w")) - req.stdin.write(self.req_conf) - req.stdin.close() - if req.wait(): - raise subprocess.CalledProcessError(req.returncode, self.req_cmd) - if x509.wait(): - raise subprocess.CalledProcessError(x509.returncode, self.x509_cmd) - self.log("Created %s and %s, chmoding %s" % ( - self.apache_cer, self.apache_key, self.apache_key)) - os.chmod(self.apache_key, 0600) - - _vhost = None - - @property - def vhost(self): - if self._vhost is None: - allow = allow_22_template if self.args.apache_version <= 22 else allow_24_template - self._vhost = vhost_template % dict(rpki.autoconf.__dict__, fqdn = fqdn, allow = allow) - return self._vhost - - @property - def name_virtual_host(self): - return name_virtual_host_template if self.args.apache_version <= 22 else "" - - @property - def too_complex(self): - return textwrap.dedent('''\ - # It looks like you already have HTTPS enabled in your - # Apache configuration, which makes your configuration too - # complex for us to enable support for the RPKI GUI automatically. - # - # To enable support, take a look at %s - # and copy what you need from that file into %s, - # paying attention to the comments which mark the bits that - # you might (or might not) need to change or omit, depending - # on the details of your particular Apache configuration. - ''' % (self.apache_conf_sample, self.apache_conf)) - - def install(self): - with open(self.apache_conf_sample, "w") as f: - self.log("Writing %s" % f.name) - f.write(self.apache_conf_preface) - f.write(self.name_virtual_host) - f.write(self.vhost) - if not os.path.exists(self.apache_conf): - self.unlink(self.apache_conf) - with open(self.apache_conf, "w") as f: - self.log("Writing %s" % f.name) - if self.test_url("https://%s/" % fqdn): - f.write(self.too_complex) - sys.stdout.write(self.too_complex) - else: - if not self.test_tcp("localhost", 443): + """ + Abstract base class representing an operating system platform. + """ + + apache_cer = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.cer") + apache_key = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.key") + + apache_conf = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.conf") + apache_conf_sample = apache_conf + ".sample" + + apache_conf_preface = "" + + def __init__(self, args): + self.args = args + self.log("RPKI Apache configuration: platform \"%s\", action \"%s\"" % ( + self.__class__.__name__, args.action)) + getattr(self, args.action)() + + def log(self, msg): + if self.args.verbose: + print msg + + def run(self, *cmd, **kwargs): + self.log("Running %s" % " ".join(cmd)) + subprocess.check_call(cmd, **kwargs) + + req_cmd = ("openssl", "req", "-new", + "-config", "/dev/stdin", + "-out", "/dev/stdout", + "-keyout", apache_key, + "-newkey", "rsa:2048") + + x509_cmd = ("openssl", "x509", "-req", "-sha256", + "-signkey", apache_key, + "-in", "/dev/stdin", + "-out", apache_cer, + "-days", "3650") + + req_conf = '''\ + [req] + default_bits = 2048 + default_md = sha256 + distinguished_name = req_dn + prompt = no + encrypt_key = no + [req_dn] + CN = %s + ''' % fqdn + + def unlink(self, fn, silent = False): + if os.path.lexists(fn): + if not silent: + self.log("Removing %s" % fn) + os.unlink(fn) + elif not silent: + self.log("Would have removed %s if it existed" % fn) + + def del_certs(self, silent = False): + self.unlink(self.apache_cer, silent) + self.unlink(self.apache_key, silent) + + def add_certs(self): + if os.path.exists(self.apache_cer) and os.path.exists(self.apache_key): + return + self.del_certs() + req = subprocess.Popen(self.req_cmd, + stdin = subprocess.PIPE, + stdout = subprocess.PIPE, + stderr = open("/dev/null", "w")) + x509 = subprocess.Popen(self.x509_cmd, + stdin = req.stdout, + stderr = open("/dev/null", "w")) + req.stdin.write(self.req_conf) + req.stdin.close() + if req.wait(): + raise subprocess.CalledProcessError(req.returncode, self.req_cmd) + if x509.wait(): + raise subprocess.CalledProcessError(x509.returncode, self.x509_cmd) + self.log("Created %s and %s, chmoding %s" % ( + self.apache_cer, self.apache_key, self.apache_key)) + os.chmod(self.apache_key, 0600) + + _vhost = None + + @property + def vhost(self): + if self._vhost is None: + allow = allow_22_template if self.args.apache_version <= 22 else allow_24_template + self._vhost = vhost_template % dict(rpki.autoconf.__dict__, fqdn = fqdn, allow = allow) + return self._vhost + + @property + def name_virtual_host(self): + return name_virtual_host_template if self.args.apache_version <= 22 else "" + + @property + def too_complex(self): + return textwrap.dedent('''\ + # It looks like you already have HTTPS enabled in your + # Apache configuration, which makes your configuration too + # complex for us to enable support for the RPKI GUI automatically. + # + # To enable support, take a look at %s + # and copy what you need from that file into %s, + # paying attention to the comments which mark the bits that + # you might (or might not) need to change or omit, depending + # on the details of your particular Apache configuration. + ''' % (self.apache_conf_sample, self.apache_conf)) + + def install(self): + with open(self.apache_conf_sample, "w") as f: + self.log("Writing %s" % f.name) f.write(self.apache_conf_preface) f.write(self.name_virtual_host) - f.write(self.vhost) - if not os.path.exists(self.apache_conf_target): - self.unlink(self.apache_conf_target) - self.log("Symlinking %s to %s" % ( - self.apache_conf_target, self.apache_conf)) - os.symlink(self.apache_conf, self.apache_conf_target) - self.add_certs() - self.enable() - self.restart() - - def enable(self): - pass - - def disable(self): - pass - - def remove(self): - try: - same = open(self.apache_conf, "r").read() == open(self.apache_conf_sample, "r").read() - except: # pylint: disable=W0702 - same = False - self.unlink(self.apache_conf_sample) - if same: - self.unlink(self.apache_conf) - self.unlink(self.apache_conf_target) - self.disable() - self.restart() - - def purge(self): - self.remove() - self.unlink(self.apache_conf) - self.del_certs() - - @staticmethod - def test_url(url = "https://localhost/"): - try: - urllib2.urlopen(url).close() - except IOError: - return False - else: - return True - - @staticmethod - def test_tcp(host = "localhost", port = 443, family = socket.AF_UNSPEC, proto = socket.SOCK_STREAM): - try: - addrinfo = socket.getaddrinfo(host, port, family, proto) - except socket.error: - return False - for af, socktype, proto, canon, sa in addrinfo: # pylint: disable=W0612 - try: - s = socket.socket(af, socktype, proto) - s.connect(sa) - s.close() - except socket.error: - continue - else: - return True - return False + f.write(self.vhost) + if not os.path.exists(self.apache_conf): + self.unlink(self.apache_conf) + with open(self.apache_conf, "w") as f: + self.log("Writing %s" % f.name) + if self.test_url("https://%s/" % fqdn): + f.write(self.too_complex) + sys.stdout.write(self.too_complex) + else: + if not self.test_tcp("localhost", 443): + f.write(self.apache_conf_preface) + f.write(self.name_virtual_host) + f.write(self.vhost) + if not os.path.exists(self.apache_conf_target): + self.unlink(self.apache_conf_target) + self.log("Symlinking %s to %s" % ( + self.apache_conf_target, self.apache_conf)) + os.symlink(self.apache_conf, self.apache_conf_target) + self.add_certs() + self.enable() + self.restart() + + def enable(self): + pass + + def disable(self): + pass + + def remove(self): + try: + same = open(self.apache_conf, "r").read() == open(self.apache_conf_sample, "r").read() + except: # pylint: disable=W0702 + same = False + self.unlink(self.apache_conf_sample) + if same: + self.unlink(self.apache_conf) + self.unlink(self.apache_conf_target) + self.disable() + self.restart() + + def purge(self): + self.remove() + self.unlink(self.apache_conf) + self.del_certs() + + @staticmethod + def test_url(url = "https://localhost/"): + try: + urllib2.urlopen(url).close() + except IOError: + return False + else: + return True + + @staticmethod + def test_tcp(host = "localhost", port = 443, family = socket.AF_UNSPEC, proto = socket.SOCK_STREAM): + try: + addrinfo = socket.getaddrinfo(host, port, family, proto) + except socket.error: + return False + for af, socktype, proto, canon, sa in addrinfo: # pylint: disable=W0612 + try: + s = socket.socket(af, socktype, proto) + s.connect(sa) + s.close() + except socket.error: + continue + else: + return True + return False class FreeBSD(Platform): - """ - FreeBSD. - """ + """ + FreeBSD. + """ - # On FreeBSD we have to ask httpd what version it is before we know - # where to put files or what to call the service. In FreeBSD's makefiles, - # this value is called APACHE_VERSION, and is calculated thusly: - # - # httpd -V | sed -ne 's/^Server version: Apache\/\([0-9]\)\.\([0-9]*\).*/\1\2/p' + # On FreeBSD we have to ask httpd what version it is before we know + # where to put files or what to call the service. In FreeBSD's makefiles, + # this value is called APACHE_VERSION, and is calculated thusly: + # + # httpd -V | sed -ne 's/^Server version: Apache\/\([0-9]\)\.\([0-9]*\).*/\1\2/p' - _apache_name = None + _apache_name = None - @property - def apache_name(self): - if self._apache_name is None: - self._apache_name = "apache%s" % self.args.apache_version - return self._apache_name + @property + def apache_name(self): + if self._apache_name is None: + self._apache_name = "apache%s" % self.args.apache_version + return self._apache_name - @property - def apache_conf_target(self): - return "/usr/local/etc/%s/Includes/rpki.conf" % self.apache_name + @property + def apache_conf_target(self): + return "/usr/local/etc/%s/Includes/rpki.conf" % self.apache_name - apache_conf_preface = textwrap.dedent('''\ - # These directives tell Apache to listen on the HTTPS port - # and to enable name-based virtual hosting. If you already - # have HTTPS enabled elsewhere in your configuration, you may - # need to remove these. + apache_conf_preface = textwrap.dedent('''\ + # These directives tell Apache to listen on the HTTPS port + # and to enable name-based virtual hosting. If you already + # have HTTPS enabled elsewhere in your configuration, you may + # need to remove these. - Listen [::]:443 - Listen 0.0.0.0:443 - ''') + Listen [::]:443 + Listen 0.0.0.0:443 + ''') - def restart(self): - self.run("service", self.apache_name, "restart") + def restart(self): + self.run("service", self.apache_name, "restart") class Debian(Platform): - """ - Debian and related platforms like Ubuntu. - """ + """ + Debian and related platforms like Ubuntu. + """ - @property - def distribution_version(self): - return tuple(int(v) for v in platform.linux_distribution()[1].split(".")) + @property + def distribution_version(self): + return tuple(int(v) for v in platform.linux_distribution()[1].split(".")) - # As of Wheezy, Debian still wants the configuration filename - # without the .conf suffix. Keep an eye on this, as it may change - # in future releases. - # - @property - def apache_conf_target(self): - return "/etc/apache2/sites-available/rpki" - - snake_oil_cer = "/etc/ssl/certs/ssl-cert-snakeoil.pem" - snake_oil_key = "/etc/ssl/private/ssl-cert-snakeoil.key" - - def add_certs(self): - if not os.path.exists(self.snake_oil_cer) or not os.path.exists(self.snake_oil_key): - return Platform.add_certs(self) - if not os.path.exists(self.apache_cer): - self.unlink(self.apache_cer) - os.symlink(self.snake_oil_cer, self.apache_cer) - if not os.path.exists(self.apache_key): - self.unlink(self.apache_key) - os.symlink(self.snake_oil_key, self.apache_key) - - def enable(self): - self.run("a2enmod", "ssl") - self.run("a2enmod", "expires") - self.run("a2ensite", "rpki") + # As of Wheezy, Debian still wants the configuration filename + # without the .conf suffix. Keep an eye on this, as it may change + # in future releases. # - # In light of BREACH and CRIME attacks, mod_deflate is looking - # like a bad idea, so make sure it's off. - self.run("a2dismod", "deflate") - - def disable(self): - self.run("a2dissite", "rpki") - - def restart(self): - self.run("service", "apache2", "restart") + @property + def apache_conf_target(self): + return "/etc/apache2/sites-available/rpki" + + snake_oil_cer = "/etc/ssl/certs/ssl-cert-snakeoil.pem" + snake_oil_key = "/etc/ssl/private/ssl-cert-snakeoil.key" + + def add_certs(self): + if not os.path.exists(self.snake_oil_cer) or not os.path.exists(self.snake_oil_key): + return Platform.add_certs(self) + if not os.path.exists(self.apache_cer): + self.unlink(self.apache_cer) + os.symlink(self.snake_oil_cer, self.apache_cer) + if not os.path.exists(self.apache_key): + self.unlink(self.apache_key) + os.symlink(self.snake_oil_key, self.apache_key) + + def enable(self): + self.run("a2enmod", "ssl") + self.run("a2enmod", "expires") + self.run("a2ensite", "rpki") + # + # In light of BREACH and CRIME attacks, mod_deflate is looking + # like a bad idea, so make sure it's off. + self.run("a2dismod", "deflate") + + def disable(self): + self.run("a2dissite", "rpki") + + def restart(self): + self.run("service", "apache2", "restart") class Ubuntu(Debian): - # On Ubuntu, the filename must end in .conf on Trusty and must not - # end in .conf on Precise. - @property - def apache_conf_target(self): - if self.distribution_version >= (14, 0): - return "/etc/apache2/sites-available/rpki.conf" - else: - return "/etc/apache2/sites-available/rpki" + # On Ubuntu, the filename must end in .conf on Trusty and must not + # end in .conf on Precise. + @property + def apache_conf_target(self): + if self.distribution_version >= (14, 0): + return "/etc/apache2/sites-available/rpki.conf" + else: + return "/etc/apache2/sites-available/rpki" class NIY(Platform): - def __init__(self, args): - super(NIY, self).__init__(args) - raise NotImplementedError("Platform %s not implemented yet, sorry" % self.__class__.__name__) + def __init__(self, args): + super(NIY, self).__init__(args) + raise NotImplementedError("Platform %s not implemented yet, sorry" % self.__class__.__name__) class Redhat(NIY): - """ - Redhat family of Linux distributions (Fedora, CentOS). - """ + """ + Redhat family of Linux distributions (Fedora, CentOS). + """ class Darwin(NIY): - """ - Mac OS X (aka Darwin). - """ + """ + Mac OS X (aka Darwin). + """ def main(): - """ - Generate and (de)install configuration suitable for using Apache httpd - to drive the RPKI web interface under WSGI. - """ - - parser = argparse.ArgumentParser(description = __doc__) - group1 = parser.add_mutually_exclusive_group() - group2 = parser.add_mutually_exclusive_group() - - parser.add_argument("-v", "--verbose", - help = "whistle while you work", action = "store_true") - parser.add_argument("--apache-version", - help = "Apache version (default " + rpki.autoconf.APACHE_VERSION + ")", - type = int, default = rpki.autoconf.APACHE_VERSION) - - group1.add_argument("--freebsd", - help = "configure for FreeBSD", - action = "store_const", dest = "platform", const = FreeBSD) - group1.add_argument("--debian", - help = "configure for Debian", - action = "store_const", dest = "platform", const = Debian) - group1.add_argument("--ubuntu", - help = "configure for Ubuntu", - action = "store_const", dest = "platform", const = Ubuntu) - group1.add_argument("--redhat", "--fedora", "--centos", - help = "configure for Redhat/Fedora/CentOS", - action = "store_const", dest = "platform", const = Redhat) - group1.add_argument("--macosx", "--darwin", - help = "configure for Mac OS X (Darwin)", - action = "store_const", dest = "platform", const = Darwin) - group1.add_argument("--guess", - help = "guess which platform configuration to use", - action = "store_const", dest = "platform", const = Guess) - - group2.add_argument("-i", "--install", - help = "install configuration", - action = "store_const", dest = "action", const = "install") - group2.add_argument("-r", "--remove", "--deinstall", "--uninstall", - help = "remove configuration", - action = "store_const", dest = "action", const = "remove") - group2.add_argument("-P", "--purge", - help = "remove configuration with extreme prejudice", - action = "store_const", dest = "action", const = "purge") - - parser.set_defaults(platform = Guess, action = "install") - args = parser.parse_args() - - try: - args.platform(args) - except Exception, e: - sys.exit(str(e)) + """ + Generate and (de)install configuration suitable for using Apache httpd + to drive the RPKI web interface under WSGI. + """ + + parser = argparse.ArgumentParser(description = __doc__) + group1 = parser.add_mutually_exclusive_group() + group2 = parser.add_mutually_exclusive_group() + + parser.add_argument("-v", "--verbose", + help = "whistle while you work", action = "store_true") + parser.add_argument("--apache-version", + help = "Apache version (default " + rpki.autoconf.APACHE_VERSION + ")", + type = int, default = rpki.autoconf.APACHE_VERSION) + + group1.add_argument("--freebsd", + help = "configure for FreeBSD", + action = "store_const", dest = "platform", const = FreeBSD) + group1.add_argument("--debian", + help = "configure for Debian", + action = "store_const", dest = "platform", const = Debian) + group1.add_argument("--ubuntu", + help = "configure for Ubuntu", + action = "store_const", dest = "platform", const = Ubuntu) + group1.add_argument("--redhat", "--fedora", "--centos", + help = "configure for Redhat/Fedora/CentOS", + action = "store_const", dest = "platform", const = Redhat) + group1.add_argument("--macosx", "--darwin", + help = "configure for Mac OS X (Darwin)", + action = "store_const", dest = "platform", const = Darwin) + group1.add_argument("--guess", + help = "guess which platform configuration to use", + action = "store_const", dest = "platform", const = Guess) + + group2.add_argument("-i", "--install", + help = "install configuration", + action = "store_const", dest = "action", const = "install") + group2.add_argument("-r", "--remove", "--deinstall", "--uninstall", + help = "remove configuration", + action = "store_const", dest = "action", const = "remove") + group2.add_argument("-P", "--purge", + help = "remove configuration with extreme prejudice", + action = "store_const", dest = "action", const = "purge") + + parser.set_defaults(platform = Guess, action = "install") + args = parser.parse_args() + + try: + args.platform(args) + except Exception, e: + sys.exit(str(e)) if __name__ == "__main__": - main() + main() diff --git a/ca/rpkigui-import-routes b/ca/rpkigui-import-routes index 0fbe0126..fb8e381e 100755 --- a/ca/rpkigui-import-routes +++ b/ca/rpkigui-import-routes @@ -110,4 +110,3 @@ automatically.""") except Exception as e: logging.exception(e) sys.exit(1) - diff --git a/ca/tests/bgpsec-yaml.py b/ca/tests/bgpsec-yaml.py index d33184bf..500d2b9d 100755 --- a/ca/tests/bgpsec-yaml.py +++ b/ca/tests/bgpsec-yaml.py @@ -30,28 +30,28 @@ root = "Root" class Kid(object): - def __init__(self, i): - self.name = "ISP-%03d" % i - self.ipv4 = "10.%d.0.0/16" % i - self.asn = i - self.router_id = i * 10000 - - @property - def declare(self): - return dict(name = self.name, - ipv4 = self.ipv4, - asn = self.asn, - hosted_by = root, - roa_request = [dict(asn = self.asn, ipv4 = self.ipv4)], - router_cert = [dict(asn = self.asn, router_id = self.router_id)]) - - @property - def del_routercert(self): - return dict(name = self.name, router_cert_del = [dict(asn = self.asn, router_id = self.router_id)]) - - @property - def add_routercert(self): - return dict(name = self.name, router_cert_add = [dict(asn = self.asn, router_id = self.router_id)]) + def __init__(self, i): + self.name = "ISP-%03d" % i + self.ipv4 = "10.%d.0.0/16" % i + self.asn = i + self.router_id = i * 10000 + + @property + def declare(self): + return dict(name = self.name, + ipv4 = self.ipv4, + asn = self.asn, + hosted_by = root, + roa_request = [dict(asn = self.asn, ipv4 = self.ipv4)], + router_cert = [dict(asn = self.asn, router_id = self.router_id)]) + + @property + def del_routercert(self): + return dict(name = self.name, router_cert_del = [dict(asn = self.asn, router_id = self.router_id)]) + + @property + def add_routercert(self): + return dict(name = self.name, router_cert_add = [dict(asn = self.asn, router_id = self.router_id)]) kids = [Kid(n + 1) for n in xrange(200)] @@ -72,14 +72,14 @@ docs.append([shell_first, gym = kids[50:70] for kid in gym: - docs.append([shell_next, - kid.del_routercert, - sleeper]) + docs.append([shell_next, + kid.del_routercert, + sleeper]) for kid in gym: - docs.append([shell_next, - kid.add_routercert, - sleeper]) + docs.append([shell_next, + kid.add_routercert, + sleeper]) print '''\ # This configuration was generated by a script. Edit at your own risk. diff --git a/ca/tests/myrpki-xml-parse-test.py b/ca/tests/myrpki-xml-parse-test.py index 9db7ec57..d915ea5b 100644 --- a/ca/tests/myrpki-xml-parse-test.py +++ b/ca/tests/myrpki-xml-parse-test.py @@ -25,77 +25,77 @@ relaxng = lxml.etree.RelaxNG(file = "myrpki.rng") tree = lxml.etree.parse("myrpki.xml").getroot() if False: - print lxml.etree.tostring(tree, pretty_print = True, encoding = "us-ascii", xml_declaration = True) + print lxml.etree.tostring(tree, pretty_print = True, encoding = "us-ascii", xml_declaration = True) relaxng.assertValid(tree) def showitems(y): - if False: - for k, v in y.items(): - if v: - print " ", k, v + if False: + for k, v in y.items(): + if v: + print " ", k, v def tag(t): - return "{http://www.hactrn.net/uris/rpki/myrpki/}" + t + return "{http://www.hactrn.net/uris/rpki/myrpki/}" + t print "My handle:", tree.get("handle") print "Children:" for x in tree.getiterator(tag("child")): - print " ", x - print " Handle:", x.get("handle") - print " ASNS: ", rpki.resource_set.resource_set_as(x.get("asns")) - print " IPv4: ", rpki.resource_set.resource_set_ipv4(x.get("v4")) - print " Valid: ", x.get("valid_until") - showitems(x) + print " ", x + print " Handle:", x.get("handle") + print " ASNS: ", rpki.resource_set.resource_set_as(x.get("asns")) + print " IPv4: ", rpki.resource_set.resource_set_ipv4(x.get("v4")) + print " Valid: ", x.get("valid_until") + showitems(x) print print "ROA requests:" for x in tree.getiterator(tag("roa_request")): - print " ", x - print " ASN: ", x.get("asn") - print " IPv4:", rpki.resource_set.roa_prefix_set_ipv4(x.get("v4")) - print " IPv6:", rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")) - showitems(x) + print " ", x + print " ASN: ", x.get("asn") + print " IPv4:", rpki.resource_set.roa_prefix_set_ipv4(x.get("v4")) + print " IPv6:", rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")) + showitems(x) print def showpem(label, b64, kind): - cmd = ("openssl", kind, "-noout", "-text", "-inform", "DER") - if kind == "x509": - cmd += ("-certopt", "no_pubkey,no_sigdump") - p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE) - text = p.communicate(input = base64.b64decode(b64))[0] - if p.returncode != 0: - raise subprocess.CalledProcessError(returncode = p.returncode, cmd = cmd) - print label, text + cmd = ("openssl", kind, "-noout", "-text", "-inform", "DER") + if kind == "x509": + cmd += ("-certopt", "no_pubkey,no_sigdump") + p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE) + text = p.communicate(input = base64.b64decode(b64))[0] + if p.returncode != 0: + raise subprocess.CalledProcessError(returncode = p.returncode, cmd = cmd) + print label, text for x in tree.getiterator(tag("child")): - cert = x.findtext(tag("bpki_certificate")) - if cert: - showpem("Child", cert, "x509") + cert = x.findtext(tag("bpki_certificate")) + if cert: + showpem("Child", cert, "x509") for x in tree.getiterator(tag("parent")): - print "Parent URI:", x.get("service_uri") - cert = x.findtext(tag("bpki_certificate")) - if cert: - showpem("Parent", cert, "x509") + print "Parent URI:", x.get("service_uri") + cert = x.findtext(tag("bpki_certificate")) + if cert: + showpem("Parent", cert, "x509") ca = tree.findtext(tag("bpki_ca_certificate")) if ca: - showpem("CA", ca, "x509") + showpem("CA", ca, "x509") bsc = tree.findtext(tag("bpki_bsc_certificate")) if bsc: - showpem("BSC EE", bsc, "x509") + showpem("BSC EE", bsc, "x509") repo = tree.findtext(tag("bpki_repository_certificate")) if repo: - showpem("Repository", repo, "x509") + showpem("Repository", repo, "x509") req = tree.findtext(tag("bpki_bsc_pkcs10")) if req: - showpem("BSC EE", req, "req") + showpem("BSC EE", req, "req") crl = tree.findtext(tag("bpki_crl")) if crl: - showpem("CA", crl, "crl") + showpem("CA", crl, "crl") diff --git a/ca/tests/old_irdbd.py b/ca/tests/old_irdbd.py index d66e683e..d26c3476 100644 --- a/ca/tests/old_irdbd.py +++ b/ca/tests/old_irdbd.py @@ -15,5 +15,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.old_irdbd - rpki.old_irdbd.main() + import rpki.old_irdbd + rpki.old_irdbd.main() diff --git a/ca/tests/smoketest.py b/ca/tests/smoketest.py index 3960981a..9d82c640 100644 --- a/ca/tests/smoketest.py +++ b/ca/tests/smoketest.py @@ -76,14 +76,14 @@ yaml_script = [y for y in yaml.safe_load_all(args.yaml_file)] # Define port allocator early, so we can use it while reading config def allocate_port(): - """ - Allocate a TCP port number. - """ + """ + Allocate a TCP port number. + """ - global base_port - p = base_port - base_port += 1 - return p + global base_port + p = base_port + base_port += 1 + return p # Most filenames in the following are relative to the working directory. @@ -139,14 +139,14 @@ pubd_last_cms_time = None ecdsa_params = None class CantRekeyYAMLLeaf(Exception): - """ - Can't rekey YAML leaf. - """ + """ + Can't rekey YAML leaf. + """ class CouldntIssueBSCEECertificate(Exception): - """ - Couldn't issue BSC EE certificate - """ + """ + Couldn't issue BSC EE certificate + """ sql_conversions = MySQLdb.converters.conversions.copy() sql_conversions.update({ @@ -154,202 +154,202 @@ sql_conversions.update({ MySQLdb.converters.FIELD_TYPE.DATETIME : rpki.sundial.datetime.DateTime_or_None }) def main(): - """ - Main program. - """ - - rpki.log.init(smoketest_name, argparse.Namespace(log_level = logging.DEBUG, - log_handler = lambda: logging.StreamHandler(sys.stdout))) - logger.info("Starting") - - rpki.http.http_client.timeout = rpki.sundial.timedelta(hours = 1) - - pubd_process = None - rootd_process = None - rsyncd_process = None - - rpki_sql = mangle_sql(rpki_sql_file) - irdb_sql = mangle_sql(irdb_sql_file) - pubd_sql = mangle_sql(pub_sql_file) - - logger.info("Initializing test directory") - - # Connect to test directory, creating it if necessary - try: - os.chdir(smoketest_dir) - except OSError: - os.makedirs(smoketest_dir) - os.chdir(smoketest_dir) - - # Now that we're in the right directory, we can figure out whether - # we have a private openssl executable to use - global prog_openssl - if not os.path.exists(prog_openssl): - prog_openssl = "openssl" - - # Discard everything but keys, which take a while to generate. - # Apparently os.walk() can't tell the difference between directories - # and symlinks to directories, so we have to handle both. - for root, dirs, files in os.walk(".", topdown = False): - for fn in files: - if not fn.endswith(".key"): - os.remove(os.path.join(root, fn)) - for d in dirs: - try: - os.rmdir(os.path.join(root, d)) - except OSError, e: - if e.errno == errno.ENOTDIR: - os.remove(os.path.join(root, d)) - else: - raise - - logger.info("Reading master YAML configuration") - y = yaml_script.pop(0) + """ + Main program. + """ - logger.info("Constructing internal allocation database") - db = allocation_db(y) + rpki.log.init(smoketest_name, argparse.Namespace(log_level = logging.DEBUG, + log_handler = lambda: logging.StreamHandler(sys.stdout))) + logger.info("Starting") - logger.info("Constructing BPKI keys and certs for rootd") - setup_bpki_cert_chain(rootd_name, ee = ("RPKI",)) + rpki.http.http_client.timeout = rpki.sundial.timedelta(hours = 1) - logger.info("Constructing BPKI keys and certs for pubd") - setup_bpki_cert_chain(pubd_name, ee = ("PUBD", "IRBE")) + pubd_process = None + rootd_process = None + rsyncd_process = None + rpki_sql = mangle_sql(rpki_sql_file) + irdb_sql = mangle_sql(irdb_sql_file) + pubd_sql = mangle_sql(pub_sql_file) - for a in db: - a.setup_bpki_certs() + logger.info("Initializing test directory") - setup_publication(pubd_sql, db.root.irdb_db_name) - setup_rootd(db.root, y.get("rootd", {}), db) - setup_rsyncd() - setup_rcynic() + # Connect to test directory, creating it if necessary + try: + os.chdir(smoketest_dir) + except OSError: + os.makedirs(smoketest_dir) + os.chdir(smoketest_dir) + + # Now that we're in the right directory, we can figure out whether + # we have a private openssl executable to use + global prog_openssl + if not os.path.exists(prog_openssl): + prog_openssl = "openssl" + + # Discard everything but keys, which take a while to generate. + # Apparently os.walk() can't tell the difference between directories + # and symlinks to directories, so we have to handle both. + for root, dirs, files in os.walk(".", topdown = False): + for fn in files: + if not fn.endswith(".key"): + os.remove(os.path.join(root, fn)) + for d in dirs: + try: + os.rmdir(os.path.join(root, d)) + except OSError, e: + if e.errno == errno.ENOTDIR: + os.remove(os.path.join(root, d)) + else: + raise + + logger.info("Reading master YAML configuration") + y = yaml_script.pop(0) + + logger.info("Constructing internal allocation database") + db = allocation_db(y) + + logger.info("Constructing BPKI keys and certs for rootd") + setup_bpki_cert_chain(rootd_name, ee = ("RPKI",)) + + logger.info("Constructing BPKI keys and certs for pubd") + setup_bpki_cert_chain(pubd_name, ee = ("PUBD", "IRBE")) + + + for a in db: + a.setup_bpki_certs() + + setup_publication(pubd_sql, db.root.irdb_db_name) + setup_rootd(db.root, y.get("rootd", {}), db) + setup_rsyncd() + setup_rcynic() - for a in db.engines: - a.setup_conf_file() - a.setup_sql(rpki_sql, irdb_sql) - a.sync_sql() + for a in db.engines: + a.setup_conf_file() + a.setup_sql(rpki_sql, irdb_sql) + a.sync_sql() - try: + try: - logger.info("Starting rootd") - rootd_process = subprocess.Popen((prog_python, prog_rootd, "--foreground", "--log-stdout", "--log-level", "debug"), - env = dict(os.environ, RPKI_CONF = rootd_name + ".conf")) + logger.info("Starting rootd") + rootd_process = subprocess.Popen((prog_python, prog_rootd, "--foreground", "--log-stdout", "--log-level", "debug"), + env = dict(os.environ, RPKI_CONF = rootd_name + ".conf")) - logger.info("Starting pubd") - pubd_process = subprocess.Popen((prog_python, prog_pubd, "--foreground", "--log-stdout", "--log-level", "debug") + - (("-p", pubd_name + ".prof") if args.profile else ()), - env = dict(os.environ, RPKI_CONF = pubd_name + ".conf")) + logger.info("Starting pubd") + pubd_process = subprocess.Popen((prog_python, prog_pubd, "--foreground", "--log-stdout", "--log-level", "debug") + + (("-p", pubd_name + ".prof") if args.profile else ()), + env = dict(os.environ, RPKI_CONF = pubd_name + ".conf")) - logger.info("Starting rsyncd") - rsyncd_process = subprocess.Popen((prog_rsyncd, "--daemon", "--no-detach", "--config", rsyncd_name + ".conf")) + logger.info("Starting rsyncd") + rsyncd_process = subprocess.Popen((prog_rsyncd, "--daemon", "--no-detach", "--config", rsyncd_name + ".conf")) - # Start rpkid and irdbd instances - for a in db.engines: - a.run_daemons() + # Start rpkid and irdbd instances + for a in db.engines: + a.run_daemons() - # From this point on we'll be running event-driven, so the rest of - # the code until final exit is all closures. + # From this point on we'll be running event-driven, so the rest of + # the code until final exit is all closures. - def start(): - rpki.async.iterator(db.engines, create_rpki_objects, create_pubd_objects) + def start(): + rpki.async.iterator(db.engines, create_rpki_objects, create_pubd_objects) - def create_rpki_objects(iterator, a): - a.create_rpki_objects(iterator) + def create_rpki_objects(iterator, a): + a.create_rpki_objects(iterator) - def create_pubd_objects(): - call_pubd([rpki.publication_control.client_elt.make_pdu(action = "create", - client_handle = db.root.client_handle + "-" + rootd_name, - base_uri = rootd_sia, - bpki_cert = cross_certify(rootd_name + "-TA", pubd_name + "-TA"))], - cb = lambda ignored: yaml_loop()) + def create_pubd_objects(): + call_pubd([rpki.publication_control.client_elt.make_pdu(action = "create", + client_handle = db.root.client_handle + "-" + rootd_name, + base_uri = rootd_sia, + bpki_cert = cross_certify(rootd_name + "-TA", pubd_name + "-TA"))], + cb = lambda ignored: yaml_loop()) - def yaml_loop(): + def yaml_loop(): - # This is probably where we should be updating expired BPKI - # objects, particular CRLs + # This is probably where we should be updating expired BPKI + # objects, particular CRLs - logger.info("Running cron for all RPKI engines") - rpki.async.iterator(db.engines, run_cron, run_yaml) + logger.info("Running cron for all RPKI engines") + rpki.async.iterator(db.engines, run_cron, run_yaml) - def run_cron(iterator, a): - a.run_cron(iterator) + def run_cron(iterator, a): + a.run_cron(iterator) - def run_yaml(): + def run_yaml(): - # Run rcynic to check results - run_rcynic() + # Run rcynic to check results + run_rcynic() - # Apply next delta if we have one; otherwise, we're done. - if yaml_script: - logger.info("Applying deltas") - db.apply_delta(yaml_script.pop(0), apply_delta_done) - else: - logger.info("No more deltas to apply, done") - rpki.async.exit_event_loop() + # Apply next delta if we have one; otherwise, we're done. + if yaml_script: + logger.info("Applying deltas") + db.apply_delta(yaml_script.pop(0), apply_delta_done) + else: + logger.info("No more deltas to apply, done") + rpki.async.exit_event_loop() - def apply_delta_done(): + def apply_delta_done(): - # Resync IRDBs - for a in db.engines: - a.sync_sql() + # Resync IRDBs + for a in db.engines: + a.sync_sql() - # Loop until we run out of control YAML - yaml_loop() + # Loop until we run out of control YAML + yaml_loop() - logger.info("Sleeping %d seconds while daemons start up", startup_delay) - rpki.async.timer(start).set(rpki.sundial.timedelta(seconds = startup_delay)) - rpki.async.event_loop() + logger.info("Sleeping %d seconds while daemons start up", startup_delay) + rpki.async.timer(start).set(rpki.sundial.timedelta(seconds = startup_delay)) + rpki.async.event_loop() - # At this point we have gone into event-driven code. + # At this point we have gone into event-driven code. - logger.info("Event loop exited normally") + logger.info("Event loop exited normally") - except Exception, e: - logger.exception("Event loop exited with an exception") + except Exception, e: + logger.exception("Event loop exited with an exception") - finally: - logger.info("Cleaning up") - for a in db.engines: - a.kill_daemons() - for proc, name in ((rootd_process, "rootd"), - (pubd_process, "pubd"), - (rsyncd_process, "rsyncd")): - # pylint: disable=E1103 - if proc is not None and proc.poll() is None: - logger.info("Killing %s, pid %s", name, proc.pid) - try: - proc.terminate() - except OSError: - pass - if proc is not None: - logger.info("Daemon %s, pid %s exited with code %s", name, proc.pid, proc.wait()) + finally: + logger.info("Cleaning up") + for a in db.engines: + a.kill_daemons() + for proc, name in ((rootd_process, "rootd"), + (pubd_process, "pubd"), + (rsyncd_process, "rsyncd")): + # pylint: disable=E1103 + if proc is not None and proc.poll() is None: + logger.info("Killing %s, pid %s", name, proc.pid) + try: + proc.terminate() + except OSError: + pass + if proc is not None: + logger.info("Daemon %s, pid %s exited with code %s", name, proc.pid, proc.wait()) def cmd_sleep(cb, interval): - """ - Set an alarm, then wait for it to go off. - """ + """ + Set an alarm, then wait for it to go off. + """ - howlong = rpki.sundial.timedelta.parse(interval) - logger.info("Sleeping %r", howlong) - rpki.async.timer(cb).set(howlong) + howlong = rpki.sundial.timedelta.parse(interval) + logger.info("Sleeping %r", howlong) + rpki.async.timer(cb).set(howlong) def cmd_shell(cb, *cmd): - """ - Run a shell command. - """ + """ + Run a shell command. + """ - cmd = " ".join(cmd) - status = subprocess.call(cmd, shell = True) - logger.info("Shell command returned status %d", status) - cb() + cmd = " ".join(cmd) + status = subprocess.call(cmd, shell = True) + logger.info("Shell command returned status %d", status) + cb() def cmd_echo(cb, *words): - """ - Echo some text to the log. - """ + """ + Echo some text to the log. + """ - logger.info(" ".join(words)) - cb() + logger.info(" ".join(words)) + cb() ## @var cmds # Dispatch table for commands embedded in delta sections @@ -359,1019 +359,1019 @@ cmds = { "sleep" : cmd_sleep, "echo" : cmd_echo } class roa_request(object): - """ - Representation for a roa_request object. - """ - - def __init__(self, asn, ipv4, ipv6): - self.asn = asn - self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None - self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - - def __eq__(self, other): - return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - - def __hash__(self): - v4 = tuple(self.v4) if self.v4 is not None else None - v6 = tuple(self.v6) if self.v6 is not None else None - return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - - def __str__(self): - if self.v4 and self.v6: s = str(self.v4) + "," + str(self.v6) - elif self.v4: s = str(self.v4) - else: s = str(self.v6) - return "%s: %s" % (self.asn, s) - - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("ipv4"), yaml.get("ipv6")) - -class router_cert(object): - """ - Representation for a router_cert object. - """ - - _ecparams = None - _keypair = None - _pkcs10 = None - _gski = None - - @classmethod - def ecparams(cls): - if cls._ecparams is None: - cls._ecparams = rpki.x509.KeyParams.generateEC() - return cls._ecparams - - def __init__(self, asn, router_id): - self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) - self.router_id = router_id - self.cn = "ROUTER-%08x" % self.asn[0].min - self.sn = "%08x" % self.router_id - self.eku = rpki.oids.id_kp_bgpsec_router - - @property - def keypair(self): - if self._keypair is None: - self._keypair = rpki.x509.ECDSA.generate(self.ecparams()) - return self._keypair - - @property - def pkcs10(self): - if self._pkcs10 is None: - self._pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) - return self._pkcs10 - - @property - def gski(self): - if self._gski is None: - self._gski = self.pkcs10.gSKI() - return self._gski - - def __eq__(self, other): - return self.asn == other.asn and self.sn == other.sn - - def __hash__(self): - return tuple(self.asn).__hash__() + self.cn.__hash__() + self.sn.__hash__() - - def __str__(self): - return "%s: %s,%s: %s" % (self.asn, self.cn, self.sn, self.gski) - - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("router_id")) - -class allocation_db(list): - """ - Representation of all the entities and allocations in the test - system. Almost everything is generated out of this database. - """ - - def __init__(self, yaml): - """ - Initialize database from the (first) YAML document. """ - - list.__init__(self) - self.root = allocation(yaml, self) - assert self.root.is_root - if self.root.crl_interval is None: - self.root.crl_interval = rpki.sundial.timedelta.parse(cfg.get("crl_interval", "1d")).convert_to_seconds() - if self.root.regen_margin is None: - self.root.regen_margin = rpki.sundial.timedelta.parse(cfg.get("regen_margin", "1d")).convert_to_seconds() - for a in self: - if a.sia_base is None: - a.sia_base = (rootd_sia + "root/trunk/" if a.is_root else a.parent.sia_base) + a.name + "/" - if a.base.valid_until is None: - a.base.valid_until = a.parent.base.valid_until - if a.crl_interval is None: - a.crl_interval = a.parent.crl_interval - if a.regen_margin is None: - a.regen_margin = a.parent.regen_margin - a.client_handle = "/".join(a.sia_base.split("/")[4:]).rstrip("/") - self.root.closure() - self.map = dict((a.name, a) for a in self) - self.engines = [a for a in self if a.is_engine] - for i, a in enumerate(self.engines): - a.set_engine_number(i) - for a in self: - if a.is_hosted: - a.hosted_by = self.map[a.hosted_by] - a.hosted_by.hosts.append(a) - assert a.is_twig, "%s is not twig" % a.name - assert not a.hosted_by.is_hosted, "%s is hosted by a hosted entity" % a.name - - def apply_delta(self, delta, cb): + Representation for a roa_request object. """ - Apply a delta or run a command. - """ - - def loop(iterator, d): - if isinstance(d, str): - c = d.split() - cmds[c[0]](iterator, *c[1:]) - else: - self.map[d["name"]].apply_delta(d, iterator) - def done(): - self.root.closure() - cb() + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - if delta is None: - cb() - else: - rpki.async.iterator(delta, loop, done) + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - def dump(self): - """ - Print content of the database. - """ + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - for a in self: - print a + def __str__(self): + if self.v4 and self.v6: s = str(self.v4) + "," + str(self.v6) + elif self.v4: s = str(self.v4) + else: s = str(self.v6) + return "%s: %s" % (self.asn, s) -class allocation(object): - - parent = None - irdb_db_name = None - irdb_port = None - rpki_db_name = None - rpki_port = None - crl_interval = None - regen_margin = None - last_cms_time = None - rpkid_process = None - irdbd_process = None - - def __init__(self, yaml, db, parent = None): - """ - Initialize one entity and insert it into the database. - """ + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("ipv4"), yaml.get("ipv6")) - db.append(self) - self.name = yaml["name"] - self.parent = parent - self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] - valid_until = None - if "valid_until" in yaml: - valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) - if valid_until is None and "valid_for" in yaml: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) - self.base = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as(yaml.get("asn")), - v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), - v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), - valid_until = valid_until) - self.sia_base = yaml.get("sia_base") - if "crl_interval" in yaml: - self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() - if "regen_margin" in yaml: - self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() - self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] - for r in self.roa_requests: - if r.v4: - self.base.v4 |= r.v4.to_resource_set() - if r.v6: - self.base.v6 |= r.v6.to_resource_set() - self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] - for r in self.router_certs: - self.base.asn |= r.asn - self.hosted_by = yaml.get("hosted_by") - self.extra_conf = yaml.get("extra_conf", []) - self.hosts = [] - - def closure(self): +class router_cert(object): """ - Compute the transitive resource closure. + Representation for a router_cert object. """ - resources = self.base - for kid in self.kids: - resources |= kid.closure() - self.resources = resources - return resources + _ecparams = None + _keypair = None + _pkcs10 = None + _gski = None + + @classmethod + def ecparams(cls): + if cls._ecparams is None: + cls._ecparams = rpki.x509.KeyParams.generateEC() + return cls._ecparams + + def __init__(self, asn, router_id): + self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) + self.router_id = router_id + self.cn = "ROUTER-%08x" % self.asn[0].min + self.sn = "%08x" % self.router_id + self.eku = rpki.oids.id_kp_bgpsec_router + + @property + def keypair(self): + if self._keypair is None: + self._keypair = rpki.x509.ECDSA.generate(self.ecparams()) + return self._keypair + + @property + def pkcs10(self): + if self._pkcs10 is None: + self._pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) + return self._pkcs10 + + @property + def gski(self): + if self._gski is None: + self._gski = self.pkcs10.gSKI() + return self._gski + + def __eq__(self, other): + return self.asn == other.asn and self.sn == other.sn + + def __hash__(self): + return tuple(self.asn).__hash__() + self.cn.__hash__() + self.sn.__hash__() + + def __str__(self): + return "%s: %s,%s: %s" % (self.asn, self.cn, self.sn, self.gski) + + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("router_id")) - def apply_delta(self, yaml, cb): +class allocation_db(list): """ - Apply deltas to this entity. + Representation of all the entities and allocations in the test + system. Almost everything is generated out of this database. """ - logger.info("Applying delta: %s", yaml) - - def loop(iterator, kv): - if kv[0] == "name": - iterator() - else: - getattr(self, "apply_" + kv[0])(kv[1], iterator) - - rpki.async.iterator(yaml.items(), loop, cb) - - def apply_add_as(self, text, cb): - self.base.asn |= rpki.resource_set.resource_set_as(text) - cb() - - def apply_add_v4(self, text, cb): - self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) - cb() - - def apply_add_v6(self, text, cb): - self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) - cb() - - def apply_sub_as(self, text, cb): - self.base.asn |= rpki.resource_set.resource_set_as(text) - cb() + def __init__(self, yaml): + """ + Initialize database from the (first) YAML document. + """ + + list.__init__(self) + self.root = allocation(yaml, self) + assert self.root.is_root + if self.root.crl_interval is None: + self.root.crl_interval = rpki.sundial.timedelta.parse(cfg.get("crl_interval", "1d")).convert_to_seconds() + if self.root.regen_margin is None: + self.root.regen_margin = rpki.sundial.timedelta.parse(cfg.get("regen_margin", "1d")).convert_to_seconds() + for a in self: + if a.sia_base is None: + a.sia_base = (rootd_sia + "root/trunk/" if a.is_root else a.parent.sia_base) + a.name + "/" + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + a.client_handle = "/".join(a.sia_base.split("/")[4:]).rstrip("/") + self.root.closure() + self.map = dict((a.name, a) for a in self) + self.engines = [a for a in self if a.is_engine] + for i, a in enumerate(self.engines): + a.set_engine_number(i) + for a in self: + if a.is_hosted: + a.hosted_by = self.map[a.hosted_by] + a.hosted_by.hosts.append(a) + assert a.is_twig, "%s is not twig" % a.name + assert not a.hosted_by.is_hosted, "%s is hosted by a hosted entity" % a.name + + def apply_delta(self, delta, cb): + """ + Apply a delta or run a command. + """ + + def loop(iterator, d): + if isinstance(d, str): + c = d.split() + cmds[c[0]](iterator, *c[1:]) + else: + self.map[d["name"]].apply_delta(d, iterator) + + def done(): + self.root.closure() + cb() + + if delta is None: + cb() + else: + rpki.async.iterator(delta, loop, done) - def apply_sub_v4(self, text, cb): - self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) - cb() + def dump(self): + """ + Print content of the database. + """ - def apply_sub_v6(self, text, cb): - self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) - cb() + for a in self: + print a - def apply_valid_until(self, stamp, cb): - self.base.valid_until = rpki.sundial.datetime.from_datetime(stamp) - cb() +class allocation(object): - def apply_valid_for(self, text, cb): - self.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(text) - cb() + parent = None + irdb_db_name = None + irdb_port = None + rpki_db_name = None + rpki_port = None + crl_interval = None + regen_margin = None + last_cms_time = None + rpkid_process = None + irdbd_process = None + + def __init__(self, yaml, db, parent = None): + """ + Initialize one entity and insert it into the database. + """ + + db.append(self) + self.name = yaml["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] + valid_until = None + if "valid_until" in yaml: + valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) + if valid_until is None and "valid_for" in yaml: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(yaml.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), + valid_until = valid_until) + self.sia_base = yaml.get("sia_base") + if "crl_interval" in yaml: + self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() + if "regen_margin" in yaml: + self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() + self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] + for r in self.roa_requests: + if r.v4: + self.base.v4 |= r.v4.to_resource_set() + if r.v6: + self.base.v6 |= r.v6.to_resource_set() + self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] + for r in self.router_certs: + self.base.asn |= r.asn + self.hosted_by = yaml.get("hosted_by") + self.extra_conf = yaml.get("extra_conf", []) + self.hosts = [] + + def closure(self): + """ + Compute the transitive resource closure. + """ + + resources = self.base + for kid in self.kids: + resources |= kid.closure() + self.resources = resources + return resources + + def apply_delta(self, yaml, cb): + """ + Apply deltas to this entity. + """ + + logger.info("Applying delta: %s", yaml) + + def loop(iterator, kv): + if kv[0] == "name": + iterator() + else: + getattr(self, "apply_" + kv[0])(kv[1], iterator) + + rpki.async.iterator(yaml.items(), loop, cb) + + def apply_add_as(self, text, cb): + self.base.asn |= rpki.resource_set.resource_set_as(text) + cb() + + def apply_add_v4(self, text, cb): + self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) + cb() + + def apply_add_v6(self, text, cb): + self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) + cb() + + def apply_sub_as(self, text, cb): + self.base.asn |= rpki.resource_set.resource_set_as(text) + cb() + + def apply_sub_v4(self, text, cb): + self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) + cb() + + def apply_sub_v6(self, text, cb): + self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) + cb() + + def apply_valid_until(self, stamp, cb): + self.base.valid_until = rpki.sundial.datetime.from_datetime(stamp) + cb() + + def apply_valid_for(self, text, cb): + self.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(text) + cb() + + def apply_valid_add(self, text, cb): + self.base.valid_until += rpki.sundial.timedelta.parse(text) + cb() + + def apply_valid_sub(self, text, cb): + self.base.valid_until -= rpki.sundial.timedelta.parse(text) + cb() + + def apply_roa_request_add(self, yaml, cb): + for y in yaml: + r = roa_request.parse(y) + if r not in self.roa_requests: + self.roa_requests.append(r) + cb() + + def apply_roa_request_del(self, yaml, cb): + for y in yaml: + r = roa_request.parse(y) + if r in self.roa_requests: + self.roa_requests.remove(r) + cb() + + def apply_router_cert_add(self, yaml, cb): + for y in yaml: + r = router_cert.parse(y) + if r not in self.router_certs: + self.router_certs.append(r) + cb() + + def apply_router_cert_del(self, yaml, cb): + for y in yaml: + r = router_cert.parse(y) + if r in self.router_certs: + self.router_certs.remove(r) + cb() + + def apply_rekey(self, target, cb): + + def done(e): + if isinstance(e, Exception): + logger.exception("Exception while rekeying %s", self.name) + raise e + cb() + + if target is None: + logger.info("Rekeying <tenant/> %s", self.name) + self.call_rpkid([rpki.left_right.self_elt.make_pdu( + action = "set", self_handle = self.name, rekey = "yes")], cb = done) + else: + logger.info("Rekeying <parent/> %s %s", self.name, target) + self.call_rpkid([rpki.left_right.parent_elt.make_pdu( + action = "set", self_handle = self.name, parent_handle = target, rekey = "yes")], cb = done) + + def apply_revoke(self, target, cb): + + def done(e): + if isinstance(e, Exception): + logger.exception("Exception while revoking %s", self.name) + raise e + cb() + + if target is None: + logger.info("Revoking <tenant/> %s", self.name) + self.call_rpkid([rpki.left_right.self_elt.make_pdu( + action = "set", self_handle = self.name, revoke = "yes")], cb = done) + else: + logger.info("Revoking <parent/> %s %s", self.name, target) + self.call_rpkid([rpki.left_right.parent_elt.make_pdu( + action = "set", self_handle = self.name, parent_handle = target, revoke = "yes")], cb = done) + + def __str__(self): + s = self.name + "\n" + if self.resources.asn: s += " ASN: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.sia_base: s += " SIA: %s\n" % self.sia_base + return s + "Until: %s\n" % self.resources.valid_until + + + @property + def is_root(self): + return self.parent is None + + @property + def is_twig(self): + return not self.is_root + + @property + def is_hosted(self): + return self.hosted_by is not None + + @property + def is_engine(self): + return not self.is_hosted + + def set_engine_number(self, n): + """ + Set the engine number for this entity. + """ + + self.irdb_db_name = "irdb%d" % n + self.irdb_port = allocate_port() + self.rpki_db_name = "rpki%d" % n + self.rpki_port = allocate_port() + + def get_rpki_port(self): + """ + Get rpki port to use for this entity. + """ + + if self.is_hosted: + assert self.hosted_by.rpki_port is not None + return self.hosted_by.rpki_port + else: + assert self.rpki_port is not None + return self.rpki_port + + def setup_bpki_certs(self): + """ + Create BPKI certificates for this entity. + """ + + logger.info("Constructing BPKI keys and certs for %s", self.name) + setup_bpki_cert_chain(name = self.name, + ee = ("RPKI", "IRDB", "IRBE"), + ca = ("SELF",)) + self.rpkid_ta = rpki.x509.X509(PEM_file = self.name + "-TA.cer") + self.irbe_key = rpki.x509.RSA( PEM_file = self.name + "-IRBE.key") + self.irbe_cert = rpki.x509.X509(PEM_file = self.name + "-IRBE.cer") + self.rpkid_cert = rpki.x509.X509(PEM_file = self.name + "-RPKI.cer") + + def setup_conf_file(self): + """ + Write config files for this entity. + """ + + logger.info("Writing config files for %s", self.name) + assert self.rpki_port is not None + d = dict(my_name = self.name, + irdb_db_name = self.irdb_db_name, + irdb_db_pass = irdb_db_pass, + irdb_port = self.irdb_port, + rpki_db_name = self.rpki_db_name, + rpki_db_pass = rpki_db_pass, + rpki_port = self.rpki_port) + f = open(self.name + ".conf", "w") + f.write(conf_fmt_1 % d) + for line in self.extra_conf: + f.write(line + "\n") + f.close() + + def setup_sql(self, rpki_sql, irdb_sql): + """ + Set up this entity's IRDB. + """ + + logger.info("Setting up MySQL for %s", self.name) + db = MySQLdb.connect(user = "rpki", db = self.rpki_db_name, passwd = rpki_db_pass, + conv = sql_conversions) + cur = db.cursor() + db.autocommit(True) + for sql in rpki_sql: + try: + cur.execute(sql) + except: + if "DROP TABLE IF EXISTS" not in sql.upper(): + raise + db.close() + db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, + conv = sql_conversions) + cur = db.cursor() + db.autocommit(True) + for sql in irdb_sql: + try: + cur.execute(sql) + except: + if "DROP TABLE IF EXISTS" not in sql.upper(): + raise + for s in [self] + self.hosts: + for kid in s.kids: + cur.execute("INSERT registrant (registrant_handle, registry_handle, valid_until) VALUES (%s, %s, %s)", + (kid.name, s.name, kid.resources.valid_until)) + db.close() + + def sync_sql(self): + """ + Whack this entity's IRDB to match our master database. We do this + once during setup, then do it again every time we apply a delta to + this entity. + """ + + logger.info("Updating MySQL data for IRDB %s", self.name) + db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, + conv = sql_conversions) + cur = db.cursor() + db.autocommit(True) + cur.execute("DELETE FROM registrant_asn") + cur.execute("DELETE FROM registrant_net") + cur.execute("DELETE FROM roa_request_prefix") + cur.execute("DELETE FROM roa_request") + cur.execute("DELETE FROM ee_certificate_asn") + cur.execute("DELETE FROM ee_certificate_net") + cur.execute("DELETE FROM ee_certificate") + + for s in [self] + self.hosts: + for kid in s.kids: + cur.execute("SELECT registrant_id FROM registrant WHERE registrant_handle = %s AND registry_handle = %s", + (kid.name, s.name)) + registrant_id = cur.fetchone()[0] + for as_range in kid.resources.asn: + cur.execute("INSERT registrant_asn (start_as, end_as, registrant_id) VALUES (%s, %s, %s)", + (as_range.min, as_range.max, registrant_id)) + for v4_range in kid.resources.v4: + cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 4, %s)", + (v4_range.min, v4_range.max, registrant_id)) + for v6_range in kid.resources.v6: + cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 6, %s)", + (v6_range.min, v6_range.max, registrant_id)) + cur.execute("UPDATE registrant SET valid_until = %s WHERE registrant_id = %s", + (kid.resources.valid_until, registrant_id)) + for r in s.roa_requests: + cur.execute("INSERT roa_request (self_handle, asn) VALUES (%s, %s)", + (s.name, r.asn)) + roa_request_id = cur.lastrowid + for version, prefix_set in ((4, r.v4), (6, r.v6)): + if prefix_set: + cur.executemany("INSERT roa_request_prefix " + "(roa_request_id, prefix, prefixlen, max_prefixlen, version) " + "VALUES (%s, %s, %s, %s, %s)", + ((roa_request_id, x.prefix, x.prefixlen, x.max_prefixlen, version) + for x in prefix_set)) + for r in s.router_certs: + cur.execute("INSERT ee_certificate (self_handle, pkcs10, gski, cn, sn, eku, valid_until) " + "VALUES (%s, %s, %s, %s, %s, %s, %s)", + (s.name, r.pkcs10.get_DER(), r.gski, r.cn, r.sn, r.eku, s.resources.valid_until)) + ee_certificate_id = cur.lastrowid + cur.executemany("INSERT ee_certificate_asn (ee_certificate_id, start_as, end_as) VALUES (%s, %s, %s)", + ((ee_certificate_id, a.min, a.max) for a in r.asn)) + db.close() + + def run_daemons(self): + """ + Run daemons for this entity. + """ + + logger.info("Running daemons for %s", self.name) + env = dict(os.environ, RPKI_CONF = self.name + ".conf") + self.rpkid_process = subprocess.Popen((prog_python, prog_rpkid, "--foreground", "--log-stdout", "--log-level", "debug") + + (("--profile", self.name + ".prof") if args.profile else ()), + env = env) + self.irdbd_process = subprocess.Popen((prog_python, prog_irdbd, "--foreground", "--log-stdout", "--log-level", "debug"), + env = env) + + def kill_daemons(self): + """ + Kill daemons for this entity. + """ + + # pylint: disable=E1103 + for proc, name in ((self.rpkid_process, "rpkid"), + (self.irdbd_process, "irdbd")): + if proc is not None and proc.poll() is None: + logger.info("Killing daemon %s pid %s for %s", name, proc.pid, self.name) + try: + proc.terminate() + except OSError: + pass + if proc is not None: + logger.info("Daemon %s pid %s for %s exited with code %s", + name, proc.pid, self.name, proc.wait()) + + def call_rpkid(self, pdus, cb): + """ + Send a left-right message to this entity's RPKI daemon and return + the response. + + If this entity is hosted (does not run its own RPKI daemon), all + of this happens with the hosting RPKI daemon. + """ + + logger.info("Calling rpkid for %s", self.name) + + if self.is_hosted: + logger.info("rpkid %s is hosted by rpkid %s, switching", self.name, self.hosted_by.name) + self = self.hosted_by + assert not self.is_hosted + + assert isinstance(pdus, (list, tuple)) + assert self.rpki_port is not None + + q_msg = rpki.left_right.msg.query(*pdus) + q_cms = rpki.left_right.cms_msg_saxify() + q_der = q_cms.wrap(q_msg, self.irbe_key, self.irbe_cert) + q_url = "http://localhost:%d/left-right" % self.rpki_port + + logger.debug(q_cms.pretty_print_content()) + + def done(r_der): + logger.info("Callback from rpkid %s", self.name) + r_cms = rpki.left_right.cms_msg_saxify(DER = r_der) + r_msg = r_cms.unwrap((self.rpkid_ta, self.rpkid_cert)) + self.last_cms_time = r_cms.check_replay(self.last_cms_time, q_url) + logger.debug(r_cms.pretty_print_content()) + assert r_msg.is_reply + for r_pdu in r_msg: + assert not isinstance(r_pdu, rpki.left_right.report_error_elt) + cb(r_msg) + + def lose(e): + raise + + rpki.http.client( + url = q_url, + msg = q_der, + callback = done, + errback = lose) + + def cross_certify(self, certificant, reverse = False): + """ + Cross-certify and return the resulting certificate. + """ + + if reverse: + certifier = certificant + certificant = self.name + "-SELF" + else: + certifier = self.name + "-SELF" + return cross_certify(certificant, certifier) + + def create_rpki_objects(self, cb): + """ + Create RPKI engine objects for this engine. + + Root node of the engine tree is special, it too has a parent but + that one is the magic self-signed micro engine. + + The rest of this is straightforward. There are a lot of objects + to create, but we can do batch them all into one honking PDU, then + issue one more PDU to set BSC EE certificates based on the PKCS + #10 requests we get back when we tell rpkid to generate BSC keys. + """ + + assert not self.is_hosted + + selves = [self] + self.hosts + + rpkid_pdus = [] + pubd_pdus = [] + + for i, s in enumerate(selves): + logger.info("Creating RPKI objects for [%d] %s", i, s.name) + + rpkid_pdus.append(rpki.left_right.self_elt.make_pdu( + action = "create", + self_handle = s.name, + crl_interval = s.crl_interval, + regen_margin = s.regen_margin, + bpki_cert = (s.cross_certify(s.hosted_by.name + "-TA", reverse = True) + if s.is_hosted else + rpki.x509.X509(Auto_file = s.name + "-SELF.cer")))) + + rpkid_pdus.append(rpki.left_right.bsc_elt.make_pdu( + action = "create", + self_handle = s.name, + bsc_handle = "b", + generate_keypair = True)) + + pubd_pdus.append(rpki.publication_control.client_elt.make_pdu( + action = "create", + client_handle = s.client_handle, + base_uri = s.sia_base, + bpki_cert = s.cross_certify(pubd_name + "-TA", reverse = True))) + + rpkid_pdus.append(rpki.left_right.repository_elt.make_pdu( + action = "create", + self_handle = s.name, + bsc_handle = "b", + repository_handle = "r", + bpki_cert = s.cross_certify(pubd_name + "-TA"), + peer_contact_uri = "http://localhost:%d/client/%s" % (pubd_port, s.client_handle))) + + for k in s.kids: + rpkid_pdus.append(rpki.left_right.child_elt.make_pdu( + action = "create", + self_handle = s.name, + child_handle = k.name, + bsc_handle = "b", + bpki_cert = s.cross_certify(k.name + "-SELF"))) + + if s.is_root: + rootd_cert = s.cross_certify(rootd_name + "-TA") + rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( + action = "create", + self_handle = s.name, + parent_handle = "rootd", + bsc_handle = "b", + repository_handle = "r", + sia_base = s.sia_base, + bpki_cert = rootd_cert, + sender_name = s.name, + recipient_name = "rootd", + peer_contact_uri = "http://localhost:%s/" % rootd_port)) + else: + rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( + action = "create", + self_handle = s.name, + parent_handle = s.parent.name, + bsc_handle = "b", + repository_handle = "r", + sia_base = s.sia_base, + bpki_cert = s.cross_certify(s.parent.name + "-SELF"), + sender_name = s.name, + recipient_name = s.parent.name, + peer_contact_uri = "http://localhost:%s/up-down/%s/%s" % (s.parent.get_rpki_port(), + s.parent.name, s.name))) + + def one(): + call_pubd(pubd_pdus, cb = two) + + def two(vals): + self.call_rpkid(rpkid_pdus, cb = three) + + def three(vals): + + bsc_dict = dict((b.self_handle, b) for b in vals if isinstance(b, rpki.left_right.bsc_elt)) + + bsc_pdus = [] + + for s in selves: + b = bsc_dict[s.name] + + logger.info("Issuing BSC EE cert for %s", s.name) + cmd = (prog_openssl, "x509", "-req", "-sha256", "-extfile", s.name + "-RPKI.conf", + "-extensions", "req_x509_ext", "-days", "30", + "-CA", s.name + "-SELF.cer", "-CAkey", s.name + "-SELF.key", "-CAcreateserial", "-text") + signer = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + signed = signer.communicate(input = b.pkcs10_request.get_PEM()) + if not signed[0]: + logger.warning(signed[1]) + raise CouldntIssueBSCEECertificate("Couldn't issue BSC EE certificate") + s.bsc_ee = rpki.x509.X509(PEM = signed[0]) + s.bsc_crl = rpki.x509.CRL(PEM_file = s.name + "-SELF.crl") + logger.info("BSC EE cert for %s SKI %s", s.name, s.bsc_ee.hSKI()) + + bsc_pdus.append(rpki.left_right.bsc_elt.make_pdu( + action = "set", + self_handle = s.name, + bsc_handle = "b", + signing_cert = s.bsc_ee, + signing_cert_crl = s.bsc_crl)) + + self.call_rpkid(bsc_pdus, cb = four) + + def four(vals): + cb() + + one() + + def setup_yaml_leaf(self): + """ + Generate certificates and write YAML scripts for leaf nodes. + + We're cheating a bit here: properly speaking, we can't generate + issue or revoke requests without knowing the class, which is + generated on the fly, but at the moment the test case is + simplistic enough that the class will always be "1", so we just + wire in that value for now. + + Well, ok, we just broke that assumption. Now we do something even + nastier, just to eke a bit more life out of this kludge. This + really needs to be rewritten, but it may require a different tool + than testpoke. + """ + + if not os.path.exists(self.name + ".key"): + logger.info("Generating RPKI key for %s", self.name) + subprocess.check_call((prog_openssl, "genrsa", "-out", self.name + ".key", "2048" ), + stdout = subprocess.PIPE, stderr = subprocess.STDOUT) + ski = rpki.x509.RSA(PEM_file = self.name + ".key").gSKI() + + if self.parent.is_hosted: + parent_host = self.parent.hosted_by.name + else: + parent_host = self.parent.name - def apply_valid_add(self, text, cb): - self.base.valid_until += rpki.sundial.timedelta.parse(text) - cb() + self.cross_certify(self.parent.name + "-SELF") + self.cross_certify(parent_host + "-TA") - def apply_valid_sub(self, text, cb): - self.base.valid_until -= rpki.sundial.timedelta.parse(text) - cb() + def run_cron(self, cb): + """ + Trigger cron run for this engine. + """ - def apply_roa_request_add(self, yaml, cb): - for y in yaml: - r = roa_request.parse(y) - if r not in self.roa_requests: - self.roa_requests.append(r) - cb() + logger.info("Running cron for %s", self.name) - def apply_roa_request_del(self, yaml, cb): - for y in yaml: - r = roa_request.parse(y) - if r in self.roa_requests: - self.roa_requests.remove(r) - cb() + assert self.rpki_port is not None - def apply_router_cert_add(self, yaml, cb): - for y in yaml: - r = router_cert.parse(y) - if r not in self.router_certs: - self.router_certs.append(r) - cb() + def done(result): + assert result == "OK", 'Expected "OK" result from cronjob, got %r' % result + cb() - def apply_router_cert_del(self, yaml, cb): - for y in yaml: - r = router_cert.parse(y) - if r in self.router_certs: - self.router_certs.remove(r) - cb() + rpki.http.client( + url = "http://localhost:%d/cronjob" % self.rpki_port, + msg = "Run cron now, please", + callback = done, + errback = done) - def apply_rekey(self, target, cb): - - def done(e): - if isinstance(e, Exception): - logger.exception("Exception while rekeying %s", self.name) - raise e - cb() - - if target is None: - logger.info("Rekeying <tenant/> %s", self.name) - self.call_rpkid([rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.name, rekey = "yes")], cb = done) - else: - logger.info("Rekeying <parent/> %s %s", self.name, target) - self.call_rpkid([rpki.left_right.parent_elt.make_pdu( - action = "set", self_handle = self.name, parent_handle = target, rekey = "yes")], cb = done) - - def apply_revoke(self, target, cb): - - def done(e): - if isinstance(e, Exception): - logger.exception("Exception while revoking %s", self.name) - raise e - cb() - - if target is None: - logger.info("Revoking <tenant/> %s", self.name) - self.call_rpkid([rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.name, revoke = "yes")], cb = done) - else: - logger.info("Revoking <parent/> %s %s", self.name, target) - self.call_rpkid([rpki.left_right.parent_elt.make_pdu( - action = "set", self_handle = self.name, parent_handle = target, revoke = "yes")], cb = done) - - def __str__(self): - s = self.name + "\n" - if self.resources.asn: s += " ASN: %s\n" % self.resources.asn - if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 - if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 - if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) - if self.parent: s += " Up: %s\n" % self.parent.name - if self.sia_base: s += " SIA: %s\n" % self.sia_base - return s + "Until: %s\n" % self.resources.valid_until - - - @property - def is_root(self): - return self.parent is None - - @property - def is_twig(self): - return not self.is_root - - @property - def is_hosted(self): - return self.hosted_by is not None - - @property - def is_engine(self): - return not self.is_hosted - - def set_engine_number(self, n): - """ - Set the engine number for this entity. - """ + def run_yaml(self): + """ + Run YAML scripts for this leaf entity. Since we're not bothering + to check the class list returned by the list command, the issue + command may fail, so we treat failure of the list command as an + error, but only issue a warning when issue fails. + """ - self.irdb_db_name = "irdb%d" % n - self.irdb_port = allocate_port() - self.rpki_db_name = "rpki%d" % n - self.rpki_port = allocate_port() + logger.info("Running YAML for %s", self.name) + subprocess.check_call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "list")) + if subprocess.call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "issue")) != 0: + logger.warning("YAML issue command failed for %s, continuing", self.name) - def get_rpki_port(self): +def setup_bpki_cert_chain(name, ee = (), ca = ()): """ - Get rpki port to use for this entity. + Build a set of BPKI certificates. """ - if self.is_hosted: - assert self.hosted_by.rpki_port is not None - return self.hosted_by.rpki_port - else: - assert self.rpki_port is not None - return self.rpki_port + s = "exec >/dev/null 2>&1\n" + #s = "set -x\n" + for kind in ("TA",) + ee + ca: + d = dict(name = name, + kind = kind, + ca = "false" if kind in ee else "true", + openssl = prog_openssl) + f = open("%(name)s-%(kind)s.conf" % d, "w") + f.write(bpki_cert_fmt_1 % d) + f.close() + if not os.path.exists("%(name)s-%(kind)s.key" % d): + s += bpki_cert_fmt_2 % d + s += bpki_cert_fmt_3 % d + d = dict(name = name, + openssl = prog_openssl) + s += bpki_cert_fmt_4 % d + for kind in ee + ca: + d["kind"] = kind + s += bpki_cert_fmt_5 % d + for kind in ("TA",) + ca: + d["kind"] = kind + s += bpki_cert_fmt_6 % d + subprocess.check_call(s, shell = True) - def setup_bpki_certs(self): +def setup_rootd(rpkid, rootd_yaml, db): """ - Create BPKI certificates for this entity. + Write the config files for rootd. """ - logger.info("Constructing BPKI keys and certs for %s", self.name) - setup_bpki_cert_chain(name = self.name, - ee = ("RPKI", "IRDB", "IRBE"), - ca = ("SELF",)) - self.rpkid_ta = rpki.x509.X509(PEM_file = self.name + "-TA.cer") - self.irbe_key = rpki.x509.RSA( PEM_file = self.name + "-IRBE.key") - self.irbe_cert = rpki.x509.X509(PEM_file = self.name + "-IRBE.cer") - self.rpkid_cert = rpki.x509.X509(PEM_file = self.name + "-RPKI.cer") + rpkid.cross_certify(rootd_name + "-TA", reverse = True) + cross_certify(pubd_name + "-TA", rootd_name + "-TA") + logger.info("Writing config files for %s", rootd_name) + d = dict(rootd_name = rootd_name, + rootd_port = rootd_port, + rpkid_name = rpkid.name, + pubd_name = pubd_name, + rootd_sia = rootd_sia, + rsyncd_dir = rsyncd_dir, + openssl = prog_openssl, + lifetime = rootd_yaml.get("lifetime", "30d"), + pubd_port = pubd_port, + rootd_handle = db.root.client_handle + "-" + rootd_name) + f = open(rootd_name + ".conf", "w") + f.write(rootd_fmt_1 % d) + f.close() + s = "exec >/dev/null 2>&1\n" + #s = "set -x\n" + if not os.path.exists("root.key"): + s += rootd_fmt_2 % d + s += rootd_fmt_3 % d + subprocess.check_call(s, shell = True) - def setup_conf_file(self): +def setup_rcynic(): """ - Write config files for this entity. + Write the config file for rcynic. """ - logger.info("Writing config files for %s", self.name) - assert self.rpki_port is not None - d = dict(my_name = self.name, - irdb_db_name = self.irdb_db_name, - irdb_db_pass = irdb_db_pass, - irdb_port = self.irdb_port, - rpki_db_name = self.rpki_db_name, - rpki_db_pass = rpki_db_pass, - rpki_port = self.rpki_port) - f = open(self.name + ".conf", "w") - f.write(conf_fmt_1 % d) - for line in self.extra_conf: - f.write(line + "\n") + logger.info("Config file for rcynic") + d = dict(rcynic_name = rcynic_name, + rootd_name = rootd_name, + rootd_sia = rootd_sia) + f = open(rcynic_name + ".conf", "w") + f.write(rcynic_fmt_1 % d) f.close() - def setup_sql(self, rpki_sql, irdb_sql): +def setup_rsyncd(): """ - Set up this entity's IRDB. + Write the config file for rsyncd. """ - logger.info("Setting up MySQL for %s", self.name) - db = MySQLdb.connect(user = "rpki", db = self.rpki_db_name, passwd = rpki_db_pass, - conv = sql_conversions) - cur = db.cursor() - db.autocommit(True) - for sql in rpki_sql: - try: - cur.execute(sql) - except: - if "DROP TABLE IF EXISTS" not in sql.upper(): - raise - db.close() - db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, - conv = sql_conversions) - cur = db.cursor() - db.autocommit(True) - for sql in irdb_sql: - try: - cur.execute(sql) - except: - if "DROP TABLE IF EXISTS" not in sql.upper(): - raise - for s in [self] + self.hosts: - for kid in s.kids: - cur.execute("INSERT registrant (registrant_handle, registry_handle, valid_until) VALUES (%s, %s, %s)", - (kid.name, s.name, kid.resources.valid_until)) - db.close() + logger.info("Config file for rsyncd") + d = dict(rsyncd_name = rsyncd_name, + rsyncd_port = rsyncd_port, + rsyncd_module = rsyncd_module, + rsyncd_dir = rsyncd_dir) + f = open(rsyncd_name + ".conf", "w") + f.write(rsyncd_fmt_1 % d) + f.close() - def sync_sql(self): +def setup_publication(pubd_sql, irdb_db_name): """ - Whack this entity's IRDB to match our master database. We do this - once during setup, then do it again every time we apply a delta to - this entity. + Set up publication daemon. """ - logger.info("Updating MySQL data for IRDB %s", self.name) - db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, + logger.info("Configure publication daemon") + publication_dir = os.getcwd() + "/publication" + assert rootd_sia.startswith("rsync://") + global rsyncd_dir + rsyncd_dir = publication_dir + "/".join(rootd_sia.split("/")[4:]) + if not rsyncd_dir.endswith("/"): + rsyncd_dir += "/" + os.makedirs(rsyncd_dir + "root/trunk") + db = MySQLdb.connect(db = pubd_db_name, user = pubd_db_user, passwd = pubd_db_pass, conv = sql_conversions) cur = db.cursor() db.autocommit(True) - cur.execute("DELETE FROM registrant_asn") - cur.execute("DELETE FROM registrant_net") - cur.execute("DELETE FROM roa_request_prefix") - cur.execute("DELETE FROM roa_request") - cur.execute("DELETE FROM ee_certificate_asn") - cur.execute("DELETE FROM ee_certificate_net") - cur.execute("DELETE FROM ee_certificate") - - for s in [self] + self.hosts: - for kid in s.kids: - cur.execute("SELECT registrant_id FROM registrant WHERE registrant_handle = %s AND registry_handle = %s", - (kid.name, s.name)) - registrant_id = cur.fetchone()[0] - for as_range in kid.resources.asn: - cur.execute("INSERT registrant_asn (start_as, end_as, registrant_id) VALUES (%s, %s, %s)", - (as_range.min, as_range.max, registrant_id)) - for v4_range in kid.resources.v4: - cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 4, %s)", - (v4_range.min, v4_range.max, registrant_id)) - for v6_range in kid.resources.v6: - cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 6, %s)", - (v6_range.min, v6_range.max, registrant_id)) - cur.execute("UPDATE registrant SET valid_until = %s WHERE registrant_id = %s", - (kid.resources.valid_until, registrant_id)) - for r in s.roa_requests: - cur.execute("INSERT roa_request (self_handle, asn) VALUES (%s, %s)", - (s.name, r.asn)) - roa_request_id = cur.lastrowid - for version, prefix_set in ((4, r.v4), (6, r.v6)): - if prefix_set: - cur.executemany("INSERT roa_request_prefix " - "(roa_request_id, prefix, prefixlen, max_prefixlen, version) " - "VALUES (%s, %s, %s, %s, %s)", - ((roa_request_id, x.prefix, x.prefixlen, x.max_prefixlen, version) - for x in prefix_set)) - for r in s.router_certs: - cur.execute("INSERT ee_certificate (self_handle, pkcs10, gski, cn, sn, eku, valid_until) " - "VALUES (%s, %s, %s, %s, %s, %s, %s)", - (s.name, r.pkcs10.get_DER(), r.gski, r.cn, r.sn, r.eku, s.resources.valid_until)) - ee_certificate_id = cur.lastrowid - cur.executemany("INSERT ee_certificate_asn (ee_certificate_id, start_as, end_as) VALUES (%s, %s, %s)", - ((ee_certificate_id, a.min, a.max) for a in r.asn)) + for sql in pubd_sql: + try: + cur.execute(sql) + except: + if "DROP TABLE IF EXISTS" not in sql.upper(): + raise db.close() + d = dict(pubd_name = pubd_name, + pubd_port = pubd_port, + pubd_db_name = pubd_db_name, + pubd_db_user = pubd_db_user, + pubd_db_pass = pubd_db_pass, + pubd_dir = rsyncd_dir, + irdb_db_name = irdb_db_name, + irdb_db_pass = irdb_db_pass) + f = open(pubd_name + ".conf", "w") + f.write(pubd_fmt_1 % d) + f.close() + global pubd_ta + global pubd_irbe_key + global pubd_irbe_cert + global pubd_pubd_cert + pubd_ta = rpki.x509.X509(Auto_file = pubd_name + "-TA.cer") + pubd_irbe_key = rpki.x509.RSA( Auto_file = pubd_name + "-IRBE.key") + pubd_irbe_cert = rpki.x509.X509(Auto_file = pubd_name + "-IRBE.cer") + pubd_pubd_cert = rpki.x509.X509(Auto_file = pubd_name + "-PUBD.cer") - def run_daemons(self): - """ - Run daemons for this entity. - """ - - logger.info("Running daemons for %s", self.name) - env = dict(os.environ, RPKI_CONF = self.name + ".conf") - self.rpkid_process = subprocess.Popen((prog_python, prog_rpkid, "--foreground", "--log-stdout", "--log-level", "debug") + - (("--profile", self.name + ".prof") if args.profile else ()), - env = env) - self.irdbd_process = subprocess.Popen((prog_python, prog_irdbd, "--foreground", "--log-stdout", "--log-level", "debug"), - env = env) - - def kill_daemons(self): - """ - Kill daemons for this entity. - """ - - # pylint: disable=E1103 - for proc, name in ((self.rpkid_process, "rpkid"), - (self.irdbd_process, "irdbd")): - if proc is not None and proc.poll() is None: - logger.info("Killing daemon %s pid %s for %s", name, proc.pid, self.name) - try: - proc.terminate() - except OSError: - pass - if proc is not None: - logger.info("Daemon %s pid %s for %s exited with code %s", - name, proc.pid, self.name, proc.wait()) - - def call_rpkid(self, pdus, cb): +def call_pubd(pdus, cb): """ - Send a left-right message to this entity's RPKI daemon and return + Send a publication control message to publication daemon and return the response. - - If this entity is hosted (does not run its own RPKI daemon), all - of this happens with the hosting RPKI daemon. """ - logger.info("Calling rpkid for %s", self.name) - - if self.is_hosted: - logger.info("rpkid %s is hosted by rpkid %s, switching", self.name, self.hosted_by.name) - self = self.hosted_by - assert not self.is_hosted - - assert isinstance(pdus, (list, tuple)) - assert self.rpki_port is not None - - q_msg = rpki.left_right.msg.query(*pdus) - q_cms = rpki.left_right.cms_msg_saxify() - q_der = q_cms.wrap(q_msg, self.irbe_key, self.irbe_cert) - q_url = "http://localhost:%d/left-right" % self.rpki_port + logger.info("Calling pubd") + q_msg = rpki.publication_control.msg.query(*pdus) + q_cms = rpki.publication_control.cms_msg_saxify() + q_der = q_cms.wrap(q_msg, pubd_irbe_key, pubd_irbe_cert) + q_url = "http://localhost:%d/control" % pubd_port logger.debug(q_cms.pretty_print_content()) - def done(r_der): - logger.info("Callback from rpkid %s", self.name) - r_cms = rpki.left_right.cms_msg_saxify(DER = r_der) - r_msg = r_cms.unwrap((self.rpkid_ta, self.rpkid_cert)) - self.last_cms_time = r_cms.check_replay(self.last_cms_time, q_url) - logger.debug(r_cms.pretty_print_content()) - assert r_msg.is_reply - for r_pdu in r_msg: - assert not isinstance(r_pdu, rpki.left_right.report_error_elt) - cb(r_msg) + def call_pubd_cb(r_der): + global pubd_last_cms_time + r_cms = rpki.publication_control.cms_msg_saxify(DER = r_der) + r_msg = r_cms.unwrap((pubd_ta, pubd_pubd_cert)) + pubd_last_cms_time = r_cms.check_replay(pubd_last_cms_time, q_url) + logger.debug(r_cms.pretty_print_content()) + assert r_msg.is_reply + for r_pdu in r_msg: + r_pdu.raise_if_error() + cb(r_msg) - def lose(e): - raise + def call_pubd_eb(e): + logger.exception("Problem calling pubd") rpki.http.client( - url = q_url, - msg = q_der, - callback = done, - errback = lose) + url = q_url, + msg = q_der, + callback = call_pubd_cb, + errback = call_pubd_eb) + - def cross_certify(self, certificant, reverse = False): +def cross_certify(certificant, certifier): """ Cross-certify and return the resulting certificate. """ - if reverse: - certifier = certificant - certificant = self.name + "-SELF" - else: - certifier = self.name + "-SELF" - return cross_certify(certificant, certifier) - - def create_rpki_objects(self, cb): - """ - Create RPKI engine objects for this engine. + certfile = certifier + "-" + certificant + ".cer" - Root node of the engine tree is special, it too has a parent but - that one is the magic self-signed micro engine. - - The rest of this is straightforward. There are a lot of objects - to create, but we can do batch them all into one honking PDU, then - issue one more PDU to set BSC EE certificates based on the PKCS - #10 requests we get back when we tell rpkid to generate BSC keys. - """ + logger.info("Cross certifying %s into %s's BPKI (%s)", certificant, certifier, certfile) - assert not self.is_hosted - - selves = [self] + self.hosts - - rpkid_pdus = [] - pubd_pdus = [] - - for i, s in enumerate(selves): - logger.info("Creating RPKI objects for [%d] %s", i, s.name) - - rpkid_pdus.append(rpki.left_right.self_elt.make_pdu( - action = "create", - self_handle = s.name, - crl_interval = s.crl_interval, - regen_margin = s.regen_margin, - bpki_cert = (s.cross_certify(s.hosted_by.name + "-TA", reverse = True) - if s.is_hosted else - rpki.x509.X509(Auto_file = s.name + "-SELF.cer")))) - - rpkid_pdus.append(rpki.left_right.bsc_elt.make_pdu( - action = "create", - self_handle = s.name, - bsc_handle = "b", - generate_keypair = True)) - - pubd_pdus.append(rpki.publication_control.client_elt.make_pdu( - action = "create", - client_handle = s.client_handle, - base_uri = s.sia_base, - bpki_cert = s.cross_certify(pubd_name + "-TA", reverse = True))) - - rpkid_pdus.append(rpki.left_right.repository_elt.make_pdu( - action = "create", - self_handle = s.name, - bsc_handle = "b", - repository_handle = "r", - bpki_cert = s.cross_certify(pubd_name + "-TA"), - peer_contact_uri = "http://localhost:%d/client/%s" % (pubd_port, s.client_handle))) - - for k in s.kids: - rpkid_pdus.append(rpki.left_right.child_elt.make_pdu( - action = "create", - self_handle = s.name, - child_handle = k.name, - bsc_handle = "b", - bpki_cert = s.cross_certify(k.name + "-SELF"))) - - if s.is_root: - rootd_cert = s.cross_certify(rootd_name + "-TA") - rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( - action = "create", - self_handle = s.name, - parent_handle = "rootd", - bsc_handle = "b", - repository_handle = "r", - sia_base = s.sia_base, - bpki_cert = rootd_cert, - sender_name = s.name, - recipient_name = "rootd", - peer_contact_uri = "http://localhost:%s/" % rootd_port)) - else: - rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( - action = "create", - self_handle = s.name, - parent_handle = s.parent.name, - bsc_handle = "b", - repository_handle = "r", - sia_base = s.sia_base, - bpki_cert = s.cross_certify(s.parent.name + "-SELF"), - sender_name = s.name, - recipient_name = s.parent.name, - peer_contact_uri = "http://localhost:%s/up-down/%s/%s" % (s.parent.get_rpki_port(), - s.parent.name, s.name))) - - def one(): - call_pubd(pubd_pdus, cb = two) - - def two(vals): - self.call_rpkid(rpkid_pdus, cb = three) - - def three(vals): - - bsc_dict = dict((b.self_handle, b) for b in vals if isinstance(b, rpki.left_right.bsc_elt)) - - bsc_pdus = [] - - for s in selves: - b = bsc_dict[s.name] - - logger.info("Issuing BSC EE cert for %s", s.name) - cmd = (prog_openssl, "x509", "-req", "-sha256", "-extfile", s.name + "-RPKI.conf", - "-extensions", "req_x509_ext", "-days", "30", - "-CA", s.name + "-SELF.cer", "-CAkey", s.name + "-SELF.key", "-CAcreateserial", "-text") - signer = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) - signed = signer.communicate(input = b.pkcs10_request.get_PEM()) - if not signed[0]: - logger.warning(signed[1]) - raise CouldntIssueBSCEECertificate("Couldn't issue BSC EE certificate") - s.bsc_ee = rpki.x509.X509(PEM = signed[0]) - s.bsc_crl = rpki.x509.CRL(PEM_file = s.name + "-SELF.crl") - logger.info("BSC EE cert for %s SKI %s", s.name, s.bsc_ee.hSKI()) - - bsc_pdus.append(rpki.left_right.bsc_elt.make_pdu( - action = "set", - self_handle = s.name, - bsc_handle = "b", - signing_cert = s.bsc_ee, - signing_cert_crl = s.bsc_crl)) - - self.call_rpkid(bsc_pdus, cb = four) - - def four(vals): - cb() - - one() - - def setup_yaml_leaf(self): - """ - Generate certificates and write YAML scripts for leaf nodes. - - We're cheating a bit here: properly speaking, we can't generate - issue or revoke requests without knowing the class, which is - generated on the fly, but at the moment the test case is - simplistic enough that the class will always be "1", so we just - wire in that value for now. - - Well, ok, we just broke that assumption. Now we do something even - nastier, just to eke a bit more life out of this kludge. This - really needs to be rewritten, but it may require a different tool - than testpoke. - """ + child = rpki.x509.X509(Auto_file = certificant + ".cer") + parent = rpki.x509.X509(Auto_file = certifier + ".cer") + keypair = rpki.x509.RSA(Auto_file = certifier + ".key") + serial_file = certifier + ".srl" - if not os.path.exists(self.name + ".key"): - logger.info("Generating RPKI key for %s", self.name) - subprocess.check_call((prog_openssl, "genrsa", "-out", self.name + ".key", "2048" ), - stdout = subprocess.PIPE, stderr = subprocess.STDOUT) - ski = rpki.x509.RSA(PEM_file = self.name + ".key").gSKI() + now = rpki.sundial.now() + notAfter = now + rpki.sundial.timedelta(days = 30) - if self.parent.is_hosted: - parent_host = self.parent.hosted_by.name - else: - parent_host = self.parent.name + try: + with open(serial_file, "r") as f: + serial = int(f.read().splitlines()[0], 16) + except IOError: + serial = 1 - self.cross_certify(self.parent.name + "-SELF") - self.cross_certify(parent_host + "-TA") + x = parent.bpki_cross_certify( + keypair = keypair, + source_cert = child, + serial = serial, + notAfter = notAfter, + now = now) - def run_cron(self, cb): - """ - Trigger cron run for this engine. - """ + with open(serial_file, "w") as f: + f.write("%02x\n" % (serial + 1)) - logger.info("Running cron for %s", self.name) + with open(certfile, "w") as f: + f.write(x.get_PEM()) - assert self.rpki_port is not None + logger.debug("Cross certified %s:", certfile) + logger.debug(" Issuer %s [%s]", x.getIssuer(), x.hAKI()) + logger.debug(" Subject %s [%s]", x.getSubject(), x.hSKI()) + return x - def done(result): - assert result == "OK", 'Expected "OK" result from cronjob, got %r' % result - cb() - - rpki.http.client( - url = "http://localhost:%d/cronjob" % self.rpki_port, - msg = "Run cron now, please", - callback = done, - errback = done) +last_rcynic_run = None - def run_yaml(self): +def run_rcynic(): """ - Run YAML scripts for this leaf entity. Since we're not bothering - to check the class list returned by the list command, the issue - command may fail, so we treat failure of the list command as an - error, but only issue a warning when issue fails. + Run rcynic to see whether what was published makes sense. """ - logger.info("Running YAML for %s", self.name) - subprocess.check_call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "list")) - if subprocess.call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "issue")) != 0: - logger.warning("YAML issue command failed for %s, continuing", self.name) + logger.info("Running rcynic") + env = os.environ.copy() + env["TZ"] = "" + global last_rcynic_run + if int(time.time()) == last_rcynic_run: + time.sleep(1) + subprocess.check_call((prog_rcynic, "-c", rcynic_name + ".conf"), env = env) + subprocess.call(rcynic_stats, shell = True, env = env) + last_rcynic_run = int(time.time()) + os.link("%s.xml" % rcynic_name, "%s.%s.xml" % (rcynic_name, last_rcynic_run)) -def setup_bpki_cert_chain(name, ee = (), ca = ()): - """ - Build a set of BPKI certificates. - """ +def mangle_sql(filename): + """ + Mangle an SQL file into a sequence of SQL statements. + """ - s = "exec >/dev/null 2>&1\n" - #s = "set -x\n" - for kind in ("TA",) + ee + ca: - d = dict(name = name, - kind = kind, - ca = "false" if kind in ee else "true", - openssl = prog_openssl) - f = open("%(name)s-%(kind)s.conf" % d, "w") - f.write(bpki_cert_fmt_1 % d) + words = [] + f = open(filename) + for line in f: + words.extend(line.partition("--")[0].split()) f.close() - if not os.path.exists("%(name)s-%(kind)s.key" % d): - s += bpki_cert_fmt_2 % d - s += bpki_cert_fmt_3 % d - d = dict(name = name, - openssl = prog_openssl) - s += bpki_cert_fmt_4 % d - for kind in ee + ca: - d["kind"] = kind - s += bpki_cert_fmt_5 % d - for kind in ("TA",) + ca: - d["kind"] = kind - s += bpki_cert_fmt_6 % d - subprocess.check_call(s, shell = True) - -def setup_rootd(rpkid, rootd_yaml, db): - """ - Write the config files for rootd. - """ - - rpkid.cross_certify(rootd_name + "-TA", reverse = True) - cross_certify(pubd_name + "-TA", rootd_name + "-TA") - logger.info("Writing config files for %s", rootd_name) - d = dict(rootd_name = rootd_name, - rootd_port = rootd_port, - rpkid_name = rpkid.name, - pubd_name = pubd_name, - rootd_sia = rootd_sia, - rsyncd_dir = rsyncd_dir, - openssl = prog_openssl, - lifetime = rootd_yaml.get("lifetime", "30d"), - pubd_port = pubd_port, - rootd_handle = db.root.client_handle + "-" + rootd_name) - f = open(rootd_name + ".conf", "w") - f.write(rootd_fmt_1 % d) - f.close() - s = "exec >/dev/null 2>&1\n" - #s = "set -x\n" - if not os.path.exists("root.key"): - s += rootd_fmt_2 % d - s += rootd_fmt_3 % d - subprocess.check_call(s, shell = True) - -def setup_rcynic(): - """ - Write the config file for rcynic. - """ - - logger.info("Config file for rcynic") - d = dict(rcynic_name = rcynic_name, - rootd_name = rootd_name, - rootd_sia = rootd_sia) - f = open(rcynic_name + ".conf", "w") - f.write(rcynic_fmt_1 % d) - f.close() - -def setup_rsyncd(): - """ - Write the config file for rsyncd. - """ - - logger.info("Config file for rsyncd") - d = dict(rsyncd_name = rsyncd_name, - rsyncd_port = rsyncd_port, - rsyncd_module = rsyncd_module, - rsyncd_dir = rsyncd_dir) - f = open(rsyncd_name + ".conf", "w") - f.write(rsyncd_fmt_1 % d) - f.close() - -def setup_publication(pubd_sql, irdb_db_name): - """ - Set up publication daemon. - """ - - logger.info("Configure publication daemon") - publication_dir = os.getcwd() + "/publication" - assert rootd_sia.startswith("rsync://") - global rsyncd_dir - rsyncd_dir = publication_dir + "/".join(rootd_sia.split("/")[4:]) - if not rsyncd_dir.endswith("/"): - rsyncd_dir += "/" - os.makedirs(rsyncd_dir + "root/trunk") - db = MySQLdb.connect(db = pubd_db_name, user = pubd_db_user, passwd = pubd_db_pass, - conv = sql_conversions) - cur = db.cursor() - db.autocommit(True) - for sql in pubd_sql: - try: - cur.execute(sql) - except: - if "DROP TABLE IF EXISTS" not in sql.upper(): - raise - db.close() - d = dict(pubd_name = pubd_name, - pubd_port = pubd_port, - pubd_db_name = pubd_db_name, - pubd_db_user = pubd_db_user, - pubd_db_pass = pubd_db_pass, - pubd_dir = rsyncd_dir, - irdb_db_name = irdb_db_name, - irdb_db_pass = irdb_db_pass) - f = open(pubd_name + ".conf", "w") - f.write(pubd_fmt_1 % d) - f.close() - global pubd_ta - global pubd_irbe_key - global pubd_irbe_cert - global pubd_pubd_cert - pubd_ta = rpki.x509.X509(Auto_file = pubd_name + "-TA.cer") - pubd_irbe_key = rpki.x509.RSA( Auto_file = pubd_name + "-IRBE.key") - pubd_irbe_cert = rpki.x509.X509(Auto_file = pubd_name + "-IRBE.cer") - pubd_pubd_cert = rpki.x509.X509(Auto_file = pubd_name + "-PUBD.cer") - -def call_pubd(pdus, cb): - """ - Send a publication control message to publication daemon and return - the response. - """ - - logger.info("Calling pubd") - q_msg = rpki.publication_control.msg.query(*pdus) - q_cms = rpki.publication_control.cms_msg_saxify() - q_der = q_cms.wrap(q_msg, pubd_irbe_key, pubd_irbe_cert) - q_url = "http://localhost:%d/control" % pubd_port - - logger.debug(q_cms.pretty_print_content()) - - def call_pubd_cb(r_der): - global pubd_last_cms_time - r_cms = rpki.publication_control.cms_msg_saxify(DER = r_der) - r_msg = r_cms.unwrap((pubd_ta, pubd_pubd_cert)) - pubd_last_cms_time = r_cms.check_replay(pubd_last_cms_time, q_url) - logger.debug(r_cms.pretty_print_content()) - assert r_msg.is_reply - for r_pdu in r_msg: - r_pdu.raise_if_error() - cb(r_msg) - - def call_pubd_eb(e): - logger.exception("Problem calling pubd") - - rpki.http.client( - url = q_url, - msg = q_der, - callback = call_pubd_cb, - errback = call_pubd_eb) - - -def cross_certify(certificant, certifier): - """ - Cross-certify and return the resulting certificate. - """ - - certfile = certifier + "-" + certificant + ".cer" - - logger.info("Cross certifying %s into %s's BPKI (%s)", certificant, certifier, certfile) - - child = rpki.x509.X509(Auto_file = certificant + ".cer") - parent = rpki.x509.X509(Auto_file = certifier + ".cer") - keypair = rpki.x509.RSA(Auto_file = certifier + ".key") - serial_file = certifier + ".srl" - - now = rpki.sundial.now() - notAfter = now + rpki.sundial.timedelta(days = 30) - - try: - with open(serial_file, "r") as f: - serial = int(f.read().splitlines()[0], 16) - except IOError: - serial = 1 - - x = parent.bpki_cross_certify( - keypair = keypair, - source_cert = child, - serial = serial, - notAfter = notAfter, - now = now) - - with open(serial_file, "w") as f: - f.write("%02x\n" % (serial + 1)) - - with open(certfile, "w") as f: - f.write(x.get_PEM()) - - logger.debug("Cross certified %s:", certfile) - logger.debug(" Issuer %s [%s]", x.getIssuer(), x.hAKI()) - logger.debug(" Subject %s [%s]", x.getSubject(), x.hSKI()) - return x - -last_rcynic_run = None - -def run_rcynic(): - """ - Run rcynic to see whether what was published makes sense. - """ - - logger.info("Running rcynic") - env = os.environ.copy() - env["TZ"] = "" - global last_rcynic_run - if int(time.time()) == last_rcynic_run: - time.sleep(1) - subprocess.check_call((prog_rcynic, "-c", rcynic_name + ".conf"), env = env) - subprocess.call(rcynic_stats, shell = True, env = env) - last_rcynic_run = int(time.time()) - os.link("%s.xml" % rcynic_name, "%s.%s.xml" % (rcynic_name, last_rcynic_run)) - -def mangle_sql(filename): - """ - Mangle an SQL file into a sequence of SQL statements. - """ - - words = [] - f = open(filename) - for line in f: - words.extend(line.partition("--")[0].split()) - f.close() - return " ".join(words).strip(";").split(";") + return " ".join(words).strip(";").split(";") bpki_cert_fmt_1 = '''\ [req] diff --git a/ca/tests/sql-cleaner.py b/ca/tests/sql-cleaner.py index 828100a4..c518b77b 100644 --- a/ca/tests/sql-cleaner.py +++ b/ca/tests/sql-cleaner.py @@ -25,27 +25,27 @@ cfg = rpki.config.parser(section = "yamltest", allow_missing = True) for name in ("rpkid", "irdbd", "pubd"): - username = cfg.get("%s_sql_username" % name, name[:4]) - password = cfg.get("%s_sql_password" % name, "fnord") + username = cfg.get("%s_sql_username" % name, name[:4]) + password = cfg.get("%s_sql_password" % name, "fnord") - db = MySQLdb.connect(user = username, passwd = password) - cur = db.cursor() + db = MySQLdb.connect(user = username, passwd = password) + cur = db.cursor() - cur.execute("SHOW DATABASES") + cur.execute("SHOW DATABASES") - databases = [r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()] + databases = [r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()] - for database in databases: + for database in databases: - cur.execute("USE " + database) + cur.execute("USE " + database) - cur.execute("SHOW TABLES") - tables = [r[0] for r in cur.fetchall()] + cur.execute("SHOW TABLES") + tables = [r[0] for r in cur.fetchall()] - cur.execute("SET foreign_key_checks = 0") - for table in tables: - cur.execute("DROP TABLE %s" % table) - cur.execute("SET foreign_key_checks = 1") + cur.execute("SET foreign_key_checks = 0") + for table in tables: + cur.execute("DROP TABLE %s" % table) + cur.execute("SET foreign_key_checks = 1") - cur.close() - db.close() + cur.close() + db.close() diff --git a/ca/tests/sql-dumper.py b/ca/tests/sql-dumper.py index d0fe3489..af24f2d4 100644 --- a/ca/tests/sql-dumper.py +++ b/ca/tests/sql-dumper.py @@ -26,18 +26,18 @@ cfg = rpki.config.parser(section = "yamltest", allow_missing = True) for name in ("rpkid", "irdbd", "pubd"): - username = cfg.get("%s_sql_username" % name, name[:4]) - password = cfg.get("%s_sql_password" % name, "fnord") + username = cfg.get("%s_sql_username" % name, name[:4]) + password = cfg.get("%s_sql_password" % name, "fnord") - cmd = ["mysqldump", "-u", username, "-p" + password, "--databases"] + cmd = ["mysqldump", "-u", username, "-p" + password, "--databases"] - db = MySQLdb.connect(user = username, passwd = password) - cur = db.cursor() + db = MySQLdb.connect(user = username, passwd = password) + cur = db.cursor() - cur.execute("SHOW DATABASES") - cmd.extend(r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()) + cur.execute("SHOW DATABASES") + cmd.extend(r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()) - cur.close() - db.close() + cur.close() + db.close() - subprocess.check_call(cmd, stdout = open("backup.%s.sql" % name, "w")) + subprocess.check_call(cmd, stdout = open("backup.%s.sql" % name, "w")) diff --git a/ca/tests/test-rrdp.py b/ca/tests/test-rrdp.py index 1a9db929..97797444 100755 --- a/ca/tests/test-rrdp.py +++ b/ca/tests/test-rrdp.py @@ -38,22 +38,22 @@ parser.add_argument("--dry-run", action = "store_true") args = parser.parse_args() def log(msg): - sys.stdout.write(msg + "\n") - sys.stdout.flush() + sys.stdout.write(msg + "\n") + sys.stdout.flush() def run(*argv): - log("Running: " + " ".join(argv)) - if not args.dry_run: - subprocess.check_call(argv) + log("Running: " + " ".join(argv)) + if not args.dry_run: + subprocess.check_call(argv) def dataglob(pattern): - return glob.iglob(os.path.join(("smoketest.dir" if args.use_smoketest else "yamltest.dir/RIR"), pattern)) + return glob.iglob(os.path.join(("smoketest.dir" if args.use_smoketest else "yamltest.dir/RIR"), pattern)) def snapshot_to_serial(fn): - return int(os.path.splitext(os.path.basename(fn))[0]) + return int(os.path.splitext(os.path.basename(fn))[0]) def delta_to_serial(fn): - return int(os.path.splitext(os.path.basename(fn))[0]) + return int(os.path.splitext(os.path.basename(fn))[0]) top = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "..", "..")) @@ -62,62 +62,62 @@ rcynic = os.path.join(top, "rp/rcynic/rcynic") rcynic_text = os.path.join(top, "rp/rcynic/rcynic-text") with open("rcynic-rrdp.conf", "w") as f: - f.write(textwrap.dedent('''# Automatically generated for RRDP tests, do not edit. - [rcynic] - xml-summary = rcynic.xml - jitter = 0 - use-links = yes - use-syslog = no - use-stderr = yes - log-level = log_debug - run-rsync = no - ''')) - if args.use_smoketest: - f.write("trust-anchor = smoketest.dir/root.cer\n") - else: - f.write("trust-anchor = yamltest.dir/RIR/publication/RIR-root/root.cer\n") + f.write(textwrap.dedent('''# Automatically generated for RRDP tests, do not edit. + [rcynic] + xml-summary = rcynic.xml + jitter = 0 + use-links = yes + use-syslog = no + use-stderr = yes + log-level = log_debug + run-rsync = no + ''')) + if args.use_smoketest: + f.write("trust-anchor = smoketest.dir/root.cer\n") + else: + f.write("trust-anchor = yamltest.dir/RIR/publication/RIR-root/root.cer\n") if args.skip_daemons: - log("--skip-daemons specified, so running neither smoketest nor yamltest") + log("--skip-daemons specified, so running neither smoketest nor yamltest") elif args.use_smoketest: - run("python", "smoketest.py", args.yaml_file) + run("python", "smoketest.py", args.yaml_file) else: - run("python", "sql-cleaner.py") - class GotSIGUSR1(Exception): - pass - def handle_sigusr1(signum, frame): - raise GotSIGUSR1 - old_sigusr1 = signal.signal(signal.SIGUSR1, handle_sigusr1) - cmd = ("python", "yamltest.py", args.yaml_file, "--notify-when-startup-complete", str(os.getpid())) - log("Running: " + " ".join(cmd)) - yamltest = subprocess.Popen(cmd) - log("Waiting for SIGUSR1 from yamltest") - try: - while True: - signal.pause() - except GotSIGUSR1: - signal.signal(signal.SIGUSR1, old_sigusr1) - log("Sleeping %s" % args.delay) - time.sleep(args.delay) - yamltest.terminate() + run("python", "sql-cleaner.py") + class GotSIGUSR1(Exception): + pass + def handle_sigusr1(signum, frame): + raise GotSIGUSR1 + old_sigusr1 = signal.signal(signal.SIGUSR1, handle_sigusr1) + cmd = ("python", "yamltest.py", args.yaml_file, "--notify-when-startup-complete", str(os.getpid())) + log("Running: " + " ".join(cmd)) + yamltest = subprocess.Popen(cmd) + log("Waiting for SIGUSR1 from yamltest") + try: + while True: + signal.pause() + except GotSIGUSR1: + signal.signal(signal.SIGUSR1, old_sigusr1) + log("Sleeping %s" % args.delay) + time.sleep(args.delay) + yamltest.terminate() snapshots = dict((snapshot_to_serial(fn), fn) for fn in dataglob("rrdp-publication/*/snapshot/*.xml")) deltas = dict((delta_to_serial(fn), fn) for fn in dataglob("rrdp-publication/*/deltas/*.xml")) for snapshot in sorted(snapshots): - time.sleep(1) - run("rm", "-rf", "rcynic-data") - run(rrdp_test_tool, snapshots[snapshot]) - run(rcynic, "-c", "rcynic-rrdp.conf") - run(rcynic_text, "rcynic.xml") - - for delta in sorted(deltas): - if delta > snapshot: - time.sleep(1) - run(rrdp_test_tool, deltas[delta]) - run(rcynic, "-c", "rcynic-rrdp.conf") - run(rcynic_text, "rcynic.xml") - - if not args.exhaustive: - break + time.sleep(1) + run("rm", "-rf", "rcynic-data") + run(rrdp_test_tool, snapshots[snapshot]) + run(rcynic, "-c", "rcynic-rrdp.conf") + run(rcynic_text, "rcynic.xml") + + for delta in sorted(deltas): + if delta > snapshot: + time.sleep(1) + run(rrdp_test_tool, deltas[delta]) + run(rcynic, "-c", "rcynic-rrdp.conf") + run(rcynic_text, "rcynic.xml") + + if not args.exhaustive: + break diff --git a/ca/tests/testpoke.py b/ca/tests/testpoke.py index 68f967b9..60cc5690 100644 --- a/ca/tests/testpoke.py +++ b/ca/tests/testpoke.py @@ -58,80 +58,80 @@ yaml_data = yaml.load(args.yaml) yaml_cmd = args.request if yaml_cmd is None and len(yaml_data["requests"]) == 1: - yaml_cmd = yaml_data["requests"].keys()[0] + yaml_cmd = yaml_data["requests"].keys()[0] yaml_req = yaml_data["requests"][yaml_cmd] def get_PEM(name, cls, y = yaml_data): - if name in y: - return cls(PEM = y[name]) - if name + "-file" in y: - return cls(PEM_file = y[name + "-file"]) - return None + if name in y: + return cls(PEM = y[name]) + if name + "-file" in y: + return cls(PEM_file = y[name + "-file"]) + return None def get_PEM_chain(name, cert = None): - chain = [] - if cert is not None: - chain.append(cert) - if name in yaml_data: - chain.extend(rpki.x509.X509(PEM = x) for x in yaml_data[name]) - elif name + "-file" in yaml_data: - chain.extend(rpki.x509.X509(PEM_file = x) for x in yaml_data[name + "-file"]) - return chain + chain = [] + if cert is not None: + chain.append(cert) + if name in yaml_data: + chain.extend(rpki.x509.X509(PEM = x) for x in yaml_data[name]) + elif name + "-file" in yaml_data: + chain.extend(rpki.x509.X509(PEM_file = x) for x in yaml_data[name + "-file"]) + return chain def query_up_down(q_pdu): - q_msg = rpki.up_down.message_pdu.make_query( - payload = q_pdu, - sender = yaml_data["sender-id"], - recipient = yaml_data["recipient-id"]) - q_der = rpki.up_down.cms_msg_saxify().wrap(q_msg, cms_key, cms_certs, cms_crl) - - def done(r_der): - global last_cms_timestamp - r_cms = rpki.up_down.cms_msg_saxify(DER = r_der) - r_msg = r_cms.unwrap([cms_ta] + cms_ca_certs) - last_cms_timestamp = r_cms.check_replay(last_cms_timestamp) - print r_cms.pretty_print_content() - try: - r_msg.payload.check_response() - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: - fail(e) - - rpki.http.want_persistent_client = False - - rpki.http.client( - msg = q_der, - url = yaml_data["posturl"], - callback = done, - errback = fail, - content_type = rpki.up_down.content_type) + q_msg = rpki.up_down.message_pdu.make_query( + payload = q_pdu, + sender = yaml_data["sender-id"], + recipient = yaml_data["recipient-id"]) + q_der = rpki.up_down.cms_msg_saxify().wrap(q_msg, cms_key, cms_certs, cms_crl) + + def done(r_der): + global last_cms_timestamp + r_cms = rpki.up_down.cms_msg_saxify(DER = r_der) + r_msg = r_cms.unwrap([cms_ta] + cms_ca_certs) + last_cms_timestamp = r_cms.check_replay(last_cms_timestamp) + print r_cms.pretty_print_content() + try: + r_msg.payload.check_response() + except (rpki.async.ExitNow, SystemExit): + raise + except Exception, e: + fail(e) + + rpki.http.want_persistent_client = False + + rpki.http.client( + msg = q_der, + url = yaml_data["posturl"], + callback = done, + errback = fail, + content_type = rpki.up_down.content_type) def do_list(): - query_up_down(rpki.up_down.list_pdu()) + query_up_down(rpki.up_down.list_pdu()) def do_issue(): - q_pdu = rpki.up_down.issue_pdu() - req_key = get_PEM("cert-request-key", rpki.x509.RSA, yaml_req) or cms_key - q_pdu.class_name = yaml_req["class"] - q_pdu.pkcs10 = rpki.x509.PKCS10.create( - keypair = req_key, - is_ca = True, - caRepository = yaml_req["sia"][0], - rpkiManifest = yaml_req["sia"][0] + req_key.gSKI() + ".mft") - query_up_down(q_pdu) + q_pdu = rpki.up_down.issue_pdu() + req_key = get_PEM("cert-request-key", rpki.x509.RSA, yaml_req) or cms_key + q_pdu.class_name = yaml_req["class"] + q_pdu.pkcs10 = rpki.x509.PKCS10.create( + keypair = req_key, + is_ca = True, + caRepository = yaml_req["sia"][0], + rpkiManifest = yaml_req["sia"][0] + req_key.gSKI() + ".mft") + query_up_down(q_pdu) def do_revoke(): - q_pdu = rpki.up_down.revoke_pdu() - q_pdu.class_name = yaml_req["class"] - q_pdu.ski = yaml_req["ski"] - query_up_down(q_pdu) + q_pdu = rpki.up_down.revoke_pdu() + q_pdu.class_name = yaml_req["class"] + q_pdu.ski = yaml_req["ski"] + query_up_down(q_pdu) dispatch = { "list" : do_list, "issue" : do_issue, "revoke" : do_revoke } def fail(e): # pylint: disable=W0621 - sys.exit("Testpoke failed: %s" % e) + sys.exit("Testpoke failed: %s" % e) cms_ta = get_PEM("cms-ca-cert", rpki.x509.X509) cms_cert = get_PEM("cms-cert", rpki.x509.X509) @@ -143,7 +143,7 @@ cms_ca_certs = get_PEM_chain("cms-ca-certs") last_cms_timestamp = None try: - dispatch[yaml_req["type"]]() - rpki.async.event_loop() + dispatch[yaml_req["type"]]() + rpki.async.event_loop() except Exception, e: - fail(e) + fail(e) diff --git a/ca/tests/xml-parse-test.py b/ca/tests/xml-parse-test.py index 90c80775..f24d5683 100644 --- a/ca/tests/xml-parse-test.py +++ b/ca/tests/xml-parse-test.py @@ -39,70 +39,70 @@ import rpki.relaxng verbose = False def test(fileglob, rng, parser, encoding, tester = None): - files = glob.glob(fileglob) - files.sort() - for f in files: - print "<!--", f, "-->" - elt_in = lxml.etree.parse(f).getroot() - if verbose: - print "<!-- Input -->" - print lxml.etree.tostring(elt_in, pretty_print = True, encoding = encoding, xml_declaration = True) - rng.assertValid(elt_in) - parsed = parser.fromXML(elt_in) - elt_out = parsed.toXML() - if verbose: - print "<!-- Output -->" - print lxml.etree.tostring(elt_out, pretty_print = True, encoding = encoding, xml_declaration = True) - rng.assertValid(elt_out) - if tester: - tester(elt_in, elt_out, parsed) - if verbose: - print + files = glob.glob(fileglob) + files.sort() + for f in files: + print "<!--", f, "-->" + elt_in = lxml.etree.parse(f).getroot() + if verbose: + print "<!-- Input -->" + print lxml.etree.tostring(elt_in, pretty_print = True, encoding = encoding, xml_declaration = True) + rng.assertValid(elt_in) + parsed = parser.fromXML(elt_in) + elt_out = parsed.toXML() + if verbose: + print "<!-- Output -->" + print lxml.etree.tostring(elt_out, pretty_print = True, encoding = encoding, xml_declaration = True) + rng.assertValid(elt_out) + if tester: + tester(elt_in, elt_out, parsed) + if verbose: + print def pprint(pairs): - if verbose: - for thing, name in pairs: - if thing is not None: - print "[%s]" % name - print thing.get_POW().pprint() + if verbose: + for thing, name in pairs: + if thing is not None: + print "[%s]" % name + print thing.get_POW().pprint() def ud_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.up_down.message_pdu) - if isinstance(msg.payload, rpki.up_down.list_response_pdu): - for c in msg.payload.classes: - pprint([(c.certs[i].cert, ("%s certificate #%d" % (c.class_name, i))) for i in xrange(len(c.certs))] + [(c.issuer, ("%s issuer" % c.class_name))]) + assert isinstance(msg, rpki.up_down.message_pdu) + if isinstance(msg.payload, rpki.up_down.list_response_pdu): + for c in msg.payload.classes: + pprint([(c.certs[i].cert, ("%s certificate #%d" % (c.class_name, i))) for i in xrange(len(c.certs))] + [(c.issuer, ("%s issuer" % c.class_name))]) def lr_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.left_right.msg) - for obj in msg: - if isinstance(obj, rpki.left_right.self_elt): - pprint(((obj.bpki_cert, "BPKI cert"), - (obj.bpki_glue, "BPKI glue"))) - if isinstance(obj, rpki.left_right.bsc_elt): - pprint(((obj.signing_cert, "Signing certificate"), - (obj.signing_cert_crl, "Signing certificate CRL"))) - # (obj.pkcs10_request, "PKCS #10 request") - if isinstance(obj, rpki.left_right.parent_elt): - pprint(((obj.bpki_cert, "BPKI certificate"), - (obj.bpki_glue, "BPKI glue"))) - if isinstance(obj, (rpki.left_right.child_elt, rpki.left_right.repository_elt)): - pprint(((obj.bpki_cert, "BPKI certificate"), - (obj.bpki_glue, "BPKI glue"))) + assert isinstance(msg, rpki.left_right.msg) + for obj in msg: + if isinstance(obj, rpki.left_right.self_elt): + pprint(((obj.bpki_cert, "BPKI cert"), + (obj.bpki_glue, "BPKI glue"))) + if isinstance(obj, rpki.left_right.bsc_elt): + pprint(((obj.signing_cert, "Signing certificate"), + (obj.signing_cert_crl, "Signing certificate CRL"))) + # (obj.pkcs10_request, "PKCS #10 request") + if isinstance(obj, rpki.left_right.parent_elt): + pprint(((obj.bpki_cert, "BPKI certificate"), + (obj.bpki_glue, "BPKI glue"))) + if isinstance(obj, (rpki.left_right.child_elt, rpki.left_right.repository_elt)): + pprint(((obj.bpki_cert, "BPKI certificate"), + (obj.bpki_glue, "BPKI glue"))) def pp_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.publication.msg) - for obj in msg: - if isinstance(obj, rpki.publication.publish_elt): - pprint(((obj.payload, "Publish object"),)) - if isinstance(obj, rpki.publication.withdraw_elt): - pprint(((None, "Withdraw object"),)) + assert isinstance(msg, rpki.publication.msg) + for obj in msg: + if isinstance(obj, rpki.publication.publish_elt): + pprint(((obj.payload, "Publish object"),)) + if isinstance(obj, rpki.publication.withdraw_elt): + pprint(((None, "Withdraw object"),)) def pc_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.publication_control.msg) - for obj in msg: - if isinstance(obj, rpki.publication_control.client_elt): - pprint(((obj.bpki_cert, "BPKI cert"), - (obj.bpki_glue, "BPKI glue"))) + assert isinstance(msg, rpki.publication_control.msg) + for obj in msg: + if isinstance(obj, rpki.publication_control.client_elt): + pprint(((obj.bpki_cert, "BPKI cert"), + (obj.bpki_glue, "BPKI glue"))) test(fileglob = "up-down-protocol-samples/*.xml", rng = rpki.relaxng.up_down, diff --git a/ca/tests/yamlconf.py b/ca/tests/yamlconf.py index 13456377..52c4da26 100644 --- a/ca/tests/yamlconf.py +++ b/ca/tests/yamlconf.py @@ -75,787 +75,787 @@ config_overrides = { "pubd_sql_username" : "pubd", "pubd_sql_password" : "fnord" } def cleanpath(*names): - return os.path.normpath(os.path.join(*names)) + return os.path.normpath(os.path.join(*names)) this_dir = os.getcwd() test_dir = None rpki_conf = None class roa_request(object): - """ - Representation of a ROA request. - """ - - def __init__(self, asn, ipv4, ipv6): - self.asn = asn - self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None - self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - - def __eq__(self, other): - return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - - def __hash__(self): - v4 = tuple(self.v4) if self.v4 is not None else None - v6 = tuple(self.v6) if self.v6 is not None else None - return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - - def __str__(self): - if self.v4 and self.v6: - return "%s: %s,%s" % (self.asn, self.v4, self.v6) - else: - return "%s: %s" % (self.asn, self.v4 or self.v6) + """ + Representation of a ROA request. + """ - @classmethod - def parse(cls, y): - return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None -class router_cert(object): - """ - Representation for a router_cert object. - """ + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - _ecparams = None + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - @classmethod - def ecparams(cls): - if cls._ecparams is None: - cls._ecparams = rpki.x509.KeyParams.generateEC() - return cls._ecparams + def __str__(self): + if self.v4 and self.v6: + return "%s: %s,%s" % (self.asn, self.v4, self.v6) + else: + return "%s: %s" % (self.asn, self.v4 or self.v6) - def __init__(self, asn, router_id): - self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) - self.router_id = router_id - self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) - self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) - self.gski = self.pkcs10.gSKI() + @classmethod + def parse(cls, y): + return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) - def __eq__(self, other): - return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski +class router_cert(object): + """ + Representation for a router_cert object. + """ - def __hash__(self): - return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() + _ecparams = None - def __str__(self): - return "%s: %s: %s" % (self.asn, self.router_id, self.gski) + @classmethod + def ecparams(cls): + if cls._ecparams is None: + cls._ecparams = rpki.x509.KeyParams.generateEC() + return cls._ecparams - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("router_id")) + def __init__(self, asn, router_id): + self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) + self.router_id = router_id + self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) + self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) + self.gski = self.pkcs10.gSKI() + def __eq__(self, other): + return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski -class allocation_db(list): - """ - Allocation database. - """ - - def __init__(self, y): - list.__init__(self) - self.root = allocation(y, self) - assert self.root.is_root - if self.root.crl_interval is None: - self.root.crl_interval = 60 * 60 - if self.root.regen_margin is None: - self.root.regen_margin = 24 * 60 * 60 - if self.root.base.valid_until is None: - self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) - for a in self: - if a.base.valid_until is None: - a.base.valid_until = a.parent.base.valid_until - if a.crl_interval is None: - a.crl_interval = a.parent.crl_interval - if a.regen_margin is None: - a.regen_margin = a.parent.regen_margin - self.root.closure() - self.map = dict((a.name, a) for a in self) - for a in self: - if a.is_hosted: - a.hosted_by = self.map[a.hosted_by] - a.hosted_by.hosts.append(a) - assert not a.is_root and not a.hosted_by.is_hosted - - def dump(self): - for a in self: - a.dump() + def __hash__(self): + return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() + def __str__(self): + return "%s: %s: %s" % (self.asn, self.router_id, self.gski) -class allocation(object): - """ - One entity in our allocation database. Every entity in the database - is assumed to hold resources. Entities that don't have the - hosted_by property run their own copies of rpkid, irdbd, and pubd. - """ - - base_port = 4400 - base_engine = -1 - parent = None - crl_interval = None - regen_margin = None - engine = -1 - rpkid_port = 4404 - irdbd_port = 4403 - pubd_port = 4402 - rootd_port = 4401 - rsync_port = 873 - - @classmethod - def allocate_port(cls): - cls.base_port += 1 - return cls.base_port - - @classmethod - def allocate_engine(cls): - cls.base_engine += 1 - return cls.base_engine - - def __init__(self, y, db, parent = None): - db.append(self) - self.name = y["name"] - self.parent = parent - self.kids = [allocation(k, db, self) for k in y.get("kids", ())] - valid_until = None - if "valid_until" in y: - valid_until = rpki.sundial.datetime.from_datetime(y.get("valid_until")) - if valid_until is None and "valid_for" in y: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(y["valid_for"]) - self.base = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as(y.get("asn")), - v4 = rpki.resource_set.resource_set_ipv4(y.get("ipv4")), - v6 = rpki.resource_set.resource_set_ipv6(y.get("ipv6")), - valid_until = valid_until) - if "crl_interval" in y: - self.crl_interval = rpki.sundial.timedelta.parse(y["crl_interval"]).convert_to_seconds() - if "regen_margin" in y: - self.regen_margin = rpki.sundial.timedelta.parse(y["regen_margin"]).convert_to_seconds() - if "ghostbusters" in y: - self.ghostbusters = y.get("ghostbusters") - elif "ghostbuster" in y: - self.ghostbusters = [y.get("ghostbuster")] - else: - self.ghostbusters = [] - self.roa_requests = [roa_request.parse(r) for r in y.get("roa_request", ())] - self.router_certs = [router_cert.parse(r) for r in y.get("router_cert", ())] - for r in self.roa_requests: - if r.v4: - self.base.v4 |= r.v4.to_resource_set() - if r.v6: - self.base.v6 |= r.v6.to_resource_set() - for r in self.router_certs: - self.base.asn |= r.asn - self.hosted_by = y.get("hosted_by") - self.hosts = [] - if not self.is_hosted: - self.engine = self.allocate_engine() - if loopback and not self.is_hosted: - self.rpkid_port = self.allocate_port() - self.irdbd_port = self.allocate_port() - if loopback and self.runs_pubd: - self.pubd_port = self.allocate_port() - self.rsync_port = self.allocate_port() - if loopback and self.is_root: - self.rootd_port = self.allocate_port() - - def closure(self): - resources = self.base - for kid in self.kids: - resources |= kid.closure() - self.resources = resources - return resources - - @property - def hostname(self): - if loopback: - return "localhost" - elif dns_suffix: - return self.name + "." + dns_suffix.lstrip(".") - else: - return self.name + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("router_id")) - @property - def rsync_server(self): - if loopback: - return "%s:%s" % (self.pubd.hostname, self.pubd.rsync_port) - else: - return self.pubd.hostname - def dump(self): - if not quiet: - print str(self) - - def __str__(self): - s = self.name + ":\n" - if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn - if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 - if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 - if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) - if self.parent: s += " Up: %s\n" % self.parent.name - if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name - if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) - for r in self.roa_requests: s += " ROA: %s\n" % r - if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port - if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port - if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port - if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port - if self.is_root: s += " TPort: %s\n" % self.rootd_port - return s + " Until: %s\n" % self.resources.valid_until - - @property - def is_root(self): - return self.parent is None - - @property - def is_hosted(self): - return self.hosted_by is not None - - @property - def runs_pubd(self): - return self.is_root or not (self.is_hosted or only_one_pubd) - - def path(self, *names): - return cleanpath(test_dir, self.host.name, *names) - - def csvout(self, fn): - path = self.path(fn) - if not quiet: - print "Writing", path - return rpki.csv_utils.csv_writer(path) - - def up_down_url(self): - return "http://%s:%d/up-down/%s/%s" % (self.parent.host.hostname, - self.parent.host.rpkid_port, - self.parent.name, - self.name) - - def dump_asns(self, fn): - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, a) for a in k.resources.asn) - - def dump_prefixes(self, fn): - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) - - def dump_roas(self, fn): - with self.csvout(fn) as f: - for g1, r in enumerate(self.roa_requests): - f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) - for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) - - def dump_ghostbusters(self, fn): - if self.ghostbusters: - path = self.path(fn) - if not quiet: - print "Writing", path - with open(path, "w") as f: - for i, g in enumerate(self.ghostbusters): - if i > 0: - f.write("\n") - f.write(g) - - def dump_router_certificates(self, fn): - if self.router_certs: - path = self.path(fn) - if not quiet: - print "Writing", path - xmlns = rpki.relaxng.router_certificate.xmlns - xml = lxml.etree.Element(xmlns + "router_certificate_requests", - version = rpki.relaxng.router_certificate.version, - nsmap = rpki.relaxng.router_certificate.nsmap) - for r in self.router_certs: - x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", - router_id = str(r.router_id), - asn = str(r.asn), - valid_until = str(self.resources.valid_until)) - x.text = r.pkcs10.get_Base64() - rpki.relaxng.router_certificate.assertValid(xml) - lxml.etree.ElementTree(xml).write(path, pretty_print = True) - - @property - def pubd(self): - s = self - while not s.runs_pubd: - s = s.parent - return s - - @property - def client_handle(self): - path = [] - s = self - if not flat_publication: - while not s.runs_pubd: - path.append(s) - s = s.parent - path.append(s) - return ".".join(i.name for i in reversed(path)) - - @property - def host(self): - return self.hosted_by or self - - @property - def publication_base_directory(self): - if not loopback and publication_base is not None: - return publication_base - else: - return self.path("publication") +class allocation_db(list): + """ + Allocation database. + """ + + def __init__(self, y): + list.__init__(self) + self.root = allocation(y, self) + assert self.root.is_root + if self.root.crl_interval is None: + self.root.crl_interval = 60 * 60 + if self.root.regen_margin is None: + self.root.regen_margin = 24 * 60 * 60 + if self.root.base.valid_until is None: + self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) + for a in self: + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + self.root.closure() + self.map = dict((a.name, a) for a in self) + for a in self: + if a.is_hosted: + a.hosted_by = self.map[a.hosted_by] + a.hosted_by.hosts.append(a) + assert not a.is_root and not a.hosted_by.is_hosted + + def dump(self): + for a in self: + a.dump() - @property - def publication_root_directory(self): - if not loopback and publication_root is not None: - return publication_root - else: - return self.path("publication.root") - - def dump_conf(self): - - r = dict( - handle = self.name, - run_rpkid = str(not self.is_hosted), - run_pubd = str(self.runs_pubd), - run_rootd = str(self.is_root), - irdbd_sql_username = "irdb", - rpkid_sql_username = "rpki", - rpkid_server_host = self.hostname, - rpkid_server_port = str(self.rpkid_port), - irdbd_server_host = "localhost", - irdbd_server_port = str(self.irdbd_port), - rootd_server_port = str(self.rootd_port), - pubd_sql_username = "pubd", - pubd_server_host = self.pubd.hostname, - pubd_server_port = str(self.pubd.pubd_port), - publication_rsync_server = self.rsync_server) - - if loopback: - r.update( - irdbd_sql_database = self.irdb_name, - rpkid_sql_database = "rpki%d" % self.engine, - pubd_sql_database = "pubd%d" % self.engine, - bpki_servers_directory = self.path(), - publication_base_directory = self.publication_base_directory) - - r.update(config_overrides) - - with open(self.path("rpki.conf"), "w") as f: - f.write("# Automatically generated, do not edit\n") - if not quiet: - print "Writing", f.name - - section = None - for line in open(rpki_conf): - m = section_regexp.match(line) - if m: - section = m.group(1) - m = variable_regexp.match(line) - option = m.group(1) if m and section == "myrpki" else None - if option and option in r: - line = "%s = %s\n" % (option, r[option]) - f.write(line) - - def dump_rsyncd(self): - lines = [] - if self.runs_pubd: - lines.extend(( - "# Automatically generated, do not edit", - "port = %d" % self.rsync_port, - "address = %s" % self.hostname, - "log file = rsyncd.log", - "read only = yes", - "use chroot = no", - "[rpki]", - "path = %s" % self.publication_base_directory, - "comment = RPKI test")) - if self.is_root: - assert self.runs_pubd - lines.extend(( - "[root]", - "path = %s" % self.publication_root_directory, - "comment = RPKI test root")) - if lines: - with open(self.path("rsyncd.conf"), "w") as f: - if not quiet: - print "Writing", f.name - f.writelines(line + "\n" for line in lines) - - @property - def irdb_name(self): - return "irdb%d" % self.host.engine - - @property - def irdb(self): - prior_name = self.zoo.handle - return rpki.irdb.database( - self.irdb_name, - on_entry = lambda: self.zoo.reset_identity(self.name), - on_exit = lambda: self.zoo.reset_identity(prior_name)) - - def syncdb(self): - import django.core.management - assert not self.is_hosted - django.core.management.call_command( - "syncdb", - verbosity = 0, - database = self.irdb_name, - migrate = True, - load_initial_data = False, - interactive = False) - - def hire_zookeeper(self): - assert not self.is_hosted - self._zoo = rpki.irdb.Zookeeper( - cfg = rpki.config.parser(filename = self.path("rpki.conf")), - logstream = None if quiet else sys.stdout) - - @property - def zoo(self): - return self.host._zoo - - def dump_root(self): - - assert self.is_root and not self.is_hosted - - root_resources = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as("0-4294967295"), - v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), - v6 = rpki.resource_set.resource_set_ipv6("::/0")) - - root_key = rpki.x509.RSA.generate(quiet = True) - - root_uri = "rsync://%s/rpki/" % self.rsync_server - - root_sia = (root_uri, root_uri + "root.mft", None, rpki.publication.rrdp_sia_uri_kludge) - - root_cert = rpki.x509.X509.self_certify( - keypair = root_key, - subject_key = root_key.get_public(), - serial = 1, - sia = root_sia, - notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), - resources = root_resources) - - with open(self.path("root.cer"), "wb") as f: - f.write(root_cert.get_DER()) - - with open(self.path("root.key"), "wb") as f: - f.write(root_key.get_DER()) - - with open(cleanpath(test_dir, "root.tal"), "w") as f: - f.write("rsync://%s/root/root.cer\n\n" % self.rsync_server) - f.write(root_key.get_public().get_Base64()) - - def mkdir(self, *path): - path = self.path(*path) - if not quiet: - print "Creating directory", path - os.makedirs(path) - def dump_sql(self): - if not self.is_hosted: - with open(self.path("rpkid.sql"), "w") as f: +class allocation(object): + """ + One entity in our allocation database. Every entity in the database + is assumed to hold resources. Entities that don't have the + hosted_by property run their own copies of rpkid, irdbd, and pubd. + """ + + base_port = 4400 + base_engine = -1 + parent = None + crl_interval = None + regen_margin = None + engine = -1 + rpkid_port = 4404 + irdbd_port = 4403 + pubd_port = 4402 + rootd_port = 4401 + rsync_port = 873 + + @classmethod + def allocate_port(cls): + cls.base_port += 1 + return cls.base_port + + @classmethod + def allocate_engine(cls): + cls.base_engine += 1 + return cls.base_engine + + def __init__(self, y, db, parent = None): + db.append(self) + self.name = y["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in y.get("kids", ())] + valid_until = None + if "valid_until" in y: + valid_until = rpki.sundial.datetime.from_datetime(y.get("valid_until")) + if valid_until is None and "valid_for" in y: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(y["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(y.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(y.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(y.get("ipv6")), + valid_until = valid_until) + if "crl_interval" in y: + self.crl_interval = rpki.sundial.timedelta.parse(y["crl_interval"]).convert_to_seconds() + if "regen_margin" in y: + self.regen_margin = rpki.sundial.timedelta.parse(y["regen_margin"]).convert_to_seconds() + if "ghostbusters" in y: + self.ghostbusters = y.get("ghostbusters") + elif "ghostbuster" in y: + self.ghostbusters = [y.get("ghostbuster")] + else: + self.ghostbusters = [] + self.roa_requests = [roa_request.parse(r) for r in y.get("roa_request", ())] + self.router_certs = [router_cert.parse(r) for r in y.get("router_cert", ())] + for r in self.roa_requests: + if r.v4: + self.base.v4 |= r.v4.to_resource_set() + if r.v6: + self.base.v6 |= r.v6.to_resource_set() + for r in self.router_certs: + self.base.asn |= r.asn + self.hosted_by = y.get("hosted_by") + self.hosts = [] + if not self.is_hosted: + self.engine = self.allocate_engine() + if loopback and not self.is_hosted: + self.rpkid_port = self.allocate_port() + self.irdbd_port = self.allocate_port() + if loopback and self.runs_pubd: + self.pubd_port = self.allocate_port() + self.rsync_port = self.allocate_port() + if loopback and self.is_root: + self.rootd_port = self.allocate_port() + + def closure(self): + resources = self.base + for kid in self.kids: + resources |= kid.closure() + self.resources = resources + return resources + + @property + def hostname(self): + if loopback: + return "localhost" + elif dns_suffix: + return self.name + "." + dns_suffix.lstrip(".") + else: + return self.name + + @property + def rsync_server(self): + if loopback: + return "%s:%s" % (self.pubd.hostname, self.pubd.rsync_port) + else: + return self.pubd.hostname + + def dump(self): if not quiet: - print "Writing", f.name - f.write(rpki.sql_schemas.rpkid) - if self.runs_pubd: - with open(self.path("pubd.sql"), "w") as f: + print str(self) + + def __str__(self): + s = self.name + ":\n" + if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name + if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) + for r in self.roa_requests: s += " ROA: %s\n" % r + if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port + if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port + if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port + if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port + if self.is_root: s += " TPort: %s\n" % self.rootd_port + return s + " Until: %s\n" % self.resources.valid_until + + @property + def is_root(self): + return self.parent is None + + @property + def is_hosted(self): + return self.hosted_by is not None + + @property + def runs_pubd(self): + return self.is_root or not (self.is_hosted or only_one_pubd) + + def path(self, *names): + return cleanpath(test_dir, self.host.name, *names) + + def csvout(self, fn): + path = self.path(fn) if not quiet: - print "Writing", f.name - f.write(rpki.sql_schemas.pubd) - if not self.is_hosted: - username = config_overrides["irdbd_sql_username"] - password = config_overrides["irdbd_sql_password"] - cmd = ("mysqldump", "-u", username, "-p" + password, self.irdb_name) - with open(self.path("irdbd.sql"), "w") as f: + print "Writing", path + return rpki.csv_utils.csv_writer(path) + + def up_down_url(self): + return "http://%s:%d/up-down/%s/%s" % (self.parent.host.hostname, + self.parent.host.rpkid_port, + self.parent.name, + self.name) + + def dump_asns(self, fn): + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, a) for a in k.resources.asn) + + def dump_prefixes(self, fn): + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) + + def dump_roas(self, fn): + with self.csvout(fn) as f: + for g1, r in enumerate(self.roa_requests): + f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) + for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) + + def dump_ghostbusters(self, fn): + if self.ghostbusters: + path = self.path(fn) + if not quiet: + print "Writing", path + with open(path, "w") as f: + for i, g in enumerate(self.ghostbusters): + if i > 0: + f.write("\n") + f.write(g) + + def dump_router_certificates(self, fn): + if self.router_certs: + path = self.path(fn) + if not quiet: + print "Writing", path + xmlns = rpki.relaxng.router_certificate.xmlns + xml = lxml.etree.Element(xmlns + "router_certificate_requests", + version = rpki.relaxng.router_certificate.version, + nsmap = rpki.relaxng.router_certificate.nsmap) + for r in self.router_certs: + x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", + router_id = str(r.router_id), + asn = str(r.asn), + valid_until = str(self.resources.valid_until)) + x.text = r.pkcs10.get_Base64() + rpki.relaxng.router_certificate.assertValid(xml) + lxml.etree.ElementTree(xml).write(path, pretty_print = True) + + @property + def pubd(self): + s = self + while not s.runs_pubd: + s = s.parent + return s + + @property + def client_handle(self): + path = [] + s = self + if not flat_publication: + while not s.runs_pubd: + path.append(s) + s = s.parent + path.append(s) + return ".".join(i.name for i in reversed(path)) + + @property + def host(self): + return self.hosted_by or self + + @property + def publication_base_directory(self): + if not loopback and publication_base is not None: + return publication_base + else: + return self.path("publication") + + @property + def publication_root_directory(self): + if not loopback and publication_root is not None: + return publication_root + else: + return self.path("publication.root") + + def dump_conf(self): + + r = dict( + handle = self.name, + run_rpkid = str(not self.is_hosted), + run_pubd = str(self.runs_pubd), + run_rootd = str(self.is_root), + irdbd_sql_username = "irdb", + rpkid_sql_username = "rpki", + rpkid_server_host = self.hostname, + rpkid_server_port = str(self.rpkid_port), + irdbd_server_host = "localhost", + irdbd_server_port = str(self.irdbd_port), + rootd_server_port = str(self.rootd_port), + pubd_sql_username = "pubd", + pubd_server_host = self.pubd.hostname, + pubd_server_port = str(self.pubd.pubd_port), + publication_rsync_server = self.rsync_server) + + if loopback: + r.update( + irdbd_sql_database = self.irdb_name, + rpkid_sql_database = "rpki%d" % self.engine, + pubd_sql_database = "pubd%d" % self.engine, + bpki_servers_directory = self.path(), + publication_base_directory = self.publication_base_directory) + + r.update(config_overrides) + + with open(self.path("rpki.conf"), "w") as f: + f.write("# Automatically generated, do not edit\n") + if not quiet: + print "Writing", f.name + + section = None + for line in open(rpki_conf): + m = section_regexp.match(line) + if m: + section = m.group(1) + m = variable_regexp.match(line) + option = m.group(1) if m and section == "myrpki" else None + if option and option in r: + line = "%s = %s\n" % (option, r[option]) + f.write(line) + + def dump_rsyncd(self): + lines = [] + if self.runs_pubd: + lines.extend(( + "# Automatically generated, do not edit", + "port = %d" % self.rsync_port, + "address = %s" % self.hostname, + "log file = rsyncd.log", + "read only = yes", + "use chroot = no", + "[rpki]", + "path = %s" % self.publication_base_directory, + "comment = RPKI test")) + if self.is_root: + assert self.runs_pubd + lines.extend(( + "[root]", + "path = %s" % self.publication_root_directory, + "comment = RPKI test root")) + if lines: + with open(self.path("rsyncd.conf"), "w") as f: + if not quiet: + print "Writing", f.name + f.writelines(line + "\n" for line in lines) + + @property + def irdb_name(self): + return "irdb%d" % self.host.engine + + @property + def irdb(self): + prior_name = self.zoo.handle + return rpki.irdb.database( + self.irdb_name, + on_entry = lambda: self.zoo.reset_identity(self.name), + on_exit = lambda: self.zoo.reset_identity(prior_name)) + + def syncdb(self): + import django.core.management + assert not self.is_hosted + django.core.management.call_command( + "syncdb", + verbosity = 0, + database = self.irdb_name, + migrate = True, + load_initial_data = False, + interactive = False) + + def hire_zookeeper(self): + assert not self.is_hosted + self._zoo = rpki.irdb.Zookeeper( + cfg = rpki.config.parser(filename = self.path("rpki.conf")), + logstream = None if quiet else sys.stdout) + + @property + def zoo(self): + return self.host._zoo + + def dump_root(self): + + assert self.is_root and not self.is_hosted + + root_resources = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as("0-4294967295"), + v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), + v6 = rpki.resource_set.resource_set_ipv6("::/0")) + + root_key = rpki.x509.RSA.generate(quiet = True) + + root_uri = "rsync://%s/rpki/" % self.rsync_server + + root_sia = (root_uri, root_uri + "root.mft", None, rpki.publication.rrdp_sia_uri_kludge) + + root_cert = rpki.x509.X509.self_certify( + keypair = root_key, + subject_key = root_key.get_public(), + serial = 1, + sia = root_sia, + notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), + resources = root_resources) + + with open(self.path("root.cer"), "wb") as f: + f.write(root_cert.get_DER()) + + with open(self.path("root.key"), "wb") as f: + f.write(root_key.get_DER()) + + with open(cleanpath(test_dir, "root.tal"), "w") as f: + f.write("rsync://%s/root/root.cer\n\n" % self.rsync_server) + f.write(root_key.get_public().get_Base64()) + + def mkdir(self, *path): + path = self.path(*path) if not quiet: - print "Writing", f.name - subprocess.check_call(cmd, stdout = f) + print "Creating directory", path + os.makedirs(path) + + def dump_sql(self): + if not self.is_hosted: + with open(self.path("rpkid.sql"), "w") as f: + if not quiet: + print "Writing", f.name + f.write(rpki.sql_schemas.rpkid) + if self.runs_pubd: + with open(self.path("pubd.sql"), "w") as f: + if not quiet: + print "Writing", f.name + f.write(rpki.sql_schemas.pubd) + if not self.is_hosted: + username = config_overrides["irdbd_sql_username"] + password = config_overrides["irdbd_sql_password"] + cmd = ("mysqldump", "-u", username, "-p" + password, self.irdb_name) + with open(self.path("irdbd.sql"), "w") as f: + if not quiet: + print "Writing", f.name + subprocess.check_call(cmd, stdout = f) def pre_django_sql_setup(needed): - username = config_overrides["irdbd_sql_username"] - password = config_overrides["irdbd_sql_password"] - - # If we have the MySQL root password, just blow away and recreate - # the required databases. Otherwise, check for missing databases, - # then blow away all tables in the required databases. In either - # case, we assume that the Django syncdb code will populate - # databases as necessary, all we need to do here is provide empty - # databases for the Django code to fill in. + username = config_overrides["irdbd_sql_username"] + password = config_overrides["irdbd_sql_password"] + + # If we have the MySQL root password, just blow away and recreate + # the required databases. Otherwise, check for missing databases, + # then blow away all tables in the required databases. In either + # case, we assume that the Django syncdb code will populate + # databases as necessary, all we need to do here is provide empty + # databases for the Django code to fill in. + + if mysql_rootpass is not None: + if mysql_rootpass: + db = MySQLdb.connect(user = mysql_rootuser, passwd = mysql_rootpass) + else: + db = MySQLdb.connect(user = mysql_rootuser) + cur = db.cursor() + for database in needed: + try: + cur.execute("DROP DATABASE IF EXISTS %s" % database) + except: + pass + cur.execute("CREATE DATABASE %s" % database) + cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % ( + database, username), (password,)) - if mysql_rootpass is not None: - if mysql_rootpass: - db = MySQLdb.connect(user = mysql_rootuser, passwd = mysql_rootpass) else: - db = MySQLdb.connect(user = mysql_rootuser) - cur = db.cursor() - for database in needed: - try: - cur.execute("DROP DATABASE IF EXISTS %s" % database) - except: - pass - cur.execute("CREATE DATABASE %s" % database) - cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % ( - database, username), (password,)) - - else: - db = MySQLdb.connect(user = username, passwd = password) - cur = db.cursor() - cur.execute("SHOW DATABASES") - existing = set(r[0] for r in cur.fetchall()) - if needed - existing: - sys.stderr.write("The following databases are missing:\n") - for database in sorted(needed - existing): - sys.stderr.write(" %s\n" % database) - sys.stderr.write("Please create them manually or put MySQL root password in my config file\n") - sys.exit("Missing databases and MySQL root password not known, can't continue") - for database in needed: - db.select_db(database) - cur.execute("SHOW TABLES") - tables = [r[0] for r in cur.fetchall()] - cur.execute("SET foreign_key_checks = 0") - for table in tables: - cur.execute("DROP TABLE %s" % table) - cur.execute("SET foreign_key_checks = 1") - - cur.close() - db.commit() - db.close() + db = MySQLdb.connect(user = username, passwd = password) + cur = db.cursor() + cur.execute("SHOW DATABASES") + existing = set(r[0] for r in cur.fetchall()) + if needed - existing: + sys.stderr.write("The following databases are missing:\n") + for database in sorted(needed - existing): + sys.stderr.write(" %s\n" % database) + sys.stderr.write("Please create them manually or put MySQL root password in my config file\n") + sys.exit("Missing databases and MySQL root password not known, can't continue") + for database in needed: + db.select_db(database) + cur.execute("SHOW TABLES") + tables = [r[0] for r in cur.fetchall()] + cur.execute("SET foreign_key_checks = 0") + for table in tables: + cur.execute("DROP TABLE %s" % table) + cur.execute("SET foreign_key_checks = 1") + + cur.close() + db.commit() + db.close() class timestamp(object): - def __init__(self, *args): - self.count = 0 - self.start = self.tick = rpki.sundial.now() + def __init__(self, *args): + self.count = 0 + self.start = self.tick = rpki.sundial.now() - def __call__(self, *args): - now = rpki.sundial.now() - if not quiet: - print "[Count %s last %s total %s now %s]" % ( - self.count, now - self.tick, now - self.start, now) - self.tick = now - self.count += 1 + def __call__(self, *args): + now = rpki.sundial.now() + if not quiet: + print "[Count %s last %s total %s now %s]" % ( + self.count, now - self.tick, now - self.start, now) + self.tick = now + self.count += 1 def main(): - global flat_publication - global config_overrides - global only_one_pubd - global loopback - global dns_suffix - global mysql_rootuser - global mysql_rootpass - global yaml_file - global test_dir - global rpki_conf - global publication_base - global publication_root - global quiet - - os.environ["TZ"] = "UTC" - time.tzset() - - parser = argparse.ArgumentParser(description = "yamlconf") - parser.add_argument("-c", "--config", help = "configuration file") - parser.add_argument("--dns_suffix", - help = "DNS suffix to add to hostnames") - parser.add_argument("-l", "--loopback", action = "store_true", - help = "Configure for use with yamltest on localhost") - parser.add_argument("-f", "--flat_publication", action = "store_true", - help = "Use flat publication model") - parser.add_argument("-q", "--quiet", action = "store_true", - help = "Work more quietly") - parser.add_argument("--profile", - help = "Filename for profile output") - parser.add_argument("yaml_file", type = argparse.FileType("r"), - help = "YAML file describing network to build") - args = parser.parse_args() - - dns_suffix = args.dns_suffix - loopback = args.loopback - flat_publication = args.flat_publication - quiet = args.quiet - yaml_file = args.yaml_file - - rpki.log.init("yamlconf", argparse.Namespace(log_level = logging.DEBUG, - log_handler = lambda: logging.StreamHandler(sys.stdout))) - - # Allow optional config file for this tool to override default - # passwords: this is mostly so that I can show a complete working - # example without publishing my own server's passwords. - - cfg = rpki.config.parser(set_filename = args.config, section = "yamlconf", allow_missing = True) - try: - cfg.set_global_flags() - except: - pass - - # Use of "yamltest.dir" is deliberate: intent is for what we write to - # be usable with "yamltest --skip_config". - - only_one_pubd = cfg.getboolean("only_one_pubd", True) - test_dir = cfg.get("test_directory", cleanpath(this_dir, "yamltest.dir")) - rpki_conf = cfg.get("rpki_conf", cleanpath(this_dir, "..", "examples/rpki.conf")) - mysql_rootuser = cfg.get("mysql_rootuser", "root") - - try: - mysql_rootpass = cfg.get("mysql_rootpass") - except: - pass - - try: - publication_base = cfg.get("publication_base") - except: - pass - - try: - publication_root = cfg.get("publication_root") - except: - pass - - for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", - "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username"): - if cfg.has_option(k): - config_overrides[k] = cfg.get(k) - - if args.profile: - import cProfile - prof = cProfile.Profile() + global flat_publication + global config_overrides + global only_one_pubd + global loopback + global dns_suffix + global mysql_rootuser + global mysql_rootpass + global yaml_file + global test_dir + global rpki_conf + global publication_base + global publication_root + global quiet + + os.environ["TZ"] = "UTC" + time.tzset() + + parser = argparse.ArgumentParser(description = "yamlconf") + parser.add_argument("-c", "--config", help = "configuration file") + parser.add_argument("--dns_suffix", + help = "DNS suffix to add to hostnames") + parser.add_argument("-l", "--loopback", action = "store_true", + help = "Configure for use with yamltest on localhost") + parser.add_argument("-f", "--flat_publication", action = "store_true", + help = "Use flat publication model") + parser.add_argument("-q", "--quiet", action = "store_true", + help = "Work more quietly") + parser.add_argument("--profile", + help = "Filename for profile output") + parser.add_argument("yaml_file", type = argparse.FileType("r"), + help = "YAML file describing network to build") + args = parser.parse_args() + + dns_suffix = args.dns_suffix + loopback = args.loopback + flat_publication = args.flat_publication + quiet = args.quiet + yaml_file = args.yaml_file + + rpki.log.init("yamlconf", argparse.Namespace(log_level = logging.DEBUG, + log_handler = lambda: logging.StreamHandler(sys.stdout))) + + # Allow optional config file for this tool to override default + # passwords: this is mostly so that I can show a complete working + # example without publishing my own server's passwords. + + cfg = rpki.config.parser(set_filename = args.config, section = "yamlconf", allow_missing = True) try: - prof.runcall(body) - finally: - prof.dump_stats(args.profile) - if not quiet: - print - print "Dumped profile data to %s" % args.profile - else: - body() + cfg.set_global_flags() + except: + pass -def body(): + # Use of "yamltest.dir" is deliberate: intent is for what we write to + # be usable with "yamltest --skip_config". + + only_one_pubd = cfg.getboolean("only_one_pubd", True) + test_dir = cfg.get("test_directory", cleanpath(this_dir, "yamltest.dir")) + rpki_conf = cfg.get("rpki_conf", cleanpath(this_dir, "..", "examples/rpki.conf")) + mysql_rootuser = cfg.get("mysql_rootuser", "root") - global rpki + try: + mysql_rootpass = cfg.get("mysql_rootpass") + except: + pass - ts = timestamp() + try: + publication_base = cfg.get("publication_base") + except: + pass - for root, dirs, files in os.walk(test_dir, topdown = False): - for fn in files: - os.unlink(os.path.join(root, fn)) - for d in dirs: - os.rmdir(os.path.join(root, d)) + try: + publication_root = cfg.get("publication_root") + except: + pass - if not quiet: - print - print "Reading YAML", yaml_file.name + for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", + "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username"): + if cfg.has_option(k): + config_overrides[k] = cfg.get(k) + + if args.profile: + import cProfile + prof = cProfile.Profile() + try: + prof.runcall(body) + finally: + prof.dump_stats(args.profile) + if not quiet: + print + print "Dumped profile data to %s" % args.profile + else: + body() - db = allocation_db(yaml.safe_load_all(yaml_file).next()) +def body(): - # Show what we loaded + global rpki - #db.dump() + ts = timestamp() - # Do pre-Django SQL setup + for root, dirs, files in os.walk(test_dir, topdown = False): + for fn in files: + os.unlink(os.path.join(root, fn)) + for d in dirs: + os.rmdir(os.path.join(root, d)) - pre_django_sql_setup(set(d.irdb_name for d in db if not d.is_hosted)) + if not quiet: + print + print "Reading YAML", yaml_file.name - # Now ready for fun with multiple databases in Django! - # - # https://docs.djangoproject.com/en/1.4/topics/db/multi-db/ - # https://docs.djangoproject.com/en/1.4/topics/db/sql/ - # - # This program's use of the ORM is sufficiently different that it's - # not worth straining to use rpki.django_settings, so we just use - # Django's settings API directly. + db = allocation_db(yaml.safe_load_all(yaml_file).next()) - database_template = { - "ENGINE" : "django.db.backends.mysql", - "USER" : config_overrides["irdbd_sql_username"], - "PASSWORD" : config_overrides["irdbd_sql_password"], - "HOST" : "", - "PORT" : "", - "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }} + # Show what we loaded - databases = dict((d.irdb_name, dict(database_template, NAME = d.irdb_name)) - for d in db if not d.is_hosted) + #db.dump() - databases["default"] = databases[db.root.irdb_name] + # Do pre-Django SQL setup - import django - django.setup() + pre_django_sql_setup(set(d.irdb_name for d in db if not d.is_hosted)) - from django.conf import settings + # Now ready for fun with multiple databases in Django! + # + # https://docs.djangoproject.com/en/1.4/topics/db/multi-db/ + # https://docs.djangoproject.com/en/1.4/topics/db/sql/ + # + # This program's use of the ORM is sufficiently different that it's + # not worth straining to use rpki.django_settings, so we just use + # Django's settings API directly. - settings.configure( - DATABASES = databases, - DATABASE_ROUTERS = ["rpki.irdb.router.DBContextRouter"], - INSTALLED_APPS = ["rpki.irdb"]) + database_template = { + "ENGINE" : "django.db.backends.mysql", + "USER" : config_overrides["irdbd_sql_username"], + "PASSWORD" : config_overrides["irdbd_sql_password"], + "HOST" : "", + "PORT" : "", + "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }} - import rpki.irdb + databases = dict((d.irdb_name, dict(database_template, NAME = d.irdb_name)) + for d in db if not d.is_hosted) - rpki.irdb.models.ca_certificate_lifetime = rpki.sundial.timedelta(days = 3652 * 2) - rpki.irdb.models.ee_certificate_lifetime = rpki.sundial.timedelta(days = 3652) + databases["default"] = databases[db.root.irdb_name] - ts() + import django + django.setup() - for d in db: - if not quiet: - print - print "Configuring", d.name - - if not d.is_hosted: - d.mkdir() - if d.runs_pubd: - d.mkdir("publication") - if d.is_root: - d.mkdir("publication.root") - - if not d.is_hosted: - d.dump_conf() - d.dump_rsyncd() - - d.dump_asns("%s.asns.csv" % d.name) - d.dump_prefixes("%s.prefixes.csv" % d.name) - d.dump_roas("%s.roas.csv" % d.name) - d.dump_ghostbusters("%s.ghostbusters.vcard" % d.name) - d.dump_router_certificates("%s.routercerts.xml" % d.name) - - if not d.is_hosted: - if not quiet: - print "Initializing SQL" - d.syncdb() - if not quiet: - print "Hiring zookeeper" - d.hire_zookeeper() - - with d.irdb: - if not quiet: - print "Creating identity" - x = d.zoo.initialize() - - if d.is_root: - if not quiet: - print "Creating RPKI root certificate and TAL" - d.dump_root() - x = d.zoo.configure_rootd() + from django.conf import settings - else: - with d.parent.irdb: - x = d.parent.zoo.configure_child(x.file)[0] - x = d.zoo.configure_parent(x.file)[0] + settings.configure( + DATABASES = databases, + DATABASE_ROUTERS = ["rpki.irdb.router.DBContextRouter"], + INSTALLED_APPS = ["rpki.irdb"]) - with d.pubd.irdb: - x = d.pubd.zoo.configure_publication_client(x.file, flat = flat_publication)[0] - d.zoo.configure_repository(x.file) + import rpki.irdb - if loopback and not d.is_hosted: - with d.irdb: - d.zoo.write_bpki_files() + rpki.irdb.models.ca_certificate_lifetime = rpki.sundial.timedelta(days = 3652 * 2) + rpki.irdb.models.ee_certificate_lifetime = rpki.sundial.timedelta(days = 3652) ts() - if not loopback: - if not quiet: - print for d in db: - d.dump_sql() + if not quiet: + print + print "Configuring", d.name + + if not d.is_hosted: + d.mkdir() + if d.runs_pubd: + d.mkdir("publication") + if d.is_root: + d.mkdir("publication.root") + + if not d.is_hosted: + d.dump_conf() + d.dump_rsyncd() + + d.dump_asns("%s.asns.csv" % d.name) + d.dump_prefixes("%s.prefixes.csv" % d.name) + d.dump_roas("%s.roas.csv" % d.name) + d.dump_ghostbusters("%s.ghostbusters.vcard" % d.name) + d.dump_router_certificates("%s.routercerts.xml" % d.name) + + if not d.is_hosted: + if not quiet: + print "Initializing SQL" + d.syncdb() + if not quiet: + print "Hiring zookeeper" + d.hire_zookeeper() + + with d.irdb: + if not quiet: + print "Creating identity" + x = d.zoo.initialize() + + if d.is_root: + if not quiet: + print "Creating RPKI root certificate and TAL" + d.dump_root() + x = d.zoo.configure_rootd() + + else: + with d.parent.irdb: + x = d.parent.zoo.configure_child(x.file)[0] + x = d.zoo.configure_parent(x.file)[0] + + with d.pubd.irdb: + x = d.pubd.zoo.configure_publication_client(x.file, flat = flat_publication)[0] + d.zoo.configure_repository(x.file) + + if loopback and not d.is_hosted: + with d.irdb: + d.zoo.write_bpki_files() + + ts() + + if not loopback: + if not quiet: + print + for d in db: + d.dump_sql() if __name__ == "__main__": - main() + main() diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index a26fefb9..f82cc192 100644 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -65,11 +65,11 @@ section_regexp = re.compile(r"\s*\[\s*(.+?)\s*\]\s*$") variable_regexp = re.compile(r"\s*([-a-zA-Z0-9_]+)\s*=\s*(.+?)\s*$") def cleanpath(*names): - """ - Construct normalized pathnames. - """ + """ + Construct normalized pathnames. + """ - return os.path.normpath(os.path.join(*names)) + return os.path.normpath(os.path.join(*names)) # Pathnames for various things we need @@ -85,652 +85,652 @@ prog_rootd = cleanpath(ca_dir, "rootd") prog_rpki_manage = cleanpath(ca_dir, "rpki-manage") class roa_request(object): - """ - Representation of a ROA request. - """ - - def __init__(self, asn, ipv4, ipv6): - self.asn = asn - self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None - self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - - def __eq__(self, other): - return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - - def __hash__(self): - v4 = tuple(self.v4) if self.v4 is not None else None - v6 = tuple(self.v6) if self.v6 is not None else None - return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - - def __str__(self): - if self.v4 and self.v6: - return "%s: %s,%s" % (self.asn, self.v4, self.v6) - else: - return "%s: %s" % (self.asn, self.v4 or self.v6) - - @classmethod - def parse(cls, y): - """ - Parse a ROA request from YAML format. - """ - - return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) - - -class router_cert(object): - """ - Representation for a router_cert object. - """ - - _ecparams = None - - @classmethod - def ecparams(cls): - if cls._ecparams is None: - cls._ecparams = rpki.x509.KeyParams.generateEC() - return cls._ecparams - - def __init__(self, asn, router_id): - self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) - self.router_id = router_id - self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) - self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) - self.gski = self.pkcs10.gSKI() - - def __eq__(self, other): - return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski - - def __hash__(self): - return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() - - def __str__(self): - return "%s: %s: %s" % (self.asn, self.router_id, self.gski) - - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("router_id")) - -class allocation_db(list): - """ - Our allocation database. - """ - - def __init__(self, yaml): - list.__init__(self) - self.root = allocation(yaml, self) - assert self.root.is_root and not any(a.is_root for a in self if a is not self.root) and self[0] is self.root - if self.root.crl_interval is None: - self.root.crl_interval = 60 * 60 - if self.root.regen_margin is None: - self.root.regen_margin = 24 * 60 * 60 - if self.root.base.valid_until is None: - self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) - for a in self: - if a.base.valid_until is None: - a.base.valid_until = a.parent.base.valid_until - if a.crl_interval is None: - a.crl_interval = a.parent.crl_interval - if a.regen_margin is None: - a.regen_margin = a.parent.regen_margin - self.root.closure() - self.map = dict((a.name, a) for a in self) - for a in self: - if a.is_hosted: - a.hosted_by = self.map[a.hosted_by] - a.hosted_by.hosts.append(a) - assert not a.is_root and not a.hosted_by.is_hosted - - def dump(self): """ - Show contents of allocation database. + Representation of a ROA request. """ - for a in self: - a.dump() + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 -class allocation(object): - """ - One entity in our allocation database. Every entity in the database - is assumed to hold resources, so needs at least rpkic services. - Entities that don't have the hosted_by property run their own copies - of rpkid, irdbd, and pubd, so they also need myirbe services. - """ - - base_port = None - parent = None - crl_interval = None - regen_margin = None - rootd_port = None - engine = -1 - rpkid_port = -1 - irdbd_port = -1 - pubd_port = -1 - rsync_port = -1 - rootd_port = -1 - rrdp_port = -1 - rpkic_counter = 0L - - @classmethod - def allocate_port(cls): - """ - Allocate a TCP port. - """ - - cls.base_port += 1 - return cls.base_port - - base_engine = -1 - - @classmethod - def allocate_engine(cls): - """ - Allocate an engine number, mostly used to construct SQL database - names. - """ - - cls.base_engine += 1 - return cls.base_engine - - def __init__(self, yaml, db, parent = None): - db.append(self) - self.name = yaml["name"] - self.parent = parent - self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] - valid_until = None - if "valid_until" in yaml: - valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) - if valid_until is None and "valid_for" in yaml: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) - self.base = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as(yaml.get("asn")), - v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), - v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), - valid_until = valid_until) - if "crl_interval" in yaml: - self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() - if "regen_margin" in yaml: - self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() - self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] - self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] - if "ghostbusters" in yaml: - self.ghostbusters = yaml.get("ghostbusters") - elif "ghostbuster" in yaml: - self.ghostbusters = [yaml.get("ghostbuster")] - else: - self.ghostbusters = [] - for r in self.roa_requests: - if r.v4: - self.base.v4 |= r.v4.to_resource_set() - if r.v6: - self.base.v6 |= r.v6.to_resource_set() - for r in self.router_certs: - self.base.asn |= r.asn - self.hosted_by = yaml.get("hosted_by") - self.hosts = [] - if not self.is_hosted: - self.engine = self.allocate_engine() - self.rpkid_port = self.allocate_port() - self.irdbd_port = self.allocate_port() - if self.runs_pubd: - self.pubd_port = self.allocate_port() - self.rsync_port = self.allocate_port() - self.rrdp_port = self.allocate_port() - if self.is_root: - self.rootd_port = self.allocate_port() - - def closure(self): - """ - Compute resource closure of this node and its children, to avoid a - lot of tedious (and error-prone) duplication in the YAML file. - """ - - resources = self.base - for kid in self.kids: - resources |= kid.closure() - self.resources = resources - return resources - - def dump(self): - """ - Show content of this allocation node. - """ - - print str(self) - - def __str__(self): - s = self.name + ":\n" - if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn - if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 - if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 - if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) - if self.parent: s += " Up: %s\n" % self.parent.name - if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name - if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) - for r in self.roa_requests: s += " ROA: %s\n" % r - if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port - if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port - if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port - if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port - if self.is_root: s += " TPort: %s\n" % self.rootd_port - return s + " Until: %s\n" % self.resources.valid_until - - @property - def is_root(self): - """ - Is this the root node? - """ + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - return self.parent is None - - @property - def is_hosted(self): - """ - Is this entity hosted? - """ - - return self.hosted_by is not None - - @property - def runs_pubd(self): - """ - Does this entity run a pubd? - """ + def __str__(self): + if self.v4 and self.v6: + return "%s: %s,%s" % (self.asn, self.v4, self.v6) + else: + return "%s: %s" % (self.asn, self.v4 or self.v6) - return self.is_root or not (self.is_hosted or only_one_pubd) + @classmethod + def parse(cls, y): + """ + Parse a ROA request from YAML format. + """ - def path(self, *names): - """ - Construct pathnames in this entity's test directory. - """ + return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) - return cleanpath(test_dir, self.host.name, *names) - def csvout(self, fn): +class router_cert(object): """ - Open and log a CSV output file. + Representation for a router_cert object. """ - path = self.path(fn) - print "Writing", path - return rpki.csv_utils.csv_writer(path) + _ecparams = None - def up_down_url(self): - """ - Construct service URL for this node's parent. - """ + @classmethod + def ecparams(cls): + if cls._ecparams is None: + cls._ecparams = rpki.x509.KeyParams.generateEC() + return cls._ecparams - return "http://localhost:%d/up-down/%s/%s" % (self.parent.host.rpkid_port, - self.parent.name, - self.name) + def __init__(self, asn, router_id): + self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) + self.router_id = router_id + self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) + self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) + self.gski = self.pkcs10.gSKI() - def dump_asns(self): - """ - Write Autonomous System Numbers CSV file. - """ + def __eq__(self, other): + return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski - fn = "%s.asns.csv" % d.name - if not args.skip_config: - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, a) for a in k.resources.asn) - if not args.stop_after_config: - self.run_rpkic("load_asns", fn) + def __hash__(self): + return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() - def dump_prefixes(self): - """ - Write prefixes CSV file. - """ + def __str__(self): + return "%s: %s: %s" % (self.asn, self.router_id, self.gski) - fn = "%s.prefixes.csv" % d.name - if not args.skip_config: - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) - if not args.stop_after_config: - self.run_rpkic("load_prefixes", fn) + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("router_id")) - def dump_roas(self): - """ - Write ROA CSV file. - """ - - fn = "%s.roas.csv" % d.name - if not args.skip_config: - with self.csvout(fn) as f: - for g1, r in enumerate(self.roa_requests): - f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) - for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) - if not args.stop_after_config: - self.run_rpkic("load_roa_requests", fn) - - def dump_ghostbusters(self): - """ - Write Ghostbusters vCard file. +class allocation_db(list): """ + Our allocation database. + """ + + def __init__(self, yaml): + list.__init__(self) + self.root = allocation(yaml, self) + assert self.root.is_root and not any(a.is_root for a in self if a is not self.root) and self[0] is self.root + if self.root.crl_interval is None: + self.root.crl_interval = 60 * 60 + if self.root.regen_margin is None: + self.root.regen_margin = 24 * 60 * 60 + if self.root.base.valid_until is None: + self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) + for a in self: + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + self.root.closure() + self.map = dict((a.name, a) for a in self) + for a in self: + if a.is_hosted: + a.hosted_by = self.map[a.hosted_by] + a.hosted_by.hosts.append(a) + assert not a.is_root and not a.hosted_by.is_hosted + + def dump(self): + """ + Show contents of allocation database. + """ + + for a in self: + a.dump() - if self.ghostbusters: - fn = "%s.ghostbusters.vcard" % d.name - if not args.skip_config: - path = self.path(fn) - print "Writing", path - with open(path, "w") as f: - f.write("\n".join(self.ghostbusters)) - if not args.stop_after_config: - self.run_rpkic("load_ghostbuster_requests", fn) - def dump_router_certificates(self): - """ - Write EE certificates (router certificates, etc). +class allocation(object): """ + One entity in our allocation database. Every entity in the database + is assumed to hold resources, so needs at least rpkic services. + Entities that don't have the hosted_by property run their own copies + of rpkid, irdbd, and pubd, so they also need myirbe services. + """ + + base_port = None + parent = None + crl_interval = None + regen_margin = None + rootd_port = None + engine = -1 + rpkid_port = -1 + irdbd_port = -1 + pubd_port = -1 + rsync_port = -1 + rootd_port = -1 + rrdp_port = -1 + rpkic_counter = 0L + + @classmethod + def allocate_port(cls): + """ + Allocate a TCP port. + """ + + cls.base_port += 1 + return cls.base_port + + base_engine = -1 + + @classmethod + def allocate_engine(cls): + """ + Allocate an engine number, mostly used to construct SQL database + names. + """ + + cls.base_engine += 1 + return cls.base_engine + + def __init__(self, yaml, db, parent = None): + db.append(self) + self.name = yaml["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] + valid_until = None + if "valid_until" in yaml: + valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) + if valid_until is None and "valid_for" in yaml: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(yaml.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), + valid_until = valid_until) + if "crl_interval" in yaml: + self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() + if "regen_margin" in yaml: + self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() + self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] + self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] + if "ghostbusters" in yaml: + self.ghostbusters = yaml.get("ghostbusters") + elif "ghostbuster" in yaml: + self.ghostbusters = [yaml.get("ghostbuster")] + else: + self.ghostbusters = [] + for r in self.roa_requests: + if r.v4: + self.base.v4 |= r.v4.to_resource_set() + if r.v6: + self.base.v6 |= r.v6.to_resource_set() + for r in self.router_certs: + self.base.asn |= r.asn + self.hosted_by = yaml.get("hosted_by") + self.hosts = [] + if not self.is_hosted: + self.engine = self.allocate_engine() + self.rpkid_port = self.allocate_port() + self.irdbd_port = self.allocate_port() + if self.runs_pubd: + self.pubd_port = self.allocate_port() + self.rsync_port = self.allocate_port() + self.rrdp_port = self.allocate_port() + if self.is_root: + self.rootd_port = self.allocate_port() + + def closure(self): + """ + Compute resource closure of this node and its children, to avoid a + lot of tedious (and error-prone) duplication in the YAML file. + """ + + resources = self.base + for kid in self.kids: + resources |= kid.closure() + self.resources = resources + return resources + + def dump(self): + """ + Show content of this allocation node. + """ + + print str(self) + + def __str__(self): + s = self.name + ":\n" + if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name + if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) + for r in self.roa_requests: s += " ROA: %s\n" % r + if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port + if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port + if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port + if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port + if self.is_root: s += " TPort: %s\n" % self.rootd_port + return s + " Until: %s\n" % self.resources.valid_until + + @property + def is_root(self): + """ + Is this the root node? + """ + + return self.parent is None + + @property + def is_hosted(self): + """ + Is this entity hosted? + """ + + return self.hosted_by is not None + + @property + def runs_pubd(self): + """ + Does this entity run a pubd? + """ + + return self.is_root or not (self.is_hosted or only_one_pubd) + + def path(self, *names): + """ + Construct pathnames in this entity's test directory. + """ + + return cleanpath(test_dir, self.host.name, *names) + + def csvout(self, fn): + """ + Open and log a CSV output file. + """ - if self.router_certs: - fn = "%s.routercerts.xml" % d.name - if not args.skip_config: path = self.path(fn) print "Writing", path - xmlns = rpki.relaxng.router_certificate.xmlns - xml = lxml.etree.Element(xmlns + "router_certificate_requests", - version = rpki.relaxng.router_certificate.version, - nsmap = rpki.relaxng.router_certificate.nsmap) - for r in self.router_certs: - x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", - router_id = str(r.router_id), - asn = str(r.asn), - valid_until = str(self.resources.valid_until)) - x.text = r.pkcs10.get_Base64() - rpki.relaxng.router_certificate.assertValid(xml) - lxml.etree.ElementTree(xml).write(path, pretty_print = True) - if not args.stop_after_config: - self.run_rpkic("add_router_certificate_request", fn) - if not args.skip_config and args.store_router_private_keys: - path = self.path("%s.routercerts.keys" % d.name) - print "Writing", path - with open(path, "w") as f: - for r in self.router_certs: - f.write(r.keypair.get_PEM()) - - @property - def pubd(self): - """ - Walk up tree until we find somebody who runs pubd. - """ - - s = self - while not s.runs_pubd: - s = s.parent - return s - - @property - def client_handle(self): - """ - Work out what pubd configure_publication_client will call us. - """ - - path = [] - s = self - if not args.flat_publication: - while not s.runs_pubd: + return rpki.csv_utils.csv_writer(path) + + def up_down_url(self): + """ + Construct service URL for this node's parent. + """ + + return "http://localhost:%d/up-down/%s/%s" % (self.parent.host.rpkid_port, + self.parent.name, + self.name) + + def dump_asns(self): + """ + Write Autonomous System Numbers CSV file. + """ + + fn = "%s.asns.csv" % d.name + if not args.skip_config: + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, a) for a in k.resources.asn) + if not args.stop_after_config: + self.run_rpkic("load_asns", fn) + + def dump_prefixes(self): + """ + Write prefixes CSV file. + """ + + fn = "%s.prefixes.csv" % d.name + if not args.skip_config: + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) + if not args.stop_after_config: + self.run_rpkic("load_prefixes", fn) + + def dump_roas(self): + """ + Write ROA CSV file. + """ + + fn = "%s.roas.csv" % d.name + if not args.skip_config: + with self.csvout(fn) as f: + for g1, r in enumerate(self.roa_requests): + f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) + for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) + if not args.stop_after_config: + self.run_rpkic("load_roa_requests", fn) + + def dump_ghostbusters(self): + """ + Write Ghostbusters vCard file. + """ + + if self.ghostbusters: + fn = "%s.ghostbusters.vcard" % d.name + if not args.skip_config: + path = self.path(fn) + print "Writing", path + with open(path, "w") as f: + f.write("\n".join(self.ghostbusters)) + if not args.stop_after_config: + self.run_rpkic("load_ghostbuster_requests", fn) + + def dump_router_certificates(self): + """ + Write EE certificates (router certificates, etc). + """ + + if self.router_certs: + fn = "%s.routercerts.xml" % d.name + if not args.skip_config: + path = self.path(fn) + print "Writing", path + xmlns = rpki.relaxng.router_certificate.xmlns + xml = lxml.etree.Element(xmlns + "router_certificate_requests", + version = rpki.relaxng.router_certificate.version, + nsmap = rpki.relaxng.router_certificate.nsmap) + for r in self.router_certs: + x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", + router_id = str(r.router_id), + asn = str(r.asn), + valid_until = str(self.resources.valid_until)) + x.text = r.pkcs10.get_Base64() + rpki.relaxng.router_certificate.assertValid(xml) + lxml.etree.ElementTree(xml).write(path, pretty_print = True) + if not args.stop_after_config: + self.run_rpkic("add_router_certificate_request", fn) + if not args.skip_config and args.store_router_private_keys: + path = self.path("%s.routercerts.keys" % d.name) + print "Writing", path + with open(path, "w") as f: + for r in self.router_certs: + f.write(r.keypair.get_PEM()) + + @property + def pubd(self): + """ + Walk up tree until we find somebody who runs pubd. + """ + + s = self + while not s.runs_pubd: + s = s.parent + return s + + @property + def client_handle(self): + """ + Work out what pubd configure_publication_client will call us. + """ + + path = [] + s = self + if not args.flat_publication: + while not s.runs_pubd: + path.append(s) + s = s.parent path.append(s) - s = s.parent - path.append(s) - return ".".join(i.name for i in reversed(path)) - - @property - def host(self): - return self.hosted_by or self - - def dump_conf(self): - """ - Write configuration file for OpenSSL and RPKI tools. - """ - - r = dict( - handle = self.name, - run_rpkid = str(not self.is_hosted), - run_pubd = str(self.runs_pubd), - run_rootd = str(self.is_root), - rpkid_server_host = "localhost", - rpkid_server_port = str(self.rpkid_port), - irdbd_server_host = "localhost", - irdbd_server_port = str(self.irdbd_port), - rootd_server_port = str(self.rootd_port), - pubd_server_host = "localhost", - pubd_server_port = str(self.pubd.pubd_port), - publication_rsync_server = "localhost:%s" % self.pubd.rsync_port, - publication_rrdp_notification_uri = "http://localhost:%s/rrdp/notify.xml" % self.pubd.rrdp_port, - bpki_servers_directory = self.path(), - publication_base_directory = self.path("publication"), - rrdp_publication_base_directory = self.path("rrdp-publication"), - shared_sql_engine = args.sql_engine, - shared_sql_password = "fnord", - irdbd_sql_username = "irdb", - rpkid_sql_username = "rpki", - pubd_sql_username = "pubd") - - if args.sql_engine == "sqlite3": - r.update( - irdbd_sql_database = self.path("irdb.sqlite3"), - rpkid_sql_database = self.path("rpkidb.sqlite3"), - pubd_sql_database = self.path("pubdb.sqlite3")) - else: - r.update( - irdbd_sql_database = "irdb%d" % self.engine, - rpkid_sql_database = "rpki%d" % self.engine, - pubd_sql_database = "pubd%d" % self.engine) - - r.update(config_overrides) - - with open(self.path("rpki.conf"), "w") as f: - f.write("# Automatically generated, do not edit\n") - print "Writing", f.name - - section = None - for line in open(cleanpath(ca_dir, "examples/rpki.conf")): - m = section_regexp.match(line) - if m: - section = m.group(1) - m = variable_regexp.match(line) - option = m.group(1) if m and section == "myrpki" else None - if option and option in r: - line = "%s = %s\n" % (option, r[option]) - f.write(line) - - def dump_rsyncd(self): - """ - Write rsyncd configuration file. - """ - - if self.runs_pubd: - with open(self.path("rsyncd.conf"), "w") as f: - print "Writing", f.name - f.writelines(s + "\n" for s in - ("# Automatically generated, do not edit", - "port = %d" % self.rsync_port, - "address = localhost", - "[rpki]", - "log file = rsyncd.log", - "read only = yes", - "use chroot = no", - "path = %s" % self.path("publication"), - "comment = RPKI test", - "[root]", - "log file = rsyncd_root.log", - "read only = yes", - "use chroot = no", - "path = %s" % self.path("publication.root"), - "comment = RPKI test root")) - - @classmethod - def next_rpkic_counter(cls): - cls.rpkic_counter += 10000 - return str(cls.rpkic_counter) - - def run_rpkic(self, *argv): - """ - Run rpkic for this entity. - """ - - cmd = [prog_rpkic, "-i", self.name] - if args.profile: - cmd.append("--profile") - cmd.append(self.path("rpkic.%s.prof" % rpki.sundial.now())) - cmd.extend(str(a) for a in argv if a is not None) - print 'Running "%s"' % " ".join(cmd) - env = dict(os.environ, - YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(), - RPKI_CONF = self.path("rpki.conf")) - subprocess.check_call(cmd, cwd = self.host.path(), env = env) - - def syncdb(self): - """ - Run whatever Django ORM commands are necessary to set up the - database this week. - """ - - # Fork a sub-process for each syncdb/migrate run, because it's - # easier than figuring out how to change Django settings after - # initialization. - - def sync_settings(settings, verbosity = 1): - - if verbosity > 0: - print "Running Django setup for", self.name - - pid = os.fork() - - if pid == 0: - logging.getLogger().setLevel(logging.WARNING) - - os.environ.update(RPKI_CONF = self.path("rpki.conf"), - DJANGO_SETTINGS_MODULE = "rpki.django_settings." + settings) - - import django - django.setup() - - import django.core.management - django.core.management.call_command("migrate", verbosity = verbosity, no_color = True, - load_initial_data = False, interactive = False) - - if settings in ("gui", "irdb"): - from django.contrib.auth.models import User - User.objects.create_superuser("root", "root@example.org", "fnord") - - sys.exit(0) - - elif os.waitpid(pid, 0)[1]: - raise RuntimeError("Django setup failed for %s %s" % (self.name, settings)) - - for settings in ("rpkid", "pubd", "gui"): - sync_settings(settings) - - def run_python_daemon(self, prog): - """ - Start a Python daemon and return a subprocess.Popen object - representing the running daemon. - """ - - basename = os.path.splitext(os.path.basename(prog))[0] - cmd = [prog, "--foreground", "--log-level", "debug", - "--log-file", self.path(basename + ".log")] - if args.profile and basename != "rootd": - cmd.extend(( - "--profile", self.path(basename + ".prof"))) - env = dict(os.environ, RPKI_CONF = self.path("rpki.conf")) - p = subprocess.Popen(cmd, cwd = self.path(), env = env) - print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) - return p - - def run_rpkid(self): - """ - Run rpkid. - """ - - return self.run_python_daemon(prog_rpkid) - - def run_irdbd(self): - """ - Run irdbd. - """ - - return self.run_python_daemon(prog_irdbd) - - def run_pubd(self): - """ - Run pubd. - """ - - return self.run_python_daemon(prog_pubd) - - def run_rootd(self): - """ - Run rootd. - """ - - return self.run_python_daemon(prog_rootd) - - def run_rsyncd(self): - """ - Run rsyncd. - """ - - p = subprocess.Popen(("rsync", "--daemon", "--no-detach", "--config", "rsyncd.conf"), - cwd = self.path()) - print "Running rsyncd for %s: pid %d process %r" % (self.name, p.pid, p) - return p + return ".".join(i.name for i in reversed(path)) + + @property + def host(self): + return self.hosted_by or self + + def dump_conf(self): + """ + Write configuration file for OpenSSL and RPKI tools. + """ + + r = dict( + handle = self.name, + run_rpkid = str(not self.is_hosted), + run_pubd = str(self.runs_pubd), + run_rootd = str(self.is_root), + rpkid_server_host = "localhost", + rpkid_server_port = str(self.rpkid_port), + irdbd_server_host = "localhost", + irdbd_server_port = str(self.irdbd_port), + rootd_server_port = str(self.rootd_port), + pubd_server_host = "localhost", + pubd_server_port = str(self.pubd.pubd_port), + publication_rsync_server = "localhost:%s" % self.pubd.rsync_port, + publication_rrdp_notification_uri = "http://localhost:%s/rrdp/notify.xml" % self.pubd.rrdp_port, + bpki_servers_directory = self.path(), + publication_base_directory = self.path("publication"), + rrdp_publication_base_directory = self.path("rrdp-publication"), + shared_sql_engine = args.sql_engine, + shared_sql_password = "fnord", + irdbd_sql_username = "irdb", + rpkid_sql_username = "rpki", + pubd_sql_username = "pubd") + + if args.sql_engine == "sqlite3": + r.update( + irdbd_sql_database = self.path("irdb.sqlite3"), + rpkid_sql_database = self.path("rpkidb.sqlite3"), + pubd_sql_database = self.path("pubdb.sqlite3")) + else: + r.update( + irdbd_sql_database = "irdb%d" % self.engine, + rpkid_sql_database = "rpki%d" % self.engine, + pubd_sql_database = "pubd%d" % self.engine) + + r.update(config_overrides) + + with open(self.path("rpki.conf"), "w") as f: + f.write("# Automatically generated, do not edit\n") + print "Writing", f.name + + section = None + for line in open(cleanpath(ca_dir, "examples/rpki.conf")): + m = section_regexp.match(line) + if m: + section = m.group(1) + m = variable_regexp.match(line) + option = m.group(1) if m and section == "myrpki" else None + if option and option in r: + line = "%s = %s\n" % (option, r[option]) + f.write(line) + + def dump_rsyncd(self): + """ + Write rsyncd configuration file. + """ + + if self.runs_pubd: + with open(self.path("rsyncd.conf"), "w") as f: + print "Writing", f.name + f.writelines(s + "\n" for s in + ("# Automatically generated, do not edit", + "port = %d" % self.rsync_port, + "address = localhost", + "[rpki]", + "log file = rsyncd.log", + "read only = yes", + "use chroot = no", + "path = %s" % self.path("publication"), + "comment = RPKI test", + "[root]", + "log file = rsyncd_root.log", + "read only = yes", + "use chroot = no", + "path = %s" % self.path("publication.root"), + "comment = RPKI test root")) + + @classmethod + def next_rpkic_counter(cls): + cls.rpkic_counter += 10000 + return str(cls.rpkic_counter) + + def run_rpkic(self, *argv): + """ + Run rpkic for this entity. + """ + + cmd = [prog_rpkic, "-i", self.name] + if args.profile: + cmd.append("--profile") + cmd.append(self.path("rpkic.%s.prof" % rpki.sundial.now())) + cmd.extend(str(a) for a in argv if a is not None) + print 'Running "%s"' % " ".join(cmd) + env = dict(os.environ, + YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(), + RPKI_CONF = self.path("rpki.conf")) + subprocess.check_call(cmd, cwd = self.host.path(), env = env) + + def syncdb(self): + """ + Run whatever Django ORM commands are necessary to set up the + database this week. + """ + + # Fork a sub-process for each syncdb/migrate run, because it's + # easier than figuring out how to change Django settings after + # initialization. + + def sync_settings(settings, verbosity = 1): + + if verbosity > 0: + print "Running Django setup for", self.name + + pid = os.fork() + + if pid == 0: + logging.getLogger().setLevel(logging.WARNING) + + os.environ.update(RPKI_CONF = self.path("rpki.conf"), + DJANGO_SETTINGS_MODULE = "rpki.django_settings." + settings) + + import django + django.setup() + + import django.core.management + django.core.management.call_command("migrate", verbosity = verbosity, no_color = True, + load_initial_data = False, interactive = False) + + if settings in ("gui", "irdb"): + from django.contrib.auth.models import User + User.objects.create_superuser("root", "root@example.org", "fnord") + + sys.exit(0) + + elif os.waitpid(pid, 0)[1]: + raise RuntimeError("Django setup failed for %s %s" % (self.name, settings)) + + for settings in ("rpkid", "pubd", "gui"): + sync_settings(settings) + + def run_python_daemon(self, prog): + """ + Start a Python daemon and return a subprocess.Popen object + representing the running daemon. + """ + + basename = os.path.splitext(os.path.basename(prog))[0] + cmd = [prog, "--foreground", "--log-level", "debug", + "--log-file", self.path(basename + ".log")] + if args.profile and basename != "rootd": + cmd.extend(( + "--profile", self.path(basename + ".prof"))) + env = dict(os.environ, RPKI_CONF = self.path("rpki.conf")) + p = subprocess.Popen(cmd, cwd = self.path(), env = env) + print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) + return p + + def run_rpkid(self): + """ + Run rpkid. + """ + + return self.run_python_daemon(prog_rpkid) + + def run_irdbd(self): + """ + Run irdbd. + """ + + return self.run_python_daemon(prog_irdbd) + + def run_pubd(self): + """ + Run pubd. + """ + + return self.run_python_daemon(prog_pubd) + + def run_rootd(self): + """ + Run rootd. + """ + + return self.run_python_daemon(prog_rootd) + + def run_rsyncd(self): + """ + Run rsyncd. + """ + + p = subprocess.Popen(("rsync", "--daemon", "--no-detach", "--config", "rsyncd.conf"), + cwd = self.path()) + print "Running rsyncd for %s: pid %d process %r" % (self.name, p.pid, p) + return p - def run_gui(self): - """ - Start an instance of the RPKI GUI under the Django test server and - return a subprocess.Popen object representing the running daemon. - """ + def run_gui(self): + """ + Start an instance of the RPKI GUI under the Django test server and + return a subprocess.Popen object representing the running daemon. + """ - port = 8000 + self.engine - cmd = (prog_rpki_manage, "runserver", str(port)) - env = dict(os.environ, - RPKI_CONF = self.path("rpki.conf"), - RPKI_DJANGO_DEBUG = "yes", - ALLOW_PLAIN_HTTP_FOR_TESTING = "I solemnly swear that I am not running this in production") - p = subprocess.Popen(cmd, cwd = self.path(), env = env, - stdout = open(self.path("gui.log"), "w"), stderr = subprocess.STDOUT) - print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) - return p + port = 8000 + self.engine + cmd = (prog_rpki_manage, "runserver", str(port)) + env = dict(os.environ, + RPKI_CONF = self.path("rpki.conf"), + RPKI_DJANGO_DEBUG = "yes", + ALLOW_PLAIN_HTTP_FOR_TESTING = "I solemnly swear that I am not running this in production") + p = subprocess.Popen(cmd, cwd = self.path(), env = env, + stdout = open(self.path("gui.log"), "w"), stderr = subprocess.STDOUT) + print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) + return p def create_root_certificate(db_root): - print "Creating rootd RPKI root certificate" + print "Creating rootd RPKI root certificate" - root_resources = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as("0-4294967295"), - v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), - v6 = rpki.resource_set.resource_set_ipv6("::/0")) + root_resources = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as("0-4294967295"), + v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), + v6 = rpki.resource_set.resource_set_ipv6("::/0")) - root_key = rpki.x509.RSA.generate(quiet = True) + root_key = rpki.x509.RSA.generate(quiet = True) - root_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name) + root_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name) - rrdp_uri = "http://localhost:%s/rrdp/notify.xml" % db.root.pubd.rrdp_port + rrdp_uri = "http://localhost:%s/rrdp/notify.xml" % db.root.pubd.rrdp_port - root_sia = (root_uri + "/", root_uri + "/root.mft", None, rrdp_uri) + root_sia = (root_uri + "/", root_uri + "/root.mft", None, rrdp_uri) - root_cert = rpki.x509.X509.self_certify( - keypair = root_key, - subject_key = root_key.get_public(), - serial = 1, - sia = root_sia, - notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), - resources = root_resources) + root_cert = rpki.x509.X509.self_certify( + keypair = root_key, + subject_key = root_key.get_public(), + serial = 1, + sia = root_sia, + notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), + resources = root_resources) - with open(db_root.path("root.cer"), "wb") as f: - f.write(root_cert.get_DER()) + with open(db_root.path("root.cer"), "wb") as f: + f.write(root_cert.get_DER()) - with open(db_root.path("root.key"), "wb") as f: - f.write(root_key.get_DER()) + with open(db_root.path("root.key"), "wb") as f: + f.write(root_key.get_DER()) - with open(os.path.join(test_dir, "root.tal"), "w") as f: - f.write(root_uri + ".cer\n\n") - f.write(root_key.get_public().get_Base64()) + with open(os.path.join(test_dir, "root.tal"), "w") as f: + f.write(root_uri + ".cer\n\n") + f.write(root_key.get_public().get_Base64()) logger = logging.getLogger(__name__) @@ -772,247 +772,247 @@ args = parser.parse_args() try: - if args.pidfile is not None: - with open(args.pidfile, "w") as f: - print "Writing pidfile", f.name - f.write("%s\n" % os.getpid()) + if args.pidfile is not None: + with open(args.pidfile, "w") as f: + print "Writing pidfile", f.name + f.write("%s\n" % os.getpid()) - rpki.log.init("yamltest", argparse.Namespace(log_level = logging.DEBUG, - log_handler = lambda: logging.StreamHandler(sys.stdout))) + rpki.log.init("yamltest", argparse.Namespace(log_level = logging.DEBUG, + log_handler = lambda: logging.StreamHandler(sys.stdout))) - # Allow optional config file for this tool to override default - # passwords: this is mostly so that I can show a complete working - # example without publishing my own server's passwords. + # Allow optional config file for this tool to override default + # passwords: this is mostly so that I can show a complete working + # example without publishing my own server's passwords. - cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True) + cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True) - only_one_pubd = cfg.getboolean("only_one_pubd", True) - allocation.base_port = cfg.getint("base_port", 4400) + only_one_pubd = cfg.getboolean("only_one_pubd", True) + allocation.base_port = cfg.getint("base_port", 4400) - config_overrides = dict( - (k, cfg.get(k)) - for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", - "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username") - if cfg.has_option(k)) + config_overrides = dict( + (k, cfg.get(k)) + for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", + "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username") + if cfg.has_option(k)) - # Start clean, maybe + # Start clean, maybe - if not args.skip_config: - for root, dirs, files in os.walk(test_dir, topdown = False): - for fn in files: - os.unlink(os.path.join(root, fn)) - for d in dirs: - os.rmdir(os.path.join(root, d)) + if not args.skip_config: + for root, dirs, files in os.walk(test_dir, topdown = False): + for fn in files: + os.unlink(os.path.join(root, fn)) + for d in dirs: + os.rmdir(os.path.join(root, d)) - # Read first YAML doc in file and process as compact description of - # test layout and resource allocations. Ignore subsequent YAML docs, - # they're for smoketest.py, not this script. + # Read first YAML doc in file and process as compact description of + # test layout and resource allocations. Ignore subsequent YAML docs, + # they're for smoketest.py, not this script. - db = allocation_db(yaml.safe_load_all(args.yaml_file).next()) + db = allocation_db(yaml.safe_load_all(args.yaml_file).next()) - # Show what we loaded + # Show what we loaded - #db.dump() + #db.dump() - if args.skip_config: + if args.skip_config: - print "Skipping pre-daemon configuration, assuming you already did that" + print "Skipping pre-daemon configuration, assuming you already did that" - else: + else: - # Set up each entity in our test, create publication directories, - # and initialize server BPKI. + # Set up each entity in our test, create publication directories, + # and initialize server BPKI. - for d in db: - if not d.is_hosted: - print "Initializing", d.name - os.makedirs(d.path()) - d.dump_conf() - if d.runs_pubd: - os.makedirs(d.path("publication")) - d.dump_rsyncd() - if d.is_root: - os.makedirs(d.path("publication.root")) - d.syncdb() - d.run_rpkic("initialize_server_bpki") - print + for d in db: + if not d.is_hosted: + print "Initializing", d.name + os.makedirs(d.path()) + d.dump_conf() + if d.runs_pubd: + os.makedirs(d.path("publication")) + d.dump_rsyncd() + if d.is_root: + os.makedirs(d.path("publication.root")) + d.syncdb() + d.run_rpkic("initialize_server_bpki") + print - # Initialize resource holding BPKI and generate self-descriptor - # for each entity. + # Initialize resource holding BPKI and generate self-descriptor + # for each entity. - for d in db: - d.run_rpkic("create_identity", d.name) + for d in db: + d.run_rpkic("create_identity", d.name) - # Create RPKI root certificate. + # Create RPKI root certificate. - create_root_certificate(db.root) + create_root_certificate(db.root) - # Set up rootd. + # Set up rootd. - db.root.run_rpkic("configure_root") + db.root.run_rpkic("configure_root") - # From here on we need to pay attention to initialization order. We - # used to do all the pre-configure_daemons stuff before running any - # of the daemons, but that doesn't work right in hosted cases, so we - # have to interleave configuration with starting daemons, just as - # one would in the real world for this sort of thing. + # From here on we need to pay attention to initialization order. We + # used to do all the pre-configure_daemons stuff before running any + # of the daemons, but that doesn't work right in hosted cases, so we + # have to interleave configuration with starting daemons, just as + # one would in the real world for this sort of thing. - progs = [] + progs = [] - try: + try: - for d in db: + for d in db: - if not d.is_hosted: - print - print "Running daemons for", d.name - if d.is_root: - progs.append(d.run_rootd()) - progs.append(d.run_irdbd()) - progs.append(d.run_rpkid()) - if d.runs_pubd: - progs.append(d.run_pubd()) - progs.append(d.run_rsyncd()) - if args.run_gui: - progs.append(d.run_gui()) + if not d.is_hosted: + print + print "Running daemons for", d.name + if d.is_root: + progs.append(d.run_rootd()) + progs.append(d.run_irdbd()) + progs.append(d.run_rpkid()) + if d.runs_pubd: + progs.append(d.run_pubd()) + progs.append(d.run_rsyncd()) + if args.run_gui: + progs.append(d.run_gui()) - if args.synchronize or not args.skip_config: + if args.synchronize or not args.skip_config: - print - print "Giving daemons time to start up" - time.sleep(20) - assert all(p.poll() is None for p in progs) + print + print "Giving daemons time to start up" + time.sleep(20) + assert all(p.poll() is None for p in progs) - if args.skip_config: + if args.skip_config: - print - print "Skipping configure_*, you'll have to do that yourself if needed" + print + print "Skipping configure_*, you'll have to do that yourself if needed" - else: + else: + + for d in db: + + print + print "Configuring", d.name + print + if d.is_root: + assert not d.is_hosted + d.run_rpkic("configure_publication_client", + "--flat" if args.flat_publication else None, + d.path("%s.%s.repository-request.xml" % (d.name, d.name))) + print + d.run_rpkic("configure_repository", + d.path("%s.repository-response.xml" % d.client_handle)) + print + else: + d.parent.run_rpkic("configure_child", + "--valid_until", d.resources.valid_until, + d.path("%s.identity.xml" % d.name)) + print + d.run_rpkic("configure_parent", + d.parent.path("%s.%s.parent-response.xml" % (d.parent.name, d.name))) + print + d.pubd.run_rpkic("configure_publication_client", + "--flat" if args.flat_publication else None, + d.path("%s.%s.repository-request.xml" % (d.name, d.parent.name))) + print + d.run_rpkic("configure_repository", + d.pubd.path("%s.repository-response.xml" % d.client_handle)) + print + + print + print "Done with initial configuration" + print + + if args.synchronize: + print + print "Synchronizing" + print + for d in db: + if not d.is_hosted: + d.run_rpkic("synchronize") + + if args.synchronize or not args.skip_config: + print + print "Loading CSV files" + print + for d in db: + d.dump_asns() + d.dump_prefixes() + d.dump_roas() + d.dump_ghostbusters() + d.dump_router_certificates() - for d in db: + if args.run_gui: + print + print 'GUI user "root", password "fnord"' + for d in db: + if not d.is_hosted: + url = "http://127.0.0.1:%d/rpki/" % (8000 + d.engine) + print "GUI URL", url, "for", d.name + if args.browser: + if d is db.root: + webbrowser.open_new(url) + else: + webbrowser.open_new_tab(url) + time.sleep(2) + + # Wait until something terminates. + + if not args.stop_after_config or args.keep_going: + if args.notify_when_startup_complete: + print + print "Sending SIGUSR1 to process", args.notify_when_startup_complete + os.kill(args.notify_when_startup_complete, signal.SIGUSR1) + print + print "Waiting for daemons to exit" + signal.signal(signal.SIGCHLD, lambda *dont_care: None) + while (any(p.poll() is None for p in progs) + if args.keep_going else + all(p.poll() is None for p in progs)): + signal.pause() + + finally: print - print "Configuring", d.name + print "Shutting down" print - if d.is_root: - assert not d.is_hosted - d.run_rpkic("configure_publication_client", - "--flat" if args.flat_publication else None, - d.path("%s.%s.repository-request.xml" % (d.name, d.name))) - print - d.run_rpkic("configure_repository", - d.path("%s.repository-response.xml" % d.client_handle)) - print + + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + if args.profile: + how_long = 300 else: - d.parent.run_rpkic("configure_child", - "--valid_until", d.resources.valid_until, - d.path("%s.identity.xml" % d.name)) - print - d.run_rpkic("configure_parent", - d.parent.path("%s.%s.parent-response.xml" % (d.parent.name, d.name))) - print - d.pubd.run_rpkic("configure_publication_client", - "--flat" if args.flat_publication else None, - d.path("%s.%s.repository-request.xml" % (d.name, d.parent.name))) - print - d.run_rpkic("configure_repository", - d.pubd.path("%s.repository-response.xml" % d.client_handle)) - print - - print - print "Done with initial configuration" - print - - if args.synchronize: - print - print "Synchronizing" - print - for d in db: - if not d.is_hosted: - d.run_rpkic("synchronize") - - if args.synchronize or not args.skip_config: - print - print "Loading CSV files" - print - for d in db: - d.dump_asns() - d.dump_prefixes() - d.dump_roas() - d.dump_ghostbusters() - d.dump_router_certificates() - - if args.run_gui: - print - print 'GUI user "root", password "fnord"' - for d in db: - if not d.is_hosted: - url = "http://127.0.0.1:%d/rpki/" % (8000 + d.engine) - print "GUI URL", url, "for", d.name - if args.browser: - if d is db.root: - webbrowser.open_new(url) - else: - webbrowser.open_new_tab(url) - time.sleep(2) - - # Wait until something terminates. - - if not args.stop_after_config or args.keep_going: - if args.notify_when_startup_complete: - print - print "Sending SIGUSR1 to process", args.notify_when_startup_complete - os.kill(args.notify_when_startup_complete, signal.SIGUSR1) - print - print "Waiting for daemons to exit" - signal.signal(signal.SIGCHLD, lambda *dont_care: None) - while (any(p.poll() is None for p in progs) - if args.keep_going else - all(p.poll() is None for p in progs)): - signal.pause() - - finally: - - print - print "Shutting down" - print - - signal.signal(signal.SIGCHLD, signal.SIG_DFL) - - if args.profile: - how_long = 300 - else: - how_long = 30 + how_long = 30 - how_often = how_long / 2 + how_often = how_long / 2 - for i in xrange(how_long): - if i % how_often == 0: - for p in progs: - if p.poll() is None: - print "Politely nudging pid %d" % p.pid - p.terminate() - print - if all(p.poll() is not None for p in progs): - break - time.sleep(1) + for i in xrange(how_long): + if i % how_often == 0: + for p in progs: + if p.poll() is None: + print "Politely nudging pid %d" % p.pid + p.terminate() + print + if all(p.poll() is not None for p in progs): + break + time.sleep(1) - for p in progs: - if p.poll() is None: - print "Pulling the plug on pid %d" % p.pid - p.kill() + for p in progs: + if p.poll() is None: + print "Pulling the plug on pid %d" % p.pid + p.kill() - for p in progs: - print "Program pid %d %r returned %d" % (p.pid, p, p.wait()) + for p in progs: + print "Program pid %d %r returned %d" % (p.pid, p, p.wait()) except Exception, e: - print "Blowing out on exception", str(e) - raise + print "Blowing out on exception", str(e) + raise finally: - if args.pidfile is not None and os.path.exists(args.pidfile): - os.unlink(args.pidfile) + if args.pidfile is not None and os.path.exists(args.pidfile): + os.unlink(args.pidfile) # Local Variables: # indent-tabs-mode: nil |