diff options
| author | Rob Austein <sra@hactrn.net> | 2015-10-26 06:29:00 +0000 |
|---|---|---|
| committer | Rob Austein <sra@hactrn.net> | 2015-10-26 06:29:00 +0000 |
| commit | b46deb1417dc3596e9ac9fe2fe8cc0b7f42457e7 (patch) | |
| tree | ca0dc0276d1adc168bc3337ce0564c4ec4957c1b /ca | |
| parent | 397beaf6d9900dc3b3cb612c89ebf1d57b1d16f6 (diff) | |
"Any programmer who fails to comply with the standard naming, formatting,
or commenting conventions should be shot. If it so happens that it is
inconvenient to shoot him, then he is to be politely requested to recode
his program in adherence to the above standard."
-- Michael Spier, Digital Equipment Corporation
svn path=/branches/tk705/; revision=6152
Diffstat (limited to 'ca')
| -rwxr-xr-x | ca/irbe_cli | 422 | ||||
| -rwxr-xr-x | ca/irdbd | 4 | ||||
| -rwxr-xr-x | ca/pubd | 4 | ||||
| -rwxr-xr-x | ca/rootd | 4 | ||||
| -rwxr-xr-x | ca/rpki-confgen | 455 | ||||
| -rwxr-xr-x | ca/rpki-sql-backup | 14 | ||||
| -rwxr-xr-x | ca/rpki-sql-setup | 378 | ||||
| -rwxr-xr-x | ca/rpki-start-servers | 36 | ||||
| -rwxr-xr-x | ca/rpkic | 4 | ||||
| -rwxr-xr-x | ca/rpkid | 4 | ||||
| -rwxr-xr-x | ca/rpkigui-apache-conf-gen | 672 | ||||
| -rwxr-xr-x | ca/rpkigui-import-routes | 1 | ||||
| -rwxr-xr-x | ca/tests/bgpsec-yaml.py | 56 | ||||
| -rw-r--r-- | ca/tests/myrpki-xml-parse-test.py | 74 | ||||
| -rw-r--r-- | ca/tests/old_irdbd.py | 4 | ||||
| -rw-r--r-- | ca/tests/smoketest.py | 2196 | ||||
| -rw-r--r-- | ca/tests/sql-cleaner.py | 32 | ||||
| -rw-r--r-- | ca/tests/sql-dumper.py | 20 | ||||
| -rwxr-xr-x | ca/tests/test-rrdp.py | 114 | ||||
| -rw-r--r-- | ca/tests/testpoke.py | 118 | ||||
| -rw-r--r-- | ca/tests/xml-parse-test.py | 108 | ||||
| -rw-r--r-- | ca/tests/yamlconf.py | 1426 | ||||
| -rw-r--r-- | ca/tests/yamltest.py | 1606 |
23 files changed, 3878 insertions, 3874 deletions
diff --git a/ca/irbe_cli b/ca/irbe_cli index ebd5d2a5..59039bf9 100755 --- a/ca/irbe_cli +++ b/ca/irbe_cli @@ -51,231 +51,231 @@ import rpki.publication pem_out = None class UsageWrapper(textwrap.TextWrapper): - """ - Call interface around Python textwrap.Textwrapper class. - """ - - def __call__(self, *args): """ - Format arguments, with TextWrapper indentation. + Call interface around Python textwrap.Textwrapper class. """ - return self.fill(textwrap.dedent(" ".join(args))) + + def __call__(self, *args): + """ + Format arguments, with TextWrapper indentation. + """ + return self.fill(textwrap.dedent(" ".join(args))) usage_fill = UsageWrapper(subsequent_indent = " " * 4) class reply_elt_mixin(object): - """ - Protocol mix-in for printout of reply PDUs. - """ - - is_cmd = False - - def client_reply_decode(self): - pass - - def client_reply_show(self): - print self.element_name - for i in self.attributes + self.elements: - if getattr(self, i) is not None: - print " %s: %s" % (i, getattr(self, i)) - -class cmd_elt_mixin(reply_elt_mixin): - """ - Protocol mix-in for command line client element PDUs. - """ - - is_cmd = True - - ## @var excludes - # XML attributes and elements that should not be allowed as command - # line arguments. - excludes = () - - @classmethod - def usage(cls): """ - Generate usage message for this PDU. + Protocol mix-in for printout of reply PDUs. """ - args = " ".join("--" + x + "=" for x in cls.attributes + cls.elements if x not in cls.excludes) - bools = " ".join("--" + x for x in cls.booleans) - if args and bools: - return args + " " + bools - else: - return args or bools - def client_getopt(self, argv): - """ - Parse options for this class. - """ - # pylint: disable=W0621 - opts, argv = getopt.getopt(argv, "", [x + "=" for x in self.attributes + self.elements if x not in self.excludes] + list(self.booleans)) - for o, a in opts: - o = o[2:] - handler = getattr(self, "client_query_" + o, None) - if handler is not None: - handler(a) - elif o in self.booleans: - setattr(self, o, True) - else: - assert o in self.attributes - setattr(self, o, a) - return argv - - def client_query_bpki_cert(self, arg): - """ - Special handler for --bpki_cert option. - """ - self.bpki_cert = rpki.x509.X509(Auto_file = arg) + is_cmd = False - def client_query_glue(self, arg): - """ - Special handler for --bpki_glue option. - """ - self.bpki_glue = rpki.x509.X509(Auto_file = arg) + def client_reply_decode(self): + pass - def client_query_bpki_cms_cert(self, arg): - """ - Special handler for --bpki_cms_cert option. - """ - self.bpki_cms_cert = rpki.x509.X509(Auto_file = arg) + def client_reply_show(self): + print self.element_name + for i in self.attributes + self.elements: + if getattr(self, i) is not None: + print " %s: %s" % (i, getattr(self, i)) - def client_query_cms_glue(self, arg): +class cmd_elt_mixin(reply_elt_mixin): """ - Special handler for --bpki_cms_glue option. + Protocol mix-in for command line client element PDUs. """ - self.bpki_cms_glue = rpki.x509.X509(Auto_file = arg) -class cmd_msg_mixin(object): - """ - Protocol mix-in for command line client message PDUs. - """ + is_cmd = True + + ## @var excludes + # XML attributes and elements that should not be allowed as command + # line arguments. + excludes = () + + @classmethod + def usage(cls): + """ + Generate usage message for this PDU. + """ + args = " ".join("--" + x + "=" for x in cls.attributes + cls.elements if x not in cls.excludes) + bools = " ".join("--" + x for x in cls.booleans) + if args and bools: + return args + " " + bools + else: + return args or bools + + def client_getopt(self, argv): + """ + Parse options for this class. + """ + # pylint: disable=W0621 + opts, argv = getopt.getopt(argv, "", [x + "=" for x in self.attributes + self.elements if x not in self.excludes] + list(self.booleans)) + for o, a in opts: + o = o[2:] + handler = getattr(self, "client_query_" + o, None) + if handler is not None: + handler(a) + elif o in self.booleans: + setattr(self, o, True) + else: + assert o in self.attributes + setattr(self, o, a) + return argv + + def client_query_bpki_cert(self, arg): + """ + Special handler for --bpki_cert option. + """ + self.bpki_cert = rpki.x509.X509(Auto_file = arg) + + def client_query_glue(self, arg): + """ + Special handler for --bpki_glue option. + """ + self.bpki_glue = rpki.x509.X509(Auto_file = arg) + + def client_query_bpki_cms_cert(self, arg): + """ + Special handler for --bpki_cms_cert option. + """ + self.bpki_cms_cert = rpki.x509.X509(Auto_file = arg) + + def client_query_cms_glue(self, arg): + """ + Special handler for --bpki_cms_glue option. + """ + self.bpki_cms_glue = rpki.x509.X509(Auto_file = arg) - @classmethod - def usage(cls): +class cmd_msg_mixin(object): """ - Generate usage message for this PDU. + Protocol mix-in for command line client message PDUs. """ - for k, v in cls.pdus.items(): - if v.is_cmd: - print usage_fill(k, v.usage()) + + @classmethod + def usage(cls): + """ + Generate usage message for this PDU. + """ + for k, v in cls.pdus.items(): + if v.is_cmd: + print usage_fill(k, v.usage()) # left-right protcol class left_right_msg(cmd_msg_mixin, rpki.left_right.msg): - class self_elt(cmd_elt_mixin, rpki.left_right.self_elt): - pass + class self_elt(cmd_elt_mixin, rpki.left_right.self_elt): + pass - class bsc_elt(cmd_elt_mixin, rpki.left_right.bsc_elt): + class bsc_elt(cmd_elt_mixin, rpki.left_right.bsc_elt): - excludes = ("pkcs10_request",) + excludes = ("pkcs10_request",) - def client_query_signing_cert(self, arg): - """--signing_cert option.""" - self.signing_cert = rpki.x509.X509(Auto_file = arg) + def client_query_signing_cert(self, arg): + """--signing_cert option.""" + self.signing_cert = rpki.x509.X509(Auto_file = arg) - def client_query_signing_cert_crl(self, arg): - """--signing_cert_crl option.""" - self.signing_cert_crl = rpki.x509.CRL(Auto_file = arg) + def client_query_signing_cert_crl(self, arg): + """--signing_cert_crl option.""" + self.signing_cert_crl = rpki.x509.CRL(Auto_file = arg) - def client_reply_decode(self): - global pem_out - if pem_out is not None and self.pkcs10_request is not None: - if isinstance(pem_out, str): - pem_out = open(pem_out, "w") - pem_out.write(self.pkcs10_request.get_PEM()) + def client_reply_decode(self): + global pem_out + if pem_out is not None and self.pkcs10_request is not None: + if isinstance(pem_out, str): + pem_out = open(pem_out, "w") + pem_out.write(self.pkcs10_request.get_PEM()) - class parent_elt(cmd_elt_mixin, rpki.left_right.parent_elt): - pass + class parent_elt(cmd_elt_mixin, rpki.left_right.parent_elt): + pass - class child_elt(cmd_elt_mixin, rpki.left_right.child_elt): - pass + class child_elt(cmd_elt_mixin, rpki.left_right.child_elt): + pass - class repository_elt(cmd_elt_mixin, rpki.left_right.repository_elt): - pass + class repository_elt(cmd_elt_mixin, rpki.left_right.repository_elt): + pass - class list_published_objects_elt(cmd_elt_mixin, rpki.left_right.list_published_objects_elt): - excludes = ("uri",) + class list_published_objects_elt(cmd_elt_mixin, rpki.left_right.list_published_objects_elt): + excludes = ("uri",) - class list_received_resources_elt(cmd_elt_mixin, rpki.left_right.list_received_resources_elt): - excludes = ("parent_handle", "notBefore", "notAfter", "uri", "sia_uri", "aia_uri", "asn", "ipv4", "ipv6") + class list_received_resources_elt(cmd_elt_mixin, rpki.left_right.list_received_resources_elt): + excludes = ("parent_handle", "notBefore", "notAfter", "uri", "sia_uri", "aia_uri", "asn", "ipv4", "ipv6") - class report_error_elt(reply_elt_mixin, rpki.left_right.report_error_elt): - pass + class report_error_elt(reply_elt_mixin, rpki.left_right.report_error_elt): + pass - pdus = dict((x.element_name, x) - for x in (self_elt, bsc_elt, parent_elt, child_elt, repository_elt, - list_published_objects_elt, list_received_resources_elt, report_error_elt)) + pdus = dict((x.element_name, x) + for x in (self_elt, bsc_elt, parent_elt, child_elt, repository_elt, + list_published_objects_elt, list_received_resources_elt, report_error_elt)) class left_right_sax_handler(rpki.left_right.sax_handler): # pylint: disable=W0232 - pdu = left_right_msg + pdu = left_right_msg class left_right_cms_msg(rpki.left_right.cms_msg): - saxify = left_right_sax_handler.saxify + saxify = left_right_sax_handler.saxify # Publication protocol class publication_msg(cmd_msg_mixin, rpki.publication.msg): - class config_elt(cmd_elt_mixin, rpki.publication.config_elt): + class config_elt(cmd_elt_mixin, rpki.publication.config_elt): - def client_query_bpki_crl(self, arg): - """ - Special handler for --bpki_crl option. - """ - self.bpki_crl = rpki.x509.CRL(Auto_file = arg) + def client_query_bpki_crl(self, arg): + """ + Special handler for --bpki_crl option. + """ + self.bpki_crl = rpki.x509.CRL(Auto_file = arg) - class client_elt(cmd_elt_mixin, rpki.publication.client_elt): - pass + class client_elt(cmd_elt_mixin, rpki.publication.client_elt): + pass - class certificate_elt(cmd_elt_mixin, rpki.publication.certificate_elt): - pass + class certificate_elt(cmd_elt_mixin, rpki.publication.certificate_elt): + pass - class crl_elt(cmd_elt_mixin, rpki.publication.crl_elt): - pass + class crl_elt(cmd_elt_mixin, rpki.publication.crl_elt): + pass - class manifest_elt(cmd_elt_mixin, rpki.publication.manifest_elt): - pass + class manifest_elt(cmd_elt_mixin, rpki.publication.manifest_elt): + pass - class roa_elt(cmd_elt_mixin, rpki.publication.roa_elt): - pass + class roa_elt(cmd_elt_mixin, rpki.publication.roa_elt): + pass - class report_error_elt(reply_elt_mixin, rpki.publication.report_error_elt): - pass + class report_error_elt(reply_elt_mixin, rpki.publication.report_error_elt): + pass - class ghostbuster_elt(cmd_elt_mixin, rpki.publication.ghostbuster_elt): - pass + class ghostbuster_elt(cmd_elt_mixin, rpki.publication.ghostbuster_elt): + pass - pdus = dict((x.element_name, x) - for x in (config_elt, client_elt, certificate_elt, crl_elt, - manifest_elt, roa_elt, report_error_elt, - ghostbuster_elt)) + pdus = dict((x.element_name, x) + for x in (config_elt, client_elt, certificate_elt, crl_elt, + manifest_elt, roa_elt, report_error_elt, + ghostbuster_elt)) class publication_sax_handler(rpki.publication.sax_handler): # pylint: disable=W0232 - pdu = publication_msg + pdu = publication_msg class publication_cms_msg(rpki.publication.cms_msg): - saxify = publication_sax_handler.saxify + saxify = publication_sax_handler.saxify # Usage top_opts = ["config=", "help", "pem_out=", "quiet", "verbose"] def usage(code = 1): - if __doc__ is not None: - print __doc__.strip() + if __doc__ is not None: + print __doc__.strip() + print + print "Usage:" + print + print "# Top-level options:" + print usage_fill(*["--" + x for x in top_opts]) print - print "Usage:" - print - print "# Top-level options:" - print usage_fill(*["--" + x for x in top_opts]) - print - print "# left-right protocol:" - left_right_msg.usage() - print - print "# publication protocol:" - publication_msg.usage() - sys.exit(code) + print "# left-right protocol:" + left_right_msg.usage() + print + print "# publication protocol:" + publication_msg.usage() + sys.exit(code) # Main program @@ -284,26 +284,26 @@ rpki.log.init("irbe_cli") argv = sys.argv[1:] if not argv: - usage(0) + usage(0) cfg_file = None verbose = True opts, argv = getopt.getopt(argv, "c:hpqv?", top_opts) for o, a in opts: - if o in ("-?", "-h", "--help"): - usage(0) - elif o in ("-c", "--config"): - cfg_file = a - elif o in ("-p", "--pem_out"): - pem_out = a - elif o in ("-q", "--quiet"): - verbose = False - elif o in ("-v", "--verbose"): - verbose = True + if o in ("-?", "-h", "--help"): + usage(0) + elif o in ("-c", "--config"): + cfg_file = a + elif o in ("-p", "--pem_out"): + pem_out = a + elif o in ("-q", "--quiet"): + verbose = False + elif o in ("-v", "--verbose"): + verbose = True if not argv: - usage(1) + usage(1) cfg = rpki.config.parser(set_filename = cfg_file, section = "irbe_cli") @@ -311,16 +311,16 @@ q_msg_left_right = [] q_msg_publication = [] while argv: - if argv[0] in left_right_msg.pdus: - q_pdu = left_right_msg.pdus[argv[0]]() - q_msg = q_msg_left_right - elif argv[0] in publication_msg.pdus: - q_pdu = publication_msg.pdus[argv[0]]() - q_msg = q_msg_publication - else: - usage(1) - argv = q_pdu.client_getopt(argv[1:]) - q_msg.append(q_pdu) + if argv[0] in left_right_msg.pdus: + q_pdu = left_right_msg.pdus[argv[0]]() + q_msg = q_msg_left_right + elif argv[0] in publication_msg.pdus: + q_pdu = publication_msg.pdus[argv[0]]() + q_msg = q_msg_publication + else: + usage(1) + argv = q_pdu.client_getopt(argv[1:]) + q_msg.append(q_pdu) os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb") @@ -331,36 +331,36 @@ irbe = server_ca.ee_certificates.get(purpose = "irbe") if q_msg_left_right: - rpkid = server_ca.ee_certificates.get(purpose = "rpkid") + rpkid = server_ca.ee_certificates.get(purpose = "rpkid") - rpkid_url = "http://%s:%s/left-right/" % ( - cfg.get("server-host", section = "rpkid"), - cfg.get("server-port", section = "rpkid")) + rpkid_url = "http://%s:%s/left-right/" % ( + cfg.get("server-host", section = "rpkid"), + cfg.get("server-port", section = "rpkid")) - rpki.http_simple.client( - proto_cms_msg = left_right_cms_msg, - client_key = irbe.private_key, - client_cert = irbe.certificate, - server_ta = server_ca.certificate, - server_cert = rpkid.certificate, - url = rpkid_url, - debug = verbose, - q_msg = left_right_msg(*q_msg_left_right)) + rpki.http_simple.client( + proto_cms_msg = left_right_cms_msg, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = server_ca.certificate, + server_cert = rpkid.certificate, + url = rpkid_url, + debug = verbose, + q_msg = left_right_msg(*q_msg_left_right)) if q_msg_publication: - pubd = server_ca.ee_certificates.get(purpose = "pubd") - - pubd_url = "http://%s:%s/control/" % ( - cfg.get("server-host", section = "pubd"), - cfg.get("server-port", section = "pubd")) - - rpki.http_simple.client( - proto_cms_msg = publication_cms_msg, - client_key = irbe.private_key, - client_cert = irbe.certificate, - server_ta = server_ca.certificate, - server_cert = pubd.certificate, - url = pubd_url, - debug = verbose, - q_msg = publication_msg(*q_msg_publication)) + pubd = server_ca.ee_certificates.get(purpose = "pubd") + + pubd_url = "http://%s:%s/control/" % ( + cfg.get("server-host", section = "pubd"), + cfg.get("server-port", section = "pubd")) + + rpki.http_simple.client( + proto_cms_msg = publication_cms_msg, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = server_ca.certificate, + server_cert = pubd.certificate, + url = pubd_url, + debug = verbose, + q_msg = publication_msg(*q_msg_publication)) @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.irdbd - rpki.irdbd.main() + import rpki.irdbd + rpki.irdbd.main() @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.pubd - rpki.pubd.main() + import rpki.pubd + rpki.pubd.main() @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.rootd - rpki.rootd.main() + import rpki.rootd + rpki.rootd.main() diff --git a/ca/rpki-confgen b/ca/rpki-confgen index 07c87f0f..beb8506d 100755 --- a/ca/rpki-confgen +++ b/ca/rpki-confgen @@ -38,254 +38,259 @@ xml8_wrapper = textwrap.TextWrapper(initial_indent = space8, subsequent_indent = class Option(object): - def __init__(self, name, value, doc): - self.name = name - self.value = value - self.doc = doc - - @property - def width(self): - return len(self.name) - - def to_xml(self): - x = Element("option", name = self.name) - if self.value is not None: - x.set("value", self.value) - for d in self.doc: - SubElement(x, "doc").text = "\n" + xml8_wrapper.fill(d) + "\n" + space6 - return x - - def to_wiki(self, f): - f.write("\n== %s == #%s\n" % (self.name, self.name)) - for d in self.doc: - f.write("\n%s\n" % wiki_wrapper.fill(d)) - if self.value is None: - f.write("\n%s\n" % wiki_wrapper.fill("No default value.")) - else: - f.write("\n{{{\n#!ini\n%s = %s\n}}}\n" % (self.name, self.value)) - - def to_conf(self, f, width): - for i, d in enumerate(self.doc): - f.write("%s\n%s\n" % ("" if i == 0 else "#", conf_wrapper.fill(d))) - if self.value is None: - f.write("\n#%-*s = ???\n" % (width - 1, self.name)) - else: - f.write("\n%-*s = %s\n" % (width, self.name, self.value)) + def __init__(self, name, value, doc): + self.name = name + self.value = value + self.doc = doc + + @property + def width(self): + return len(self.name) + + def to_xml(self): + x = Element("option", name = self.name) + if self.value is not None: + x.set("value", self.value) + for d in self.doc: + SubElement(x, "doc").text = "\n" + xml8_wrapper.fill(d) + "\n" + space6 + return x + + def to_wiki(self, f): + f.write("\n== %s == #%s\n" % (self.name, self.name)) + for d in self.doc: + f.write("\n%s\n" % wiki_wrapper.fill(d)) + if self.value is None: + f.write("\n%s\n" % wiki_wrapper.fill("No default value.")) + else: + f.write("\n{{{\n#!ini\n%s = %s\n}}}\n" % (self.name, self.value)) + + def to_conf(self, f, width): + for i, d in enumerate(self.doc): + f.write("%s\n%s\n" % ("" if i == 0 else "#", conf_wrapper.fill(d))) + if self.value is None: + f.write("\n#%-*s = ???\n" % (width - 1, self.name)) + else: + f.write("\n%-*s = %s\n" % (width, self.name, self.value)) class Section(object): - def __init__(self, name): - self.name = name - self.doc = [] - self.options = [] - - @property - def width(self): - return max(o.width for o in self.options) - - @classmethod - def from_xml(cls, elt): - self = cls(name = elt.get("name")) - for x in elt.iterchildren("doc"): - self.doc.append(" ".join(x.text.split())) - for x in elt.iterchildren("option"): - self.options.append(Option(name = x.get("name"), value = x.get("value"), - doc = [" ".join(d.text.split()) - for d in x.iterchildren("doc")])) - return self - - def to_xml(self): - x = Element("section", name = self.name) - for d in self.doc: - SubElement(x, "doc").text = "\n" + xml6_wrapper.fill(d) + "\n" + space4 - x.extend(o.to_xml() for o in self.options) - return x - - def to_wiki(self, f): - f.write("\n= [%s] section = #%s\n" % (self.name, self.name)) - for d in self.doc: - f.write("\n%s\n" % wiki_wrapper.fill(d)) - for o in self.options: - o.to_wiki(f) - - def to_conf(self, f, width): - f.write("\n" + "#" * 78 + "\n\n[" + self.name + "]\n") - if self.doc: - f.write("\n##") - for i, d in enumerate(self.doc): - f.write("%s\n%s\n" % ("" if i == 0 else "#", conf_wrapper.fill(d))) - f.write("##\n") - for o in self.options: - o.to_conf(f, width) + def __init__(self, name): + self.name = name + self.doc = [] + self.options = [] + + @property + def width(self): + return max(o.width for o in self.options) + + @classmethod + def from_xml(cls, elt): + self = cls(name = elt.get("name")) + for x in elt.iterchildren("doc"): + self.doc.append(" ".join(x.text.split())) + for x in elt.iterchildren("option"): + self.options.append(Option(name = x.get("name"), value = x.get("value"), + doc = [" ".join(d.text.split()) + for d in x.iterchildren("doc")])) + return self + + def to_xml(self): + x = Element("section", name = self.name) + for d in self.doc: + SubElement(x, "doc").text = "\n" + xml6_wrapper.fill(d) + "\n" + space4 + x.extend(o.to_xml() for o in self.options) + return x + + def to_wiki(self, f): + f.write("\n= [%s] section = #%s\n" % (self.name, self.name)) + for d in self.doc: + f.write("\n%s\n" % wiki_wrapper.fill(d)) + for o in self.options: + o.to_wiki(f) + + def to_conf(self, f, width): + f.write("\n" + "#" * 78 + "\n\n[" + self.name + "]\n") + if self.doc: + f.write("\n##") + for i, d in enumerate(self.doc): + f.write("%s\n%s\n" % ("" if i == 0 else "#", conf_wrapper.fill(d))) + f.write("##\n") + for o in self.options: + o.to_conf(f, width) def wiki_header(f, ident, toc): - f.write("\n".join(( - "{{{", - "#!comment", - "", - star78, - "THIS PAGE WAS GENERATED AUTOMATICALLY, DO NOT EDIT.", - "", - "Generated from " + ident, - " by $Id$", - star78, - "", - "}}}", - ""))) - if toc is not None: - f.write("[[TracNav(%s)]]\n" % toc) - f.write("[[PageOutline]]\n") + f.write("\n".join(( + "{{{", + "#!comment", + "", + star78, + "THIS PAGE WAS GENERATED AUTOMATICALLY, DO NOT EDIT.", + "", + "Generated from " + ident, + " by $Id$", + star78, + "", + "}}}", + ""))) + if toc is not None: + f.write("[[TracNav(%s)]]\n" % toc) + f.write("[[PageOutline]]\n") def conf_header(f, ident): - f.write("\n".join(( - "# Automatically generated. Edit as needed, but be careful of overwriting.", - "#", - "# Generated from " + ident, - "# by $Id$", - ""))) + f.write("\n".join(( + "# Automatically generated. Edit as needed, but be careful of overwriting.", + "#", + "# Generated from " + ident, + "# by $Id$", + ""))) # http://stackoverflow.com/questions/9027028/argparse-argument-order class CustomAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string = None): - if not "ordered_args" in namespace: - namespace.ordered_args = [] - namespace.ordered_args.append((self.dest, values)) + def __call__(self, parser, namespace, values, option_string = None): + if not "ordered_args" in namespace: + namespace.ordered_args = [] + namespace.ordered_args.append((self.dest, values)) class CustomFlagAction(argparse.Action): - def __init__(self, option_strings, dest, default = None, - required = False, help = None): # pylint: disable=W0622 - super(CustomFlagAction, self).__init__( - option_strings = option_strings, dest = dest, nargs = 0, - const = None, default = default, required = required, help = help) - def __call__(self, parser, namespace, values, option_string = None): - if not "ordered_args" in namespace: - namespace.ordered_args = [] - namespace.ordered_args.append((self.dest, None)) + def __init__(self, option_strings, dest, default = None, + required = False, help = None): # pylint: disable=W0622 + super(CustomFlagAction, self).__init__( + option_strings = option_strings, + dest = dest, + nargs = 0, + const = None, + default = default, + required = required, + help = help) + def __call__(self, parser, namespace, values, option_string = None): + if not "ordered_args" in namespace: + namespace.ordered_args = [] + namespace.ordered_args.append((self.dest, None)) class main(object): - def __init__(self): - self.sections = [] - self.section_map = None - self.option_map = None - self.ident = None - self.toc = None - - parser = argparse.ArgumentParser(description = __doc__) - parser.add_argument("--read-xml", metavar = "FILE", action = CustomAction, - required = True, type = argparse.FileType("r"), - help = "XML input file defining sections and options") - parser.add_argument("--write-xml", metavar = "FILE", action = CustomAction, - help = "XML file to write") - parser.add_argument("--write-wiki", metavar = "FILE", action = CustomAction, - help = "TracWiki file to write") - parser.add_argument("--write-conf", metavar = "FILE", action = CustomAction, - help = "rpki.conf configuration file to write") - parser.add_argument("--set", metavar = "VARVAL", action = CustomAction, - help = "variable setting in form \"VAR=VAL\"") - parser.add_argument("--pwgen", metavar = "VAR", action = CustomAction, - help = "set variable to generated password") - parser.add_argument("--toc", metavar = "TRACNAV", action = CustomAction, - help = "set TOC value to use with TracNav plugin") - parser.add_argument("--autoconf", action = CustomFlagAction, - help = "configure [autoconf] section") - args = parser.parse_args() - - for cmd, arg in args.ordered_args: - getattr(self, "do_" + cmd)(arg) - - - def do_read_xml(self, arg): - self.option_map = None - root = ElementTree(file = arg).getroot() - self.ident = root.get("ident") - self.sections.extend(Section.from_xml(x) for x in root) - self.option_map = {} - self.section_map = {} - for section in self.sections: - if section.name in self.section_map: - sys.exit("Duplicate section %s" % section.name) - self.section_map[section.name] = section - for option in section.options: - name = (section.name, option.name) - if name in self.option_map: - sys.exit("Duplicate option %s::%s" % name) - self.option_map[name] = option - - - def do_set(self, arg): - try: - name, value = arg.split("=", 1) - section, option = name.split("::") - except ValueError: - sys.exit("Couldn't parse --set specification \"%s\"" % arg) - name = (section, option) - if name not in self.option_map: - sys.exit("Couldn't find option %s::%s" % name) - self.option_map[name].value = value - - - def do_pwgen(self, arg): - try: - section, option = arg.split("::") - except ValueError: - sys.exit("Couldn't parse --pwgen specification \"%s\"" % arg) - name = (section, option) - if name not in self.option_map: - sys.exit("Couldn't find option %s::%s" % name) - self.option_map[name].value = base64.urlsafe_b64encode(os.urandom(66)) - - - def do_autoconf(self, ignored): - try: - import rpki.autoconf - for option in self.section_map["autoconf"].options: + def __init__(self): + self.sections = [] + self.section_map = None + self.option_map = None + self.ident = None + self.toc = None + + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("--read-xml", metavar = "FILE", action = CustomAction, + required = True, type = argparse.FileType("r"), + help = "XML input file defining sections and options") + parser.add_argument("--write-xml", metavar = "FILE", action = CustomAction, + help = "XML file to write") + parser.add_argument("--write-wiki", metavar = "FILE", action = CustomAction, + help = "TracWiki file to write") + parser.add_argument("--write-conf", metavar = "FILE", action = CustomAction, + help = "rpki.conf configuration file to write") + parser.add_argument("--set", metavar = "VARVAL", action = CustomAction, + help = "variable setting in form \"VAR=VAL\"") + parser.add_argument("--pwgen", metavar = "VAR", action = CustomAction, + help = "set variable to generated password") + parser.add_argument("--toc", metavar = "TRACNAV", action = CustomAction, + help = "set TOC value to use with TracNav plugin") + parser.add_argument("--autoconf", action = CustomFlagAction, + help = "configure [autoconf] section") + args = parser.parse_args() + + for cmd, arg in args.ordered_args: + getattr(self, "do_" + cmd)(arg) + + + def do_read_xml(self, arg): + self.option_map = None + root = ElementTree(file = arg).getroot() + self.ident = root.get("ident") + self.sections.extend(Section.from_xml(x) for x in root) + self.option_map = {} + self.section_map = {} + for section in self.sections: + if section.name in self.section_map: + sys.exit("Duplicate section %s" % section.name) + self.section_map[section.name] = section + for option in section.options: + name = (section.name, option.name) + if name in self.option_map: + sys.exit("Duplicate option %s::%s" % name) + self.option_map[name] = option + + + def do_set(self, arg): try: - option.value = getattr(rpki.autoconf, option.name) - except AttributeError: - pass - except ImportError: - sys.exit("rpki.autoconf module is not available") - except KeyError: - sys.exit("Couldn't find autoconf section") + name, value = arg.split("=", 1) + section, option = name.split("::") + except ValueError: + sys.exit("Couldn't parse --set specification \"%s\"" % arg) + name = (section, option) + if name not in self.option_map: + sys.exit("Couldn't find option %s::%s" % name) + self.option_map[name].value = value - def do_write_xml(self, arg): - x = Element("configuration", ident = self.ident) - x.extend(s.to_xml() for s in self.sections) - ElementTree(x).write(arg, pretty_print = True, encoding = "us-ascii") - - - def do_write_wiki(self, arg): - if "%" in arg: - for section in self.sections: - with open(arg % section.name, "w") as f: - wiki_header(f, self.ident, self.toc) - section.to_wiki(f) - else: - with open(arg, "w") as f: - for i, section in enumerate(self.sections): - if i == 0: - wiki_header(f, self.ident, self.toc) - else: - f.write("\f\n") - section.to_wiki(f) - - - def do_write_conf(self, arg): - with open(arg, "w") as f: - conf_header(f, self.ident) - width = max(s.width for s in self.sections) - for section in self.sections: - section.to_conf(f, width) + def do_pwgen(self, arg): + try: + section, option = arg.split("::") + except ValueError: + sys.exit("Couldn't parse --pwgen specification \"%s\"" % arg) + name = (section, option) + if name not in self.option_map: + sys.exit("Couldn't find option %s::%s" % name) + self.option_map[name].value = base64.urlsafe_b64encode(os.urandom(66)) - def do_toc(self, arg): - self.toc = arg + def do_autoconf(self, ignored): + try: + import rpki.autoconf + for option in self.section_map["autoconf"].options: + try: + option.value = getattr(rpki.autoconf, option.name) + except AttributeError: + pass + except ImportError: + sys.exit("rpki.autoconf module is not available") + except KeyError: + sys.exit("Couldn't find autoconf section") + + + def do_write_xml(self, arg): + x = Element("configuration", ident = self.ident) + x.extend(s.to_xml() for s in self.sections) + ElementTree(x).write(arg, pretty_print = True, encoding = "us-ascii") + + + def do_write_wiki(self, arg): + if "%" in arg: + for section in self.sections: + with open(arg % section.name, "w") as f: + wiki_header(f, self.ident, self.toc) + section.to_wiki(f) + else: + with open(arg, "w") as f: + for i, section in enumerate(self.sections): + if i == 0: + wiki_header(f, self.ident, self.toc) + else: + f.write("\f\n") + section.to_wiki(f) + + + def do_write_conf(self, arg): + with open(arg, "w") as f: + conf_header(f, self.ident) + width = max(s.width for s in self.sections) + for section in self.sections: + section.to_conf(f, width) + + + def do_toc(self, arg): + self.toc = arg if __name__ == "__main__": - main() + main() diff --git a/ca/rpki-sql-backup b/ca/rpki-sql-backup index 02835956..986e10b1 100755 --- a/ca/rpki-sql-backup +++ b/ca/rpki-sql-backup @@ -44,10 +44,10 @@ args = parser.parse_args() cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") for name in ("rpkid", "irdbd", "pubd"): - if cfg.getboolean("start_" + name, False): - subprocess.check_call( - ("mysqldump", "--add-drop-database", - "-u", cfg.get("sql-username", section = name), - "-p" + cfg.get("sql-password", section = name), - "-B", cfg.get("sql-database", section = name)), - stdout = args.output) + if cfg.getboolean("start_" + name, False): + subprocess.check_call( + ("mysqldump", "--add-drop-database", + "-u", cfg.get("sql-username", section = name), + "-p" + cfg.get("sql-password", section = name), + "-B", cfg.get("sql-database", section = name)), + stdout = args.output) diff --git a/ca/rpki-sql-setup b/ca/rpki-sql-setup index b209fec7..c399fcfe 100755 --- a/ca/rpki-sql-setup +++ b/ca/rpki-sql-setup @@ -39,218 +39,218 @@ ER_NO_SUCH_TABLE = 1146 # See mysqld_ername.h class RootDB(object): - """ - Class to wrap MySQL actions that require root-equivalent access so - we can defer such actions until we're sure they're really needed. - Overall goal here is to prompt the user for the root password once - at most, and not at all when not necessary. - """ - - def __init__(self, mysql_defaults = None): - self.initialized = False - self.mysql_defaults = mysql_defaults - - def __getattr__(self, name): - if self.initialized: - raise AttributeError - if self.mysql_defaults is None: - self.db = MySQLdb.connect(db = "mysql", - user = "root", - passwd = getpass.getpass("Please enter your MySQL root password: ")) - else: - mysql_cfg = rpki.config.parser(set_filename = self.mysql_defaults, section = "client") - self.db = MySQLdb.connect(db = "mysql", - user = mysql_cfg.get("user"), - passwd = mysql_cfg.get("password")) - self.cur = self.db.cursor() - self.cur.execute("SHOW DATABASES") - self.databases = set(d[0] for d in self.cur.fetchall()) - self.initialized = True - return getattr(self, name) - - def close(self): - if self.initialized: - self.db.close() + """ + Class to wrap MySQL actions that require root-equivalent access so + we can defer such actions until we're sure they're really needed. + Overall goal here is to prompt the user for the root password once + at most, and not at all when not necessary. + """ + + def __init__(self, mysql_defaults = None): + self.initialized = False + self.mysql_defaults = mysql_defaults + + def __getattr__(self, name): + if self.initialized: + raise AttributeError + if self.mysql_defaults is None: + self.db = MySQLdb.connect(db = "mysql", + user = "root", + passwd = getpass.getpass("Please enter your MySQL root password: ")) + else: + mysql_cfg = rpki.config.parser(set_filename = self.mysql_defaults, section = "client") + self.db = MySQLdb.connect(db = "mysql", + user = mysql_cfg.get("user"), + passwd = mysql_cfg.get("password")) + self.cur = self.db.cursor() + self.cur.execute("SHOW DATABASES") + self.databases = set(d[0] for d in self.cur.fetchall()) + self.initialized = True + return getattr(self, name) + + def close(self): + if self.initialized: + self.db.close() class UserDB(object): - """ - Class to wrap MySQL access parameters for a particular database. - - NB: The SQL definitions for the upgrade_version table is embedded in - this class rather than being declared in any of the .sql files. - This is deliberate: nothing but the upgrade system should ever touch - this table, and it's simpler to keep everything in one place. - - We have to be careful about SQL commits here, because CREATE TABLE - implies an automatic commit. So presence of the magic table per se - isn't significant, only its content (or lack thereof). - """ - - upgrade_version_table_schema = """ - CREATE TABLE upgrade_version ( - version TEXT NOT NULL, - updated DATETIME NOT NULL - ) ENGINE=InnoDB + """ + Class to wrap MySQL access parameters for a particular database. + + NB: The SQL definitions for the upgrade_version table is embedded in + this class rather than being declared in any of the .sql files. + This is deliberate: nothing but the upgrade system should ever touch + this table, and it's simpler to keep everything in one place. + + We have to be careful about SQL commits here, because CREATE TABLE + implies an automatic commit. So presence of the magic table per se + isn't significant, only its content (or lack thereof). """ - def __init__(self, name): - self.name = name - self.database = cfg.get("sql-database", section = name) - self.username = cfg.get("sql-username", section = name) - self.password = cfg.get("sql-password", section = name) - self.db = None - self.cur = None - - def open(self): - self.db = MySQLdb.connect(db = self.database, user = self.username, passwd = self.password) - self.db.autocommit(False) - self.cur = self.db.cursor() - - def close(self): - if self.cur is not None: - self.cur.close() - self.cur = None - if self.db is not None: - self.db.commit() - self.db.close() - self.db = None - - @property - def exists_and_accessible(self): - try: - MySQLdb.connect(db = self.database, user = self.username, passwd = self.password).close() - except: # pylint: disable=W0702 - return False - else: - return True - - @property - def version(self): - try: - self.cur.execute("SELECT version FROM upgrade_version") - v = self.cur.fetchone() - return Version(None if v is None else v[0]) - except _mysql_exceptions.ProgrammingError, e: - if e.args[0] != ER_NO_SUCH_TABLE: - raise - log("Creating upgrade_version table in %s" % self.name) - self.cur.execute(self.upgrade_version_table_schema) - return Version(None) - - @version.setter - def version(self, v): - if v > self.version: - self.cur.execute("DELETE FROM upgrade_version") - self.cur.execute("INSERT upgrade_version (version, updated) VALUES (%s, %s)", (v, datetime.datetime.now())) - self.db.commit() - log("Updated %s to %s" % (self.name, v)) + upgrade_version_table_schema = """ + CREATE TABLE upgrade_version ( + version TEXT NOT NULL, + updated DATETIME NOT NULL + ) ENGINE=InnoDB + """ + + def __init__(self, name): + self.name = name + self.database = cfg.get("sql-database", section = name) + self.username = cfg.get("sql-username", section = name) + self.password = cfg.get("sql-password", section = name) + self.db = None + self.cur = None + + def open(self): + self.db = MySQLdb.connect(db = self.database, user = self.username, passwd = self.password) + self.db.autocommit(False) + self.cur = self.db.cursor() + + def close(self): + if self.cur is not None: + self.cur.close() + self.cur = None + if self.db is not None: + self.db.commit() + self.db.close() + self.db = None + + @property + def exists_and_accessible(self): + try: + MySQLdb.connect(db = self.database, user = self.username, passwd = self.password).close() + except: # pylint: disable=W0702 + return False + else: + return True + + @property + def version(self): + try: + self.cur.execute("SELECT version FROM upgrade_version") + v = self.cur.fetchone() + return Version(None if v is None else v[0]) + except _mysql_exceptions.ProgrammingError, e: + if e.args[0] != ER_NO_SUCH_TABLE: + raise + log("Creating upgrade_version table in %s" % self.name) + self.cur.execute(self.upgrade_version_table_schema) + return Version(None) + + @version.setter + def version(self, v): + if v > self.version: + self.cur.execute("DELETE FROM upgrade_version") + self.cur.execute("INSERT upgrade_version (version, updated) VALUES (%s, %s)", (v, datetime.datetime.now())) + self.db.commit() + log("Updated %s to %s" % (self.name, v)) class Version(object): - """ - A version number. This is a class in its own right to force the - comparision and string I/O behavior we want. - """ + """ + A version number. This is a class in its own right to force the + comparision and string I/O behavior we want. + """ - def __init__(self, v): - if v is None: - v = "0.0" - self.v = tuple(v.lower().split(".")) + def __init__(self, v): + if v is None: + v = "0.0" + self.v = tuple(v.lower().split(".")) - def __str__(self): - return ".".join(self.v) + def __str__(self): + return ".".join(self.v) - def __cmp__(self, other): - return cmp(self.v, other.v) + def __cmp__(self, other): + return cmp(self.v, other.v) class Upgrade(object): - """ - One upgrade script. Really, just its filename and the Version - object we parse from its filename, we don't need to read the script - itself except when applying it, but we do need to sort all the - available upgrade scripts into version order. - """ - - @classmethod - def load_all(cls, name, dn): - g = os.path.join(dn, "upgrade-%s-to-*.py" % name) - for fn in glob.iglob(g): - yield cls(g, fn) - - def __init__(self, g, fn): - head, sep, tail = g.partition("*") # pylint: disable=W0612 - self.fn = fn - self.version = Version(fn[len(head):-len(tail)]) - - def __cmp__(self, other): - return cmp(self.version, other.version) - - def apply(self, db): - # db is an argument here primarily so the script we exec can get at it - log("Applying %s to %s" % (self.fn, db.name)) - with open(self.fn, "r") as f: - exec f # pylint: disable=W0122 + """ + One upgrade script. Really, just its filename and the Version + object we parse from its filename, we don't need to read the script + itself except when applying it, but we do need to sort all the + available upgrade scripts into version order. + """ + + @classmethod + def load_all(cls, name, dn): + g = os.path.join(dn, "upgrade-%s-to-*.py" % name) + for fn in glob.iglob(g): + yield cls(g, fn) + + def __init__(self, g, fn): + head, sep, tail = g.partition("*") # pylint: disable=W0612 + self.fn = fn + self.version = Version(fn[len(head):-len(tail)]) + + def __cmp__(self, other): + return cmp(self.version, other.version) + + def apply(self, db): + # db is an argument here primarily so the script we exec can get at it + log("Applying %s to %s" % (self.fn, db.name)) + with open(self.fn, "r") as f: + exec f # pylint: disable=W0122 def do_drop(name): - db = UserDB(name) - if db.database in root.databases: - log("DROP DATABASE %s" % db.database) - root.cur.execute("DROP DATABASE %s" % db.database) - root.db.commit() + db = UserDB(name) + if db.database in root.databases: + log("DROP DATABASE %s" % db.database) + root.cur.execute("DROP DATABASE %s" % db.database) + root.db.commit() def do_create(name): - db = UserDB(name) - log("CREATE DATABASE %s" % db.database) - root.cur.execute("CREATE DATABASE %s" % db.database) - log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) - root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), - (db.password,)) - root.db.commit() - db.open() - db.version = current_version - db.close() + db = UserDB(name) + log("CREATE DATABASE %s" % db.database) + root.cur.execute("CREATE DATABASE %s" % db.database) + log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) + root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), + (db.password,)) + root.db.commit() + db.open() + db.version = current_version + db.close() def do_script_drop(name): - db = UserDB(name) - print "DROP DATABASE IF EXISTS %s;" % db.database + db = UserDB(name) + print "DROP DATABASE IF EXISTS %s;" % db.database def do_drop_and_create(name): - do_drop(name) - do_create(name) + do_drop(name) + do_create(name) def do_fix_grants(name): - db = UserDB(name) - if not db.exists_and_accessible: - log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) - root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), - (db.password,)) - root.db.commit() + db = UserDB(name) + if not db.exists_and_accessible: + log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) + root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), + (db.password,)) + root.db.commit() def do_create_if_missing(name): - db = UserDB(name) - if not db.exists_and_accessible: - do_create(name) + db = UserDB(name) + if not db.exists_and_accessible: + do_create(name) def do_apply_upgrades(name): - upgrades = sorted(Upgrade.load_all(name, args.upgrade_scripts)) - if upgrades: - db = UserDB(name) - db.open() - log("Current version of %s is %s" % (db.name, db.version)) - for upgrade in upgrades: - if upgrade.version > db.version: - upgrade.apply(db) - db.version = upgrade.version - db.version = current_version - db.close() + upgrades = sorted(Upgrade.load_all(name, args.upgrade_scripts)) + if upgrades: + db = UserDB(name) + db.open() + log("Current version of %s is %s" % (db.name, db.version)) + for upgrade in upgrades: + if upgrade.version > db.version: + upgrade.apply(db) + db.version = upgrade.version + db.version = current_version + db.close() def log(text): - if args.verbose: - print "#", text + if args.verbose: + print "#", text parser = argparse.ArgumentParser(description = """\ Automated setup of all SQL stuff used by the RPKI CA tools. Pulls @@ -291,13 +291,13 @@ parser.set_defaults(dispatch = do_create_if_missing) args = parser.parse_args() try: - cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") - root = RootDB(args.mysql_defaults) - current_version = Version(rpki.version.VERSION) - for program_name in ("irdbd", "rpkid", "pubd"): - if cfg.getboolean("start_" + program_name, False): - args.dispatch(program_name) - root.close() + cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") + root = RootDB(args.mysql_defaults) + current_version = Version(rpki.version.VERSION) + for program_name in ("irdbd", "rpkid", "pubd"): + if cfg.getboolean("start_" + program_name, False): + args.dispatch(program_name) + root.close() except Exception, e: - #raise - sys.exit(str(e)) + #raise + sys.exit(str(e)) diff --git a/ca/rpki-start-servers b/ca/rpki-start-servers index f1f70aa8..e01f4f9b 100755 --- a/ca/rpki-start-servers +++ b/ca/rpki-start-servers @@ -38,9 +38,9 @@ os.environ["TZ"] = "UTC" time.tzset() def non_negative_integer(s): - if int(s) < 0: - raise ValueError - return s + if int(s) < 0: + raise ValueError + return s parser = argparse.ArgumentParser(description = __doc__) parser.add_argument("-c", "--config", @@ -67,21 +67,21 @@ args = parser.parse_args() cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") def run(name, old_flag = None): - if cfg.getboolean("start_" + name, cfg.getboolean("run_" + name if old_flag is None else old_flag, False)): - # pylint: disable=E1103 - log_file = os.path.join(args.log_directory, name + ".log") - cmd = (os.path.join(rpki.autoconf.libexecdir, name), "--log-level", args.log_level) - if args.log_file: - cmd += ("--log-file", log_file) - elif args.log_rotating_file_kbytes: - cmd += ("--log-rotating-file", log_file, args.log_rotating_file_kbytes, args.log_backup_count) - elif args.log_rotating_file_hours: - cmd += ("--log-timed-rotating-file", log_file, args.log_rotating_file_hours, args.log_backup_count) - else: - cmd += ("--log-syslog", args.log_syslog) - proc = subprocess.Popen(cmd) - if proc.wait() != 0: - sys.exit("Problem starting %s, pid %s" % (name, proc.pid)) + if cfg.getboolean("start_" + name, cfg.getboolean("run_" + name if old_flag is None else old_flag, False)): + # pylint: disable=E1103 + log_file = os.path.join(args.log_directory, name + ".log") + cmd = (os.path.join(rpki.autoconf.libexecdir, name), "--log-level", args.log_level) + if args.log_file: + cmd += ("--log-file", log_file) + elif args.log_rotating_file_kbytes: + cmd += ("--log-rotating-file", log_file, args.log_rotating_file_kbytes, args.log_backup_count) + elif args.log_rotating_file_hours: + cmd += ("--log-timed-rotating-file", log_file, args.log_rotating_file_hours, args.log_backup_count) + else: + cmd += ("--log-syslog", args.log_syslog) + proc = subprocess.Popen(cmd) + if proc.wait() != 0: + sys.exit("Problem starting %s, pid %s" % (name, proc.pid)) run("irdbd", "run_rpkid") run("rpkid") @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.rpkic - rpki.rpkic.main() + import rpki.rpkic + rpki.rpkic.main() @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.rpkid - rpki.rpkid.main() + import rpki.rpkid + rpki.rpkid.main() diff --git a/ca/rpkigui-apache-conf-gen b/ca/rpkigui-apache-conf-gen index 8ac9c94a..a6a58c2c 100755 --- a/ca/rpkigui-apache-conf-gen +++ b/ca/rpkigui-apache-conf-gen @@ -188,370 +188,370 @@ NameVirtualHost *:443 ''' def Guess(args): - """ - Guess what platform this is and dispatch to platform constructor. - """ - - system = platform.system() - if system == "FreeBSD": - return FreeBSD(args) - if system == "Darwin": - return Darwin(args) - if system == "Linux": - distro = platform.linux_distribution()[0].lower() - if distro == "debian": - return Debian(args) - if distro == "ubuntu": - return Ubuntu(args) - if distro in ("fedora", "centos"): - return Redhat(args) - raise NotImplementedError("Can't guess what platform this is, sorry") + """ + Guess what platform this is and dispatch to platform constructor. + """ + + system = platform.system() + if system == "FreeBSD": + return FreeBSD(args) + if system == "Darwin": + return Darwin(args) + if system == "Linux": + distro = platform.linux_distribution()[0].lower() + if distro == "debian": + return Debian(args) + if distro == "ubuntu": + return Ubuntu(args) + if distro in ("fedora", "centos"): + return Redhat(args) + raise NotImplementedError("Can't guess what platform this is, sorry") class Platform(object): - """ - Abstract base class representing an operating system platform. - """ - - apache_cer = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.cer") - apache_key = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.key") - - apache_conf = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.conf") - apache_conf_sample = apache_conf + ".sample" - - apache_conf_preface = "" - - def __init__(self, args): - self.args = args - self.log("RPKI Apache configuration: platform \"%s\", action \"%s\"" % ( - self.__class__.__name__, args.action)) - getattr(self, args.action)() - - def log(self, msg): - if self.args.verbose: - print msg - - def run(self, *cmd, **kwargs): - self.log("Running %s" % " ".join(cmd)) - subprocess.check_call(cmd, **kwargs) - - req_cmd = ("openssl", "req", "-new", - "-config", "/dev/stdin", - "-out", "/dev/stdout", - "-keyout", apache_key, - "-newkey", "rsa:2048") - - x509_cmd = ("openssl", "x509", "-req", "-sha256", - "-signkey", apache_key, - "-in", "/dev/stdin", - "-out", apache_cer, - "-days", "3650") - - req_conf = '''\ - [req] - default_bits = 2048 - default_md = sha256 - distinguished_name = req_dn - prompt = no - encrypt_key = no - [req_dn] - CN = %s - ''' % fqdn - - def unlink(self, fn, silent = False): - if os.path.lexists(fn): - if not silent: - self.log("Removing %s" % fn) - os.unlink(fn) - elif not silent: - self.log("Would have removed %s if it existed" % fn) - - def del_certs(self, silent = False): - self.unlink(self.apache_cer, silent) - self.unlink(self.apache_key, silent) - - def add_certs(self): - if os.path.exists(self.apache_cer) and os.path.exists(self.apache_key): - return - self.del_certs() - req = subprocess.Popen(self.req_cmd, - stdin = subprocess.PIPE, - stdout = subprocess.PIPE, - stderr = open("/dev/null", "w")) - x509 = subprocess.Popen(self.x509_cmd, - stdin = req.stdout, - stderr = open("/dev/null", "w")) - req.stdin.write(self.req_conf) - req.stdin.close() - if req.wait(): - raise subprocess.CalledProcessError(req.returncode, self.req_cmd) - if x509.wait(): - raise subprocess.CalledProcessError(x509.returncode, self.x509_cmd) - self.log("Created %s and %s, chmoding %s" % ( - self.apache_cer, self.apache_key, self.apache_key)) - os.chmod(self.apache_key, 0600) - - _vhost = None - - @property - def vhost(self): - if self._vhost is None: - allow = allow_22_template if self.args.apache_version <= 22 else allow_24_template - self._vhost = vhost_template % dict(rpki.autoconf.__dict__, fqdn = fqdn, allow = allow) - return self._vhost - - @property - def name_virtual_host(self): - return name_virtual_host_template if self.args.apache_version <= 22 else "" - - @property - def too_complex(self): - return textwrap.dedent('''\ - # It looks like you already have HTTPS enabled in your - # Apache configuration, which makes your configuration too - # complex for us to enable support for the RPKI GUI automatically. - # - # To enable support, take a look at %s - # and copy what you need from that file into %s, - # paying attention to the comments which mark the bits that - # you might (or might not) need to change or omit, depending - # on the details of your particular Apache configuration. - ''' % (self.apache_conf_sample, self.apache_conf)) - - def install(self): - with open(self.apache_conf_sample, "w") as f: - self.log("Writing %s" % f.name) - f.write(self.apache_conf_preface) - f.write(self.name_virtual_host) - f.write(self.vhost) - if not os.path.exists(self.apache_conf): - self.unlink(self.apache_conf) - with open(self.apache_conf, "w") as f: - self.log("Writing %s" % f.name) - if self.test_url("https://%s/" % fqdn): - f.write(self.too_complex) - sys.stdout.write(self.too_complex) - else: - if not self.test_tcp("localhost", 443): + """ + Abstract base class representing an operating system platform. + """ + + apache_cer = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.cer") + apache_key = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.key") + + apache_conf = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.conf") + apache_conf_sample = apache_conf + ".sample" + + apache_conf_preface = "" + + def __init__(self, args): + self.args = args + self.log("RPKI Apache configuration: platform \"%s\", action \"%s\"" % ( + self.__class__.__name__, args.action)) + getattr(self, args.action)() + + def log(self, msg): + if self.args.verbose: + print msg + + def run(self, *cmd, **kwargs): + self.log("Running %s" % " ".join(cmd)) + subprocess.check_call(cmd, **kwargs) + + req_cmd = ("openssl", "req", "-new", + "-config", "/dev/stdin", + "-out", "/dev/stdout", + "-keyout", apache_key, + "-newkey", "rsa:2048") + + x509_cmd = ("openssl", "x509", "-req", "-sha256", + "-signkey", apache_key, + "-in", "/dev/stdin", + "-out", apache_cer, + "-days", "3650") + + req_conf = '''\ + [req] + default_bits = 2048 + default_md = sha256 + distinguished_name = req_dn + prompt = no + encrypt_key = no + [req_dn] + CN = %s + ''' % fqdn + + def unlink(self, fn, silent = False): + if os.path.lexists(fn): + if not silent: + self.log("Removing %s" % fn) + os.unlink(fn) + elif not silent: + self.log("Would have removed %s if it existed" % fn) + + def del_certs(self, silent = False): + self.unlink(self.apache_cer, silent) + self.unlink(self.apache_key, silent) + + def add_certs(self): + if os.path.exists(self.apache_cer) and os.path.exists(self.apache_key): + return + self.del_certs() + req = subprocess.Popen(self.req_cmd, + stdin = subprocess.PIPE, + stdout = subprocess.PIPE, + stderr = open("/dev/null", "w")) + x509 = subprocess.Popen(self.x509_cmd, + stdin = req.stdout, + stderr = open("/dev/null", "w")) + req.stdin.write(self.req_conf) + req.stdin.close() + if req.wait(): + raise subprocess.CalledProcessError(req.returncode, self.req_cmd) + if x509.wait(): + raise subprocess.CalledProcessError(x509.returncode, self.x509_cmd) + self.log("Created %s and %s, chmoding %s" % ( + self.apache_cer, self.apache_key, self.apache_key)) + os.chmod(self.apache_key, 0600) + + _vhost = None + + @property + def vhost(self): + if self._vhost is None: + allow = allow_22_template if self.args.apache_version <= 22 else allow_24_template + self._vhost = vhost_template % dict(rpki.autoconf.__dict__, fqdn = fqdn, allow = allow) + return self._vhost + + @property + def name_virtual_host(self): + return name_virtual_host_template if self.args.apache_version <= 22 else "" + + @property + def too_complex(self): + return textwrap.dedent('''\ + # It looks like you already have HTTPS enabled in your + # Apache configuration, which makes your configuration too + # complex for us to enable support for the RPKI GUI automatically. + # + # To enable support, take a look at %s + # and copy what you need from that file into %s, + # paying attention to the comments which mark the bits that + # you might (or might not) need to change or omit, depending + # on the details of your particular Apache configuration. + ''' % (self.apache_conf_sample, self.apache_conf)) + + def install(self): + with open(self.apache_conf_sample, "w") as f: + self.log("Writing %s" % f.name) f.write(self.apache_conf_preface) f.write(self.name_virtual_host) - f.write(self.vhost) - if not os.path.exists(self.apache_conf_target): - self.unlink(self.apache_conf_target) - self.log("Symlinking %s to %s" % ( - self.apache_conf_target, self.apache_conf)) - os.symlink(self.apache_conf, self.apache_conf_target) - self.add_certs() - self.enable() - self.restart() - - def enable(self): - pass - - def disable(self): - pass - - def remove(self): - try: - same = open(self.apache_conf, "r").read() == open(self.apache_conf_sample, "r").read() - except: # pylint: disable=W0702 - same = False - self.unlink(self.apache_conf_sample) - if same: - self.unlink(self.apache_conf) - self.unlink(self.apache_conf_target) - self.disable() - self.restart() - - def purge(self): - self.remove() - self.unlink(self.apache_conf) - self.del_certs() - - @staticmethod - def test_url(url = "https://localhost/"): - try: - urllib2.urlopen(url).close() - except IOError: - return False - else: - return True - - @staticmethod - def test_tcp(host = "localhost", port = 443, family = socket.AF_UNSPEC, proto = socket.SOCK_STREAM): - try: - addrinfo = socket.getaddrinfo(host, port, family, proto) - except socket.error: - return False - for af, socktype, proto, canon, sa in addrinfo: # pylint: disable=W0612 - try: - s = socket.socket(af, socktype, proto) - s.connect(sa) - s.close() - except socket.error: - continue - else: - return True - return False + f.write(self.vhost) + if not os.path.exists(self.apache_conf): + self.unlink(self.apache_conf) + with open(self.apache_conf, "w") as f: + self.log("Writing %s" % f.name) + if self.test_url("https://%s/" % fqdn): + f.write(self.too_complex) + sys.stdout.write(self.too_complex) + else: + if not self.test_tcp("localhost", 443): + f.write(self.apache_conf_preface) + f.write(self.name_virtual_host) + f.write(self.vhost) + if not os.path.exists(self.apache_conf_target): + self.unlink(self.apache_conf_target) + self.log("Symlinking %s to %s" % ( + self.apache_conf_target, self.apache_conf)) + os.symlink(self.apache_conf, self.apache_conf_target) + self.add_certs() + self.enable() + self.restart() + + def enable(self): + pass + + def disable(self): + pass + + def remove(self): + try: + same = open(self.apache_conf, "r").read() == open(self.apache_conf_sample, "r").read() + except: # pylint: disable=W0702 + same = False + self.unlink(self.apache_conf_sample) + if same: + self.unlink(self.apache_conf) + self.unlink(self.apache_conf_target) + self.disable() + self.restart() + + def purge(self): + self.remove() + self.unlink(self.apache_conf) + self.del_certs() + + @staticmethod + def test_url(url = "https://localhost/"): + try: + urllib2.urlopen(url).close() + except IOError: + return False + else: + return True + + @staticmethod + def test_tcp(host = "localhost", port = 443, family = socket.AF_UNSPEC, proto = socket.SOCK_STREAM): + try: + addrinfo = socket.getaddrinfo(host, port, family, proto) + except socket.error: + return False + for af, socktype, proto, canon, sa in addrinfo: # pylint: disable=W0612 + try: + s = socket.socket(af, socktype, proto) + s.connect(sa) + s.close() + except socket.error: + continue + else: + return True + return False class FreeBSD(Platform): - """ - FreeBSD. - """ + """ + FreeBSD. + """ - # On FreeBSD we have to ask httpd what version it is before we know - # where to put files or what to call the service. In FreeBSD's makefiles, - # this value is called APACHE_VERSION, and is calculated thusly: - # - # httpd -V | sed -ne 's/^Server version: Apache\/\([0-9]\)\.\([0-9]*\).*/\1\2/p' + # On FreeBSD we have to ask httpd what version it is before we know + # where to put files or what to call the service. In FreeBSD's makefiles, + # this value is called APACHE_VERSION, and is calculated thusly: + # + # httpd -V | sed -ne 's/^Server version: Apache\/\([0-9]\)\.\([0-9]*\).*/\1\2/p' - _apache_name = None + _apache_name = None - @property - def apache_name(self): - if self._apache_name is None: - self._apache_name = "apache%s" % self.args.apache_version - return self._apache_name + @property + def apache_name(self): + if self._apache_name is None: + self._apache_name = "apache%s" % self.args.apache_version + return self._apache_name - @property - def apache_conf_target(self): - return "/usr/local/etc/%s/Includes/rpki.conf" % self.apache_name + @property + def apache_conf_target(self): + return "/usr/local/etc/%s/Includes/rpki.conf" % self.apache_name - apache_conf_preface = textwrap.dedent('''\ - # These directives tell Apache to listen on the HTTPS port - # and to enable name-based virtual hosting. If you already - # have HTTPS enabled elsewhere in your configuration, you may - # need to remove these. + apache_conf_preface = textwrap.dedent('''\ + # These directives tell Apache to listen on the HTTPS port + # and to enable name-based virtual hosting. If you already + # have HTTPS enabled elsewhere in your configuration, you may + # need to remove these. - Listen [::]:443 - Listen 0.0.0.0:443 - ''') + Listen [::]:443 + Listen 0.0.0.0:443 + ''') - def restart(self): - self.run("service", self.apache_name, "restart") + def restart(self): + self.run("service", self.apache_name, "restart") class Debian(Platform): - """ - Debian and related platforms like Ubuntu. - """ + """ + Debian and related platforms like Ubuntu. + """ - @property - def distribution_version(self): - return tuple(int(v) for v in platform.linux_distribution()[1].split(".")) + @property + def distribution_version(self): + return tuple(int(v) for v in platform.linux_distribution()[1].split(".")) - # As of Wheezy, Debian still wants the configuration filename - # without the .conf suffix. Keep an eye on this, as it may change - # in future releases. - # - @property - def apache_conf_target(self): - return "/etc/apache2/sites-available/rpki" - - snake_oil_cer = "/etc/ssl/certs/ssl-cert-snakeoil.pem" - snake_oil_key = "/etc/ssl/private/ssl-cert-snakeoil.key" - - def add_certs(self): - if not os.path.exists(self.snake_oil_cer) or not os.path.exists(self.snake_oil_key): - return Platform.add_certs(self) - if not os.path.exists(self.apache_cer): - self.unlink(self.apache_cer) - os.symlink(self.snake_oil_cer, self.apache_cer) - if not os.path.exists(self.apache_key): - self.unlink(self.apache_key) - os.symlink(self.snake_oil_key, self.apache_key) - - def enable(self): - self.run("a2enmod", "ssl") - self.run("a2enmod", "expires") - self.run("a2ensite", "rpki") + # As of Wheezy, Debian still wants the configuration filename + # without the .conf suffix. Keep an eye on this, as it may change + # in future releases. # - # In light of BREACH and CRIME attacks, mod_deflate is looking - # like a bad idea, so make sure it's off. - self.run("a2dismod", "deflate") - - def disable(self): - self.run("a2dissite", "rpki") - - def restart(self): - self.run("service", "apache2", "restart") + @property + def apache_conf_target(self): + return "/etc/apache2/sites-available/rpki" + + snake_oil_cer = "/etc/ssl/certs/ssl-cert-snakeoil.pem" + snake_oil_key = "/etc/ssl/private/ssl-cert-snakeoil.key" + + def add_certs(self): + if not os.path.exists(self.snake_oil_cer) or not os.path.exists(self.snake_oil_key): + return Platform.add_certs(self) + if not os.path.exists(self.apache_cer): + self.unlink(self.apache_cer) + os.symlink(self.snake_oil_cer, self.apache_cer) + if not os.path.exists(self.apache_key): + self.unlink(self.apache_key) + os.symlink(self.snake_oil_key, self.apache_key) + + def enable(self): + self.run("a2enmod", "ssl") + self.run("a2enmod", "expires") + self.run("a2ensite", "rpki") + # + # In light of BREACH and CRIME attacks, mod_deflate is looking + # like a bad idea, so make sure it's off. + self.run("a2dismod", "deflate") + + def disable(self): + self.run("a2dissite", "rpki") + + def restart(self): + self.run("service", "apache2", "restart") class Ubuntu(Debian): - # On Ubuntu, the filename must end in .conf on Trusty and must not - # end in .conf on Precise. - @property - def apache_conf_target(self): - if self.distribution_version >= (14, 0): - return "/etc/apache2/sites-available/rpki.conf" - else: - return "/etc/apache2/sites-available/rpki" + # On Ubuntu, the filename must end in .conf on Trusty and must not + # end in .conf on Precise. + @property + def apache_conf_target(self): + if self.distribution_version >= (14, 0): + return "/etc/apache2/sites-available/rpki.conf" + else: + return "/etc/apache2/sites-available/rpki" class NIY(Platform): - def __init__(self, args): - super(NIY, self).__init__(args) - raise NotImplementedError("Platform %s not implemented yet, sorry" % self.__class__.__name__) + def __init__(self, args): + super(NIY, self).__init__(args) + raise NotImplementedError("Platform %s not implemented yet, sorry" % self.__class__.__name__) class Redhat(NIY): - """ - Redhat family of Linux distributions (Fedora, CentOS). - """ + """ + Redhat family of Linux distributions (Fedora, CentOS). + """ class Darwin(NIY): - """ - Mac OS X (aka Darwin). - """ + """ + Mac OS X (aka Darwin). + """ def main(): - """ - Generate and (de)install configuration suitable for using Apache httpd - to drive the RPKI web interface under WSGI. - """ - - parser = argparse.ArgumentParser(description = __doc__) - group1 = parser.add_mutually_exclusive_group() - group2 = parser.add_mutually_exclusive_group() - - parser.add_argument("-v", "--verbose", - help = "whistle while you work", action = "store_true") - parser.add_argument("--apache-version", - help = "Apache version (default " + rpki.autoconf.APACHE_VERSION + ")", - type = int, default = rpki.autoconf.APACHE_VERSION) - - group1.add_argument("--freebsd", - help = "configure for FreeBSD", - action = "store_const", dest = "platform", const = FreeBSD) - group1.add_argument("--debian", - help = "configure for Debian", - action = "store_const", dest = "platform", const = Debian) - group1.add_argument("--ubuntu", - help = "configure for Ubuntu", - action = "store_const", dest = "platform", const = Ubuntu) - group1.add_argument("--redhat", "--fedora", "--centos", - help = "configure for Redhat/Fedora/CentOS", - action = "store_const", dest = "platform", const = Redhat) - group1.add_argument("--macosx", "--darwin", - help = "configure for Mac OS X (Darwin)", - action = "store_const", dest = "platform", const = Darwin) - group1.add_argument("--guess", - help = "guess which platform configuration to use", - action = "store_const", dest = "platform", const = Guess) - - group2.add_argument("-i", "--install", - help = "install configuration", - action = "store_const", dest = "action", const = "install") - group2.add_argument("-r", "--remove", "--deinstall", "--uninstall", - help = "remove configuration", - action = "store_const", dest = "action", const = "remove") - group2.add_argument("-P", "--purge", - help = "remove configuration with extreme prejudice", - action = "store_const", dest = "action", const = "purge") - - parser.set_defaults(platform = Guess, action = "install") - args = parser.parse_args() - - try: - args.platform(args) - except Exception, e: - sys.exit(str(e)) + """ + Generate and (de)install configuration suitable for using Apache httpd + to drive the RPKI web interface under WSGI. + """ + + parser = argparse.ArgumentParser(description = __doc__) + group1 = parser.add_mutually_exclusive_group() + group2 = parser.add_mutually_exclusive_group() + + parser.add_argument("-v", "--verbose", + help = "whistle while you work", action = "store_true") + parser.add_argument("--apache-version", + help = "Apache version (default " + rpki.autoconf.APACHE_VERSION + ")", + type = int, default = rpki.autoconf.APACHE_VERSION) + + group1.add_argument("--freebsd", + help = "configure for FreeBSD", + action = "store_const", dest = "platform", const = FreeBSD) + group1.add_argument("--debian", + help = "configure for Debian", + action = "store_const", dest = "platform", const = Debian) + group1.add_argument("--ubuntu", + help = "configure for Ubuntu", + action = "store_const", dest = "platform", const = Ubuntu) + group1.add_argument("--redhat", "--fedora", "--centos", + help = "configure for Redhat/Fedora/CentOS", + action = "store_const", dest = "platform", const = Redhat) + group1.add_argument("--macosx", "--darwin", + help = "configure for Mac OS X (Darwin)", + action = "store_const", dest = "platform", const = Darwin) + group1.add_argument("--guess", + help = "guess which platform configuration to use", + action = "store_const", dest = "platform", const = Guess) + + group2.add_argument("-i", "--install", + help = "install configuration", + action = "store_const", dest = "action", const = "install") + group2.add_argument("-r", "--remove", "--deinstall", "--uninstall", + help = "remove configuration", + action = "store_const", dest = "action", const = "remove") + group2.add_argument("-P", "--purge", + help = "remove configuration with extreme prejudice", + action = "store_const", dest = "action", const = "purge") + + parser.set_defaults(platform = Guess, action = "install") + args = parser.parse_args() + + try: + args.platform(args) + except Exception, e: + sys.exit(str(e)) if __name__ == "__main__": - main() + main() diff --git a/ca/rpkigui-import-routes b/ca/rpkigui-import-routes index 0fbe0126..fb8e381e 100755 --- a/ca/rpkigui-import-routes +++ b/ca/rpkigui-import-routes @@ -110,4 +110,3 @@ automatically.""") except Exception as e: logging.exception(e) sys.exit(1) - diff --git a/ca/tests/bgpsec-yaml.py b/ca/tests/bgpsec-yaml.py index d33184bf..500d2b9d 100755 --- a/ca/tests/bgpsec-yaml.py +++ b/ca/tests/bgpsec-yaml.py @@ -30,28 +30,28 @@ root = "Root" class Kid(object): - def __init__(self, i): - self.name = "ISP-%03d" % i - self.ipv4 = "10.%d.0.0/16" % i - self.asn = i - self.router_id = i * 10000 - - @property - def declare(self): - return dict(name = self.name, - ipv4 = self.ipv4, - asn = self.asn, - hosted_by = root, - roa_request = [dict(asn = self.asn, ipv4 = self.ipv4)], - router_cert = [dict(asn = self.asn, router_id = self.router_id)]) - - @property - def del_routercert(self): - return dict(name = self.name, router_cert_del = [dict(asn = self.asn, router_id = self.router_id)]) - - @property - def add_routercert(self): - return dict(name = self.name, router_cert_add = [dict(asn = self.asn, router_id = self.router_id)]) + def __init__(self, i): + self.name = "ISP-%03d" % i + self.ipv4 = "10.%d.0.0/16" % i + self.asn = i + self.router_id = i * 10000 + + @property + def declare(self): + return dict(name = self.name, + ipv4 = self.ipv4, + asn = self.asn, + hosted_by = root, + roa_request = [dict(asn = self.asn, ipv4 = self.ipv4)], + router_cert = [dict(asn = self.asn, router_id = self.router_id)]) + + @property + def del_routercert(self): + return dict(name = self.name, router_cert_del = [dict(asn = self.asn, router_id = self.router_id)]) + + @property + def add_routercert(self): + return dict(name = self.name, router_cert_add = [dict(asn = self.asn, router_id = self.router_id)]) kids = [Kid(n + 1) for n in xrange(200)] @@ -72,14 +72,14 @@ docs.append([shell_first, gym = kids[50:70] for kid in gym: - docs.append([shell_next, - kid.del_routercert, - sleeper]) + docs.append([shell_next, + kid.del_routercert, + sleeper]) for kid in gym: - docs.append([shell_next, - kid.add_routercert, - sleeper]) + docs.append([shell_next, + kid.add_routercert, + sleeper]) print '''\ # This configuration was generated by a script. Edit at your own risk. diff --git a/ca/tests/myrpki-xml-parse-test.py b/ca/tests/myrpki-xml-parse-test.py index 9db7ec57..d915ea5b 100644 --- a/ca/tests/myrpki-xml-parse-test.py +++ b/ca/tests/myrpki-xml-parse-test.py @@ -25,77 +25,77 @@ relaxng = lxml.etree.RelaxNG(file = "myrpki.rng") tree = lxml.etree.parse("myrpki.xml").getroot() if False: - print lxml.etree.tostring(tree, pretty_print = True, encoding = "us-ascii", xml_declaration = True) + print lxml.etree.tostring(tree, pretty_print = True, encoding = "us-ascii", xml_declaration = True) relaxng.assertValid(tree) def showitems(y): - if False: - for k, v in y.items(): - if v: - print " ", k, v + if False: + for k, v in y.items(): + if v: + print " ", k, v def tag(t): - return "{http://www.hactrn.net/uris/rpki/myrpki/}" + t + return "{http://www.hactrn.net/uris/rpki/myrpki/}" + t print "My handle:", tree.get("handle") print "Children:" for x in tree.getiterator(tag("child")): - print " ", x - print " Handle:", x.get("handle") - print " ASNS: ", rpki.resource_set.resource_set_as(x.get("asns")) - print " IPv4: ", rpki.resource_set.resource_set_ipv4(x.get("v4")) - print " Valid: ", x.get("valid_until") - showitems(x) + print " ", x + print " Handle:", x.get("handle") + print " ASNS: ", rpki.resource_set.resource_set_as(x.get("asns")) + print " IPv4: ", rpki.resource_set.resource_set_ipv4(x.get("v4")) + print " Valid: ", x.get("valid_until") + showitems(x) print print "ROA requests:" for x in tree.getiterator(tag("roa_request")): - print " ", x - print " ASN: ", x.get("asn") - print " IPv4:", rpki.resource_set.roa_prefix_set_ipv4(x.get("v4")) - print " IPv6:", rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")) - showitems(x) + print " ", x + print " ASN: ", x.get("asn") + print " IPv4:", rpki.resource_set.roa_prefix_set_ipv4(x.get("v4")) + print " IPv6:", rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")) + showitems(x) print def showpem(label, b64, kind): - cmd = ("openssl", kind, "-noout", "-text", "-inform", "DER") - if kind == "x509": - cmd += ("-certopt", "no_pubkey,no_sigdump") - p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE) - text = p.communicate(input = base64.b64decode(b64))[0] - if p.returncode != 0: - raise subprocess.CalledProcessError(returncode = p.returncode, cmd = cmd) - print label, text + cmd = ("openssl", kind, "-noout", "-text", "-inform", "DER") + if kind == "x509": + cmd += ("-certopt", "no_pubkey,no_sigdump") + p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE) + text = p.communicate(input = base64.b64decode(b64))[0] + if p.returncode != 0: + raise subprocess.CalledProcessError(returncode = p.returncode, cmd = cmd) + print label, text for x in tree.getiterator(tag("child")): - cert = x.findtext(tag("bpki_certificate")) - if cert: - showpem("Child", cert, "x509") + cert = x.findtext(tag("bpki_certificate")) + if cert: + showpem("Child", cert, "x509") for x in tree.getiterator(tag("parent")): - print "Parent URI:", x.get("service_uri") - cert = x.findtext(tag("bpki_certificate")) - if cert: - showpem("Parent", cert, "x509") + print "Parent URI:", x.get("service_uri") + cert = x.findtext(tag("bpki_certificate")) + if cert: + showpem("Parent", cert, "x509") ca = tree.findtext(tag("bpki_ca_certificate")) if ca: - showpem("CA", ca, "x509") + showpem("CA", ca, "x509") bsc = tree.findtext(tag("bpki_bsc_certificate")) if bsc: - showpem("BSC EE", bsc, "x509") + showpem("BSC EE", bsc, "x509") repo = tree.findtext(tag("bpki_repository_certificate")) if repo: - showpem("Repository", repo, "x509") + showpem("Repository", repo, "x509") req = tree.findtext(tag("bpki_bsc_pkcs10")) if req: - showpem("BSC EE", req, "req") + showpem("BSC EE", req, "req") crl = tree.findtext(tag("bpki_crl")) if crl: - showpem("CA", crl, "crl") + showpem("CA", crl, "crl") diff --git a/ca/tests/old_irdbd.py b/ca/tests/old_irdbd.py index d66e683e..d26c3476 100644 --- a/ca/tests/old_irdbd.py +++ b/ca/tests/old_irdbd.py @@ -15,5 +15,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.old_irdbd - rpki.old_irdbd.main() + import rpki.old_irdbd + rpki.old_irdbd.main() diff --git a/ca/tests/smoketest.py b/ca/tests/smoketest.py index 3960981a..9d82c640 100644 --- a/ca/tests/smoketest.py +++ b/ca/tests/smoketest.py @@ -76,14 +76,14 @@ yaml_script = [y for y in yaml.safe_load_all(args.yaml_file)] # Define port allocator early, so we can use it while reading config def allocate_port(): - """ - Allocate a TCP port number. - """ + """ + Allocate a TCP port number. + """ - global base_port - p = base_port - base_port += 1 - return p + global base_port + p = base_port + base_port += 1 + return p # Most filenames in the following are relative to the working directory. @@ -139,14 +139,14 @@ pubd_last_cms_time = None ecdsa_params = None class CantRekeyYAMLLeaf(Exception): - """ - Can't rekey YAML leaf. - """ + """ + Can't rekey YAML leaf. + """ class CouldntIssueBSCEECertificate(Exception): - """ - Couldn't issue BSC EE certificate - """ + """ + Couldn't issue BSC EE certificate + """ sql_conversions = MySQLdb.converters.conversions.copy() sql_conversions.update({ @@ -154,202 +154,202 @@ sql_conversions.update({ MySQLdb.converters.FIELD_TYPE.DATETIME : rpki.sundial.datetime.DateTime_or_None }) def main(): - """ - Main program. - """ - - rpki.log.init(smoketest_name, argparse.Namespace(log_level = logging.DEBUG, - log_handler = lambda: logging.StreamHandler(sys.stdout))) - logger.info("Starting") - - rpki.http.http_client.timeout = rpki.sundial.timedelta(hours = 1) - - pubd_process = None - rootd_process = None - rsyncd_process = None - - rpki_sql = mangle_sql(rpki_sql_file) - irdb_sql = mangle_sql(irdb_sql_file) - pubd_sql = mangle_sql(pub_sql_file) - - logger.info("Initializing test directory") - - # Connect to test directory, creating it if necessary - try: - os.chdir(smoketest_dir) - except OSError: - os.makedirs(smoketest_dir) - os.chdir(smoketest_dir) - - # Now that we're in the right directory, we can figure out whether - # we have a private openssl executable to use - global prog_openssl - if not os.path.exists(prog_openssl): - prog_openssl = "openssl" - - # Discard everything but keys, which take a while to generate. - # Apparently os.walk() can't tell the difference between directories - # and symlinks to directories, so we have to handle both. - for root, dirs, files in os.walk(".", topdown = False): - for fn in files: - if not fn.endswith(".key"): - os.remove(os.path.join(root, fn)) - for d in dirs: - try: - os.rmdir(os.path.join(root, d)) - except OSError, e: - if e.errno == errno.ENOTDIR: - os.remove(os.path.join(root, d)) - else: - raise - - logger.info("Reading master YAML configuration") - y = yaml_script.pop(0) + """ + Main program. + """ - logger.info("Constructing internal allocation database") - db = allocation_db(y) + rpki.log.init(smoketest_name, argparse.Namespace(log_level = logging.DEBUG, + log_handler = lambda: logging.StreamHandler(sys.stdout))) + logger.info("Starting") - logger.info("Constructing BPKI keys and certs for rootd") - setup_bpki_cert_chain(rootd_name, ee = ("RPKI",)) + rpki.http.http_client.timeout = rpki.sundial.timedelta(hours = 1) - logger.info("Constructing BPKI keys and certs for pubd") - setup_bpki_cert_chain(pubd_name, ee = ("PUBD", "IRBE")) + pubd_process = None + rootd_process = None + rsyncd_process = None + rpki_sql = mangle_sql(rpki_sql_file) + irdb_sql = mangle_sql(irdb_sql_file) + pubd_sql = mangle_sql(pub_sql_file) - for a in db: - a.setup_bpki_certs() + logger.info("Initializing test directory") - setup_publication(pubd_sql, db.root.irdb_db_name) - setup_rootd(db.root, y.get("rootd", {}), db) - setup_rsyncd() - setup_rcynic() + # Connect to test directory, creating it if necessary + try: + os.chdir(smoketest_dir) + except OSError: + os.makedirs(smoketest_dir) + os.chdir(smoketest_dir) + + # Now that we're in the right directory, we can figure out whether + # we have a private openssl executable to use + global prog_openssl + if not os.path.exists(prog_openssl): + prog_openssl = "openssl" + + # Discard everything but keys, which take a while to generate. + # Apparently os.walk() can't tell the difference between directories + # and symlinks to directories, so we have to handle both. + for root, dirs, files in os.walk(".", topdown = False): + for fn in files: + if not fn.endswith(".key"): + os.remove(os.path.join(root, fn)) + for d in dirs: + try: + os.rmdir(os.path.join(root, d)) + except OSError, e: + if e.errno == errno.ENOTDIR: + os.remove(os.path.join(root, d)) + else: + raise + + logger.info("Reading master YAML configuration") + y = yaml_script.pop(0) + + logger.info("Constructing internal allocation database") + db = allocation_db(y) + + logger.info("Constructing BPKI keys and certs for rootd") + setup_bpki_cert_chain(rootd_name, ee = ("RPKI",)) + + logger.info("Constructing BPKI keys and certs for pubd") + setup_bpki_cert_chain(pubd_name, ee = ("PUBD", "IRBE")) + + + for a in db: + a.setup_bpki_certs() + + setup_publication(pubd_sql, db.root.irdb_db_name) + setup_rootd(db.root, y.get("rootd", {}), db) + setup_rsyncd() + setup_rcynic() - for a in db.engines: - a.setup_conf_file() - a.setup_sql(rpki_sql, irdb_sql) - a.sync_sql() + for a in db.engines: + a.setup_conf_file() + a.setup_sql(rpki_sql, irdb_sql) + a.sync_sql() - try: + try: - logger.info("Starting rootd") - rootd_process = subprocess.Popen((prog_python, prog_rootd, "--foreground", "--log-stdout", "--log-level", "debug"), - env = dict(os.environ, RPKI_CONF = rootd_name + ".conf")) + logger.info("Starting rootd") + rootd_process = subprocess.Popen((prog_python, prog_rootd, "--foreground", "--log-stdout", "--log-level", "debug"), + env = dict(os.environ, RPKI_CONF = rootd_name + ".conf")) - logger.info("Starting pubd") - pubd_process = subprocess.Popen((prog_python, prog_pubd, "--foreground", "--log-stdout", "--log-level", "debug") + - (("-p", pubd_name + ".prof") if args.profile else ()), - env = dict(os.environ, RPKI_CONF = pubd_name + ".conf")) + logger.info("Starting pubd") + pubd_process = subprocess.Popen((prog_python, prog_pubd, "--foreground", "--log-stdout", "--log-level", "debug") + + (("-p", pubd_name + ".prof") if args.profile else ()), + env = dict(os.environ, RPKI_CONF = pubd_name + ".conf")) - logger.info("Starting rsyncd") - rsyncd_process = subprocess.Popen((prog_rsyncd, "--daemon", "--no-detach", "--config", rsyncd_name + ".conf")) + logger.info("Starting rsyncd") + rsyncd_process = subprocess.Popen((prog_rsyncd, "--daemon", "--no-detach", "--config", rsyncd_name + ".conf")) - # Start rpkid and irdbd instances - for a in db.engines: - a.run_daemons() + # Start rpkid and irdbd instances + for a in db.engines: + a.run_daemons() - # From this point on we'll be running event-driven, so the rest of - # the code until final exit is all closures. + # From this point on we'll be running event-driven, so the rest of + # the code until final exit is all closures. - def start(): - rpki.async.iterator(db.engines, create_rpki_objects, create_pubd_objects) + def start(): + rpki.async.iterator(db.engines, create_rpki_objects, create_pubd_objects) - def create_rpki_objects(iterator, a): - a.create_rpki_objects(iterator) + def create_rpki_objects(iterator, a): + a.create_rpki_objects(iterator) - def create_pubd_objects(): - call_pubd([rpki.publication_control.client_elt.make_pdu(action = "create", - client_handle = db.root.client_handle + "-" + rootd_name, - base_uri = rootd_sia, - bpki_cert = cross_certify(rootd_name + "-TA", pubd_name + "-TA"))], - cb = lambda ignored: yaml_loop()) + def create_pubd_objects(): + call_pubd([rpki.publication_control.client_elt.make_pdu(action = "create", + client_handle = db.root.client_handle + "-" + rootd_name, + base_uri = rootd_sia, + bpki_cert = cross_certify(rootd_name + "-TA", pubd_name + "-TA"))], + cb = lambda ignored: yaml_loop()) - def yaml_loop(): + def yaml_loop(): - # This is probably where we should be updating expired BPKI - # objects, particular CRLs + # This is probably where we should be updating expired BPKI + # objects, particular CRLs - logger.info("Running cron for all RPKI engines") - rpki.async.iterator(db.engines, run_cron, run_yaml) + logger.info("Running cron for all RPKI engines") + rpki.async.iterator(db.engines, run_cron, run_yaml) - def run_cron(iterator, a): - a.run_cron(iterator) + def run_cron(iterator, a): + a.run_cron(iterator) - def run_yaml(): + def run_yaml(): - # Run rcynic to check results - run_rcynic() + # Run rcynic to check results + run_rcynic() - # Apply next delta if we have one; otherwise, we're done. - if yaml_script: - logger.info("Applying deltas") - db.apply_delta(yaml_script.pop(0), apply_delta_done) - else: - logger.info("No more deltas to apply, done") - rpki.async.exit_event_loop() + # Apply next delta if we have one; otherwise, we're done. + if yaml_script: + logger.info("Applying deltas") + db.apply_delta(yaml_script.pop(0), apply_delta_done) + else: + logger.info("No more deltas to apply, done") + rpki.async.exit_event_loop() - def apply_delta_done(): + def apply_delta_done(): - # Resync IRDBs - for a in db.engines: - a.sync_sql() + # Resync IRDBs + for a in db.engines: + a.sync_sql() - # Loop until we run out of control YAML - yaml_loop() + # Loop until we run out of control YAML + yaml_loop() - logger.info("Sleeping %d seconds while daemons start up", startup_delay) - rpki.async.timer(start).set(rpki.sundial.timedelta(seconds = startup_delay)) - rpki.async.event_loop() + logger.info("Sleeping %d seconds while daemons start up", startup_delay) + rpki.async.timer(start).set(rpki.sundial.timedelta(seconds = startup_delay)) + rpki.async.event_loop() - # At this point we have gone into event-driven code. + # At this point we have gone into event-driven code. - logger.info("Event loop exited normally") + logger.info("Event loop exited normally") - except Exception, e: - logger.exception("Event loop exited with an exception") + except Exception, e: + logger.exception("Event loop exited with an exception") - finally: - logger.info("Cleaning up") - for a in db.engines: - a.kill_daemons() - for proc, name in ((rootd_process, "rootd"), - (pubd_process, "pubd"), - (rsyncd_process, "rsyncd")): - # pylint: disable=E1103 - if proc is not None and proc.poll() is None: - logger.info("Killing %s, pid %s", name, proc.pid) - try: - proc.terminate() - except OSError: - pass - if proc is not None: - logger.info("Daemon %s, pid %s exited with code %s", name, proc.pid, proc.wait()) + finally: + logger.info("Cleaning up") + for a in db.engines: + a.kill_daemons() + for proc, name in ((rootd_process, "rootd"), + (pubd_process, "pubd"), + (rsyncd_process, "rsyncd")): + # pylint: disable=E1103 + if proc is not None and proc.poll() is None: + logger.info("Killing %s, pid %s", name, proc.pid) + try: + proc.terminate() + except OSError: + pass + if proc is not None: + logger.info("Daemon %s, pid %s exited with code %s", name, proc.pid, proc.wait()) def cmd_sleep(cb, interval): - """ - Set an alarm, then wait for it to go off. - """ + """ + Set an alarm, then wait for it to go off. + """ - howlong = rpki.sundial.timedelta.parse(interval) - logger.info("Sleeping %r", howlong) - rpki.async.timer(cb).set(howlong) + howlong = rpki.sundial.timedelta.parse(interval) + logger.info("Sleeping %r", howlong) + rpki.async.timer(cb).set(howlong) def cmd_shell(cb, *cmd): - """ - Run a shell command. - """ + """ + Run a shell command. + """ - cmd = " ".join(cmd) - status = subprocess.call(cmd, shell = True) - logger.info("Shell command returned status %d", status) - cb() + cmd = " ".join(cmd) + status = subprocess.call(cmd, shell = True) + logger.info("Shell command returned status %d", status) + cb() def cmd_echo(cb, *words): - """ - Echo some text to the log. - """ + """ + Echo some text to the log. + """ - logger.info(" ".join(words)) - cb() + logger.info(" ".join(words)) + cb() ## @var cmds # Dispatch table for commands embedded in delta sections @@ -359,1019 +359,1019 @@ cmds = { "sleep" : cmd_sleep, "echo" : cmd_echo } class roa_request(object): - """ - Representation for a roa_request object. - """ - - def __init__(self, asn, ipv4, ipv6): - self.asn = asn - self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None - self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - - def __eq__(self, other): - return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - - def __hash__(self): - v4 = tuple(self.v4) if self.v4 is not None else None - v6 = tuple(self.v6) if self.v6 is not None else None - return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - - def __str__(self): - if self.v4 and self.v6: s = str(self.v4) + "," + str(self.v6) - elif self.v4: s = str(self.v4) - else: s = str(self.v6) - return "%s: %s" % (self.asn, s) - - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("ipv4"), yaml.get("ipv6")) - -class router_cert(object): - """ - Representation for a router_cert object. - """ - - _ecparams = None - _keypair = None - _pkcs10 = None - _gski = None - - @classmethod - def ecparams(cls): - if cls._ecparams is None: - cls._ecparams = rpki.x509.KeyParams.generateEC() - return cls._ecparams - - def __init__(self, asn, router_id): - self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) - self.router_id = router_id - self.cn = "ROUTER-%08x" % self.asn[0].min - self.sn = "%08x" % self.router_id - self.eku = rpki.oids.id_kp_bgpsec_router - - @property - def keypair(self): - if self._keypair is None: - self._keypair = rpki.x509.ECDSA.generate(self.ecparams()) - return self._keypair - - @property - def pkcs10(self): - if self._pkcs10 is None: - self._pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) - return self._pkcs10 - - @property - def gski(self): - if self._gski is None: - self._gski = self.pkcs10.gSKI() - return self._gski - - def __eq__(self, other): - return self.asn == other.asn and self.sn == other.sn - - def __hash__(self): - return tuple(self.asn).__hash__() + self.cn.__hash__() + self.sn.__hash__() - - def __str__(self): - return "%s: %s,%s: %s" % (self.asn, self.cn, self.sn, self.gski) - - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("router_id")) - -class allocation_db(list): - """ - Representation of all the entities and allocations in the test - system. Almost everything is generated out of this database. - """ - - def __init__(self, yaml): - """ - Initialize database from the (first) YAML document. """ - - list.__init__(self) - self.root = allocation(yaml, self) - assert self.root.is_root - if self.root.crl_interval is None: - self.root.crl_interval = rpki.sundial.timedelta.parse(cfg.get("crl_interval", "1d")).convert_to_seconds() - if self.root.regen_margin is None: - self.root.regen_margin = rpki.sundial.timedelta.parse(cfg.get("regen_margin", "1d")).convert_to_seconds() - for a in self: - if a.sia_base is None: - a.sia_base = (rootd_sia + "root/trunk/" if a.is_root else a.parent.sia_base) + a.name + "/" - if a.base.valid_until is None: - a.base.valid_until = a.parent.base.valid_until - if a.crl_interval is None: - a.crl_interval = a.parent.crl_interval - if a.regen_margin is None: - a.regen_margin = a.parent.regen_margin - a.client_handle = "/".join(a.sia_base.split("/")[4:]).rstrip("/") - self.root.closure() - self.map = dict((a.name, a) for a in self) - self.engines = [a for a in self if a.is_engine] - for i, a in enumerate(self.engines): - a.set_engine_number(i) - for a in self: - if a.is_hosted: - a.hosted_by = self.map[a.hosted_by] - a.hosted_by.hosts.append(a) - assert a.is_twig, "%s is not twig" % a.name - assert not a.hosted_by.is_hosted, "%s is hosted by a hosted entity" % a.name - - def apply_delta(self, delta, cb): + Representation for a roa_request object. """ - Apply a delta or run a command. - """ - - def loop(iterator, d): - if isinstance(d, str): - c = d.split() - cmds[c[0]](iterator, *c[1:]) - else: - self.map[d["name"]].apply_delta(d, iterator) - def done(): - self.root.closure() - cb() + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - if delta is None: - cb() - else: - rpki.async.iterator(delta, loop, done) + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - def dump(self): - """ - Print content of the database. - """ + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - for a in self: - print a + def __str__(self): + if self.v4 and self.v6: s = str(self.v4) + "," + str(self.v6) + elif self.v4: s = str(self.v4) + else: s = str(self.v6) + return "%s: %s" % (self.asn, s) -class allocation(object): - - parent = None - irdb_db_name = None - irdb_port = None - rpki_db_name = None - rpki_port = None - crl_interval = None - regen_margin = None - last_cms_time = None - rpkid_process = None - irdbd_process = None - - def __init__(self, yaml, db, parent = None): - """ - Initialize one entity and insert it into the database. - """ + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("ipv4"), yaml.get("ipv6")) - db.append(self) - self.name = yaml["name"] - self.parent = parent - self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] - valid_until = None - if "valid_until" in yaml: - valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) - if valid_until is None and "valid_for" in yaml: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) - self.base = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as(yaml.get("asn")), - v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), - v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), - valid_until = valid_until) - self.sia_base = yaml.get("sia_base") - if "crl_interval" in yaml: - self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() - if "regen_margin" in yaml: - self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() - self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] - for r in self.roa_requests: - if r.v4: - self.base.v4 |= r.v4.to_resource_set() - if r.v6: - self.base.v6 |= r.v6.to_resource_set() - self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] - for r in self.router_certs: - self.base.asn |= r.asn - self.hosted_by = yaml.get("hosted_by") - self.extra_conf = yaml.get("extra_conf", []) - self.hosts = [] - - def closure(self): +class router_cert(object): """ - Compute the transitive resource closure. + Representation for a router_cert object. """ - resources = self.base - for kid in self.kids: - resources |= kid.closure() - self.resources = resources - return resources + _ecparams = None + _keypair = None + _pkcs10 = None + _gski = None + + @classmethod + def ecparams(cls): + if cls._ecparams is None: + cls._ecparams = rpki.x509.KeyParams.generateEC() + return cls._ecparams + + def __init__(self, asn, router_id): + self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) + self.router_id = router_id + self.cn = "ROUTER-%08x" % self.asn[0].min + self.sn = "%08x" % self.router_id + self.eku = rpki.oids.id_kp_bgpsec_router + + @property + def keypair(self): + if self._keypair is None: + self._keypair = rpki.x509.ECDSA.generate(self.ecparams()) + return self._keypair + + @property + def pkcs10(self): + if self._pkcs10 is None: + self._pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) + return self._pkcs10 + + @property + def gski(self): + if self._gski is None: + self._gski = self.pkcs10.gSKI() + return self._gski + + def __eq__(self, other): + return self.asn == other.asn and self.sn == other.sn + + def __hash__(self): + return tuple(self.asn).__hash__() + self.cn.__hash__() + self.sn.__hash__() + + def __str__(self): + return "%s: %s,%s: %s" % (self.asn, self.cn, self.sn, self.gski) + + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("router_id")) - def apply_delta(self, yaml, cb): +class allocation_db(list): """ - Apply deltas to this entity. + Representation of all the entities and allocations in the test + system. Almost everything is generated out of this database. """ - logger.info("Applying delta: %s", yaml) - - def loop(iterator, kv): - if kv[0] == "name": - iterator() - else: - getattr(self, "apply_" + kv[0])(kv[1], iterator) - - rpki.async.iterator(yaml.items(), loop, cb) - - def apply_add_as(self, text, cb): - self.base.asn |= rpki.resource_set.resource_set_as(text) - cb() - - def apply_add_v4(self, text, cb): - self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) - cb() - - def apply_add_v6(self, text, cb): - self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) - cb() - - def apply_sub_as(self, text, cb): - self.base.asn |= rpki.resource_set.resource_set_as(text) - cb() + def __init__(self, yaml): + """ + Initialize database from the (first) YAML document. + """ + + list.__init__(self) + self.root = allocation(yaml, self) + assert self.root.is_root + if self.root.crl_interval is None: + self.root.crl_interval = rpki.sundial.timedelta.parse(cfg.get("crl_interval", "1d")).convert_to_seconds() + if self.root.regen_margin is None: + self.root.regen_margin = rpki.sundial.timedelta.parse(cfg.get("regen_margin", "1d")).convert_to_seconds() + for a in self: + if a.sia_base is None: + a.sia_base = (rootd_sia + "root/trunk/" if a.is_root else a.parent.sia_base) + a.name + "/" + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + a.client_handle = "/".join(a.sia_base.split("/")[4:]).rstrip("/") + self.root.closure() + self.map = dict((a.name, a) for a in self) + self.engines = [a for a in self if a.is_engine] + for i, a in enumerate(self.engines): + a.set_engine_number(i) + for a in self: + if a.is_hosted: + a.hosted_by = self.map[a.hosted_by] + a.hosted_by.hosts.append(a) + assert a.is_twig, "%s is not twig" % a.name + assert not a.hosted_by.is_hosted, "%s is hosted by a hosted entity" % a.name + + def apply_delta(self, delta, cb): + """ + Apply a delta or run a command. + """ + + def loop(iterator, d): + if isinstance(d, str): + c = d.split() + cmds[c[0]](iterator, *c[1:]) + else: + self.map[d["name"]].apply_delta(d, iterator) + + def done(): + self.root.closure() + cb() + + if delta is None: + cb() + else: + rpki.async.iterator(delta, loop, done) - def apply_sub_v4(self, text, cb): - self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) - cb() + def dump(self): + """ + Print content of the database. + """ - def apply_sub_v6(self, text, cb): - self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) - cb() + for a in self: + print a - def apply_valid_until(self, stamp, cb): - self.base.valid_until = rpki.sundial.datetime.from_datetime(stamp) - cb() +class allocation(object): - def apply_valid_for(self, text, cb): - self.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(text) - cb() + parent = None + irdb_db_name = None + irdb_port = None + rpki_db_name = None + rpki_port = None + crl_interval = None + regen_margin = None + last_cms_time = None + rpkid_process = None + irdbd_process = None + + def __init__(self, yaml, db, parent = None): + """ + Initialize one entity and insert it into the database. + """ + + db.append(self) + self.name = yaml["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] + valid_until = None + if "valid_until" in yaml: + valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) + if valid_until is None and "valid_for" in yaml: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(yaml.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), + valid_until = valid_until) + self.sia_base = yaml.get("sia_base") + if "crl_interval" in yaml: + self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() + if "regen_margin" in yaml: + self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() + self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] + for r in self.roa_requests: + if r.v4: + self.base.v4 |= r.v4.to_resource_set() + if r.v6: + self.base.v6 |= r.v6.to_resource_set() + self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] + for r in self.router_certs: + self.base.asn |= r.asn + self.hosted_by = yaml.get("hosted_by") + self.extra_conf = yaml.get("extra_conf", []) + self.hosts = [] + + def closure(self): + """ + Compute the transitive resource closure. + """ + + resources = self.base + for kid in self.kids: + resources |= kid.closure() + self.resources = resources + return resources + + def apply_delta(self, yaml, cb): + """ + Apply deltas to this entity. + """ + + logger.info("Applying delta: %s", yaml) + + def loop(iterator, kv): + if kv[0] == "name": + iterator() + else: + getattr(self, "apply_" + kv[0])(kv[1], iterator) + + rpki.async.iterator(yaml.items(), loop, cb) + + def apply_add_as(self, text, cb): + self.base.asn |= rpki.resource_set.resource_set_as(text) + cb() + + def apply_add_v4(self, text, cb): + self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) + cb() + + def apply_add_v6(self, text, cb): + self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) + cb() + + def apply_sub_as(self, text, cb): + self.base.asn |= rpki.resource_set.resource_set_as(text) + cb() + + def apply_sub_v4(self, text, cb): + self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) + cb() + + def apply_sub_v6(self, text, cb): + self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) + cb() + + def apply_valid_until(self, stamp, cb): + self.base.valid_until = rpki.sundial.datetime.from_datetime(stamp) + cb() + + def apply_valid_for(self, text, cb): + self.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(text) + cb() + + def apply_valid_add(self, text, cb): + self.base.valid_until += rpki.sundial.timedelta.parse(text) + cb() + + def apply_valid_sub(self, text, cb): + self.base.valid_until -= rpki.sundial.timedelta.parse(text) + cb() + + def apply_roa_request_add(self, yaml, cb): + for y in yaml: + r = roa_request.parse(y) + if r not in self.roa_requests: + self.roa_requests.append(r) + cb() + + def apply_roa_request_del(self, yaml, cb): + for y in yaml: + r = roa_request.parse(y) + if r in self.roa_requests: + self.roa_requests.remove(r) + cb() + + def apply_router_cert_add(self, yaml, cb): + for y in yaml: + r = router_cert.parse(y) + if r not in self.router_certs: + self.router_certs.append(r) + cb() + + def apply_router_cert_del(self, yaml, cb): + for y in yaml: + r = router_cert.parse(y) + if r in self.router_certs: + self.router_certs.remove(r) + cb() + + def apply_rekey(self, target, cb): + + def done(e): + if isinstance(e, Exception): + logger.exception("Exception while rekeying %s", self.name) + raise e + cb() + + if target is None: + logger.info("Rekeying <tenant/> %s", self.name) + self.call_rpkid([rpki.left_right.self_elt.make_pdu( + action = "set", self_handle = self.name, rekey = "yes")], cb = done) + else: + logger.info("Rekeying <parent/> %s %s", self.name, target) + self.call_rpkid([rpki.left_right.parent_elt.make_pdu( + action = "set", self_handle = self.name, parent_handle = target, rekey = "yes")], cb = done) + + def apply_revoke(self, target, cb): + + def done(e): + if isinstance(e, Exception): + logger.exception("Exception while revoking %s", self.name) + raise e + cb() + + if target is None: + logger.info("Revoking <tenant/> %s", self.name) + self.call_rpkid([rpki.left_right.self_elt.make_pdu( + action = "set", self_handle = self.name, revoke = "yes")], cb = done) + else: + logger.info("Revoking <parent/> %s %s", self.name, target) + self.call_rpkid([rpki.left_right.parent_elt.make_pdu( + action = "set", self_handle = self.name, parent_handle = target, revoke = "yes")], cb = done) + + def __str__(self): + s = self.name + "\n" + if self.resources.asn: s += " ASN: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.sia_base: s += " SIA: %s\n" % self.sia_base + return s + "Until: %s\n" % self.resources.valid_until + + + @property + def is_root(self): + return self.parent is None + + @property + def is_twig(self): + return not self.is_root + + @property + def is_hosted(self): + return self.hosted_by is not None + + @property + def is_engine(self): + return not self.is_hosted + + def set_engine_number(self, n): + """ + Set the engine number for this entity. + """ + + self.irdb_db_name = "irdb%d" % n + self.irdb_port = allocate_port() + self.rpki_db_name = "rpki%d" % n + self.rpki_port = allocate_port() + + def get_rpki_port(self): + """ + Get rpki port to use for this entity. + """ + + if self.is_hosted: + assert self.hosted_by.rpki_port is not None + return self.hosted_by.rpki_port + else: + assert self.rpki_port is not None + return self.rpki_port + + def setup_bpki_certs(self): + """ + Create BPKI certificates for this entity. + """ + + logger.info("Constructing BPKI keys and certs for %s", self.name) + setup_bpki_cert_chain(name = self.name, + ee = ("RPKI", "IRDB", "IRBE"), + ca = ("SELF",)) + self.rpkid_ta = rpki.x509.X509(PEM_file = self.name + "-TA.cer") + self.irbe_key = rpki.x509.RSA( PEM_file = self.name + "-IRBE.key") + self.irbe_cert = rpki.x509.X509(PEM_file = self.name + "-IRBE.cer") + self.rpkid_cert = rpki.x509.X509(PEM_file = self.name + "-RPKI.cer") + + def setup_conf_file(self): + """ + Write config files for this entity. + """ + + logger.info("Writing config files for %s", self.name) + assert self.rpki_port is not None + d = dict(my_name = self.name, + irdb_db_name = self.irdb_db_name, + irdb_db_pass = irdb_db_pass, + irdb_port = self.irdb_port, + rpki_db_name = self.rpki_db_name, + rpki_db_pass = rpki_db_pass, + rpki_port = self.rpki_port) + f = open(self.name + ".conf", "w") + f.write(conf_fmt_1 % d) + for line in self.extra_conf: + f.write(line + "\n") + f.close() + + def setup_sql(self, rpki_sql, irdb_sql): + """ + Set up this entity's IRDB. + """ + + logger.info("Setting up MySQL for %s", self.name) + db = MySQLdb.connect(user = "rpki", db = self.rpki_db_name, passwd = rpki_db_pass, + conv = sql_conversions) + cur = db.cursor() + db.autocommit(True) + for sql in rpki_sql: + try: + cur.execute(sql) + except: + if "DROP TABLE IF EXISTS" not in sql.upper(): + raise + db.close() + db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, + conv = sql_conversions) + cur = db.cursor() + db.autocommit(True) + for sql in irdb_sql: + try: + cur.execute(sql) + except: + if "DROP TABLE IF EXISTS" not in sql.upper(): + raise + for s in [self] + self.hosts: + for kid in s.kids: + cur.execute("INSERT registrant (registrant_handle, registry_handle, valid_until) VALUES (%s, %s, %s)", + (kid.name, s.name, kid.resources.valid_until)) + db.close() + + def sync_sql(self): + """ + Whack this entity's IRDB to match our master database. We do this + once during setup, then do it again every time we apply a delta to + this entity. + """ + + logger.info("Updating MySQL data for IRDB %s", self.name) + db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, + conv = sql_conversions) + cur = db.cursor() + db.autocommit(True) + cur.execute("DELETE FROM registrant_asn") + cur.execute("DELETE FROM registrant_net") + cur.execute("DELETE FROM roa_request_prefix") + cur.execute("DELETE FROM roa_request") + cur.execute("DELETE FROM ee_certificate_asn") + cur.execute("DELETE FROM ee_certificate_net") + cur.execute("DELETE FROM ee_certificate") + + for s in [self] + self.hosts: + for kid in s.kids: + cur.execute("SELECT registrant_id FROM registrant WHERE registrant_handle = %s AND registry_handle = %s", + (kid.name, s.name)) + registrant_id = cur.fetchone()[0] + for as_range in kid.resources.asn: + cur.execute("INSERT registrant_asn (start_as, end_as, registrant_id) VALUES (%s, %s, %s)", + (as_range.min, as_range.max, registrant_id)) + for v4_range in kid.resources.v4: + cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 4, %s)", + (v4_range.min, v4_range.max, registrant_id)) + for v6_range in kid.resources.v6: + cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 6, %s)", + (v6_range.min, v6_range.max, registrant_id)) + cur.execute("UPDATE registrant SET valid_until = %s WHERE registrant_id = %s", + (kid.resources.valid_until, registrant_id)) + for r in s.roa_requests: + cur.execute("INSERT roa_request (self_handle, asn) VALUES (%s, %s)", + (s.name, r.asn)) + roa_request_id = cur.lastrowid + for version, prefix_set in ((4, r.v4), (6, r.v6)): + if prefix_set: + cur.executemany("INSERT roa_request_prefix " + "(roa_request_id, prefix, prefixlen, max_prefixlen, version) " + "VALUES (%s, %s, %s, %s, %s)", + ((roa_request_id, x.prefix, x.prefixlen, x.max_prefixlen, version) + for x in prefix_set)) + for r in s.router_certs: + cur.execute("INSERT ee_certificate (self_handle, pkcs10, gski, cn, sn, eku, valid_until) " + "VALUES (%s, %s, %s, %s, %s, %s, %s)", + (s.name, r.pkcs10.get_DER(), r.gski, r.cn, r.sn, r.eku, s.resources.valid_until)) + ee_certificate_id = cur.lastrowid + cur.executemany("INSERT ee_certificate_asn (ee_certificate_id, start_as, end_as) VALUES (%s, %s, %s)", + ((ee_certificate_id, a.min, a.max) for a in r.asn)) + db.close() + + def run_daemons(self): + """ + Run daemons for this entity. + """ + + logger.info("Running daemons for %s", self.name) + env = dict(os.environ, RPKI_CONF = self.name + ".conf") + self.rpkid_process = subprocess.Popen((prog_python, prog_rpkid, "--foreground", "--log-stdout", "--log-level", "debug") + + (("--profile", self.name + ".prof") if args.profile else ()), + env = env) + self.irdbd_process = subprocess.Popen((prog_python, prog_irdbd, "--foreground", "--log-stdout", "--log-level", "debug"), + env = env) + + def kill_daemons(self): + """ + Kill daemons for this entity. + """ + + # pylint: disable=E1103 + for proc, name in ((self.rpkid_process, "rpkid"), + (self.irdbd_process, "irdbd")): + if proc is not None and proc.poll() is None: + logger.info("Killing daemon %s pid %s for %s", name, proc.pid, self.name) + try: + proc.terminate() + except OSError: + pass + if proc is not None: + logger.info("Daemon %s pid %s for %s exited with code %s", + name, proc.pid, self.name, proc.wait()) + + def call_rpkid(self, pdus, cb): + """ + Send a left-right message to this entity's RPKI daemon and return + the response. + + If this entity is hosted (does not run its own RPKI daemon), all + of this happens with the hosting RPKI daemon. + """ + + logger.info("Calling rpkid for %s", self.name) + + if self.is_hosted: + logger.info("rpkid %s is hosted by rpkid %s, switching", self.name, self.hosted_by.name) + self = self.hosted_by + assert not self.is_hosted + + assert isinstance(pdus, (list, tuple)) + assert self.rpki_port is not None + + q_msg = rpki.left_right.msg.query(*pdus) + q_cms = rpki.left_right.cms_msg_saxify() + q_der = q_cms.wrap(q_msg, self.irbe_key, self.irbe_cert) + q_url = "http://localhost:%d/left-right" % self.rpki_port + + logger.debug(q_cms.pretty_print_content()) + + def done(r_der): + logger.info("Callback from rpkid %s", self.name) + r_cms = rpki.left_right.cms_msg_saxify(DER = r_der) + r_msg = r_cms.unwrap((self.rpkid_ta, self.rpkid_cert)) + self.last_cms_time = r_cms.check_replay(self.last_cms_time, q_url) + logger.debug(r_cms.pretty_print_content()) + assert r_msg.is_reply + for r_pdu in r_msg: + assert not isinstance(r_pdu, rpki.left_right.report_error_elt) + cb(r_msg) + + def lose(e): + raise + + rpki.http.client( + url = q_url, + msg = q_der, + callback = done, + errback = lose) + + def cross_certify(self, certificant, reverse = False): + """ + Cross-certify and return the resulting certificate. + """ + + if reverse: + certifier = certificant + certificant = self.name + "-SELF" + else: + certifier = self.name + "-SELF" + return cross_certify(certificant, certifier) + + def create_rpki_objects(self, cb): + """ + Create RPKI engine objects for this engine. + + Root node of the engine tree is special, it too has a parent but + that one is the magic self-signed micro engine. + + The rest of this is straightforward. There are a lot of objects + to create, but we can do batch them all into one honking PDU, then + issue one more PDU to set BSC EE certificates based on the PKCS + #10 requests we get back when we tell rpkid to generate BSC keys. + """ + + assert not self.is_hosted + + selves = [self] + self.hosts + + rpkid_pdus = [] + pubd_pdus = [] + + for i, s in enumerate(selves): + logger.info("Creating RPKI objects for [%d] %s", i, s.name) + + rpkid_pdus.append(rpki.left_right.self_elt.make_pdu( + action = "create", + self_handle = s.name, + crl_interval = s.crl_interval, + regen_margin = s.regen_margin, + bpki_cert = (s.cross_certify(s.hosted_by.name + "-TA", reverse = True) + if s.is_hosted else + rpki.x509.X509(Auto_file = s.name + "-SELF.cer")))) + + rpkid_pdus.append(rpki.left_right.bsc_elt.make_pdu( + action = "create", + self_handle = s.name, + bsc_handle = "b", + generate_keypair = True)) + + pubd_pdus.append(rpki.publication_control.client_elt.make_pdu( + action = "create", + client_handle = s.client_handle, + base_uri = s.sia_base, + bpki_cert = s.cross_certify(pubd_name + "-TA", reverse = True))) + + rpkid_pdus.append(rpki.left_right.repository_elt.make_pdu( + action = "create", + self_handle = s.name, + bsc_handle = "b", + repository_handle = "r", + bpki_cert = s.cross_certify(pubd_name + "-TA"), + peer_contact_uri = "http://localhost:%d/client/%s" % (pubd_port, s.client_handle))) + + for k in s.kids: + rpkid_pdus.append(rpki.left_right.child_elt.make_pdu( + action = "create", + self_handle = s.name, + child_handle = k.name, + bsc_handle = "b", + bpki_cert = s.cross_certify(k.name + "-SELF"))) + + if s.is_root: + rootd_cert = s.cross_certify(rootd_name + "-TA") + rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( + action = "create", + self_handle = s.name, + parent_handle = "rootd", + bsc_handle = "b", + repository_handle = "r", + sia_base = s.sia_base, + bpki_cert = rootd_cert, + sender_name = s.name, + recipient_name = "rootd", + peer_contact_uri = "http://localhost:%s/" % rootd_port)) + else: + rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( + action = "create", + self_handle = s.name, + parent_handle = s.parent.name, + bsc_handle = "b", + repository_handle = "r", + sia_base = s.sia_base, + bpki_cert = s.cross_certify(s.parent.name + "-SELF"), + sender_name = s.name, + recipient_name = s.parent.name, + peer_contact_uri = "http://localhost:%s/up-down/%s/%s" % (s.parent.get_rpki_port(), + s.parent.name, s.name))) + + def one(): + call_pubd(pubd_pdus, cb = two) + + def two(vals): + self.call_rpkid(rpkid_pdus, cb = three) + + def three(vals): + + bsc_dict = dict((b.self_handle, b) for b in vals if isinstance(b, rpki.left_right.bsc_elt)) + + bsc_pdus = [] + + for s in selves: + b = bsc_dict[s.name] + + logger.info("Issuing BSC EE cert for %s", s.name) + cmd = (prog_openssl, "x509", "-req", "-sha256", "-extfile", s.name + "-RPKI.conf", + "-extensions", "req_x509_ext", "-days", "30", + "-CA", s.name + "-SELF.cer", "-CAkey", s.name + "-SELF.key", "-CAcreateserial", "-text") + signer = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + signed = signer.communicate(input = b.pkcs10_request.get_PEM()) + if not signed[0]: + logger.warning(signed[1]) + raise CouldntIssueBSCEECertificate("Couldn't issue BSC EE certificate") + s.bsc_ee = rpki.x509.X509(PEM = signed[0]) + s.bsc_crl = rpki.x509.CRL(PEM_file = s.name + "-SELF.crl") + logger.info("BSC EE cert for %s SKI %s", s.name, s.bsc_ee.hSKI()) + + bsc_pdus.append(rpki.left_right.bsc_elt.make_pdu( + action = "set", + self_handle = s.name, + bsc_handle = "b", + signing_cert = s.bsc_ee, + signing_cert_crl = s.bsc_crl)) + + self.call_rpkid(bsc_pdus, cb = four) + + def four(vals): + cb() + + one() + + def setup_yaml_leaf(self): + """ + Generate certificates and write YAML scripts for leaf nodes. + + We're cheating a bit here: properly speaking, we can't generate + issue or revoke requests without knowing the class, which is + generated on the fly, but at the moment the test case is + simplistic enough that the class will always be "1", so we just + wire in that value for now. + + Well, ok, we just broke that assumption. Now we do something even + nastier, just to eke a bit more life out of this kludge. This + really needs to be rewritten, but it may require a different tool + than testpoke. + """ + + if not os.path.exists(self.name + ".key"): + logger.info("Generating RPKI key for %s", self.name) + subprocess.check_call((prog_openssl, "genrsa", "-out", self.name + ".key", "2048" ), + stdout = subprocess.PIPE, stderr = subprocess.STDOUT) + ski = rpki.x509.RSA(PEM_file = self.name + ".key").gSKI() + + if self.parent.is_hosted: + parent_host = self.parent.hosted_by.name + else: + parent_host = self.parent.name - def apply_valid_add(self, text, cb): - self.base.valid_until += rpki.sundial.timedelta.parse(text) - cb() + self.cross_certify(self.parent.name + "-SELF") + self.cross_certify(parent_host + "-TA") - def apply_valid_sub(self, text, cb): - self.base.valid_until -= rpki.sundial.timedelta.parse(text) - cb() + def run_cron(self, cb): + """ + Trigger cron run for this engine. + """ - def apply_roa_request_add(self, yaml, cb): - for y in yaml: - r = roa_request.parse(y) - if r not in self.roa_requests: - self.roa_requests.append(r) - cb() + logger.info("Running cron for %s", self.name) - def apply_roa_request_del(self, yaml, cb): - for y in yaml: - r = roa_request.parse(y) - if r in self.roa_requests: - self.roa_requests.remove(r) - cb() + assert self.rpki_port is not None - def apply_router_cert_add(self, yaml, cb): - for y in yaml: - r = router_cert.parse(y) - if r not in self.router_certs: - self.router_certs.append(r) - cb() + def done(result): + assert result == "OK", 'Expected "OK" result from cronjob, got %r' % result + cb() - def apply_router_cert_del(self, yaml, cb): - for y in yaml: - r = router_cert.parse(y) - if r in self.router_certs: - self.router_certs.remove(r) - cb() + rpki.http.client( + url = "http://localhost:%d/cronjob" % self.rpki_port, + msg = "Run cron now, please", + callback = done, + errback = done) - def apply_rekey(self, target, cb): - - def done(e): - if isinstance(e, Exception): - logger.exception("Exception while rekeying %s", self.name) - raise e - cb() - - if target is None: - logger.info("Rekeying <tenant/> %s", self.name) - self.call_rpkid([rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.name, rekey = "yes")], cb = done) - else: - logger.info("Rekeying <parent/> %s %s", self.name, target) - self.call_rpkid([rpki.left_right.parent_elt.make_pdu( - action = "set", self_handle = self.name, parent_handle = target, rekey = "yes")], cb = done) - - def apply_revoke(self, target, cb): - - def done(e): - if isinstance(e, Exception): - logger.exception("Exception while revoking %s", self.name) - raise e - cb() - - if target is None: - logger.info("Revoking <tenant/> %s", self.name) - self.call_rpkid([rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.name, revoke = "yes")], cb = done) - else: - logger.info("Revoking <parent/> %s %s", self.name, target) - self.call_rpkid([rpki.left_right.parent_elt.make_pdu( - action = "set", self_handle = self.name, parent_handle = target, revoke = "yes")], cb = done) - - def __str__(self): - s = self.name + "\n" - if self.resources.asn: s += " ASN: %s\n" % self.resources.asn - if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 - if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 - if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) - if self.parent: s += " Up: %s\n" % self.parent.name - if self.sia_base: s += " SIA: %s\n" % self.sia_base - return s + "Until: %s\n" % self.resources.valid_until - - - @property - def is_root(self): - return self.parent is None - - @property - def is_twig(self): - return not self.is_root - - @property - def is_hosted(self): - return self.hosted_by is not None - - @property - def is_engine(self): - return not self.is_hosted - - def set_engine_number(self, n): - """ - Set the engine number for this entity. - """ + def run_yaml(self): + """ + Run YAML scripts for this leaf entity. Since we're not bothering + to check the class list returned by the list command, the issue + command may fail, so we treat failure of the list command as an + error, but only issue a warning when issue fails. + """ - self.irdb_db_name = "irdb%d" % n - self.irdb_port = allocate_port() - self.rpki_db_name = "rpki%d" % n - self.rpki_port = allocate_port() + logger.info("Running YAML for %s", self.name) + subprocess.check_call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "list")) + if subprocess.call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "issue")) != 0: + logger.warning("YAML issue command failed for %s, continuing", self.name) - def get_rpki_port(self): +def setup_bpki_cert_chain(name, ee = (), ca = ()): """ - Get rpki port to use for this entity. + Build a set of BPKI certificates. """ - if self.is_hosted: - assert self.hosted_by.rpki_port is not None - return self.hosted_by.rpki_port - else: - assert self.rpki_port is not None - return self.rpki_port + s = "exec >/dev/null 2>&1\n" + #s = "set -x\n" + for kind in ("TA",) + ee + ca: + d = dict(name = name, + kind = kind, + ca = "false" if kind in ee else "true", + openssl = prog_openssl) + f = open("%(name)s-%(kind)s.conf" % d, "w") + f.write(bpki_cert_fmt_1 % d) + f.close() + if not os.path.exists("%(name)s-%(kind)s.key" % d): + s += bpki_cert_fmt_2 % d + s += bpki_cert_fmt_3 % d + d = dict(name = name, + openssl = prog_openssl) + s += bpki_cert_fmt_4 % d + for kind in ee + ca: + d["kind"] = kind + s += bpki_cert_fmt_5 % d + for kind in ("TA",) + ca: + d["kind"] = kind + s += bpki_cert_fmt_6 % d + subprocess.check_call(s, shell = True) - def setup_bpki_certs(self): +def setup_rootd(rpkid, rootd_yaml, db): """ - Create BPKI certificates for this entity. + Write the config files for rootd. """ - logger.info("Constructing BPKI keys and certs for %s", self.name) - setup_bpki_cert_chain(name = self.name, - ee = ("RPKI", "IRDB", "IRBE"), - ca = ("SELF",)) - self.rpkid_ta = rpki.x509.X509(PEM_file = self.name + "-TA.cer") - self.irbe_key = rpki.x509.RSA( PEM_file = self.name + "-IRBE.key") - self.irbe_cert = rpki.x509.X509(PEM_file = self.name + "-IRBE.cer") - self.rpkid_cert = rpki.x509.X509(PEM_file = self.name + "-RPKI.cer") + rpkid.cross_certify(rootd_name + "-TA", reverse = True) + cross_certify(pubd_name + "-TA", rootd_name + "-TA") + logger.info("Writing config files for %s", rootd_name) + d = dict(rootd_name = rootd_name, + rootd_port = rootd_port, + rpkid_name = rpkid.name, + pubd_name = pubd_name, + rootd_sia = rootd_sia, + rsyncd_dir = rsyncd_dir, + openssl = prog_openssl, + lifetime = rootd_yaml.get("lifetime", "30d"), + pubd_port = pubd_port, + rootd_handle = db.root.client_handle + "-" + rootd_name) + f = open(rootd_name + ".conf", "w") + f.write(rootd_fmt_1 % d) + f.close() + s = "exec >/dev/null 2>&1\n" + #s = "set -x\n" + if not os.path.exists("root.key"): + s += rootd_fmt_2 % d + s += rootd_fmt_3 % d + subprocess.check_call(s, shell = True) - def setup_conf_file(self): +def setup_rcynic(): """ - Write config files for this entity. + Write the config file for rcynic. """ - logger.info("Writing config files for %s", self.name) - assert self.rpki_port is not None - d = dict(my_name = self.name, - irdb_db_name = self.irdb_db_name, - irdb_db_pass = irdb_db_pass, - irdb_port = self.irdb_port, - rpki_db_name = self.rpki_db_name, - rpki_db_pass = rpki_db_pass, - rpki_port = self.rpki_port) - f = open(self.name + ".conf", "w") - f.write(conf_fmt_1 % d) - for line in self.extra_conf: - f.write(line + "\n") + logger.info("Config file for rcynic") + d = dict(rcynic_name = rcynic_name, + rootd_name = rootd_name, + rootd_sia = rootd_sia) + f = open(rcynic_name + ".conf", "w") + f.write(rcynic_fmt_1 % d) f.close() - def setup_sql(self, rpki_sql, irdb_sql): +def setup_rsyncd(): """ - Set up this entity's IRDB. + Write the config file for rsyncd. """ - logger.info("Setting up MySQL for %s", self.name) - db = MySQLdb.connect(user = "rpki", db = self.rpki_db_name, passwd = rpki_db_pass, - conv = sql_conversions) - cur = db.cursor() - db.autocommit(True) - for sql in rpki_sql: - try: - cur.execute(sql) - except: - if "DROP TABLE IF EXISTS" not in sql.upper(): - raise - db.close() - db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, - conv = sql_conversions) - cur = db.cursor() - db.autocommit(True) - for sql in irdb_sql: - try: - cur.execute(sql) - except: - if "DROP TABLE IF EXISTS" not in sql.upper(): - raise - for s in [self] + self.hosts: - for kid in s.kids: - cur.execute("INSERT registrant (registrant_handle, registry_handle, valid_until) VALUES (%s, %s, %s)", - (kid.name, s.name, kid.resources.valid_until)) - db.close() + logger.info("Config file for rsyncd") + d = dict(rsyncd_name = rsyncd_name, + rsyncd_port = rsyncd_port, + rsyncd_module = rsyncd_module, + rsyncd_dir = rsyncd_dir) + f = open(rsyncd_name + ".conf", "w") + f.write(rsyncd_fmt_1 % d) + f.close() - def sync_sql(self): +def setup_publication(pubd_sql, irdb_db_name): """ - Whack this entity's IRDB to match our master database. We do this - once during setup, then do it again every time we apply a delta to - this entity. + Set up publication daemon. """ - logger.info("Updating MySQL data for IRDB %s", self.name) - db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, + logger.info("Configure publication daemon") + publication_dir = os.getcwd() + "/publication" + assert rootd_sia.startswith("rsync://") + global rsyncd_dir + rsyncd_dir = publication_dir + "/".join(rootd_sia.split("/")[4:]) + if not rsyncd_dir.endswith("/"): + rsyncd_dir += "/" + os.makedirs(rsyncd_dir + "root/trunk") + db = MySQLdb.connect(db = pubd_db_name, user = pubd_db_user, passwd = pubd_db_pass, conv = sql_conversions) cur = db.cursor() db.autocommit(True) - cur.execute("DELETE FROM registrant_asn") - cur.execute("DELETE FROM registrant_net") - cur.execute("DELETE FROM roa_request_prefix") - cur.execute("DELETE FROM roa_request") - cur.execute("DELETE FROM ee_certificate_asn") - cur.execute("DELETE FROM ee_certificate_net") - cur.execute("DELETE FROM ee_certificate") - - for s in [self] + self.hosts: - for kid in s.kids: - cur.execute("SELECT registrant_id FROM registrant WHERE registrant_handle = %s AND registry_handle = %s", - (kid.name, s.name)) - registrant_id = cur.fetchone()[0] - for as_range in kid.resources.asn: - cur.execute("INSERT registrant_asn (start_as, end_as, registrant_id) VALUES (%s, %s, %s)", - (as_range.min, as_range.max, registrant_id)) - for v4_range in kid.resources.v4: - cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 4, %s)", - (v4_range.min, v4_range.max, registrant_id)) - for v6_range in kid.resources.v6: - cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 6, %s)", - (v6_range.min, v6_range.max, registrant_id)) - cur.execute("UPDATE registrant SET valid_until = %s WHERE registrant_id = %s", - (kid.resources.valid_until, registrant_id)) - for r in s.roa_requests: - cur.execute("INSERT roa_request (self_handle, asn) VALUES (%s, %s)", - (s.name, r.asn)) - roa_request_id = cur.lastrowid - for version, prefix_set in ((4, r.v4), (6, r.v6)): - if prefix_set: - cur.executemany("INSERT roa_request_prefix " - "(roa_request_id, prefix, prefixlen, max_prefixlen, version) " - "VALUES (%s, %s, %s, %s, %s)", - ((roa_request_id, x.prefix, x.prefixlen, x.max_prefixlen, version) - for x in prefix_set)) - for r in s.router_certs: - cur.execute("INSERT ee_certificate (self_handle, pkcs10, gski, cn, sn, eku, valid_until) " - "VALUES (%s, %s, %s, %s, %s, %s, %s)", - (s.name, r.pkcs10.get_DER(), r.gski, r.cn, r.sn, r.eku, s.resources.valid_until)) - ee_certificate_id = cur.lastrowid - cur.executemany("INSERT ee_certificate_asn (ee_certificate_id, start_as, end_as) VALUES (%s, %s, %s)", - ((ee_certificate_id, a.min, a.max) for a in r.asn)) + for sql in pubd_sql: + try: + cur.execute(sql) + except: + if "DROP TABLE IF EXISTS" not in sql.upper(): + raise db.close() + d = dict(pubd_name = pubd_name, + pubd_port = pubd_port, + pubd_db_name = pubd_db_name, + pubd_db_user = pubd_db_user, + pubd_db_pass = pubd_db_pass, + pubd_dir = rsyncd_dir, + irdb_db_name = irdb_db_name, + irdb_db_pass = irdb_db_pass) + f = open(pubd_name + ".conf", "w") + f.write(pubd_fmt_1 % d) + f.close() + global pubd_ta + global pubd_irbe_key + global pubd_irbe_cert + global pubd_pubd_cert + pubd_ta = rpki.x509.X509(Auto_file = pubd_name + "-TA.cer") + pubd_irbe_key = rpki.x509.RSA( Auto_file = pubd_name + "-IRBE.key") + pubd_irbe_cert = rpki.x509.X509(Auto_file = pubd_name + "-IRBE.cer") + pubd_pubd_cert = rpki.x509.X509(Auto_file = pubd_name + "-PUBD.cer") - def run_daemons(self): - """ - Run daemons for this entity. - """ - - logger.info("Running daemons for %s", self.name) - env = dict(os.environ, RPKI_CONF = self.name + ".conf") - self.rpkid_process = subprocess.Popen((prog_python, prog_rpkid, "--foreground", "--log-stdout", "--log-level", "debug") + - (("--profile", self.name + ".prof") if args.profile else ()), - env = env) - self.irdbd_process = subprocess.Popen((prog_python, prog_irdbd, "--foreground", "--log-stdout", "--log-level", "debug"), - env = env) - - def kill_daemons(self): - """ - Kill daemons for this entity. - """ - - # pylint: disable=E1103 - for proc, name in ((self.rpkid_process, "rpkid"), - (self.irdbd_process, "irdbd")): - if proc is not None and proc.poll() is None: - logger.info("Killing daemon %s pid %s for %s", name, proc.pid, self.name) - try: - proc.terminate() - except OSError: - pass - if proc is not None: - logger.info("Daemon %s pid %s for %s exited with code %s", - name, proc.pid, self.name, proc.wait()) - - def call_rpkid(self, pdus, cb): +def call_pubd(pdus, cb): """ - Send a left-right message to this entity's RPKI daemon and return + Send a publication control message to publication daemon and return the response. - - If this entity is hosted (does not run its own RPKI daemon), all - of this happens with the hosting RPKI daemon. """ - logger.info("Calling rpkid for %s", self.name) - - if self.is_hosted: - logger.info("rpkid %s is hosted by rpkid %s, switching", self.name, self.hosted_by.name) - self = self.hosted_by - assert not self.is_hosted - - assert isinstance(pdus, (list, tuple)) - assert self.rpki_port is not None - - q_msg = rpki.left_right.msg.query(*pdus) - q_cms = rpki.left_right.cms_msg_saxify() - q_der = q_cms.wrap(q_msg, self.irbe_key, self.irbe_cert) - q_url = "http://localhost:%d/left-right" % self.rpki_port + logger.info("Calling pubd") + q_msg = rpki.publication_control.msg.query(*pdus) + q_cms = rpki.publication_control.cms_msg_saxify() + q_der = q_cms.wrap(q_msg, pubd_irbe_key, pubd_irbe_cert) + q_url = "http://localhost:%d/control" % pubd_port logger.debug(q_cms.pretty_print_content()) - def done(r_der): - logger.info("Callback from rpkid %s", self.name) - r_cms = rpki.left_right.cms_msg_saxify(DER = r_der) - r_msg = r_cms.unwrap((self.rpkid_ta, self.rpkid_cert)) - self.last_cms_time = r_cms.check_replay(self.last_cms_time, q_url) - logger.debug(r_cms.pretty_print_content()) - assert r_msg.is_reply - for r_pdu in r_msg: - assert not isinstance(r_pdu, rpki.left_right.report_error_elt) - cb(r_msg) + def call_pubd_cb(r_der): + global pubd_last_cms_time + r_cms = rpki.publication_control.cms_msg_saxify(DER = r_der) + r_msg = r_cms.unwrap((pubd_ta, pubd_pubd_cert)) + pubd_last_cms_time = r_cms.check_replay(pubd_last_cms_time, q_url) + logger.debug(r_cms.pretty_print_content()) + assert r_msg.is_reply + for r_pdu in r_msg: + r_pdu.raise_if_error() + cb(r_msg) - def lose(e): - raise + def call_pubd_eb(e): + logger.exception("Problem calling pubd") rpki.http.client( - url = q_url, - msg = q_der, - callback = done, - errback = lose) + url = q_url, + msg = q_der, + callback = call_pubd_cb, + errback = call_pubd_eb) + - def cross_certify(self, certificant, reverse = False): +def cross_certify(certificant, certifier): """ Cross-certify and return the resulting certificate. """ - if reverse: - certifier = certificant - certificant = self.name + "-SELF" - else: - certifier = self.name + "-SELF" - return cross_certify(certificant, certifier) - - def create_rpki_objects(self, cb): - """ - Create RPKI engine objects for this engine. + certfile = certifier + "-" + certificant + ".cer" - Root node of the engine tree is special, it too has a parent but - that one is the magic self-signed micro engine. - - The rest of this is straightforward. There are a lot of objects - to create, but we can do batch them all into one honking PDU, then - issue one more PDU to set BSC EE certificates based on the PKCS - #10 requests we get back when we tell rpkid to generate BSC keys. - """ + logger.info("Cross certifying %s into %s's BPKI (%s)", certificant, certifier, certfile) - assert not self.is_hosted - - selves = [self] + self.hosts - - rpkid_pdus = [] - pubd_pdus = [] - - for i, s in enumerate(selves): - logger.info("Creating RPKI objects for [%d] %s", i, s.name) - - rpkid_pdus.append(rpki.left_right.self_elt.make_pdu( - action = "create", - self_handle = s.name, - crl_interval = s.crl_interval, - regen_margin = s.regen_margin, - bpki_cert = (s.cross_certify(s.hosted_by.name + "-TA", reverse = True) - if s.is_hosted else - rpki.x509.X509(Auto_file = s.name + "-SELF.cer")))) - - rpkid_pdus.append(rpki.left_right.bsc_elt.make_pdu( - action = "create", - self_handle = s.name, - bsc_handle = "b", - generate_keypair = True)) - - pubd_pdus.append(rpki.publication_control.client_elt.make_pdu( - action = "create", - client_handle = s.client_handle, - base_uri = s.sia_base, - bpki_cert = s.cross_certify(pubd_name + "-TA", reverse = True))) - - rpkid_pdus.append(rpki.left_right.repository_elt.make_pdu( - action = "create", - self_handle = s.name, - bsc_handle = "b", - repository_handle = "r", - bpki_cert = s.cross_certify(pubd_name + "-TA"), - peer_contact_uri = "http://localhost:%d/client/%s" % (pubd_port, s.client_handle))) - - for k in s.kids: - rpkid_pdus.append(rpki.left_right.child_elt.make_pdu( - action = "create", - self_handle = s.name, - child_handle = k.name, - bsc_handle = "b", - bpki_cert = s.cross_certify(k.name + "-SELF"))) - - if s.is_root: - rootd_cert = s.cross_certify(rootd_name + "-TA") - rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( - action = "create", - self_handle = s.name, - parent_handle = "rootd", - bsc_handle = "b", - repository_handle = "r", - sia_base = s.sia_base, - bpki_cert = rootd_cert, - sender_name = s.name, - recipient_name = "rootd", - peer_contact_uri = "http://localhost:%s/" % rootd_port)) - else: - rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( - action = "create", - self_handle = s.name, - parent_handle = s.parent.name, - bsc_handle = "b", - repository_handle = "r", - sia_base = s.sia_base, - bpki_cert = s.cross_certify(s.parent.name + "-SELF"), - sender_name = s.name, - recipient_name = s.parent.name, - peer_contact_uri = "http://localhost:%s/up-down/%s/%s" % (s.parent.get_rpki_port(), - s.parent.name, s.name))) - - def one(): - call_pubd(pubd_pdus, cb = two) - - def two(vals): - self.call_rpkid(rpkid_pdus, cb = three) - - def three(vals): - - bsc_dict = dict((b.self_handle, b) for b in vals if isinstance(b, rpki.left_right.bsc_elt)) - - bsc_pdus = [] - - for s in selves: - b = bsc_dict[s.name] - - logger.info("Issuing BSC EE cert for %s", s.name) - cmd = (prog_openssl, "x509", "-req", "-sha256", "-extfile", s.name + "-RPKI.conf", - "-extensions", "req_x509_ext", "-days", "30", - "-CA", s.name + "-SELF.cer", "-CAkey", s.name + "-SELF.key", "-CAcreateserial", "-text") - signer = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) - signed = signer.communicate(input = b.pkcs10_request.get_PEM()) - if not signed[0]: - logger.warning(signed[1]) - raise CouldntIssueBSCEECertificate("Couldn't issue BSC EE certificate") - s.bsc_ee = rpki.x509.X509(PEM = signed[0]) - s.bsc_crl = rpki.x509.CRL(PEM_file = s.name + "-SELF.crl") - logger.info("BSC EE cert for %s SKI %s", s.name, s.bsc_ee.hSKI()) - - bsc_pdus.append(rpki.left_right.bsc_elt.make_pdu( - action = "set", - self_handle = s.name, - bsc_handle = "b", - signing_cert = s.bsc_ee, - signing_cert_crl = s.bsc_crl)) - - self.call_rpkid(bsc_pdus, cb = four) - - def four(vals): - cb() - - one() - - def setup_yaml_leaf(self): - """ - Generate certificates and write YAML scripts for leaf nodes. - - We're cheating a bit here: properly speaking, we can't generate - issue or revoke requests without knowing the class, which is - generated on the fly, but at the moment the test case is - simplistic enough that the class will always be "1", so we just - wire in that value for now. - - Well, ok, we just broke that assumption. Now we do something even - nastier, just to eke a bit more life out of this kludge. This - really needs to be rewritten, but it may require a different tool - than testpoke. - """ + child = rpki.x509.X509(Auto_file = certificant + ".cer") + parent = rpki.x509.X509(Auto_file = certifier + ".cer") + keypair = rpki.x509.RSA(Auto_file = certifier + ".key") + serial_file = certifier + ".srl" - if not os.path.exists(self.name + ".key"): - logger.info("Generating RPKI key for %s", self.name) - subprocess.check_call((prog_openssl, "genrsa", "-out", self.name + ".key", "2048" ), - stdout = subprocess.PIPE, stderr = subprocess.STDOUT) - ski = rpki.x509.RSA(PEM_file = self.name + ".key").gSKI() + now = rpki.sundial.now() + notAfter = now + rpki.sundial.timedelta(days = 30) - if self.parent.is_hosted: - parent_host = self.parent.hosted_by.name - else: - parent_host = self.parent.name + try: + with open(serial_file, "r") as f: + serial = int(f.read().splitlines()[0], 16) + except IOError: + serial = 1 - self.cross_certify(self.parent.name + "-SELF") - self.cross_certify(parent_host + "-TA") + x = parent.bpki_cross_certify( + keypair = keypair, + source_cert = child, + serial = serial, + notAfter = notAfter, + now = now) - def run_cron(self, cb): - """ - Trigger cron run for this engine. - """ + with open(serial_file, "w") as f: + f.write("%02x\n" % (serial + 1)) - logger.info("Running cron for %s", self.name) + with open(certfile, "w") as f: + f.write(x.get_PEM()) - assert self.rpki_port is not None + logger.debug("Cross certified %s:", certfile) + logger.debug(" Issuer %s [%s]", x.getIssuer(), x.hAKI()) + logger.debug(" Subject %s [%s]", x.getSubject(), x.hSKI()) + return x - def done(result): - assert result == "OK", 'Expected "OK" result from cronjob, got %r' % result - cb() - - rpki.http.client( - url = "http://localhost:%d/cronjob" % self.rpki_port, - msg = "Run cron now, please", - callback = done, - errback = done) +last_rcynic_run = None - def run_yaml(self): +def run_rcynic(): """ - Run YAML scripts for this leaf entity. Since we're not bothering - to check the class list returned by the list command, the issue - command may fail, so we treat failure of the list command as an - error, but only issue a warning when issue fails. + Run rcynic to see whether what was published makes sense. """ - logger.info("Running YAML for %s", self.name) - subprocess.check_call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "list")) - if subprocess.call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "issue")) != 0: - logger.warning("YAML issue command failed for %s, continuing", self.name) + logger.info("Running rcynic") + env = os.environ.copy() + env["TZ"] = "" + global last_rcynic_run + if int(time.time()) == last_rcynic_run: + time.sleep(1) + subprocess.check_call((prog_rcynic, "-c", rcynic_name + ".conf"), env = env) + subprocess.call(rcynic_stats, shell = True, env = env) + last_rcynic_run = int(time.time()) + os.link("%s.xml" % rcynic_name, "%s.%s.xml" % (rcynic_name, last_rcynic_run)) -def setup_bpki_cert_chain(name, ee = (), ca = ()): - """ - Build a set of BPKI certificates. - """ +def mangle_sql(filename): + """ + Mangle an SQL file into a sequence of SQL statements. + """ - s = "exec >/dev/null 2>&1\n" - #s = "set -x\n" - for kind in ("TA",) + ee + ca: - d = dict(name = name, - kind = kind, - ca = "false" if kind in ee else "true", - openssl = prog_openssl) - f = open("%(name)s-%(kind)s.conf" % d, "w") - f.write(bpki_cert_fmt_1 % d) + words = [] + f = open(filename) + for line in f: + words.extend(line.partition("--")[0].split()) f.close() - if not os.path.exists("%(name)s-%(kind)s.key" % d): - s += bpki_cert_fmt_2 % d - s += bpki_cert_fmt_3 % d - d = dict(name = name, - openssl = prog_openssl) - s += bpki_cert_fmt_4 % d - for kind in ee + ca: - d["kind"] = kind - s += bpki_cert_fmt_5 % d - for kind in ("TA",) + ca: - d["kind"] = kind - s += bpki_cert_fmt_6 % d - subprocess.check_call(s, shell = True) - -def setup_rootd(rpkid, rootd_yaml, db): - """ - Write the config files for rootd. - """ - - rpkid.cross_certify(rootd_name + "-TA", reverse = True) - cross_certify(pubd_name + "-TA", rootd_name + "-TA") - logger.info("Writing config files for %s", rootd_name) - d = dict(rootd_name = rootd_name, - rootd_port = rootd_port, - rpkid_name = rpkid.name, - pubd_name = pubd_name, - rootd_sia = rootd_sia, - rsyncd_dir = rsyncd_dir, - openssl = prog_openssl, - lifetime = rootd_yaml.get("lifetime", "30d"), - pubd_port = pubd_port, - rootd_handle = db.root.client_handle + "-" + rootd_name) - f = open(rootd_name + ".conf", "w") - f.write(rootd_fmt_1 % d) - f.close() - s = "exec >/dev/null 2>&1\n" - #s = "set -x\n" - if not os.path.exists("root.key"): - s += rootd_fmt_2 % d - s += rootd_fmt_3 % d - subprocess.check_call(s, shell = True) - -def setup_rcynic(): - """ - Write the config file for rcynic. - """ - - logger.info("Config file for rcynic") - d = dict(rcynic_name = rcynic_name, - rootd_name = rootd_name, - rootd_sia = rootd_sia) - f = open(rcynic_name + ".conf", "w") - f.write(rcynic_fmt_1 % d) - f.close() - -def setup_rsyncd(): - """ - Write the config file for rsyncd. - """ - - logger.info("Config file for rsyncd") - d = dict(rsyncd_name = rsyncd_name, - rsyncd_port = rsyncd_port, - rsyncd_module = rsyncd_module, - rsyncd_dir = rsyncd_dir) - f = open(rsyncd_name + ".conf", "w") - f.write(rsyncd_fmt_1 % d) - f.close() - -def setup_publication(pubd_sql, irdb_db_name): - """ - Set up publication daemon. - """ - - logger.info("Configure publication daemon") - publication_dir = os.getcwd() + "/publication" - assert rootd_sia.startswith("rsync://") - global rsyncd_dir - rsyncd_dir = publication_dir + "/".join(rootd_sia.split("/")[4:]) - if not rsyncd_dir.endswith("/"): - rsyncd_dir += "/" - os.makedirs(rsyncd_dir + "root/trunk") - db = MySQLdb.connect(db = pubd_db_name, user = pubd_db_user, passwd = pubd_db_pass, - conv = sql_conversions) - cur = db.cursor() - db.autocommit(True) - for sql in pubd_sql: - try: - cur.execute(sql) - except: - if "DROP TABLE IF EXISTS" not in sql.upper(): - raise - db.close() - d = dict(pubd_name = pubd_name, - pubd_port = pubd_port, - pubd_db_name = pubd_db_name, - pubd_db_user = pubd_db_user, - pubd_db_pass = pubd_db_pass, - pubd_dir = rsyncd_dir, - irdb_db_name = irdb_db_name, - irdb_db_pass = irdb_db_pass) - f = open(pubd_name + ".conf", "w") - f.write(pubd_fmt_1 % d) - f.close() - global pubd_ta - global pubd_irbe_key - global pubd_irbe_cert - global pubd_pubd_cert - pubd_ta = rpki.x509.X509(Auto_file = pubd_name + "-TA.cer") - pubd_irbe_key = rpki.x509.RSA( Auto_file = pubd_name + "-IRBE.key") - pubd_irbe_cert = rpki.x509.X509(Auto_file = pubd_name + "-IRBE.cer") - pubd_pubd_cert = rpki.x509.X509(Auto_file = pubd_name + "-PUBD.cer") - -def call_pubd(pdus, cb): - """ - Send a publication control message to publication daemon and return - the response. - """ - - logger.info("Calling pubd") - q_msg = rpki.publication_control.msg.query(*pdus) - q_cms = rpki.publication_control.cms_msg_saxify() - q_der = q_cms.wrap(q_msg, pubd_irbe_key, pubd_irbe_cert) - q_url = "http://localhost:%d/control" % pubd_port - - logger.debug(q_cms.pretty_print_content()) - - def call_pubd_cb(r_der): - global pubd_last_cms_time - r_cms = rpki.publication_control.cms_msg_saxify(DER = r_der) - r_msg = r_cms.unwrap((pubd_ta, pubd_pubd_cert)) - pubd_last_cms_time = r_cms.check_replay(pubd_last_cms_time, q_url) - logger.debug(r_cms.pretty_print_content()) - assert r_msg.is_reply - for r_pdu in r_msg: - r_pdu.raise_if_error() - cb(r_msg) - - def call_pubd_eb(e): - logger.exception("Problem calling pubd") - - rpki.http.client( - url = q_url, - msg = q_der, - callback = call_pubd_cb, - errback = call_pubd_eb) - - -def cross_certify(certificant, certifier): - """ - Cross-certify and return the resulting certificate. - """ - - certfile = certifier + "-" + certificant + ".cer" - - logger.info("Cross certifying %s into %s's BPKI (%s)", certificant, certifier, certfile) - - child = rpki.x509.X509(Auto_file = certificant + ".cer") - parent = rpki.x509.X509(Auto_file = certifier + ".cer") - keypair = rpki.x509.RSA(Auto_file = certifier + ".key") - serial_file = certifier + ".srl" - - now = rpki.sundial.now() - notAfter = now + rpki.sundial.timedelta(days = 30) - - try: - with open(serial_file, "r") as f: - serial = int(f.read().splitlines()[0], 16) - except IOError: - serial = 1 - - x = parent.bpki_cross_certify( - keypair = keypair, - source_cert = child, - serial = serial, - notAfter = notAfter, - now = now) - - with open(serial_file, "w") as f: - f.write("%02x\n" % (serial + 1)) - - with open(certfile, "w") as f: - f.write(x.get_PEM()) - - logger.debug("Cross certified %s:", certfile) - logger.debug(" Issuer %s [%s]", x.getIssuer(), x.hAKI()) - logger.debug(" Subject %s [%s]", x.getSubject(), x.hSKI()) - return x - -last_rcynic_run = None - -def run_rcynic(): - """ - Run rcynic to see whether what was published makes sense. - """ - - logger.info("Running rcynic") - env = os.environ.copy() - env["TZ"] = "" - global last_rcynic_run - if int(time.time()) == last_rcynic_run: - time.sleep(1) - subprocess.check_call((prog_rcynic, "-c", rcynic_name + ".conf"), env = env) - subprocess.call(rcynic_stats, shell = True, env = env) - last_rcynic_run = int(time.time()) - os.link("%s.xml" % rcynic_name, "%s.%s.xml" % (rcynic_name, last_rcynic_run)) - -def mangle_sql(filename): - """ - Mangle an SQL file into a sequence of SQL statements. - """ - - words = [] - f = open(filename) - for line in f: - words.extend(line.partition("--")[0].split()) - f.close() - return " ".join(words).strip(";").split(";") + return " ".join(words).strip(";").split(";") bpki_cert_fmt_1 = '''\ [req] diff --git a/ca/tests/sql-cleaner.py b/ca/tests/sql-cleaner.py index 828100a4..c518b77b 100644 --- a/ca/tests/sql-cleaner.py +++ b/ca/tests/sql-cleaner.py @@ -25,27 +25,27 @@ cfg = rpki.config.parser(section = "yamltest", allow_missing = True) for name in ("rpkid", "irdbd", "pubd"): - username = cfg.get("%s_sql_username" % name, name[:4]) - password = cfg.get("%s_sql_password" % name, "fnord") + username = cfg.get("%s_sql_username" % name, name[:4]) + password = cfg.get("%s_sql_password" % name, "fnord") - db = MySQLdb.connect(user = username, passwd = password) - cur = db.cursor() + db = MySQLdb.connect(user = username, passwd = password) + cur = db.cursor() - cur.execute("SHOW DATABASES") + cur.execute("SHOW DATABASES") - databases = [r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()] + databases = [r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()] - for database in databases: + for database in databases: - cur.execute("USE " + database) + cur.execute("USE " + database) - cur.execute("SHOW TABLES") - tables = [r[0] for r in cur.fetchall()] + cur.execute("SHOW TABLES") + tables = [r[0] for r in cur.fetchall()] - cur.execute("SET foreign_key_checks = 0") - for table in tables: - cur.execute("DROP TABLE %s" % table) - cur.execute("SET foreign_key_checks = 1") + cur.execute("SET foreign_key_checks = 0") + for table in tables: + cur.execute("DROP TABLE %s" % table) + cur.execute("SET foreign_key_checks = 1") - cur.close() - db.close() + cur.close() + db.close() diff --git a/ca/tests/sql-dumper.py b/ca/tests/sql-dumper.py index d0fe3489..af24f2d4 100644 --- a/ca/tests/sql-dumper.py +++ b/ca/tests/sql-dumper.py @@ -26,18 +26,18 @@ cfg = rpki.config.parser(section = "yamltest", allow_missing = True) for name in ("rpkid", "irdbd", "pubd"): - username = cfg.get("%s_sql_username" % name, name[:4]) - password = cfg.get("%s_sql_password" % name, "fnord") + username = cfg.get("%s_sql_username" % name, name[:4]) + password = cfg.get("%s_sql_password" % name, "fnord") - cmd = ["mysqldump", "-u", username, "-p" + password, "--databases"] + cmd = ["mysqldump", "-u", username, "-p" + password, "--databases"] - db = MySQLdb.connect(user = username, passwd = password) - cur = db.cursor() + db = MySQLdb.connect(user = username, passwd = password) + cur = db.cursor() - cur.execute("SHOW DATABASES") - cmd.extend(r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()) + cur.execute("SHOW DATABASES") + cmd.extend(r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()) - cur.close() - db.close() + cur.close() + db.close() - subprocess.check_call(cmd, stdout = open("backup.%s.sql" % name, "w")) + subprocess.check_call(cmd, stdout = open("backup.%s.sql" % name, "w")) diff --git a/ca/tests/test-rrdp.py b/ca/tests/test-rrdp.py index 1a9db929..97797444 100755 --- a/ca/tests/test-rrdp.py +++ b/ca/tests/test-rrdp.py @@ -38,22 +38,22 @@ parser.add_argument("--dry-run", action = "store_true") args = parser.parse_args() def log(msg): - sys.stdout.write(msg + "\n") - sys.stdout.flush() + sys.stdout.write(msg + "\n") + sys.stdout.flush() def run(*argv): - log("Running: " + " ".join(argv)) - if not args.dry_run: - subprocess.check_call(argv) + log("Running: " + " ".join(argv)) + if not args.dry_run: + subprocess.check_call(argv) def dataglob(pattern): - return glob.iglob(os.path.join(("smoketest.dir" if args.use_smoketest else "yamltest.dir/RIR"), pattern)) + return glob.iglob(os.path.join(("smoketest.dir" if args.use_smoketest else "yamltest.dir/RIR"), pattern)) def snapshot_to_serial(fn): - return int(os.path.splitext(os.path.basename(fn))[0]) + return int(os.path.splitext(os.path.basename(fn))[0]) def delta_to_serial(fn): - return int(os.path.splitext(os.path.basename(fn))[0]) + return int(os.path.splitext(os.path.basename(fn))[0]) top = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "..", "..")) @@ -62,62 +62,62 @@ rcynic = os.path.join(top, "rp/rcynic/rcynic") rcynic_text = os.path.join(top, "rp/rcynic/rcynic-text") with open("rcynic-rrdp.conf", "w") as f: - f.write(textwrap.dedent('''# Automatically generated for RRDP tests, do not edit. - [rcynic] - xml-summary = rcynic.xml - jitter = 0 - use-links = yes - use-syslog = no - use-stderr = yes - log-level = log_debug - run-rsync = no - ''')) - if args.use_smoketest: - f.write("trust-anchor = smoketest.dir/root.cer\n") - else: - f.write("trust-anchor = yamltest.dir/RIR/publication/RIR-root/root.cer\n") + f.write(textwrap.dedent('''# Automatically generated for RRDP tests, do not edit. + [rcynic] + xml-summary = rcynic.xml + jitter = 0 + use-links = yes + use-syslog = no + use-stderr = yes + log-level = log_debug + run-rsync = no + ''')) + if args.use_smoketest: + f.write("trust-anchor = smoketest.dir/root.cer\n") + else: + f.write("trust-anchor = yamltest.dir/RIR/publication/RIR-root/root.cer\n") if args.skip_daemons: - log("--skip-daemons specified, so running neither smoketest nor yamltest") + log("--skip-daemons specified, so running neither smoketest nor yamltest") elif args.use_smoketest: - run("python", "smoketest.py", args.yaml_file) + run("python", "smoketest.py", args.yaml_file) else: - run("python", "sql-cleaner.py") - class GotSIGUSR1(Exception): - pass - def handle_sigusr1(signum, frame): - raise GotSIGUSR1 - old_sigusr1 = signal.signal(signal.SIGUSR1, handle_sigusr1) - cmd = ("python", "yamltest.py", args.yaml_file, "--notify-when-startup-complete", str(os.getpid())) - log("Running: " + " ".join(cmd)) - yamltest = subprocess.Popen(cmd) - log("Waiting for SIGUSR1 from yamltest") - try: - while True: - signal.pause() - except GotSIGUSR1: - signal.signal(signal.SIGUSR1, old_sigusr1) - log("Sleeping %s" % args.delay) - time.sleep(args.delay) - yamltest.terminate() + run("python", "sql-cleaner.py") + class GotSIGUSR1(Exception): + pass + def handle_sigusr1(signum, frame): + raise GotSIGUSR1 + old_sigusr1 = signal.signal(signal.SIGUSR1, handle_sigusr1) + cmd = ("python", "yamltest.py", args.yaml_file, "--notify-when-startup-complete", str(os.getpid())) + log("Running: " + " ".join(cmd)) + yamltest = subprocess.Popen(cmd) + log("Waiting for SIGUSR1 from yamltest") + try: + while True: + signal.pause() + except GotSIGUSR1: + signal.signal(signal.SIGUSR1, old_sigusr1) + log("Sleeping %s" % args.delay) + time.sleep(args.delay) + yamltest.terminate() snapshots = dict((snapshot_to_serial(fn), fn) for fn in dataglob("rrdp-publication/*/snapshot/*.xml")) deltas = dict((delta_to_serial(fn), fn) for fn in dataglob("rrdp-publication/*/deltas/*.xml")) for snapshot in sorted(snapshots): - time.sleep(1) - run("rm", "-rf", "rcynic-data") - run(rrdp_test_tool, snapshots[snapshot]) - run(rcynic, "-c", "rcynic-rrdp.conf") - run(rcynic_text, "rcynic.xml") - - for delta in sorted(deltas): - if delta > snapshot: - time.sleep(1) - run(rrdp_test_tool, deltas[delta]) - run(rcynic, "-c", "rcynic-rrdp.conf") - run(rcynic_text, "rcynic.xml") - - if not args.exhaustive: - break + time.sleep(1) + run("rm", "-rf", "rcynic-data") + run(rrdp_test_tool, snapshots[snapshot]) + run(rcynic, "-c", "rcynic-rrdp.conf") + run(rcynic_text, "rcynic.xml") + + for delta in sorted(deltas): + if delta > snapshot: + time.sleep(1) + run(rrdp_test_tool, deltas[delta]) + run(rcynic, "-c", "rcynic-rrdp.conf") + run(rcynic_text, "rcynic.xml") + + if not args.exhaustive: + break diff --git a/ca/tests/testpoke.py b/ca/tests/testpoke.py index 68f967b9..60cc5690 100644 --- a/ca/tests/testpoke.py +++ b/ca/tests/testpoke.py @@ -58,80 +58,80 @@ yaml_data = yaml.load(args.yaml) yaml_cmd = args.request if yaml_cmd is None and len(yaml_data["requests"]) == 1: - yaml_cmd = yaml_data["requests"].keys()[0] + yaml_cmd = yaml_data["requests"].keys()[0] yaml_req = yaml_data["requests"][yaml_cmd] def get_PEM(name, cls, y = yaml_data): - if name in y: - return cls(PEM = y[name]) - if name + "-file" in y: - return cls(PEM_file = y[name + "-file"]) - return None + if name in y: + return cls(PEM = y[name]) + if name + "-file" in y: + return cls(PEM_file = y[name + "-file"]) + return None def get_PEM_chain(name, cert = None): - chain = [] - if cert is not None: - chain.append(cert) - if name in yaml_data: - chain.extend(rpki.x509.X509(PEM = x) for x in yaml_data[name]) - elif name + "-file" in yaml_data: - chain.extend(rpki.x509.X509(PEM_file = x) for x in yaml_data[name + "-file"]) - return chain + chain = [] + if cert is not None: + chain.append(cert) + if name in yaml_data: + chain.extend(rpki.x509.X509(PEM = x) for x in yaml_data[name]) + elif name + "-file" in yaml_data: + chain.extend(rpki.x509.X509(PEM_file = x) for x in yaml_data[name + "-file"]) + return chain def query_up_down(q_pdu): - q_msg = rpki.up_down.message_pdu.make_query( - payload = q_pdu, - sender = yaml_data["sender-id"], - recipient = yaml_data["recipient-id"]) - q_der = rpki.up_down.cms_msg_saxify().wrap(q_msg, cms_key, cms_certs, cms_crl) - - def done(r_der): - global last_cms_timestamp - r_cms = rpki.up_down.cms_msg_saxify(DER = r_der) - r_msg = r_cms.unwrap([cms_ta] + cms_ca_certs) - last_cms_timestamp = r_cms.check_replay(last_cms_timestamp) - print r_cms.pretty_print_content() - try: - r_msg.payload.check_response() - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: - fail(e) - - rpki.http.want_persistent_client = False - - rpki.http.client( - msg = q_der, - url = yaml_data["posturl"], - callback = done, - errback = fail, - content_type = rpki.up_down.content_type) + q_msg = rpki.up_down.message_pdu.make_query( + payload = q_pdu, + sender = yaml_data["sender-id"], + recipient = yaml_data["recipient-id"]) + q_der = rpki.up_down.cms_msg_saxify().wrap(q_msg, cms_key, cms_certs, cms_crl) + + def done(r_der): + global last_cms_timestamp + r_cms = rpki.up_down.cms_msg_saxify(DER = r_der) + r_msg = r_cms.unwrap([cms_ta] + cms_ca_certs) + last_cms_timestamp = r_cms.check_replay(last_cms_timestamp) + print r_cms.pretty_print_content() + try: + r_msg.payload.check_response() + except (rpki.async.ExitNow, SystemExit): + raise + except Exception, e: + fail(e) + + rpki.http.want_persistent_client = False + + rpki.http.client( + msg = q_der, + url = yaml_data["posturl"], + callback = done, + errback = fail, + content_type = rpki.up_down.content_type) def do_list(): - query_up_down(rpki.up_down.list_pdu()) + query_up_down(rpki.up_down.list_pdu()) def do_issue(): - q_pdu = rpki.up_down.issue_pdu() - req_key = get_PEM("cert-request-key", rpki.x509.RSA, yaml_req) or cms_key - q_pdu.class_name = yaml_req["class"] - q_pdu.pkcs10 = rpki.x509.PKCS10.create( - keypair = req_key, - is_ca = True, - caRepository = yaml_req["sia"][0], - rpkiManifest = yaml_req["sia"][0] + req_key.gSKI() + ".mft") - query_up_down(q_pdu) + q_pdu = rpki.up_down.issue_pdu() + req_key = get_PEM("cert-request-key", rpki.x509.RSA, yaml_req) or cms_key + q_pdu.class_name = yaml_req["class"] + q_pdu.pkcs10 = rpki.x509.PKCS10.create( + keypair = req_key, + is_ca = True, + caRepository = yaml_req["sia"][0], + rpkiManifest = yaml_req["sia"][0] + req_key.gSKI() + ".mft") + query_up_down(q_pdu) def do_revoke(): - q_pdu = rpki.up_down.revoke_pdu() - q_pdu.class_name = yaml_req["class"] - q_pdu.ski = yaml_req["ski"] - query_up_down(q_pdu) + q_pdu = rpki.up_down.revoke_pdu() + q_pdu.class_name = yaml_req["class"] + q_pdu.ski = yaml_req["ski"] + query_up_down(q_pdu) dispatch = { "list" : do_list, "issue" : do_issue, "revoke" : do_revoke } def fail(e): # pylint: disable=W0621 - sys.exit("Testpoke failed: %s" % e) + sys.exit("Testpoke failed: %s" % e) cms_ta = get_PEM("cms-ca-cert", rpki.x509.X509) cms_cert = get_PEM("cms-cert", rpki.x509.X509) @@ -143,7 +143,7 @@ cms_ca_certs = get_PEM_chain("cms-ca-certs") last_cms_timestamp = None try: - dispatch[yaml_req["type"]]() - rpki.async.event_loop() + dispatch[yaml_req["type"]]() + rpki.async.event_loop() except Exception, e: - fail(e) + fail(e) diff --git a/ca/tests/xml-parse-test.py b/ca/tests/xml-parse-test.py index 90c80775..f24d5683 100644 --- a/ca/tests/xml-parse-test.py +++ b/ca/tests/xml-parse-test.py @@ -39,70 +39,70 @@ import rpki.relaxng verbose = False def test(fileglob, rng, parser, encoding, tester = None): - files = glob.glob(fileglob) - files.sort() - for f in files: - print "<!--", f, "-->" - elt_in = lxml.etree.parse(f).getroot() - if verbose: - print "<!-- Input -->" - print lxml.etree.tostring(elt_in, pretty_print = True, encoding = encoding, xml_declaration = True) - rng.assertValid(elt_in) - parsed = parser.fromXML(elt_in) - elt_out = parsed.toXML() - if verbose: - print "<!-- Output -->" - print lxml.etree.tostring(elt_out, pretty_print = True, encoding = encoding, xml_declaration = True) - rng.assertValid(elt_out) - if tester: - tester(elt_in, elt_out, parsed) - if verbose: - print + files = glob.glob(fileglob) + files.sort() + for f in files: + print "<!--", f, "-->" + elt_in = lxml.etree.parse(f).getroot() + if verbose: + print "<!-- Input -->" + print lxml.etree.tostring(elt_in, pretty_print = True, encoding = encoding, xml_declaration = True) + rng.assertValid(elt_in) + parsed = parser.fromXML(elt_in) + elt_out = parsed.toXML() + if verbose: + print "<!-- Output -->" + print lxml.etree.tostring(elt_out, pretty_print = True, encoding = encoding, xml_declaration = True) + rng.assertValid(elt_out) + if tester: + tester(elt_in, elt_out, parsed) + if verbose: + print def pprint(pairs): - if verbose: - for thing, name in pairs: - if thing is not None: - print "[%s]" % name - print thing.get_POW().pprint() + if verbose: + for thing, name in pairs: + if thing is not None: + print "[%s]" % name + print thing.get_POW().pprint() def ud_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.up_down.message_pdu) - if isinstance(msg.payload, rpki.up_down.list_response_pdu): - for c in msg.payload.classes: - pprint([(c.certs[i].cert, ("%s certificate #%d" % (c.class_name, i))) for i in xrange(len(c.certs))] + [(c.issuer, ("%s issuer" % c.class_name))]) + assert isinstance(msg, rpki.up_down.message_pdu) + if isinstance(msg.payload, rpki.up_down.list_response_pdu): + for c in msg.payload.classes: + pprint([(c.certs[i].cert, ("%s certificate #%d" % (c.class_name, i))) for i in xrange(len(c.certs))] + [(c.issuer, ("%s issuer" % c.class_name))]) def lr_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.left_right.msg) - for obj in msg: - if isinstance(obj, rpki.left_right.self_elt): - pprint(((obj.bpki_cert, "BPKI cert"), - (obj.bpki_glue, "BPKI glue"))) - if isinstance(obj, rpki.left_right.bsc_elt): - pprint(((obj.signing_cert, "Signing certificate"), - (obj.signing_cert_crl, "Signing certificate CRL"))) - # (obj.pkcs10_request, "PKCS #10 request") - if isinstance(obj, rpki.left_right.parent_elt): - pprint(((obj.bpki_cert, "BPKI certificate"), - (obj.bpki_glue, "BPKI glue"))) - if isinstance(obj, (rpki.left_right.child_elt, rpki.left_right.repository_elt)): - pprint(((obj.bpki_cert, "BPKI certificate"), - (obj.bpki_glue, "BPKI glue"))) + assert isinstance(msg, rpki.left_right.msg) + for obj in msg: + if isinstance(obj, rpki.left_right.self_elt): + pprint(((obj.bpki_cert, "BPKI cert"), + (obj.bpki_glue, "BPKI glue"))) + if isinstance(obj, rpki.left_right.bsc_elt): + pprint(((obj.signing_cert, "Signing certificate"), + (obj.signing_cert_crl, "Signing certificate CRL"))) + # (obj.pkcs10_request, "PKCS #10 request") + if isinstance(obj, rpki.left_right.parent_elt): + pprint(((obj.bpki_cert, "BPKI certificate"), + (obj.bpki_glue, "BPKI glue"))) + if isinstance(obj, (rpki.left_right.child_elt, rpki.left_right.repository_elt)): + pprint(((obj.bpki_cert, "BPKI certificate"), + (obj.bpki_glue, "BPKI glue"))) def pp_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.publication.msg) - for obj in msg: - if isinstance(obj, rpki.publication.publish_elt): - pprint(((obj.payload, "Publish object"),)) - if isinstance(obj, rpki.publication.withdraw_elt): - pprint(((None, "Withdraw object"),)) + assert isinstance(msg, rpki.publication.msg) + for obj in msg: + if isinstance(obj, rpki.publication.publish_elt): + pprint(((obj.payload, "Publish object"),)) + if isinstance(obj, rpki.publication.withdraw_elt): + pprint(((None, "Withdraw object"),)) def pc_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.publication_control.msg) - for obj in msg: - if isinstance(obj, rpki.publication_control.client_elt): - pprint(((obj.bpki_cert, "BPKI cert"), - (obj.bpki_glue, "BPKI glue"))) + assert isinstance(msg, rpki.publication_control.msg) + for obj in msg: + if isinstance(obj, rpki.publication_control.client_elt): + pprint(((obj.bpki_cert, "BPKI cert"), + (obj.bpki_glue, "BPKI glue"))) test(fileglob = "up-down-protocol-samples/*.xml", rng = rpki.relaxng.up_down, diff --git a/ca/tests/yamlconf.py b/ca/tests/yamlconf.py index 13456377..52c4da26 100644 --- a/ca/tests/yamlconf.py +++ b/ca/tests/yamlconf.py @@ -75,787 +75,787 @@ config_overrides = { "pubd_sql_username" : "pubd", "pubd_sql_password" : "fnord" } def cleanpath(*names): - return os.path.normpath(os.path.join(*names)) + return os.path.normpath(os.path.join(*names)) this_dir = os.getcwd() test_dir = None rpki_conf = None class roa_request(object): - """ - Representation of a ROA request. - """ - - def __init__(self, asn, ipv4, ipv6): - self.asn = asn - self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None - self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - - def __eq__(self, other): - return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - - def __hash__(self): - v4 = tuple(self.v4) if self.v4 is not None else None - v6 = tuple(self.v6) if self.v6 is not None else None - return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - - def __str__(self): - if self.v4 and self.v6: - return "%s: %s,%s" % (self.asn, self.v4, self.v6) - else: - return "%s: %s" % (self.asn, self.v4 or self.v6) + """ + Representation of a ROA request. + """ - @classmethod - def parse(cls, y): - return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None -class router_cert(object): - """ - Representation for a router_cert object. - """ + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - _ecparams = None + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - @classmethod - def ecparams(cls): - if cls._ecparams is None: - cls._ecparams = rpki.x509.KeyParams.generateEC() - return cls._ecparams + def __str__(self): + if self.v4 and self.v6: + return "%s: %s,%s" % (self.asn, self.v4, self.v6) + else: + return "%s: %s" % (self.asn, self.v4 or self.v6) - def __init__(self, asn, router_id): - self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) - self.router_id = router_id - self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) - self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) - self.gski = self.pkcs10.gSKI() + @classmethod + def parse(cls, y): + return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) - def __eq__(self, other): - return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski +class router_cert(object): + """ + Representation for a router_cert object. + """ - def __hash__(self): - return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() + _ecparams = None - def __str__(self): - return "%s: %s: %s" % (self.asn, self.router_id, self.gski) + @classmethod + def ecparams(cls): + if cls._ecparams is None: + cls._ecparams = rpki.x509.KeyParams.generateEC() + return cls._ecparams - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("router_id")) + def __init__(self, asn, router_id): + self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) + self.router_id = router_id + self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) + self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) + self.gski = self.pkcs10.gSKI() + def __eq__(self, other): + return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski -class allocation_db(list): - """ - Allocation database. - """ - - def __init__(self, y): - list.__init__(self) - self.root = allocation(y, self) - assert self.root.is_root - if self.root.crl_interval is None: - self.root.crl_interval = 60 * 60 - if self.root.regen_margin is None: - self.root.regen_margin = 24 * 60 * 60 - if self.root.base.valid_until is None: - self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) - for a in self: - if a.base.valid_until is None: - a.base.valid_until = a.parent.base.valid_until - if a.crl_interval is None: - a.crl_interval = a.parent.crl_interval - if a.regen_margin is None: - a.regen_margin = a.parent.regen_margin - self.root.closure() - self.map = dict((a.name, a) for a in self) - for a in self: - if a.is_hosted: - a.hosted_by = self.map[a.hosted_by] - a.hosted_by.hosts.append(a) - assert not a.is_root and not a.hosted_by.is_hosted - - def dump(self): - for a in self: - a.dump() + def __hash__(self): + return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() + def __str__(self): + return "%s: %s: %s" % (self.asn, self.router_id, self.gski) -class allocation(object): - """ - One entity in our allocation database. Every entity in the database - is assumed to hold resources. Entities that don't have the - hosted_by property run their own copies of rpkid, irdbd, and pubd. - """ - - base_port = 4400 - base_engine = -1 - parent = None - crl_interval = None - regen_margin = None - engine = -1 - rpkid_port = 4404 - irdbd_port = 4403 - pubd_port = 4402 - rootd_port = 4401 - rsync_port = 873 - - @classmethod - def allocate_port(cls): - cls.base_port += 1 - return cls.base_port - - @classmethod - def allocate_engine(cls): - cls.base_engine += 1 - return cls.base_engine - - def __init__(self, y, db, parent = None): - db.append(self) - self.name = y["name"] - self.parent = parent - self.kids = [allocation(k, db, self) for k in y.get("kids", ())] - valid_until = None - if "valid_until" in y: - valid_until = rpki.sundial.datetime.from_datetime(y.get("valid_until")) - if valid_until is None and "valid_for" in y: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(y["valid_for"]) - self.base = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as(y.get("asn")), - v4 = rpki.resource_set.resource_set_ipv4(y.get("ipv4")), - v6 = rpki.resource_set.resource_set_ipv6(y.get("ipv6")), - valid_until = valid_until) - if "crl_interval" in y: - self.crl_interval = rpki.sundial.timedelta.parse(y["crl_interval"]).convert_to_seconds() - if "regen_margin" in y: - self.regen_margin = rpki.sundial.timedelta.parse(y["regen_margin"]).convert_to_seconds() - if "ghostbusters" in y: - self.ghostbusters = y.get("ghostbusters") - elif "ghostbuster" in y: - self.ghostbusters = [y.get("ghostbuster")] - else: - self.ghostbusters = [] - self.roa_requests = [roa_request.parse(r) for r in y.get("roa_request", ())] - self.router_certs = [router_cert.parse(r) for r in y.get("router_cert", ())] - for r in self.roa_requests: - if r.v4: - self.base.v4 |= r.v4.to_resource_set() - if r.v6: - self.base.v6 |= r.v6.to_resource_set() - for r in self.router_certs: - self.base.asn |= r.asn - self.hosted_by = y.get("hosted_by") - self.hosts = [] - if not self.is_hosted: - self.engine = self.allocate_engine() - if loopback and not self.is_hosted: - self.rpkid_port = self.allocate_port() - self.irdbd_port = self.allocate_port() - if loopback and self.runs_pubd: - self.pubd_port = self.allocate_port() - self.rsync_port = self.allocate_port() - if loopback and self.is_root: - self.rootd_port = self.allocate_port() - - def closure(self): - resources = self.base - for kid in self.kids: - resources |= kid.closure() - self.resources = resources - return resources - - @property - def hostname(self): - if loopback: - return "localhost" - elif dns_suffix: - return self.name + "." + dns_suffix.lstrip(".") - else: - return self.name + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("router_id")) - @property - def rsync_server(self): - if loopback: - return "%s:%s" % (self.pubd.hostname, self.pubd.rsync_port) - else: - return self.pubd.hostname - def dump(self): - if not quiet: - print str(self) - - def __str__(self): - s = self.name + ":\n" - if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn - if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 - if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 - if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) - if self.parent: s += " Up: %s\n" % self.parent.name - if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name - if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) - for r in self.roa_requests: s += " ROA: %s\n" % r - if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port - if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port - if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port - if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port - if self.is_root: s += " TPort: %s\n" % self.rootd_port - return s + " Until: %s\n" % self.resources.valid_until - - @property - def is_root(self): - return self.parent is None - - @property - def is_hosted(self): - return self.hosted_by is not None - - @property - def runs_pubd(self): - return self.is_root or not (self.is_hosted or only_one_pubd) - - def path(self, *names): - return cleanpath(test_dir, self.host.name, *names) - - def csvout(self, fn): - path = self.path(fn) - if not quiet: - print "Writing", path - return rpki.csv_utils.csv_writer(path) - - def up_down_url(self): - return "http://%s:%d/up-down/%s/%s" % (self.parent.host.hostname, - self.parent.host.rpkid_port, - self.parent.name, - self.name) - - def dump_asns(self, fn): - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, a) for a in k.resources.asn) - - def dump_prefixes(self, fn): - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) - - def dump_roas(self, fn): - with self.csvout(fn) as f: - for g1, r in enumerate(self.roa_requests): - f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) - for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) - - def dump_ghostbusters(self, fn): - if self.ghostbusters: - path = self.path(fn) - if not quiet: - print "Writing", path - with open(path, "w") as f: - for i, g in enumerate(self.ghostbusters): - if i > 0: - f.write("\n") - f.write(g) - - def dump_router_certificates(self, fn): - if self.router_certs: - path = self.path(fn) - if not quiet: - print "Writing", path - xmlns = rpki.relaxng.router_certificate.xmlns - xml = lxml.etree.Element(xmlns + "router_certificate_requests", - version = rpki.relaxng.router_certificate.version, - nsmap = rpki.relaxng.router_certificate.nsmap) - for r in self.router_certs: - x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", - router_id = str(r.router_id), - asn = str(r.asn), - valid_until = str(self.resources.valid_until)) - x.text = r.pkcs10.get_Base64() - rpki.relaxng.router_certificate.assertValid(xml) - lxml.etree.ElementTree(xml).write(path, pretty_print = True) - - @property - def pubd(self): - s = self - while not s.runs_pubd: - s = s.parent - return s - - @property - def client_handle(self): - path = [] - s = self - if not flat_publication: - while not s.runs_pubd: - path.append(s) - s = s.parent - path.append(s) - return ".".join(i.name for i in reversed(path)) - - @property - def host(self): - return self.hosted_by or self - - @property - def publication_base_directory(self): - if not loopback and publication_base is not None: - return publication_base - else: - return self.path("publication") +class allocation_db(list): + """ + Allocation database. + """ + + def __init__(self, y): + list.__init__(self) + self.root = allocation(y, self) + assert self.root.is_root + if self.root.crl_interval is None: + self.root.crl_interval = 60 * 60 + if self.root.regen_margin is None: + self.root.regen_margin = 24 * 60 * 60 + if self.root.base.valid_until is None: + self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) + for a in self: + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + self.root.closure() + self.map = dict((a.name, a) for a in self) + for a in self: + if a.is_hosted: + a.hosted_by = self.map[a.hosted_by] + a.hosted_by.hosts.append(a) + assert not a.is_root and not a.hosted_by.is_hosted + + def dump(self): + for a in self: + a.dump() - @property - def publication_root_directory(self): - if not loopback and publication_root is not None: - return publication_root - else: - return self.path("publication.root") - - def dump_conf(self): - - r = dict( - handle = self.name, - run_rpkid = str(not self.is_hosted), - run_pubd = str(self.runs_pubd), - run_rootd = str(self.is_root), - irdbd_sql_username = "irdb", - rpkid_sql_username = "rpki", - rpkid_server_host = self.hostname, - rpkid_server_port = str(self.rpkid_port), - irdbd_server_host = "localhost", - irdbd_server_port = str(self.irdbd_port), - rootd_server_port = str(self.rootd_port), - pubd_sql_username = "pubd", - pubd_server_host = self.pubd.hostname, - pubd_server_port = str(self.pubd.pubd_port), - publication_rsync_server = self.rsync_server) - - if loopback: - r.update( - irdbd_sql_database = self.irdb_name, - rpkid_sql_database = "rpki%d" % self.engine, - pubd_sql_database = "pubd%d" % self.engine, - bpki_servers_directory = self.path(), - publication_base_directory = self.publication_base_directory) - - r.update(config_overrides) - - with open(self.path("rpki.conf"), "w") as f: - f.write("# Automatically generated, do not edit\n") - if not quiet: - print "Writing", f.name - - section = None - for line in open(rpki_conf): - m = section_regexp.match(line) - if m: - section = m.group(1) - m = variable_regexp.match(line) - option = m.group(1) if m and section == "myrpki" else None - if option and option in r: - line = "%s = %s\n" % (option, r[option]) - f.write(line) - - def dump_rsyncd(self): - lines = [] - if self.runs_pubd: - lines.extend(( - "# Automatically generated, do not edit", - "port = %d" % self.rsync_port, - "address = %s" % self.hostname, - "log file = rsyncd.log", - "read only = yes", - "use chroot = no", - "[rpki]", - "path = %s" % self.publication_base_directory, - "comment = RPKI test")) - if self.is_root: - assert self.runs_pubd - lines.extend(( - "[root]", - "path = %s" % self.publication_root_directory, - "comment = RPKI test root")) - if lines: - with open(self.path("rsyncd.conf"), "w") as f: - if not quiet: - print "Writing", f.name - f.writelines(line + "\n" for line in lines) - - @property - def irdb_name(self): - return "irdb%d" % self.host.engine - - @property - def irdb(self): - prior_name = self.zoo.handle - return rpki.irdb.database( - self.irdb_name, - on_entry = lambda: self.zoo.reset_identity(self.name), - on_exit = lambda: self.zoo.reset_identity(prior_name)) - - def syncdb(self): - import django.core.management - assert not self.is_hosted - django.core.management.call_command( - "syncdb", - verbosity = 0, - database = self.irdb_name, - migrate = True, - load_initial_data = False, - interactive = False) - - def hire_zookeeper(self): - assert not self.is_hosted - self._zoo = rpki.irdb.Zookeeper( - cfg = rpki.config.parser(filename = self.path("rpki.conf")), - logstream = None if quiet else sys.stdout) - - @property - def zoo(self): - return self.host._zoo - - def dump_root(self): - - assert self.is_root and not self.is_hosted - - root_resources = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as("0-4294967295"), - v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), - v6 = rpki.resource_set.resource_set_ipv6("::/0")) - - root_key = rpki.x509.RSA.generate(quiet = True) - - root_uri = "rsync://%s/rpki/" % self.rsync_server - - root_sia = (root_uri, root_uri + "root.mft", None, rpki.publication.rrdp_sia_uri_kludge) - - root_cert = rpki.x509.X509.self_certify( - keypair = root_key, - subject_key = root_key.get_public(), - serial = 1, - sia = root_sia, - notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), - resources = root_resources) - - with open(self.path("root.cer"), "wb") as f: - f.write(root_cert.get_DER()) - - with open(self.path("root.key"), "wb") as f: - f.write(root_key.get_DER()) - - with open(cleanpath(test_dir, "root.tal"), "w") as f: - f.write("rsync://%s/root/root.cer\n\n" % self.rsync_server) - f.write(root_key.get_public().get_Base64()) - - def mkdir(self, *path): - path = self.path(*path) - if not quiet: - print "Creating directory", path - os.makedirs(path) - def dump_sql(self): - if not self.is_hosted: - with open(self.path("rpkid.sql"), "w") as f: +class allocation(object): + """ + One entity in our allocation database. Every entity in the database + is assumed to hold resources. Entities that don't have the + hosted_by property run their own copies of rpkid, irdbd, and pubd. + """ + + base_port = 4400 + base_engine = -1 + parent = None + crl_interval = None + regen_margin = None + engine = -1 + rpkid_port = 4404 + irdbd_port = 4403 + pubd_port = 4402 + rootd_port = 4401 + rsync_port = 873 + + @classmethod + def allocate_port(cls): + cls.base_port += 1 + return cls.base_port + + @classmethod + def allocate_engine(cls): + cls.base_engine += 1 + return cls.base_engine + + def __init__(self, y, db, parent = None): + db.append(self) + self.name = y["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in y.get("kids", ())] + valid_until = None + if "valid_until" in y: + valid_until = rpki.sundial.datetime.from_datetime(y.get("valid_until")) + if valid_until is None and "valid_for" in y: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(y["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(y.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(y.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(y.get("ipv6")), + valid_until = valid_until) + if "crl_interval" in y: + self.crl_interval = rpki.sundial.timedelta.parse(y["crl_interval"]).convert_to_seconds() + if "regen_margin" in y: + self.regen_margin = rpki.sundial.timedelta.parse(y["regen_margin"]).convert_to_seconds() + if "ghostbusters" in y: + self.ghostbusters = y.get("ghostbusters") + elif "ghostbuster" in y: + self.ghostbusters = [y.get("ghostbuster")] + else: + self.ghostbusters = [] + self.roa_requests = [roa_request.parse(r) for r in y.get("roa_request", ())] + self.router_certs = [router_cert.parse(r) for r in y.get("router_cert", ())] + for r in self.roa_requests: + if r.v4: + self.base.v4 |= r.v4.to_resource_set() + if r.v6: + self.base.v6 |= r.v6.to_resource_set() + for r in self.router_certs: + self.base.asn |= r.asn + self.hosted_by = y.get("hosted_by") + self.hosts = [] + if not self.is_hosted: + self.engine = self.allocate_engine() + if loopback and not self.is_hosted: + self.rpkid_port = self.allocate_port() + self.irdbd_port = self.allocate_port() + if loopback and self.runs_pubd: + self.pubd_port = self.allocate_port() + self.rsync_port = self.allocate_port() + if loopback and self.is_root: + self.rootd_port = self.allocate_port() + + def closure(self): + resources = self.base + for kid in self.kids: + resources |= kid.closure() + self.resources = resources + return resources + + @property + def hostname(self): + if loopback: + return "localhost" + elif dns_suffix: + return self.name + "." + dns_suffix.lstrip(".") + else: + return self.name + + @property + def rsync_server(self): + if loopback: + return "%s:%s" % (self.pubd.hostname, self.pubd.rsync_port) + else: + return self.pubd.hostname + + def dump(self): if not quiet: - print "Writing", f.name - f.write(rpki.sql_schemas.rpkid) - if self.runs_pubd: - with open(self.path("pubd.sql"), "w") as f: + print str(self) + + def __str__(self): + s = self.name + ":\n" + if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name + if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) + for r in self.roa_requests: s += " ROA: %s\n" % r + if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port + if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port + if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port + if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port + if self.is_root: s += " TPort: %s\n" % self.rootd_port + return s + " Until: %s\n" % self.resources.valid_until + + @property + def is_root(self): + return self.parent is None + + @property + def is_hosted(self): + return self.hosted_by is not None + + @property + def runs_pubd(self): + return self.is_root or not (self.is_hosted or only_one_pubd) + + def path(self, *names): + return cleanpath(test_dir, self.host.name, *names) + + def csvout(self, fn): + path = self.path(fn) if not quiet: - print "Writing", f.name - f.write(rpki.sql_schemas.pubd) - if not self.is_hosted: - username = config_overrides["irdbd_sql_username"] - password = config_overrides["irdbd_sql_password"] - cmd = ("mysqldump", "-u", username, "-p" + password, self.irdb_name) - with open(self.path("irdbd.sql"), "w") as f: + print "Writing", path + return rpki.csv_utils.csv_writer(path) + + def up_down_url(self): + return "http://%s:%d/up-down/%s/%s" % (self.parent.host.hostname, + self.parent.host.rpkid_port, + self.parent.name, + self.name) + + def dump_asns(self, fn): + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, a) for a in k.resources.asn) + + def dump_prefixes(self, fn): + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) + + def dump_roas(self, fn): + with self.csvout(fn) as f: + for g1, r in enumerate(self.roa_requests): + f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) + for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) + + def dump_ghostbusters(self, fn): + if self.ghostbusters: + path = self.path(fn) + if not quiet: + print "Writing", path + with open(path, "w") as f: + for i, g in enumerate(self.ghostbusters): + if i > 0: + f.write("\n") + f.write(g) + + def dump_router_certificates(self, fn): + if self.router_certs: + path = self.path(fn) + if not quiet: + print "Writing", path + xmlns = rpki.relaxng.router_certificate.xmlns + xml = lxml.etree.Element(xmlns + "router_certificate_requests", + version = rpki.relaxng.router_certificate.version, + nsmap = rpki.relaxng.router_certificate.nsmap) + for r in self.router_certs: + x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", + router_id = str(r.router_id), + asn = str(r.asn), + valid_until = str(self.resources.valid_until)) + x.text = r.pkcs10.get_Base64() + rpki.relaxng.router_certificate.assertValid(xml) + lxml.etree.ElementTree(xml).write(path, pretty_print = True) + + @property + def pubd(self): + s = self + while not s.runs_pubd: + s = s.parent + return s + + @property + def client_handle(self): + path = [] + s = self + if not flat_publication: + while not s.runs_pubd: + path.append(s) + s = s.parent + path.append(s) + return ".".join(i.name for i in reversed(path)) + + @property + def host(self): + return self.hosted_by or self + + @property + def publication_base_directory(self): + if not loopback and publication_base is not None: + return publication_base + else: + return self.path("publication") + + @property + def publication_root_directory(self): + if not loopback and publication_root is not None: + return publication_root + else: + return self.path("publication.root") + + def dump_conf(self): + + r = dict( + handle = self.name, + run_rpkid = str(not self.is_hosted), + run_pubd = str(self.runs_pubd), + run_rootd = str(self.is_root), + irdbd_sql_username = "irdb", + rpkid_sql_username = "rpki", + rpkid_server_host = self.hostname, + rpkid_server_port = str(self.rpkid_port), + irdbd_server_host = "localhost", + irdbd_server_port = str(self.irdbd_port), + rootd_server_port = str(self.rootd_port), + pubd_sql_username = "pubd", + pubd_server_host = self.pubd.hostname, + pubd_server_port = str(self.pubd.pubd_port), + publication_rsync_server = self.rsync_server) + + if loopback: + r.update( + irdbd_sql_database = self.irdb_name, + rpkid_sql_database = "rpki%d" % self.engine, + pubd_sql_database = "pubd%d" % self.engine, + bpki_servers_directory = self.path(), + publication_base_directory = self.publication_base_directory) + + r.update(config_overrides) + + with open(self.path("rpki.conf"), "w") as f: + f.write("# Automatically generated, do not edit\n") + if not quiet: + print "Writing", f.name + + section = None + for line in open(rpki_conf): + m = section_regexp.match(line) + if m: + section = m.group(1) + m = variable_regexp.match(line) + option = m.group(1) if m and section == "myrpki" else None + if option and option in r: + line = "%s = %s\n" % (option, r[option]) + f.write(line) + + def dump_rsyncd(self): + lines = [] + if self.runs_pubd: + lines.extend(( + "# Automatically generated, do not edit", + "port = %d" % self.rsync_port, + "address = %s" % self.hostname, + "log file = rsyncd.log", + "read only = yes", + "use chroot = no", + "[rpki]", + "path = %s" % self.publication_base_directory, + "comment = RPKI test")) + if self.is_root: + assert self.runs_pubd + lines.extend(( + "[root]", + "path = %s" % self.publication_root_directory, + "comment = RPKI test root")) + if lines: + with open(self.path("rsyncd.conf"), "w") as f: + if not quiet: + print "Writing", f.name + f.writelines(line + "\n" for line in lines) + + @property + def irdb_name(self): + return "irdb%d" % self.host.engine + + @property + def irdb(self): + prior_name = self.zoo.handle + return rpki.irdb.database( + self.irdb_name, + on_entry = lambda: self.zoo.reset_identity(self.name), + on_exit = lambda: self.zoo.reset_identity(prior_name)) + + def syncdb(self): + import django.core.management + assert not self.is_hosted + django.core.management.call_command( + "syncdb", + verbosity = 0, + database = self.irdb_name, + migrate = True, + load_initial_data = False, + interactive = False) + + def hire_zookeeper(self): + assert not self.is_hosted + self._zoo = rpki.irdb.Zookeeper( + cfg = rpki.config.parser(filename = self.path("rpki.conf")), + logstream = None if quiet else sys.stdout) + + @property + def zoo(self): + return self.host._zoo + + def dump_root(self): + + assert self.is_root and not self.is_hosted + + root_resources = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as("0-4294967295"), + v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), + v6 = rpki.resource_set.resource_set_ipv6("::/0")) + + root_key = rpki.x509.RSA.generate(quiet = True) + + root_uri = "rsync://%s/rpki/" % self.rsync_server + + root_sia = (root_uri, root_uri + "root.mft", None, rpki.publication.rrdp_sia_uri_kludge) + + root_cert = rpki.x509.X509.self_certify( + keypair = root_key, + subject_key = root_key.get_public(), + serial = 1, + sia = root_sia, + notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), + resources = root_resources) + + with open(self.path("root.cer"), "wb") as f: + f.write(root_cert.get_DER()) + + with open(self.path("root.key"), "wb") as f: + f.write(root_key.get_DER()) + + with open(cleanpath(test_dir, "root.tal"), "w") as f: + f.write("rsync://%s/root/root.cer\n\n" % self.rsync_server) + f.write(root_key.get_public().get_Base64()) + + def mkdir(self, *path): + path = self.path(*path) if not quiet: - print "Writing", f.name - subprocess.check_call(cmd, stdout = f) + print "Creating directory", path + os.makedirs(path) + + def dump_sql(self): + if not self.is_hosted: + with open(self.path("rpkid.sql"), "w") as f: + if not quiet: + print "Writing", f.name + f.write(rpki.sql_schemas.rpkid) + if self.runs_pubd: + with open(self.path("pubd.sql"), "w") as f: + if not quiet: + print "Writing", f.name + f.write(rpki.sql_schemas.pubd) + if not self.is_hosted: + username = config_overrides["irdbd_sql_username"] + password = config_overrides["irdbd_sql_password"] + cmd = ("mysqldump", "-u", username, "-p" + password, self.irdb_name) + with open(self.path("irdbd.sql"), "w") as f: + if not quiet: + print "Writing", f.name + subprocess.check_call(cmd, stdout = f) def pre_django_sql_setup(needed): - username = config_overrides["irdbd_sql_username"] - password = config_overrides["irdbd_sql_password"] - - # If we have the MySQL root password, just blow away and recreate - # the required databases. Otherwise, check for missing databases, - # then blow away all tables in the required databases. In either - # case, we assume that the Django syncdb code will populate - # databases as necessary, all we need to do here is provide empty - # databases for the Django code to fill in. + username = config_overrides["irdbd_sql_username"] + password = config_overrides["irdbd_sql_password"] + + # If we have the MySQL root password, just blow away and recreate + # the required databases. Otherwise, check for missing databases, + # then blow away all tables in the required databases. In either + # case, we assume that the Django syncdb code will populate + # databases as necessary, all we need to do here is provide empty + # databases for the Django code to fill in. + + if mysql_rootpass is not None: + if mysql_rootpass: + db = MySQLdb.connect(user = mysql_rootuser, passwd = mysql_rootpass) + else: + db = MySQLdb.connect(user = mysql_rootuser) + cur = db.cursor() + for database in needed: + try: + cur.execute("DROP DATABASE IF EXISTS %s" % database) + except: + pass + cur.execute("CREATE DATABASE %s" % database) + cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % ( + database, username), (password,)) - if mysql_rootpass is not None: - if mysql_rootpass: - db = MySQLdb.connect(user = mysql_rootuser, passwd = mysql_rootpass) else: - db = MySQLdb.connect(user = mysql_rootuser) - cur = db.cursor() - for database in needed: - try: - cur.execute("DROP DATABASE IF EXISTS %s" % database) - except: - pass - cur.execute("CREATE DATABASE %s" % database) - cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % ( - database, username), (password,)) - - else: - db = MySQLdb.connect(user = username, passwd = password) - cur = db.cursor() - cur.execute("SHOW DATABASES") - existing = set(r[0] for r in cur.fetchall()) - if needed - existing: - sys.stderr.write("The following databases are missing:\n") - for database in sorted(needed - existing): - sys.stderr.write(" %s\n" % database) - sys.stderr.write("Please create them manually or put MySQL root password in my config file\n") - sys.exit("Missing databases and MySQL root password not known, can't continue") - for database in needed: - db.select_db(database) - cur.execute("SHOW TABLES") - tables = [r[0] for r in cur.fetchall()] - cur.execute("SET foreign_key_checks = 0") - for table in tables: - cur.execute("DROP TABLE %s" % table) - cur.execute("SET foreign_key_checks = 1") - - cur.close() - db.commit() - db.close() + db = MySQLdb.connect(user = username, passwd = password) + cur = db.cursor() + cur.execute("SHOW DATABASES") + existing = set(r[0] for r in cur.fetchall()) + if needed - existing: + sys.stderr.write("The following databases are missing:\n") + for database in sorted(needed - existing): + sys.stderr.write(" %s\n" % database) + sys.stderr.write("Please create them manually or put MySQL root password in my config file\n") + sys.exit("Missing databases and MySQL root password not known, can't continue") + for database in needed: + db.select_db(database) + cur.execute("SHOW TABLES") + tables = [r[0] for r in cur.fetchall()] + cur.execute("SET foreign_key_checks = 0") + for table in tables: + cur.execute("DROP TABLE %s" % table) + cur.execute("SET foreign_key_checks = 1") + + cur.close() + db.commit() + db.close() class timestamp(object): - def __init__(self, *args): - self.count = 0 - self.start = self.tick = rpki.sundial.now() + def __init__(self, *args): + self.count = 0 + self.start = self.tick = rpki.sundial.now() - def __call__(self, *args): - now = rpki.sundial.now() - if not quiet: - print "[Count %s last %s total %s now %s]" % ( - self.count, now - self.tick, now - self.start, now) - self.tick = now - self.count += 1 + def __call__(self, *args): + now = rpki.sundial.now() + if not quiet: + print "[Count %s last %s total %s now %s]" % ( + self.count, now - self.tick, now - self.start, now) + self.tick = now + self.count += 1 def main(): - global flat_publication - global config_overrides - global only_one_pubd - global loopback - global dns_suffix - global mysql_rootuser - global mysql_rootpass - global yaml_file - global test_dir - global rpki_conf - global publication_base - global publication_root - global quiet - - os.environ["TZ"] = "UTC" - time.tzset() - - parser = argparse.ArgumentParser(description = "yamlconf") - parser.add_argument("-c", "--config", help = "configuration file") - parser.add_argument("--dns_suffix", - help = "DNS suffix to add to hostnames") - parser.add_argument("-l", "--loopback", action = "store_true", - help = "Configure for use with yamltest on localhost") - parser.add_argument("-f", "--flat_publication", action = "store_true", - help = "Use flat publication model") - parser.add_argument("-q", "--quiet", action = "store_true", - help = "Work more quietly") - parser.add_argument("--profile", - help = "Filename for profile output") - parser.add_argument("yaml_file", type = argparse.FileType("r"), - help = "YAML file describing network to build") - args = parser.parse_args() - - dns_suffix = args.dns_suffix - loopback = args.loopback - flat_publication = args.flat_publication - quiet = args.quiet - yaml_file = args.yaml_file - - rpki.log.init("yamlconf", argparse.Namespace(log_level = logging.DEBUG, - log_handler = lambda: logging.StreamHandler(sys.stdout))) - - # Allow optional config file for this tool to override default - # passwords: this is mostly so that I can show a complete working - # example without publishing my own server's passwords. - - cfg = rpki.config.parser(set_filename = args.config, section = "yamlconf", allow_missing = True) - try: - cfg.set_global_flags() - except: - pass - - # Use of "yamltest.dir" is deliberate: intent is for what we write to - # be usable with "yamltest --skip_config". - - only_one_pubd = cfg.getboolean("only_one_pubd", True) - test_dir = cfg.get("test_directory", cleanpath(this_dir, "yamltest.dir")) - rpki_conf = cfg.get("rpki_conf", cleanpath(this_dir, "..", "examples/rpki.conf")) - mysql_rootuser = cfg.get("mysql_rootuser", "root") - - try: - mysql_rootpass = cfg.get("mysql_rootpass") - except: - pass - - try: - publication_base = cfg.get("publication_base") - except: - pass - - try: - publication_root = cfg.get("publication_root") - except: - pass - - for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", - "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username"): - if cfg.has_option(k): - config_overrides[k] = cfg.get(k) - - if args.profile: - import cProfile - prof = cProfile.Profile() + global flat_publication + global config_overrides + global only_one_pubd + global loopback + global dns_suffix + global mysql_rootuser + global mysql_rootpass + global yaml_file + global test_dir + global rpki_conf + global publication_base + global publication_root + global quiet + + os.environ["TZ"] = "UTC" + time.tzset() + + parser = argparse.ArgumentParser(description = "yamlconf") + parser.add_argument("-c", "--config", help = "configuration file") + parser.add_argument("--dns_suffix", + help = "DNS suffix to add to hostnames") + parser.add_argument("-l", "--loopback", action = "store_true", + help = "Configure for use with yamltest on localhost") + parser.add_argument("-f", "--flat_publication", action = "store_true", + help = "Use flat publication model") + parser.add_argument("-q", "--quiet", action = "store_true", + help = "Work more quietly") + parser.add_argument("--profile", + help = "Filename for profile output") + parser.add_argument("yaml_file", type = argparse.FileType("r"), + help = "YAML file describing network to build") + args = parser.parse_args() + + dns_suffix = args.dns_suffix + loopback = args.loopback + flat_publication = args.flat_publication + quiet = args.quiet + yaml_file = args.yaml_file + + rpki.log.init("yamlconf", argparse.Namespace(log_level = logging.DEBUG, + log_handler = lambda: logging.StreamHandler(sys.stdout))) + + # Allow optional config file for this tool to override default + # passwords: this is mostly so that I can show a complete working + # example without publishing my own server's passwords. + + cfg = rpki.config.parser(set_filename = args.config, section = "yamlconf", allow_missing = True) try: - prof.runcall(body) - finally: - prof.dump_stats(args.profile) - if not quiet: - print - print "Dumped profile data to %s" % args.profile - else: - body() + cfg.set_global_flags() + except: + pass -def body(): + # Use of "yamltest.dir" is deliberate: intent is for what we write to + # be usable with "yamltest --skip_config". + + only_one_pubd = cfg.getboolean("only_one_pubd", True) + test_dir = cfg.get("test_directory", cleanpath(this_dir, "yamltest.dir")) + rpki_conf = cfg.get("rpki_conf", cleanpath(this_dir, "..", "examples/rpki.conf")) + mysql_rootuser = cfg.get("mysql_rootuser", "root") - global rpki + try: + mysql_rootpass = cfg.get("mysql_rootpass") + except: + pass - ts = timestamp() + try: + publication_base = cfg.get("publication_base") + except: + pass - for root, dirs, files in os.walk(test_dir, topdown = False): - for fn in files: - os.unlink(os.path.join(root, fn)) - for d in dirs: - os.rmdir(os.path.join(root, d)) + try: + publication_root = cfg.get("publication_root") + except: + pass - if not quiet: - print - print "Reading YAML", yaml_file.name + for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", + "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username"): + if cfg.has_option(k): + config_overrides[k] = cfg.get(k) + + if args.profile: + import cProfile + prof = cProfile.Profile() + try: + prof.runcall(body) + finally: + prof.dump_stats(args.profile) + if not quiet: + print + print "Dumped profile data to %s" % args.profile + else: + body() - db = allocation_db(yaml.safe_load_all(yaml_file).next()) +def body(): - # Show what we loaded + global rpki - #db.dump() + ts = timestamp() - # Do pre-Django SQL setup + for root, dirs, files in os.walk(test_dir, topdown = False): + for fn in files: + os.unlink(os.path.join(root, fn)) + for d in dirs: + os.rmdir(os.path.join(root, d)) - pre_django_sql_setup(set(d.irdb_name for d in db if not d.is_hosted)) + if not quiet: + print + print "Reading YAML", yaml_file.name - # Now ready for fun with multiple databases in Django! - # - # https://docs.djangoproject.com/en/1.4/topics/db/multi-db/ - # https://docs.djangoproject.com/en/1.4/topics/db/sql/ - # - # This program's use of the ORM is sufficiently different that it's - # not worth straining to use rpki.django_settings, so we just use - # Django's settings API directly. + db = allocation_db(yaml.safe_load_all(yaml_file).next()) - database_template = { - "ENGINE" : "django.db.backends.mysql", - "USER" : config_overrides["irdbd_sql_username"], - "PASSWORD" : config_overrides["irdbd_sql_password"], - "HOST" : "", - "PORT" : "", - "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }} + # Show what we loaded - databases = dict((d.irdb_name, dict(database_template, NAME = d.irdb_name)) - for d in db if not d.is_hosted) + #db.dump() - databases["default"] = databases[db.root.irdb_name] + # Do pre-Django SQL setup - import django - django.setup() + pre_django_sql_setup(set(d.irdb_name for d in db if not d.is_hosted)) - from django.conf import settings + # Now ready for fun with multiple databases in Django! + # + # https://docs.djangoproject.com/en/1.4/topics/db/multi-db/ + # https://docs.djangoproject.com/en/1.4/topics/db/sql/ + # + # This program's use of the ORM is sufficiently different that it's + # not worth straining to use rpki.django_settings, so we just use + # Django's settings API directly. - settings.configure( - DATABASES = databases, - DATABASE_ROUTERS = ["rpki.irdb.router.DBContextRouter"], - INSTALLED_APPS = ["rpki.irdb"]) + database_template = { + "ENGINE" : "django.db.backends.mysql", + "USER" : config_overrides["irdbd_sql_username"], + "PASSWORD" : config_overrides["irdbd_sql_password"], + "HOST" : "", + "PORT" : "", + "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }} - import rpki.irdb + databases = dict((d.irdb_name, dict(database_template, NAME = d.irdb_name)) + for d in db if not d.is_hosted) - rpki.irdb.models.ca_certificate_lifetime = rpki.sundial.timedelta(days = 3652 * 2) - rpki.irdb.models.ee_certificate_lifetime = rpki.sundial.timedelta(days = 3652) + databases["default"] = databases[db.root.irdb_name] - ts() + import django + django.setup() - for d in db: - if not quiet: - print - print "Configuring", d.name - - if not d.is_hosted: - d.mkdir() - if d.runs_pubd: - d.mkdir("publication") - if d.is_root: - d.mkdir("publication.root") - - if not d.is_hosted: - d.dump_conf() - d.dump_rsyncd() - - d.dump_asns("%s.asns.csv" % d.name) - d.dump_prefixes("%s.prefixes.csv" % d.name) - d.dump_roas("%s.roas.csv" % d.name) - d.dump_ghostbusters("%s.ghostbusters.vcard" % d.name) - d.dump_router_certificates("%s.routercerts.xml" % d.name) - - if not d.is_hosted: - if not quiet: - print "Initializing SQL" - d.syncdb() - if not quiet: - print "Hiring zookeeper" - d.hire_zookeeper() - - with d.irdb: - if not quiet: - print "Creating identity" - x = d.zoo.initialize() - - if d.is_root: - if not quiet: - print "Creating RPKI root certificate and TAL" - d.dump_root() - x = d.zoo.configure_rootd() + from django.conf import settings - else: - with d.parent.irdb: - x = d.parent.zoo.configure_child(x.file)[0] - x = d.zoo.configure_parent(x.file)[0] + settings.configure( + DATABASES = databases, + DATABASE_ROUTERS = ["rpki.irdb.router.DBContextRouter"], + INSTALLED_APPS = ["rpki.irdb"]) - with d.pubd.irdb: - x = d.pubd.zoo.configure_publication_client(x.file, flat = flat_publication)[0] - d.zoo.configure_repository(x.file) + import rpki.irdb - if loopback and not d.is_hosted: - with d.irdb: - d.zoo.write_bpki_files() + rpki.irdb.models.ca_certificate_lifetime = rpki.sundial.timedelta(days = 3652 * 2) + rpki.irdb.models.ee_certificate_lifetime = rpki.sundial.timedelta(days = 3652) ts() - if not loopback: - if not quiet: - print for d in db: - d.dump_sql() + if not quiet: + print + print "Configuring", d.name + + if not d.is_hosted: + d.mkdir() + if d.runs_pubd: + d.mkdir("publication") + if d.is_root: + d.mkdir("publication.root") + + if not d.is_hosted: + d.dump_conf() + d.dump_rsyncd() + + d.dump_asns("%s.asns.csv" % d.name) + d.dump_prefixes("%s.prefixes.csv" % d.name) + d.dump_roas("%s.roas.csv" % d.name) + d.dump_ghostbusters("%s.ghostbusters.vcard" % d.name) + d.dump_router_certificates("%s.routercerts.xml" % d.name) + + if not d.is_hosted: + if not quiet: + print "Initializing SQL" + d.syncdb() + if not quiet: + print "Hiring zookeeper" + d.hire_zookeeper() + + with d.irdb: + if not quiet: + print "Creating identity" + x = d.zoo.initialize() + + if d.is_root: + if not quiet: + print "Creating RPKI root certificate and TAL" + d.dump_root() + x = d.zoo.configure_rootd() + + else: + with d.parent.irdb: + x = d.parent.zoo.configure_child(x.file)[0] + x = d.zoo.configure_parent(x.file)[0] + + with d.pubd.irdb: + x = d.pubd.zoo.configure_publication_client(x.file, flat = flat_publication)[0] + d.zoo.configure_repository(x.file) + + if loopback and not d.is_hosted: + with d.irdb: + d.zoo.write_bpki_files() + + ts() + + if not loopback: + if not quiet: + print + for d in db: + d.dump_sql() if __name__ == "__main__": - main() + main() diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index a26fefb9..f82cc192 100644 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -65,11 +65,11 @@ section_regexp = re.compile(r"\s*\[\s*(.+?)\s*\]\s*$") variable_regexp = re.compile(r"\s*([-a-zA-Z0-9_]+)\s*=\s*(.+?)\s*$") def cleanpath(*names): - """ - Construct normalized pathnames. - """ + """ + Construct normalized pathnames. + """ - return os.path.normpath(os.path.join(*names)) + return os.path.normpath(os.path.join(*names)) # Pathnames for various things we need @@ -85,652 +85,652 @@ prog_rootd = cleanpath(ca_dir, "rootd") prog_rpki_manage = cleanpath(ca_dir, "rpki-manage") class roa_request(object): - """ - Representation of a ROA request. - """ - - def __init__(self, asn, ipv4, ipv6): - self.asn = asn - self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None - self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - - def __eq__(self, other): - return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - - def __hash__(self): - v4 = tuple(self.v4) if self.v4 is not None else None - v6 = tuple(self.v6) if self.v6 is not None else None - return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - - def __str__(self): - if self.v4 and self.v6: - return "%s: %s,%s" % (self.asn, self.v4, self.v6) - else: - return "%s: %s" % (self.asn, self.v4 or self.v6) - - @classmethod - def parse(cls, y): - """ - Parse a ROA request from YAML format. - """ - - return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) - - -class router_cert(object): - """ - Representation for a router_cert object. - """ - - _ecparams = None - - @classmethod - def ecparams(cls): - if cls._ecparams is None: - cls._ecparams = rpki.x509.KeyParams.generateEC() - return cls._ecparams - - def __init__(self, asn, router_id): - self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) - self.router_id = router_id - self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) - self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) - self.gski = self.pkcs10.gSKI() - - def __eq__(self, other): - return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski - - def __hash__(self): - return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() - - def __str__(self): - return "%s: %s: %s" % (self.asn, self.router_id, self.gski) - - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("router_id")) - -class allocation_db(list): - """ - Our allocation database. - """ - - def __init__(self, yaml): - list.__init__(self) - self.root = allocation(yaml, self) - assert self.root.is_root and not any(a.is_root for a in self if a is not self.root) and self[0] is self.root - if self.root.crl_interval is None: - self.root.crl_interval = 60 * 60 - if self.root.regen_margin is None: - self.root.regen_margin = 24 * 60 * 60 - if self.root.base.valid_until is None: - self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) - for a in self: - if a.base.valid_until is None: - a.base.valid_until = a.parent.base.valid_until - if a.crl_interval is None: - a.crl_interval = a.parent.crl_interval - if a.regen_margin is None: - a.regen_margin = a.parent.regen_margin - self.root.closure() - self.map = dict((a.name, a) for a in self) - for a in self: - if a.is_hosted: - a.hosted_by = self.map[a.hosted_by] - a.hosted_by.hosts.append(a) - assert not a.is_root and not a.hosted_by.is_hosted - - def dump(self): """ - Show contents of allocation database. + Representation of a ROA request. """ - for a in self: - a.dump() + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 -class allocation(object): - """ - One entity in our allocation database. Every entity in the database - is assumed to hold resources, so needs at least rpkic services. - Entities that don't have the hosted_by property run their own copies - of rpkid, irdbd, and pubd, so they also need myirbe services. - """ - - base_port = None - parent = None - crl_interval = None - regen_margin = None - rootd_port = None - engine = -1 - rpkid_port = -1 - irdbd_port = -1 - pubd_port = -1 - rsync_port = -1 - rootd_port = -1 - rrdp_port = -1 - rpkic_counter = 0L - - @classmethod - def allocate_port(cls): - """ - Allocate a TCP port. - """ - - cls.base_port += 1 - return cls.base_port - - base_engine = -1 - - @classmethod - def allocate_engine(cls): - """ - Allocate an engine number, mostly used to construct SQL database - names. - """ - - cls.base_engine += 1 - return cls.base_engine - - def __init__(self, yaml, db, parent = None): - db.append(self) - self.name = yaml["name"] - self.parent = parent - self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] - valid_until = None - if "valid_until" in yaml: - valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) - if valid_until is None and "valid_for" in yaml: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) - self.base = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as(yaml.get("asn")), - v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), - v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), - valid_until = valid_until) - if "crl_interval" in yaml: - self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() - if "regen_margin" in yaml: - self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() - self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] - self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] - if "ghostbusters" in yaml: - self.ghostbusters = yaml.get("ghostbusters") - elif "ghostbuster" in yaml: - self.ghostbusters = [yaml.get("ghostbuster")] - else: - self.ghostbusters = [] - for r in self.roa_requests: - if r.v4: - self.base.v4 |= r.v4.to_resource_set() - if r.v6: - self.base.v6 |= r.v6.to_resource_set() - for r in self.router_certs: - self.base.asn |= r.asn - self.hosted_by = yaml.get("hosted_by") - self.hosts = [] - if not self.is_hosted: - self.engine = self.allocate_engine() - self.rpkid_port = self.allocate_port() - self.irdbd_port = self.allocate_port() - if self.runs_pubd: - self.pubd_port = self.allocate_port() - self.rsync_port = self.allocate_port() - self.rrdp_port = self.allocate_port() - if self.is_root: - self.rootd_port = self.allocate_port() - - def closure(self): - """ - Compute resource closure of this node and its children, to avoid a - lot of tedious (and error-prone) duplication in the YAML file. - """ - - resources = self.base - for kid in self.kids: - resources |= kid.closure() - self.resources = resources - return resources - - def dump(self): - """ - Show content of this allocation node. - """ - - print str(self) - - def __str__(self): - s = self.name + ":\n" - if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn - if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 - if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 - if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) - if self.parent: s += " Up: %s\n" % self.parent.name - if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name - if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) - for r in self.roa_requests: s += " ROA: %s\n" % r - if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port - if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port - if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port - if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port - if self.is_root: s += " TPort: %s\n" % self.rootd_port - return s + " Until: %s\n" % self.resources.valid_until - - @property - def is_root(self): - """ - Is this the root node? - """ + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - return self.parent is None - - @property - def is_hosted(self): - """ - Is this entity hosted? - """ - - return self.hosted_by is not None - - @property - def runs_pubd(self): - """ - Does this entity run a pubd? - """ + def __str__(self): + if self.v4 and self.v6: + return "%s: %s,%s" % (self.asn, self.v4, self.v6) + else: + return "%s: %s" % (self.asn, self.v4 or self.v6) - return self.is_root or not (self.is_hosted or only_one_pubd) + @classmethod + def parse(cls, y): + """ + Parse a ROA request from YAML format. + """ - def path(self, *names): - """ - Construct pathnames in this entity's test directory. - """ + return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) - return cleanpath(test_dir, self.host.name, *names) - def csvout(self, fn): +class router_cert(object): """ - Open and log a CSV output file. + Representation for a router_cert object. """ - path = self.path(fn) - print "Writing", path - return rpki.csv_utils.csv_writer(path) + _ecparams = None - def up_down_url(self): - """ - Construct service URL for this node's parent. - """ + @classmethod + def ecparams(cls): + if cls._ecparams is None: + cls._ecparams = rpki.x509.KeyParams.generateEC() + return cls._ecparams - return "http://localhost:%d/up-down/%s/%s" % (self.parent.host.rpkid_port, - self.parent.name, - self.name) + def __init__(self, asn, router_id): + self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) + self.router_id = router_id + self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) + self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) + self.gski = self.pkcs10.gSKI() - def dump_asns(self): - """ - Write Autonomous System Numbers CSV file. - """ + def __eq__(self, other): + return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski - fn = "%s.asns.csv" % d.name - if not args.skip_config: - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, a) for a in k.resources.asn) - if not args.stop_after_config: - self.run_rpkic("load_asns", fn) + def __hash__(self): + return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() - def dump_prefixes(self): - """ - Write prefixes CSV file. - """ + def __str__(self): + return "%s: %s: %s" % (self.asn, self.router_id, self.gski) - fn = "%s.prefixes.csv" % d.name - if not args.skip_config: - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) - if not args.stop_after_config: - self.run_rpkic("load_prefixes", fn) + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("router_id")) - def dump_roas(self): - """ - Write ROA CSV file. - """ - - fn = "%s.roas.csv" % d.name - if not args.skip_config: - with self.csvout(fn) as f: - for g1, r in enumerate(self.roa_requests): - f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) - for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) - if not args.stop_after_config: - self.run_rpkic("load_roa_requests", fn) - - def dump_ghostbusters(self): - """ - Write Ghostbusters vCard file. +class allocation_db(list): """ + Our allocation database. + """ + + def __init__(self, yaml): + list.__init__(self) + self.root = allocation(yaml, self) + assert self.root.is_root and not any(a.is_root for a in self if a is not self.root) and self[0] is self.root + if self.root.crl_interval is None: + self.root.crl_interval = 60 * 60 + if self.root.regen_margin is None: + self.root.regen_margin = 24 * 60 * 60 + if self.root.base.valid_until is None: + self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) + for a in self: + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + self.root.closure() + self.map = dict((a.name, a) for a in self) + for a in self: + if a.is_hosted: + a.hosted_by = self.map[a.hosted_by] + a.hosted_by.hosts.append(a) + assert not a.is_root and not a.hosted_by.is_hosted + + def dump(self): + """ + Show contents of allocation database. + """ + + for a in self: + a.dump() - if self.ghostbusters: - fn = "%s.ghostbusters.vcard" % d.name - if not args.skip_config: - path = self.path(fn) - print "Writing", path - with open(path, "w") as f: - f.write("\n".join(self.ghostbusters)) - if not args.stop_after_config: - self.run_rpkic("load_ghostbuster_requests", fn) - def dump_router_certificates(self): - """ - Write EE certificates (router certificates, etc). +class allocation(object): """ + One entity in our allocation database. Every entity in the database + is assumed to hold resources, so needs at least rpkic services. + Entities that don't have the hosted_by property run their own copies + of rpkid, irdbd, and pubd, so they also need myirbe services. + """ + + base_port = None + parent = None + crl_interval = None + regen_margin = None + rootd_port = None + engine = -1 + rpkid_port = -1 + irdbd_port = -1 + pubd_port = -1 + rsync_port = -1 + rootd_port = -1 + rrdp_port = -1 + rpkic_counter = 0L + + @classmethod + def allocate_port(cls): + """ + Allocate a TCP port. + """ + + cls.base_port += 1 + return cls.base_port + + base_engine = -1 + + @classmethod + def allocate_engine(cls): + """ + Allocate an engine number, mostly used to construct SQL database + names. + """ + + cls.base_engine += 1 + return cls.base_engine + + def __init__(self, yaml, db, parent = None): + db.append(self) + self.name = yaml["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] + valid_until = None + if "valid_until" in yaml: + valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) + if valid_until is None and "valid_for" in yaml: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(yaml.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), + valid_until = valid_until) + if "crl_interval" in yaml: + self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() + if "regen_margin" in yaml: + self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() + self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] + self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] + if "ghostbusters" in yaml: + self.ghostbusters = yaml.get("ghostbusters") + elif "ghostbuster" in yaml: + self.ghostbusters = [yaml.get("ghostbuster")] + else: + self.ghostbusters = [] + for r in self.roa_requests: + if r.v4: + self.base.v4 |= r.v4.to_resource_set() + if r.v6: + self.base.v6 |= r.v6.to_resource_set() + for r in self.router_certs: + self.base.asn |= r.asn + self.hosted_by = yaml.get("hosted_by") + self.hosts = [] + if not self.is_hosted: + self.engine = self.allocate_engine() + self.rpkid_port = self.allocate_port() + self.irdbd_port = self.allocate_port() + if self.runs_pubd: + self.pubd_port = self.allocate_port() + self.rsync_port = self.allocate_port() + self.rrdp_port = self.allocate_port() + if self.is_root: + self.rootd_port = self.allocate_port() + + def closure(self): + """ + Compute resource closure of this node and its children, to avoid a + lot of tedious (and error-prone) duplication in the YAML file. + """ + + resources = self.base + for kid in self.kids: + resources |= kid.closure() + self.resources = resources + return resources + + def dump(self): + """ + Show content of this allocation node. + """ + + print str(self) + + def __str__(self): + s = self.name + ":\n" + if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name + if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) + for r in self.roa_requests: s += " ROA: %s\n" % r + if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port + if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port + if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port + if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port + if self.is_root: s += " TPort: %s\n" % self.rootd_port + return s + " Until: %s\n" % self.resources.valid_until + + @property + def is_root(self): + """ + Is this the root node? + """ + + return self.parent is None + + @property + def is_hosted(self): + """ + Is this entity hosted? + """ + + return self.hosted_by is not None + + @property + def runs_pubd(self): + """ + Does this entity run a pubd? + """ + + return self.is_root or not (self.is_hosted or only_one_pubd) + + def path(self, *names): + """ + Construct pathnames in this entity's test directory. + """ + + return cleanpath(test_dir, self.host.name, *names) + + def csvout(self, fn): + """ + Open and log a CSV output file. + """ - if self.router_certs: - fn = "%s.routercerts.xml" % d.name - if not args.skip_config: path = self.path(fn) print "Writing", path - xmlns = rpki.relaxng.router_certificate.xmlns - xml = lxml.etree.Element(xmlns + "router_certificate_requests", - version = rpki.relaxng.router_certificate.version, - nsmap = rpki.relaxng.router_certificate.nsmap) - for r in self.router_certs: - x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", - router_id = str(r.router_id), - asn = str(r.asn), - valid_until = str(self.resources.valid_until)) - x.text = r.pkcs10.get_Base64() - rpki.relaxng.router_certificate.assertValid(xml) - lxml.etree.ElementTree(xml).write(path, pretty_print = True) - if not args.stop_after_config: - self.run_rpkic("add_router_certificate_request", fn) - if not args.skip_config and args.store_router_private_keys: - path = self.path("%s.routercerts.keys" % d.name) - print "Writing", path - with open(path, "w") as f: - for r in self.router_certs: - f.write(r.keypair.get_PEM()) - - @property - def pubd(self): - """ - Walk up tree until we find somebody who runs pubd. - """ - - s = self - while not s.runs_pubd: - s = s.parent - return s - - @property - def client_handle(self): - """ - Work out what pubd configure_publication_client will call us. - """ - - path = [] - s = self - if not args.flat_publication: - while not s.runs_pubd: + return rpki.csv_utils.csv_writer(path) + + def up_down_url(self): + """ + Construct service URL for this node's parent. + """ + + return "http://localhost:%d/up-down/%s/%s" % (self.parent.host.rpkid_port, + self.parent.name, + self.name) + + def dump_asns(self): + """ + Write Autonomous System Numbers CSV file. + """ + + fn = "%s.asns.csv" % d.name + if not args.skip_config: + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, a) for a in k.resources.asn) + if not args.stop_after_config: + self.run_rpkic("load_asns", fn) + + def dump_prefixes(self): + """ + Write prefixes CSV file. + """ + + fn = "%s.prefixes.csv" % d.name + if not args.skip_config: + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) + if not args.stop_after_config: + self.run_rpkic("load_prefixes", fn) + + def dump_roas(self): + """ + Write ROA CSV file. + """ + + fn = "%s.roas.csv" % d.name + if not args.skip_config: + with self.csvout(fn) as f: + for g1, r in enumerate(self.roa_requests): + f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) + for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) + if not args.stop_after_config: + self.run_rpkic("load_roa_requests", fn) + + def dump_ghostbusters(self): + """ + Write Ghostbusters vCard file. + """ + + if self.ghostbusters: + fn = "%s.ghostbusters.vcard" % d.name + if not args.skip_config: + path = self.path(fn) + print "Writing", path + with open(path, "w") as f: + f.write("\n".join(self.ghostbusters)) + if not args.stop_after_config: + self.run_rpkic("load_ghostbuster_requests", fn) + + def dump_router_certificates(self): + """ + Write EE certificates (router certificates, etc). + """ + + if self.router_certs: + fn = "%s.routercerts.xml" % d.name + if not args.skip_config: + path = self.path(fn) + print "Writing", path + xmlns = rpki.relaxng.router_certificate.xmlns + xml = lxml.etree.Element(xmlns + "router_certificate_requests", + version = rpki.relaxng.router_certificate.version, + nsmap = rpki.relaxng.router_certificate.nsmap) + for r in self.router_certs: + x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", + router_id = str(r.router_id), + asn = str(r.asn), + valid_until = str(self.resources.valid_until)) + x.text = r.pkcs10.get_Base64() + rpki.relaxng.router_certificate.assertValid(xml) + lxml.etree.ElementTree(xml).write(path, pretty_print = True) + if not args.stop_after_config: + self.run_rpkic("add_router_certificate_request", fn) + if not args.skip_config and args.store_router_private_keys: + path = self.path("%s.routercerts.keys" % d.name) + print "Writing", path + with open(path, "w") as f: + for r in self.router_certs: + f.write(r.keypair.get_PEM()) + + @property + def pubd(self): + """ + Walk up tree until we find somebody who runs pubd. + """ + + s = self + while not s.runs_pubd: + s = s.parent + return s + + @property + def client_handle(self): + """ + Work out what pubd configure_publication_client will call us. + """ + + path = [] + s = self + if not args.flat_publication: + while not s.runs_pubd: + path.append(s) + s = s.parent path.append(s) - s = s.parent - path.append(s) - return ".".join(i.name for i in reversed(path)) - - @property - def host(self): - return self.hosted_by or self - - def dump_conf(self): - """ - Write configuration file for OpenSSL and RPKI tools. - """ - - r = dict( - handle = self.name, - run_rpkid = str(not self.is_hosted), - run_pubd = str(self.runs_pubd), - run_rootd = str(self.is_root), - rpkid_server_host = "localhost", - rpkid_server_port = str(self.rpkid_port), - irdbd_server_host = "localhost", - irdbd_server_port = str(self.irdbd_port), - rootd_server_port = str(self.rootd_port), - pubd_server_host = "localhost", - pubd_server_port = str(self.pubd.pubd_port), - publication_rsync_server = "localhost:%s" % self.pubd.rsync_port, - publication_rrdp_notification_uri = "http://localhost:%s/rrdp/notify.xml" % self.pubd.rrdp_port, - bpki_servers_directory = self.path(), - publication_base_directory = self.path("publication"), - rrdp_publication_base_directory = self.path("rrdp-publication"), - shared_sql_engine = args.sql_engine, - shared_sql_password = "fnord", - irdbd_sql_username = "irdb", - rpkid_sql_username = "rpki", - pubd_sql_username = "pubd") - - if args.sql_engine == "sqlite3": - r.update( - irdbd_sql_database = self.path("irdb.sqlite3"), - rpkid_sql_database = self.path("rpkidb.sqlite3"), - pubd_sql_database = self.path("pubdb.sqlite3")) - else: - r.update( - irdbd_sql_database = "irdb%d" % self.engine, - rpkid_sql_database = "rpki%d" % self.engine, - pubd_sql_database = "pubd%d" % self.engine) - - r.update(config_overrides) - - with open(self.path("rpki.conf"), "w") as f: - f.write("# Automatically generated, do not edit\n") - print "Writing", f.name - - section = None - for line in open(cleanpath(ca_dir, "examples/rpki.conf")): - m = section_regexp.match(line) - if m: - section = m.group(1) - m = variable_regexp.match(line) - option = m.group(1) if m and section == "myrpki" else None - if option and option in r: - line = "%s = %s\n" % (option, r[option]) - f.write(line) - - def dump_rsyncd(self): - """ - Write rsyncd configuration file. - """ - - if self.runs_pubd: - with open(self.path("rsyncd.conf"), "w") as f: - print "Writing", f.name - f.writelines(s + "\n" for s in - ("# Automatically generated, do not edit", - "port = %d" % self.rsync_port, - "address = localhost", - "[rpki]", - "log file = rsyncd.log", - "read only = yes", - "use chroot = no", - "path = %s" % self.path("publication"), - "comment = RPKI test", - "[root]", - "log file = rsyncd_root.log", - "read only = yes", - "use chroot = no", - "path = %s" % self.path("publication.root"), - "comment = RPKI test root")) - - @classmethod - def next_rpkic_counter(cls): - cls.rpkic_counter += 10000 - return str(cls.rpkic_counter) - - def run_rpkic(self, *argv): - """ - Run rpkic for this entity. - """ - - cmd = [prog_rpkic, "-i", self.name] - if args.profile: - cmd.append("--profile") - cmd.append(self.path("rpkic.%s.prof" % rpki.sundial.now())) - cmd.extend(str(a) for a in argv if a is not None) - print 'Running "%s"' % " ".join(cmd) - env = dict(os.environ, - YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(), - RPKI_CONF = self.path("rpki.conf")) - subprocess.check_call(cmd, cwd = self.host.path(), env = env) - - def syncdb(self): - """ - Run whatever Django ORM commands are necessary to set up the - database this week. - """ - - # Fork a sub-process for each syncdb/migrate run, because it's - # easier than figuring out how to change Django settings after - # initialization. - - def sync_settings(settings, verbosity = 1): - - if verbosity > 0: - print "Running Django setup for", self.name - - pid = os.fork() - - if pid == 0: - logging.getLogger().setLevel(logging.WARNING) - - os.environ.update(RPKI_CONF = self.path("rpki.conf"), - DJANGO_SETTINGS_MODULE = "rpki.django_settings." + settings) - - import django - django.setup() - - import django.core.management - django.core.management.call_command("migrate", verbosity = verbosity, no_color = True, - load_initial_data = False, interactive = False) - - if settings in ("gui", "irdb"): - from django.contrib.auth.models import User - User.objects.create_superuser("root", "root@example.org", "fnord") - - sys.exit(0) - - elif os.waitpid(pid, 0)[1]: - raise RuntimeError("Django setup failed for %s %s" % (self.name, settings)) - - for settings in ("rpkid", "pubd", "gui"): - sync_settings(settings) - - def run_python_daemon(self, prog): - """ - Start a Python daemon and return a subprocess.Popen object - representing the running daemon. - """ - - basename = os.path.splitext(os.path.basename(prog))[0] - cmd = [prog, "--foreground", "--log-level", "debug", - "--log-file", self.path(basename + ".log")] - if args.profile and basename != "rootd": - cmd.extend(( - "--profile", self.path(basename + ".prof"))) - env = dict(os.environ, RPKI_CONF = self.path("rpki.conf")) - p = subprocess.Popen(cmd, cwd = self.path(), env = env) - print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) - return p - - def run_rpkid(self): - """ - Run rpkid. - """ - - return self.run_python_daemon(prog_rpkid) - - def run_irdbd(self): - """ - Run irdbd. - """ - - return self.run_python_daemon(prog_irdbd) - - def run_pubd(self): - """ - Run pubd. - """ - - return self.run_python_daemon(prog_pubd) - - def run_rootd(self): - """ - Run rootd. - """ - - return self.run_python_daemon(prog_rootd) - - def run_rsyncd(self): - """ - Run rsyncd. - """ - - p = subprocess.Popen(("rsync", "--daemon", "--no-detach", "--config", "rsyncd.conf"), - cwd = self.path()) - print "Running rsyncd for %s: pid %d process %r" % (self.name, p.pid, p) - return p + return ".".join(i.name for i in reversed(path)) + + @property + def host(self): + return self.hosted_by or self + + def dump_conf(self): + """ + Write configuration file for OpenSSL and RPKI tools. + """ + + r = dict( + handle = self.name, + run_rpkid = str(not self.is_hosted), + run_pubd = str(self.runs_pubd), + run_rootd = str(self.is_root), + rpkid_server_host = "localhost", + rpkid_server_port = str(self.rpkid_port), + irdbd_server_host = "localhost", + irdbd_server_port = str(self.irdbd_port), + rootd_server_port = str(self.rootd_port), + pubd_server_host = "localhost", + pubd_server_port = str(self.pubd.pubd_port), + publication_rsync_server = "localhost:%s" % self.pubd.rsync_port, + publication_rrdp_notification_uri = "http://localhost:%s/rrdp/notify.xml" % self.pubd.rrdp_port, + bpki_servers_directory = self.path(), + publication_base_directory = self.path("publication"), + rrdp_publication_base_directory = self.path("rrdp-publication"), + shared_sql_engine = args.sql_engine, + shared_sql_password = "fnord", + irdbd_sql_username = "irdb", + rpkid_sql_username = "rpki", + pubd_sql_username = "pubd") + + if args.sql_engine == "sqlite3": + r.update( + irdbd_sql_database = self.path("irdb.sqlite3"), + rpkid_sql_database = self.path("rpkidb.sqlite3"), + pubd_sql_database = self.path("pubdb.sqlite3")) + else: + r.update( + irdbd_sql_database = "irdb%d" % self.engine, + rpkid_sql_database = "rpki%d" % self.engine, + pubd_sql_database = "pubd%d" % self.engine) + + r.update(config_overrides) + + with open(self.path("rpki.conf"), "w") as f: + f.write("# Automatically generated, do not edit\n") + print "Writing", f.name + + section = None + for line in open(cleanpath(ca_dir, "examples/rpki.conf")): + m = section_regexp.match(line) + if m: + section = m.group(1) + m = variable_regexp.match(line) + option = m.group(1) if m and section == "myrpki" else None + if option and option in r: + line = "%s = %s\n" % (option, r[option]) + f.write(line) + + def dump_rsyncd(self): + """ + Write rsyncd configuration file. + """ + + if self.runs_pubd: + with open(self.path("rsyncd.conf"), "w") as f: + print "Writing", f.name + f.writelines(s + "\n" for s in + ("# Automatically generated, do not edit", + "port = %d" % self.rsync_port, + "address = localhost", + "[rpki]", + "log file = rsyncd.log", + "read only = yes", + "use chroot = no", + "path = %s" % self.path("publication"), + "comment = RPKI test", + "[root]", + "log file = rsyncd_root.log", + "read only = yes", + "use chroot = no", + "path = %s" % self.path("publication.root"), + "comment = RPKI test root")) + + @classmethod + def next_rpkic_counter(cls): + cls.rpkic_counter += 10000 + return str(cls.rpkic_counter) + + def run_rpkic(self, *argv): + """ + Run rpkic for this entity. + """ + + cmd = [prog_rpkic, "-i", self.name] + if args.profile: + cmd.append("--profile") + cmd.append(self.path("rpkic.%s.prof" % rpki.sundial.now())) + cmd.extend(str(a) for a in argv if a is not None) + print 'Running "%s"' % " ".join(cmd) + env = dict(os.environ, + YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(), + RPKI_CONF = self.path("rpki.conf")) + subprocess.check_call(cmd, cwd = self.host.path(), env = env) + + def syncdb(self): + """ + Run whatever Django ORM commands are necessary to set up the + database this week. + """ + + # Fork a sub-process for each syncdb/migrate run, because it's + # easier than figuring out how to change Django settings after + # initialization. + + def sync_settings(settings, verbosity = 1): + + if verbosity > 0: + print "Running Django setup for", self.name + + pid = os.fork() + + if pid == 0: + logging.getLogger().setLevel(logging.WARNING) + + os.environ.update(RPKI_CONF = self.path("rpki.conf"), + DJANGO_SETTINGS_MODULE = "rpki.django_settings." + settings) + + import django + django.setup() + + import django.core.management + django.core.management.call_command("migrate", verbosity = verbosity, no_color = True, + load_initial_data = False, interactive = False) + + if settings in ("gui", "irdb"): + from django.contrib.auth.models import User + User.objects.create_superuser("root", "root@example.org", "fnord") + + sys.exit(0) + + elif os.waitpid(pid, 0)[1]: + raise RuntimeError("Django setup failed for %s %s" % (self.name, settings)) + + for settings in ("rpkid", "pubd", "gui"): + sync_settings(settings) + + def run_python_daemon(self, prog): + """ + Start a Python daemon and return a subprocess.Popen object + representing the running daemon. + """ + + basename = os.path.splitext(os.path.basename(prog))[0] + cmd = [prog, "--foreground", "--log-level", "debug", + "--log-file", self.path(basename + ".log")] + if args.profile and basename != "rootd": + cmd.extend(( + "--profile", self.path(basename + ".prof"))) + env = dict(os.environ, RPKI_CONF = self.path("rpki.conf")) + p = subprocess.Popen(cmd, cwd = self.path(), env = env) + print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) + return p + + def run_rpkid(self): + """ + Run rpkid. + """ + + return self.run_python_daemon(prog_rpkid) + + def run_irdbd(self): + """ + Run irdbd. + """ + + return self.run_python_daemon(prog_irdbd) + + def run_pubd(self): + """ + Run pubd. + """ + + return self.run_python_daemon(prog_pubd) + + def run_rootd(self): + """ + Run rootd. + """ + + return self.run_python_daemon(prog_rootd) + + def run_rsyncd(self): + """ + Run rsyncd. + """ + + p = subprocess.Popen(("rsync", "--daemon", "--no-detach", "--config", "rsyncd.conf"), + cwd = self.path()) + print "Running rsyncd for %s: pid %d process %r" % (self.name, p.pid, p) + return p - def run_gui(self): - """ - Start an instance of the RPKI GUI under the Django test server and - return a subprocess.Popen object representing the running daemon. - """ + def run_gui(self): + """ + Start an instance of the RPKI GUI under the Django test server and + return a subprocess.Popen object representing the running daemon. + """ - port = 8000 + self.engine - cmd = (prog_rpki_manage, "runserver", str(port)) - env = dict(os.environ, - RPKI_CONF = self.path("rpki.conf"), - RPKI_DJANGO_DEBUG = "yes", - ALLOW_PLAIN_HTTP_FOR_TESTING = "I solemnly swear that I am not running this in production") - p = subprocess.Popen(cmd, cwd = self.path(), env = env, - stdout = open(self.path("gui.log"), "w"), stderr = subprocess.STDOUT) - print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) - return p + port = 8000 + self.engine + cmd = (prog_rpki_manage, "runserver", str(port)) + env = dict(os.environ, + RPKI_CONF = self.path("rpki.conf"), + RPKI_DJANGO_DEBUG = "yes", + ALLOW_PLAIN_HTTP_FOR_TESTING = "I solemnly swear that I am not running this in production") + p = subprocess.Popen(cmd, cwd = self.path(), env = env, + stdout = open(self.path("gui.log"), "w"), stderr = subprocess.STDOUT) + print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) + return p def create_root_certificate(db_root): - print "Creating rootd RPKI root certificate" + print "Creating rootd RPKI root certificate" - root_resources = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as("0-4294967295"), - v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), - v6 = rpki.resource_set.resource_set_ipv6("::/0")) + root_resources = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as("0-4294967295"), + v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), + v6 = rpki.resource_set.resource_set_ipv6("::/0")) - root_key = rpki.x509.RSA.generate(quiet = True) + root_key = rpki.x509.RSA.generate(quiet = True) - root_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name) + root_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name) - rrdp_uri = "http://localhost:%s/rrdp/notify.xml" % db.root.pubd.rrdp_port + rrdp_uri = "http://localhost:%s/rrdp/notify.xml" % db.root.pubd.rrdp_port - root_sia = (root_uri + "/", root_uri + "/root.mft", None, rrdp_uri) + root_sia = (root_uri + "/", root_uri + "/root.mft", None, rrdp_uri) - root_cert = rpki.x509.X509.self_certify( - keypair = root_key, - subject_key = root_key.get_public(), - serial = 1, - sia = root_sia, - notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), - resources = root_resources) + root_cert = rpki.x509.X509.self_certify( + keypair = root_key, + subject_key = root_key.get_public(), + serial = 1, + sia = root_sia, + notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), + resources = root_resources) - with open(db_root.path("root.cer"), "wb") as f: - f.write(root_cert.get_DER()) + with open(db_root.path("root.cer"), "wb") as f: + f.write(root_cert.get_DER()) - with open(db_root.path("root.key"), "wb") as f: - f.write(root_key.get_DER()) + with open(db_root.path("root.key"), "wb") as f: + f.write(root_key.get_DER()) - with open(os.path.join(test_dir, "root.tal"), "w") as f: - f.write(root_uri + ".cer\n\n") - f.write(root_key.get_public().get_Base64()) + with open(os.path.join(test_dir, "root.tal"), "w") as f: + f.write(root_uri + ".cer\n\n") + f.write(root_key.get_public().get_Base64()) logger = logging.getLogger(__name__) @@ -772,247 +772,247 @@ args = parser.parse_args() try: - if args.pidfile is not None: - with open(args.pidfile, "w") as f: - print "Writing pidfile", f.name - f.write("%s\n" % os.getpid()) + if args.pidfile is not None: + with open(args.pidfile, "w") as f: + print "Writing pidfile", f.name + f.write("%s\n" % os.getpid()) - rpki.log.init("yamltest", argparse.Namespace(log_level = logging.DEBUG, - log_handler = lambda: logging.StreamHandler(sys.stdout))) + rpki.log.init("yamltest", argparse.Namespace(log_level = logging.DEBUG, + log_handler = lambda: logging.StreamHandler(sys.stdout))) - # Allow optional config file for this tool to override default - # passwords: this is mostly so that I can show a complete working - # example without publishing my own server's passwords. + # Allow optional config file for this tool to override default + # passwords: this is mostly so that I can show a complete working + # example without publishing my own server's passwords. - cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True) + cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True) - only_one_pubd = cfg.getboolean("only_one_pubd", True) - allocation.base_port = cfg.getint("base_port", 4400) + only_one_pubd = cfg.getboolean("only_one_pubd", True) + allocation.base_port = cfg.getint("base_port", 4400) - config_overrides = dict( - (k, cfg.get(k)) - for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", - "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username") - if cfg.has_option(k)) + config_overrides = dict( + (k, cfg.get(k)) + for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", + "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username") + if cfg.has_option(k)) - # Start clean, maybe + # Start clean, maybe - if not args.skip_config: - for root, dirs, files in os.walk(test_dir, topdown = False): - for fn in files: - os.unlink(os.path.join(root, fn)) - for d in dirs: - os.rmdir(os.path.join(root, d)) + if not args.skip_config: + for root, dirs, files in os.walk(test_dir, topdown = False): + for fn in files: + os.unlink(os.path.join(root, fn)) + for d in dirs: + os.rmdir(os.path.join(root, d)) - # Read first YAML doc in file and process as compact description of - # test layout and resource allocations. Ignore subsequent YAML docs, - # they're for smoketest.py, not this script. + # Read first YAML doc in file and process as compact description of + # test layout and resource allocations. Ignore subsequent YAML docs, + # they're for smoketest.py, not this script. - db = allocation_db(yaml.safe_load_all(args.yaml_file).next()) + db = allocation_db(yaml.safe_load_all(args.yaml_file).next()) - # Show what we loaded + # Show what we loaded - #db.dump() + #db.dump() - if args.skip_config: + if args.skip_config: - print "Skipping pre-daemon configuration, assuming you already did that" + print "Skipping pre-daemon configuration, assuming you already did that" - else: + else: - # Set up each entity in our test, create publication directories, - # and initialize server BPKI. + # Set up each entity in our test, create publication directories, + # and initialize server BPKI. - for d in db: - if not d.is_hosted: - print "Initializing", d.name - os.makedirs(d.path()) - d.dump_conf() - if d.runs_pubd: - os.makedirs(d.path("publication")) - d.dump_rsyncd() - if d.is_root: - os.makedirs(d.path("publication.root")) - d.syncdb() - d.run_rpkic("initialize_server_bpki") - print + for d in db: + if not d.is_hosted: + print "Initializing", d.name + os.makedirs(d.path()) + d.dump_conf() + if d.runs_pubd: + os.makedirs(d.path("publication")) + d.dump_rsyncd() + if d.is_root: + os.makedirs(d.path("publication.root")) + d.syncdb() + d.run_rpkic("initialize_server_bpki") + print - # Initialize resource holding BPKI and generate self-descriptor - # for each entity. + # Initialize resource holding BPKI and generate self-descriptor + # for each entity. - for d in db: - d.run_rpkic("create_identity", d.name) + for d in db: + d.run_rpkic("create_identity", d.name) - # Create RPKI root certificate. + # Create RPKI root certificate. - create_root_certificate(db.root) + create_root_certificate(db.root) - # Set up rootd. + # Set up rootd. - db.root.run_rpkic("configure_root") + db.root.run_rpkic("configure_root") - # From here on we need to pay attention to initialization order. We - # used to do all the pre-configure_daemons stuff before running any - # of the daemons, but that doesn't work right in hosted cases, so we - # have to interleave configuration with starting daemons, just as - # one would in the real world for this sort of thing. + # From here on we need to pay attention to initialization order. We + # used to do all the pre-configure_daemons stuff before running any + # of the daemons, but that doesn't work right in hosted cases, so we + # have to interleave configuration with starting daemons, just as + # one would in the real world for this sort of thing. - progs = [] + progs = [] - try: + try: - for d in db: + for d in db: - if not d.is_hosted: - print - print "Running daemons for", d.name - if d.is_root: - progs.append(d.run_rootd()) - progs.append(d.run_irdbd()) - progs.append(d.run_rpkid()) - if d.runs_pubd: - progs.append(d.run_pubd()) - progs.append(d.run_rsyncd()) - if args.run_gui: - progs.append(d.run_gui()) + if not d.is_hosted: + print + print "Running daemons for", d.name + if d.is_root: + progs.append(d.run_rootd()) + progs.append(d.run_irdbd()) + progs.append(d.run_rpkid()) + if d.runs_pubd: + progs.append(d.run_pubd()) + progs.append(d.run_rsyncd()) + if args.run_gui: + progs.append(d.run_gui()) - if args.synchronize or not args.skip_config: + if args.synchronize or not args.skip_config: - print - print "Giving daemons time to start up" - time.sleep(20) - assert all(p.poll() is None for p in progs) + print + print "Giving daemons time to start up" + time.sleep(20) + assert all(p.poll() is None for p in progs) - if args.skip_config: + if args.skip_config: - print - print "Skipping configure_*, you'll have to do that yourself if needed" + print + print "Skipping configure_*, you'll have to do that yourself if needed" - else: + else: + + for d in db: + + print + print "Configuring", d.name + print + if d.is_root: + assert not d.is_hosted + d.run_rpkic("configure_publication_client", + "--flat" if args.flat_publication else None, + d.path("%s.%s.repository-request.xml" % (d.name, d.name))) + print + d.run_rpkic("configure_repository", + d.path("%s.repository-response.xml" % d.client_handle)) + print + else: + d.parent.run_rpkic("configure_child", + "--valid_until", d.resources.valid_until, + d.path("%s.identity.xml" % d.name)) + print + d.run_rpkic("configure_parent", + d.parent.path("%s.%s.parent-response.xml" % (d.parent.name, d.name))) + print + d.pubd.run_rpkic("configure_publication_client", + "--flat" if args.flat_publication else None, + d.path("%s.%s.repository-request.xml" % (d.name, d.parent.name))) + print + d.run_rpkic("configure_repository", + d.pubd.path("%s.repository-response.xml" % d.client_handle)) + print + + print + print "Done with initial configuration" + print + + if args.synchronize: + print + print "Synchronizing" + print + for d in db: + if not d.is_hosted: + d.run_rpkic("synchronize") + + if args.synchronize or not args.skip_config: + print + print "Loading CSV files" + print + for d in db: + d.dump_asns() + d.dump_prefixes() + d.dump_roas() + d.dump_ghostbusters() + d.dump_router_certificates() - for d in db: + if args.run_gui: + print + print 'GUI user "root", password "fnord"' + for d in db: + if not d.is_hosted: + url = "http://127.0.0.1:%d/rpki/" % (8000 + d.engine) + print "GUI URL", url, "for", d.name + if args.browser: + if d is db.root: + webbrowser.open_new(url) + else: + webbrowser.open_new_tab(url) + time.sleep(2) + + # Wait until something terminates. + + if not args.stop_after_config or args.keep_going: + if args.notify_when_startup_complete: + print + print "Sending SIGUSR1 to process", args.notify_when_startup_complete + os.kill(args.notify_when_startup_complete, signal.SIGUSR1) + print + print "Waiting for daemons to exit" + signal.signal(signal.SIGCHLD, lambda *dont_care: None) + while (any(p.poll() is None for p in progs) + if args.keep_going else + all(p.poll() is None for p in progs)): + signal.pause() + + finally: print - print "Configuring", d.name + print "Shutting down" print - if d.is_root: - assert not d.is_hosted - d.run_rpkic("configure_publication_client", - "--flat" if args.flat_publication else None, - d.path("%s.%s.repository-request.xml" % (d.name, d.name))) - print - d.run_rpkic("configure_repository", - d.path("%s.repository-response.xml" % d.client_handle)) - print + + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + if args.profile: + how_long = 300 else: - d.parent.run_rpkic("configure_child", - "--valid_until", d.resources.valid_until, - d.path("%s.identity.xml" % d.name)) - print - d.run_rpkic("configure_parent", - d.parent.path("%s.%s.parent-response.xml" % (d.parent.name, d.name))) - print - d.pubd.run_rpkic("configure_publication_client", - "--flat" if args.flat_publication else None, - d.path("%s.%s.repository-request.xml" % (d.name, d.parent.name))) - print - d.run_rpkic("configure_repository", - d.pubd.path("%s.repository-response.xml" % d.client_handle)) - print - - print - print "Done with initial configuration" - print - - if args.synchronize: - print - print "Synchronizing" - print - for d in db: - if not d.is_hosted: - d.run_rpkic("synchronize") - - if args.synchronize or not args.skip_config: - print - print "Loading CSV files" - print - for d in db: - d.dump_asns() - d.dump_prefixes() - d.dump_roas() - d.dump_ghostbusters() - d.dump_router_certificates() - - if args.run_gui: - print - print 'GUI user "root", password "fnord"' - for d in db: - if not d.is_hosted: - url = "http://127.0.0.1:%d/rpki/" % (8000 + d.engine) - print "GUI URL", url, "for", d.name - if args.browser: - if d is db.root: - webbrowser.open_new(url) - else: - webbrowser.open_new_tab(url) - time.sleep(2) - - # Wait until something terminates. - - if not args.stop_after_config or args.keep_going: - if args.notify_when_startup_complete: - print - print "Sending SIGUSR1 to process", args.notify_when_startup_complete - os.kill(args.notify_when_startup_complete, signal.SIGUSR1) - print - print "Waiting for daemons to exit" - signal.signal(signal.SIGCHLD, lambda *dont_care: None) - while (any(p.poll() is None for p in progs) - if args.keep_going else - all(p.poll() is None for p in progs)): - signal.pause() - - finally: - - print - print "Shutting down" - print - - signal.signal(signal.SIGCHLD, signal.SIG_DFL) - - if args.profile: - how_long = 300 - else: - how_long = 30 + how_long = 30 - how_often = how_long / 2 + how_often = how_long / 2 - for i in xrange(how_long): - if i % how_often == 0: - for p in progs: - if p.poll() is None: - print "Politely nudging pid %d" % p.pid - p.terminate() - print - if all(p.poll() is not None for p in progs): - break - time.sleep(1) + for i in xrange(how_long): + if i % how_often == 0: + for p in progs: + if p.poll() is None: + print "Politely nudging pid %d" % p.pid + p.terminate() + print + if all(p.poll() is not None for p in progs): + break + time.sleep(1) - for p in progs: - if p.poll() is None: - print "Pulling the plug on pid %d" % p.pid - p.kill() + for p in progs: + if p.poll() is None: + print "Pulling the plug on pid %d" % p.pid + p.kill() - for p in progs: - print "Program pid %d %r returned %d" % (p.pid, p, p.wait()) + for p in progs: + print "Program pid %d %r returned %d" % (p.pid, p, p.wait()) except Exception, e: - print "Blowing out on exception", str(e) - raise + print "Blowing out on exception", str(e) + raise finally: - if args.pidfile is not None and os.path.exists(args.pidfile): - os.unlink(args.pidfile) + if args.pidfile is not None and os.path.exists(args.pidfile): + os.unlink(args.pidfile) # Local Variables: # indent-tabs-mode: nil |