diff options
author | Rob Austein <sra@hactrn.net> | 2012-05-07 02:55:00 +0000 |
---|---|---|
committer | Rob Austein <sra@hactrn.net> | 2012-05-07 02:55:00 +0000 |
commit | 9f4a94e7389f77f3286aaae0c6e7bae3a15c282c (patch) | |
tree | 23db3d7c8c7c99c816486a2e00b7b20098ec67d0 | |
parent | 88e404f1e303fc9e4b4439f301f29fc66643c21e (diff) | |
parent | 636e3c590c96e0f0ab0c95a6982a5056e2cea9fd (diff) |
Merge to trunk. Testbed changes and a few bugfixes that came along
while tracking down testbed issues. See #33.
svn path=/trunk/; revision=4473
-rw-r--r-- | rpkid/rpki/exceptions.py | 5 | ||||
-rw-r--r-- | rpkid/rpki/irdb/zookeeper.py | 56 | ||||
-rw-r--r-- | rpkid/rpki/resource_set.py | 138 | ||||
-rw-r--r-- | rpkid/rpki/rpkic.py | 32 | ||||
-rw-r--r-- | rpkid/rpki/up_down.py | 6 | ||||
-rw-r--r-- | rpkid/tests/yamltest.py | 2 | ||||
-rw-r--r-- | scripts/arin-to-csv.py | 24 | ||||
-rw-r--r-- | scripts/convert-from-entitydb-to-sql.py | 2 | ||||
-rw-r--r-- | scripts/iana-to-csv.py | 85 | ||||
-rw-r--r-- | scripts/ripe-prefixes-to-csv.awk | 6 |
10 files changed, 279 insertions, 77 deletions
diff --git a/rpkid/rpki/exceptions.py b/rpkid/rpki/exceptions.py index 4e4bc42a..21410380 100644 --- a/rpkid/rpki/exceptions.py +++ b/rpkid/rpki/exceptions.py @@ -341,3 +341,8 @@ class MultipleCMSEECert(RPKI_Exception): """ Can't have more than one CMS EE certificate in validation chain. """ + +class ResourceOverlap(RPKI_Exception): + """ + Overlapping resources in resource_set. + """ diff --git a/rpkid/rpki/irdb/zookeeper.py b/rpkid/rpki/irdb/zookeeper.py index d172e18e..17f057de 100644 --- a/rpkid/rpki/irdb/zookeeper.py +++ b/rpkid/rpki/irdb/zookeeper.py @@ -594,6 +594,18 @@ class Zookeeper(object): @django.db.transaction.commit_on_success + def delete_rootd(self): + """ + Delete rootd associated with this RPKI entity. + """ + + try: + self.resource_ca.rootd.delete() + except rpki.irdb.Rootd.DoesNotExist: + self.log("No associated rootd") + + + @django.db.transaction.commit_on_success def configure_publication_client(self, filename, sia_base = None): """ Configure publication server to know about a new client, given the @@ -774,7 +786,7 @@ class Zookeeper(object): @django.db.transaction.commit_on_success - def load_prefixes(self, filename): + def load_prefixes(self, filename, ignore_missing_children = False): """ Whack IRDB to match prefixes.csv. """ @@ -793,14 +805,19 @@ class Zookeeper(object): for version, grouped, rset in ((4, grouped4, rpki.resource_set.resource_set_ipv4), (6, grouped6, rpki.resource_set.resource_set_ipv6)): for handle, prefixes in grouped.iteritems(): - child = self.resource_ca.children.get(handle = handle) - for prefix in rset(",".join(prefixes)): - obj, created = rpki.irdb.ChildNet.objects.get_or_create( - child = child, - start_ip = str(prefix.min), - end_ip = str(prefix.max), - version = version) - primary_keys.append(obj.pk) + try: + child = self.resource_ca.children.get(handle = handle) + except rpki.irdb.Child.DoesNotExist: + if not ignore_missing_children: + raise + else: + for prefix in rset(",".join(prefixes)): + obj, created = rpki.irdb.ChildNet.objects.get_or_create( + child = child, + start_ip = str(prefix.min), + end_ip = str(prefix.max), + version = version) + primary_keys.append(obj.pk) q = rpki.irdb.ChildNet.objects q = q.filter(child__issuer__exact = self.resource_ca) @@ -809,7 +826,7 @@ class Zookeeper(object): @django.db.transaction.commit_on_success - def load_asns(self, filename): + def load_asns(self, filename, ignore_missing_children = False): """ Whack IRDB to match asns.csv. """ @@ -824,13 +841,18 @@ class Zookeeper(object): primary_keys = [] for handle, asns in grouped.iteritems(): - child = self.resource_ca.children.get(handle = handle) - for asn in rpki.resource_set.resource_set_as(",".join(asns)): - obj, created = rpki.irdb.ChildASN.objects.get_or_create( - child = child, - start_as = str(asn.min), - end_as = str(asn.max)) - primary_keys.append(obj.pk) + try: + child = self.resource_ca.children.get(handle = handle) + except rpki.irdb.Child.DoesNotExist: + if not ignore_missing_children: + raise + else: + for asn in rpki.resource_set.resource_set_as(",".join(asns)): + obj, created = rpki.irdb.ChildASN.objects.get_or_create( + child = child, + start_as = str(asn.min), + end_as = str(asn.max)) + primary_keys.append(obj.pk) q = rpki.irdb.ChildASN.objects q = q.filter(child__issuer__exact = self.resource_ca) diff --git a/rpkid/rpki/resource_set.py b/rpkid/rpki/resource_set.py index be39df75..0bc31ef2 100644 --- a/rpkid/rpki/resource_set.py +++ b/rpkid/rpki/resource_set.py @@ -301,7 +301,7 @@ class resource_set(list): canonical = False - def __init__(self, ini = None): + def __init__(self, ini = None, allow_overlap = False): """ Initialize a resource_set. """ @@ -316,24 +316,30 @@ class resource_set(list): self.parse_rfc3779_tuple(ini) elif isinstance(ini, list): self.extend(ini) - else: - assert ini is None or (isinstance(ini, str) and ini == ""), "Unexpected initializer: %s" % str(ini) - self.canonize() + elif ini is not None and ini != "": + raise ValueError("Unexpected initializer: %s" % str(ini)) + self.canonize(allow_overlap) - def canonize(self): + def canonize(self, allow_overlap = False): """ Whack this resource_set into canonical form. """ assert not self.inherit or not self if not self.canonical: self.sort() - for i in xrange(len(self) - 2, -1, -1): - if self[i].max + 1 == self[i+1].min: + i = 0 + while i + 1 < len(self): + if allow_overlap and self[i].max + 1 >= self[i+1].min: + self[i] = type(self[i])(self[i].min, max(self[i].max, self[i+1].max)) + del self[i+1] + elif self[i].max + 1 == self[i+1].min: self[i] = type(self[i])(self[i].min, self[i+1].max) - self.pop(i + 1) - if __debug__: - for i in xrange(0, len(self) - 1): - assert self[i].max < self[i+1].min, "Resource overlap: %s %s" % (self[i], self[i+1]) + del self[i+1] + else: + i += 1 + for i in xrange(0, len(self) - 1): + if self[i].max >= self[i+1].min: + raise rpki.exceptions.ResourceOverlap("Resource overlap: %s %s" % (self[i], self[i+1])) self.canonical = True def append(self, item): @@ -425,18 +431,24 @@ class resource_set(list): del set2[0] return type(self)(result) + __or__ = union + def intersection(self, other): """ Set intersection for resource sets. """ return self._comm(other)[2] + __and__ = intersection + def difference(self, other): """ Set difference for resource sets. """ return self._comm(other)[0] + __sub__ = difference + def symmetric_difference(self, other): """ Set symmetric difference (XOR) for resource sets. @@ -444,6 +456,8 @@ class resource_set(list): com = self._comm(other) return com[0].union(com[1]) + __xor__ = symmetric_difference + def contains(self, item): """ Set membership test for resource sets. @@ -468,6 +482,8 @@ class resource_set(list): hi = mid return lo < len(self) and self[lo].min <= min and self[lo].max >= max + __contains__ = contains + def issubset(self, other): """ Test whether self is a subset (possibly improper) of other. @@ -477,12 +493,26 @@ class resource_set(list): return False return True + __le__ = issubset + def issuperset(self, other): """ Test whether self is a superset (possibly improper) of other. """ return other.issubset(self) + __ge__ = issuperset + + def __lt__(self, other): + return not self.issuperset(other) + + def __gt__(self, other): + return not self.issubset(other) + + __eq__ = list.__eq__ + + __ne__ = list.__ne__ + @classmethod def from_sql(cls, sql, query, args = None): """ @@ -730,6 +760,26 @@ class resource_bag(object): return self @classmethod + def from_str(cls, text, allow_overlap = False): + """ + Parse a comma-separated text string into a resource_bag. Not + particularly efficient, fix that if and when it becomes an issue. + """ + asns = [] + v4s = [] + v6s = [] + for word in text.split(","): + if "." in word: + v4s.append(word) + elif ":" in word: + v6s.append(word) + else: + asns.append(word) + return cls(asn = resource_set_as(",".join(asns), allow_overlap) if asns else None, + v4 = resource_set_ipv4(",".join(v4s), allow_overlap) if v4s else None, + v6 = resource_set_ipv6(",".join(v6s), allow_overlap) if v6s else None) + + @classmethod def from_rfc3779_tuples(cls, exts): """ Build a resource_bag from intermediate form generated by RFC 3779 @@ -773,21 +823,49 @@ class resource_bag(object): Compute intersection with another resource_bag. valid_until attribute (if any) inherits from self. """ - return self.__class__(self.asn.intersection(other.asn), - self.v4.intersection(other.v4), - self.v6.intersection(other.v6), + return self.__class__(self.asn & other.asn, + self.v4 & other.v4, + self.v6 & other.v6, self.valid_until) + __and__ = intersection + def union(self, other): """ Compute union with another resource_bag. valid_until attribute (if any) inherits from self. """ - return self.__class__(self.asn.union(other.asn), - self.v4.union(other.v4), - self.v6.union(other.v6), + return self.__class__(self.asn | other.asn, + self.v4 | other.v4, + self.v6 | other.v6, + self.valid_until) + + __or__ = union + + def difference(self, other): + """ + Compute difference against another resource_bag. valid_until + attribute (if any) inherits from self + """ + return self.__class__(self.asn - other.asn, + self.v4 - other.v4, + self.v6 - other.v6, + self.valid_until) + + __sub__ = difference + + def symmetric_difference(self, other): + """ + Compute symmetric difference against another resource_bag. + valid_until attribute (if any) inherits from self + """ + return self.__class__(self.asn ^ other.asn, + self.v4 ^ other.v4, + self.v6 ^ other.v6, self.valid_until) + __xor__ = symmetric_difference + def __str__(self): s = "" if self.asn: @@ -1095,25 +1173,25 @@ if __name__ == "__main__": v1 = r1._comm(r2) v2 = r2._comm(r1) assert v1[0] == v2[1] and v1[1] == v2[0] and v1[2] == v2[2] - for i in r1: assert r1.contains(i) and r1.contains(i.min) and r1.contains(i.max) - for i in r2: assert r2.contains(i) and r2.contains(i.min) and r2.contains(i.max) - for i in v1[0]: assert r1.contains(i) and not r2.contains(i) - for i in v1[1]: assert not r1.contains(i) and r2.contains(i) - for i in v1[2]: assert r1.contains(i) and r2.contains(i) - v1 = r1.union(r2) - v2 = r2.union(r1) + for i in r1: assert i in r1 and i.min in r1 and i.max in r1 + for i in r2: assert i in r2 and i.min in r2 and i.max in r2 + for i in v1[0]: assert i in r1 and i not in r2 + for i in v1[1]: assert i not in r1 and i in r2 + for i in v1[2]: assert i in r1 and i in r2 + v1 = r1 | r2 + v2 = r2 | r1 assert v1 == v2 print "x|y:", v1, testprefix(v1) - v1 = r1.difference(r2) - v2 = r2.difference(r1) + v1 = r1 - r2 + v2 = r2 - r1 print "x-y:", v1, testprefix(v1) print "y-x:", v2, testprefix(v2) - v1 = r1.symmetric_difference(r2) - v2 = r2.symmetric_difference(r1) + v1 = r1 ^ r2 + v2 = r2 ^ r1 assert v1 == v2 print "x^y:", v1, testprefix(v1) - v1 = r1.intersection(r2) - v2 = r2.intersection(r1) + v1 = r1 & r2 + v2 = r2 & r1 assert v1 == v2 print "x&y:", v1, testprefix(v1) diff --git a/rpkid/rpki/rpkic.py b/rpkid/rpki/rpkic.py index a94954a2..90b75bb0 100644 --- a/rpkid/rpki/rpkic.py +++ b/rpkid/rpki/rpkic.py @@ -155,8 +155,8 @@ class main(rpki.cli.Cmd): self.stdout.write(" " * 4 + line) self.stdout.write("\n") - def irdb_handle_complete(self, klass, text, line, begidx, endidx): - return [obj.handle for obj in klass.objects.all() if obj.handle and obj.handle.startswith(text)] + def irdb_handle_complete(self, manager, text, line, begidx, endidx): + return [obj.handle for obj in manager.all() if obj.handle and obj.handle.startswith(text)] def do_select_identity(self, arg): """ @@ -169,7 +169,7 @@ class main(rpki.cli.Cmd): self.zoo.reset_identity(argv[0]) def complete_select_identity(self, *args): - return self.irdb_handle_complete(rpki.irdb.ResourceHolderCA, *args) + return self.irdb_handle_complete(rpki.irdb.ResourceHolderCA.objects, *args) def do_initialize(self, arg): @@ -249,7 +249,7 @@ class main(rpki.cli.Cmd): print "No such child \"%s\"" % arg def complete_delete_child(self, *args): - return self.irdb_handle_complete(rpki.irdb.Child, *args) + return self.irdb_handle_complete(self.zoo.resource_ca.children, *args) def do_configure_parent(self, arg): @@ -296,7 +296,19 @@ class main(rpki.cli.Cmd): print "No such parent \"%s\"" % arg def complete_delete_parent(self, *args): - return self.irdb_handle_complete(rpki.irdb.Parent, *args) + return self.irdb_handle_complete(self.zoo.resource_ca.parents, *args) + + + def do_delete_rootd(self, arg): + """ + Delete rootd associated with this RPKI entity. + """ + + try: + self.zoo.delete_rootd() + self.zoo.synchronize() + except rpki.irdb.Rootd.DoesNotExist: + print "No associated rootd" def do_configure_publication_client(self, arg): @@ -339,7 +351,7 @@ class main(rpki.cli.Cmd): print "No such client \"%s\"" % arg def complete_delete_publication_client(self, *args): - return self.irdb_handle_complete(rpki.irdb.Client, *args) + return self.irdb_handle_complete(self.zoo.resource_ca.clients, *args) def do_configure_repository(self, arg): @@ -380,7 +392,7 @@ class main(rpki.cli.Cmd): print "No such repository \"%s\"" % arg def complete_delete_repository(self, *args): - return self.irdb_handle_complete(rpki.irdb.Repository, *args) + return self.irdb_handle_complete(self.zoo.resource_ca.repositories, *args) def do_delete_self(self, arg): @@ -411,7 +423,7 @@ class main(rpki.cli.Cmd): self.zoo.synchronize(self.zoo.handle) def complete_renew_child(self, *args): - return self.irdb_handle_complete(rpki.irdb.Child, *args) + return self.irdb_handle_complete(self.zoo.resource_ca.children, *args) def do_renew_all_children(self, arg): @@ -443,7 +455,7 @@ class main(rpki.cli.Cmd): if len(argv) != 1: raise BadCommandSyntax("Need to specify prefixes.csv filename") - self.zoo.load_prefixes(argv[0]) + self.zoo.load_prefixes(argv[0], True) self.zoo.synchronize(self.zoo.handle) @@ -477,7 +489,7 @@ class main(rpki.cli.Cmd): if len(argv) != 1: raise BadCommandSyntax("Need to specify asns.csv filename") - self.zoo.load_asns(argv[0]) + self.zoo.load_asns(argv[0], True) self.zoo.synchronize(self.zoo.handle) diff --git a/rpkid/rpki/up_down.py b/rpkid/rpki/up_down.py index 0eba6b52..4fd06edd 100644 --- a/rpkid/rpki/up_down.py +++ b/rpkid/rpki/up_down.py @@ -251,14 +251,18 @@ class list_pdu(base_elt): r_msg.payload = list_response_pdu() - if irdb_resources.valid_until > rpki.sundial.now(): + if irdb_resources.valid_until < rpki.sundial.now(): + rpki.log.debug("Child %s's resources expired %s" % child.child_handle, irdb_resources.valid_until) + else: for parent in child.parents: for ca in parent.cas: ca_detail = ca.active_ca_detail if not ca_detail: + rpki.log.debug("No active ca_detail, can't issue to %s" % child.child_handle) continue resources = ca_detail.latest_ca_cert.get_3779resources().intersection(irdb_resources) if resources.empty(): + rpki.log.debug("No overlap between received resources and what child %s should get ([%s], [%s])" % (child.child_handle, ca_detail.latest_ca_cert.get_3779resources(), irdb_resources)) continue rc = class_elt() rc.class_name = str(ca.ca_id) diff --git a/rpkid/tests/yamltest.py b/rpkid/tests/yamltest.py index 32434147..12fd7f85 100644 --- a/rpkid/tests/yamltest.py +++ b/rpkid/tests/yamltest.py @@ -118,6 +118,8 @@ class allocation_db(list): self.root.crl_interval = 24 * 60 * 60 if self.root.regen_margin is None: self.root.regen_margin = 24 * 60 * 60 + if self.root.base.valid_until is None: + self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) for a in self: if a.sia_base is None: if a.runs_pubd: diff --git a/scripts/arin-to-csv.py b/scripts/arin-to-csv.py index 63368723..dcb508ff 100644 --- a/scripts/arin-to-csv.py +++ b/scripts/arin-to-csv.py @@ -56,18 +56,18 @@ def do_asn(node): find(node, tag_endAsNumber)))) erx_table = { - "AF" : "AFRINIC", - "AP" : "APNIC", - "AR" : "ARIN", - "AV" : "ARIN", - "FX" : "AFRINIC", - "LN" : "LACNIC", - "LX" : "LACNIC", - "PV" : "APNIC", - "PX" : "APNIC", - "RN" : "RIPE", - "RV" : "RIPE", - "RX" : "RIPE" } + "AF" : "afrinic", + "AP" : "apnic", + "AR" : "arin", + "AV" : "arin", + "FX" : "afrinic", + "LN" : "lacnic", + "LX" : "lacnic", + "PV" : "apnic", + "PX" : "apnic", + "RN" : "ripe", + "RV" : "ripe", + "RX" : "ripe" } def do_net(node): handle = find(node, tag_orgHandle) diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index ba073730..e1030ba9 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -95,7 +95,7 @@ tag_parent = xmlns + "parent" tag_repository = xmlns + "repository" def read_element_tree(filename, tag): - print "Reading file %s, tag %s\n" % (filename, tag) + print "Reading file %s, tag %s" % (filename, tag) e = ElementTree(file = filename).getroot() rpki.relaxng.myrpki.assertValid(e) assert e.tag == tag diff --git a/scripts/iana-to-csv.py b/scripts/iana-to-csv.py new file mode 100644 index 00000000..ee8c53d1 --- /dev/null +++ b/scripts/iana-to-csv.py @@ -0,0 +1,85 @@ +""" +Parse IANA XML data. + +$Id$ + +Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import sys +import lxml.etree +from rpki.csv_utils import csv_reader, csv_writer +from rpki.resource_set import resource_bag + +def iterate_xml(filename, tag): + return lxml.etree.parse(filename).getroot().getiterator(tag) + +def ns(tag): + return "{http://www.iana.org/assignments}" + tag + +tag_description = ns("description") +tag_designation = ns("designation") +tag_record = ns("record") +tag_number = ns("number") +tag_prefix = ns("prefix") +tag_status = ns("status") + +handles = {} +rirs = { "legacy" : resource_bag() } + +for rir in ("AfriNIC", "APNIC", "ARIN", "LACNIC", "RIPE NCC"): + handle = rir.split()[0].lower() + handles[rir] = handles["Assigned by %s" % rir] = handles["Administered by %s" % rir] = handle + rirs[handle] = resource_bag() + +asns = csv_writer("asns.csv") +prefixes = csv_writer("prefixes.csv") + +for record in iterate_xml("as-numbers.xml", tag_record): + description = record.findtext(tag_description) + if description in handles: + asns.writerow((handles[description], record.findtext(tag_number))) + +for record in iterate_xml("ipv4-address-space.xml", tag_record): + designation = record.findtext(tag_designation) + if record.findtext(tag_status) != "RESERVED": + prefix, prefixlen = [int(i) for i in record.findtext(tag_prefix).split("/")] + if prefixlen != 8: + raise ValueError("%s violated /8 assumption" % record.findtext(tag_prefix)) + rirs[handles.get(designation, "legacy")] |= resource_bag.from_str("%d.0.0.0/8" % prefix) + +for record in iterate_xml("ipv6-unicast-address-assignments.xml", tag_record): + description = record.findtext(tag_description) + if record.findtext(tag_description) in handles: + rirs[handles[description]] |= resource_bag.from_str(record.findtext(tag_prefix)) + +erx = list(csv_reader("erx.csv")) +assert all(r in rirs for r, p in erx) + +erx_overrides = resource_bag.from_str(",".join(p for r, p in erx), allow_overlap = True) + +for rir in rirs: + if rir != "legacy": + rirs[rir] -= erx_overrides + rirs[rir] |= resource_bag.from_str(",".join(p for r, p in erx if r == rir), allow_overlap = True) + +for rir, bag in rirs.iteritems(): + for p in bag.v4: + prefixes.writerow((rir, p)) + for p in bag.v6: + prefixes.writerow((rir, p)) + +asns.close() +prefixes.close() diff --git a/scripts/ripe-prefixes-to-csv.awk b/scripts/ripe-prefixes-to-csv.awk index 582d5ce7..37327484 100644 --- a/scripts/ripe-prefixes-to-csv.awk +++ b/scripts/ripe-prefixes-to-csv.awk @@ -3,13 +3,7 @@ # ftp -pa ftp://ftp.ripe.net/pub/stats/ripencc/membership/alloclist.txt -BEGIN { - translation["ie.google"] = "GoogleIreland"; -} - function done() { - if (handle in translation) - handle = translation[handle]; for (i = 1; i <= n_allocs; i++) print handle "\t" alloc[i]; n_allocs = 0; |