Yet another set of tools attempting to deal with the RIPE mess

svn path=/scripts/arin-to-csv.py; revision=3555

3 files changed, 164 insertions, 3 deletions

diff --git a/scripts/arin-to-csv.py b/scripts/arin-to-csv.py
index 98d99f9f..c862fd63 100644
--- a/scripts/arin-to-csv.py
+++ b/scripts/arin-to-csv.py
@@ -53,18 +53,38 @@ def do_asn(node):
                  "%s-%s" % (find(node, tag_startAsNumber),
                             find(node, tag_endAsNumber))))
 
+erx_table = {
+  "AF" : "AFRINIC",
+  "AP" : "APNIC",
+  "AR" : "ARIN",
+  "AV" : "ARIN",
+  "FX" : "AFRINIC",
+  "LN" : "LACNIC",
+  "LX" : "LACNIC",
+  "PV" : "APNIC",
+  "PX" : "APNIC",
+  "RN" : "RIPE",
+  "RV" : "RIPE",
+  "RX" : "RIPE" }
+
 def do_net(node):
   handle = find(node, tag_orgHandle)
   for netblock in node.iter(tag_netBlock):
-    if find(netblock, tag_type) in ("DS", "DA", "IU"):
+    tag = find(netblock, tag_type)
+    if tag in ("DS", "DA", "IU"):
       prefixes.writerow((handle,
-                         "%s-%s" % (find(netblock, tag_startAddress),
-                                    find(netblock, tag_endAddress))))
+                    "%s-%s" % (find(netblock, tag_startAddress),
+                               find(netblock, tag_endAddress))))
+    elif tag in erx_table:
+      erx.writerow((erx_table[tag],
+                    "%s-%s" % (find(netblock, tag_startAddress),
+                               find(netblock, tag_endAddress))))
 
 dispatch = { tag_asn : do_asn, tag_net : do_net }
 
 asns = rpki.myrpki.csv_writer("asns.csv")
 prefixes = rpki.myrpki.csv_writer("prefixes.csv")
+erx = rpki.myrpki.csv_writer("erx.csv")
 
 root = None
 
@@ -86,3 +106,4 @@ for event, node in lxml.etree.iterparse(sys.stdin):
 
 asns.close()
 prefixes.close()
+erx.close()
diff --git a/scripts/generate-ripe-root-cert.py b/scripts/generate-ripe-root-cert.py
new file mode 100644
index 00000000..19b6dcf8
--- /dev/null
+++ b/scripts/generate-ripe-root-cert.py
@@ -0,0 +1,57 @@
+"""
+Parse IANA XML data and write out just what we need to generate a root
+cert for Pseudo-RIPE.
+
+$Id$
+
+Copyright (C) 2010  Internet Systems Consortium ("ISC")
+
+Permission to use, copy, modify, and distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+"""
+
+import sys, lxml.etree, rpki.myrpki
+
+def ns(tag):
+  return "{http://www.iana.org/assignments}" + tag
+
+tag_registry    = ns("registry")
+tag_description = ns("description")
+tag_designation = ns("designation")
+tag_record      = ns("record")
+tag_number      = ns("number")
+tag_prefix      = ns("prefix")
+
+asn_xml  = lxml.etree.parse("as-numbers.xml").getroot()
+ipv4_xml = lxml.etree.parse("ipv4-address-space.xml").getroot()
+ipv6_xml = lxml.etree.parse("ipv6-unicast-address-assignments.xml").getroot()
+
+asns     = rpki.myrpki.csv_writer("asns.csv")
+prefixes = rpki.myrpki.csv_writer("prefixes.csv")
+
+for record in asn_xml.getiterator(tag_record):
+  if record.findtext(tag_description) == "Assigned by RIPE NCC":
+    asns.writerow(("RIPE", record.findtext(tag_number)))
+    
+for record in ipv4_xml.getiterator(tag_record):
+  if record.findtext(tag_designation) in ("RIPE NCC", "Administered by RIPE NCC"):
+    prefix = record.findtext(tag_prefix)
+    p, l = prefix.split("/")
+    assert l == "8", "Violated /8 assumption: %r" % prefix
+    prefixes.writerow(("RIPE", "%d.0.0.0/8" % int(p)))
+    
+for record in ipv6_xml.getiterator(tag_record):
+  if record.findtext(tag_description) == "RIPE NCC":
+    prefixes.writerow(("RIPE", record.findtext(tag_prefix)))
+
+asns.close()
+prefixes.close()
diff --git a/scripts/whack-ripe-asns.py b/scripts/whack-ripe-asns.py
new file mode 100644
index 00000000..b6457918
--- /dev/null
+++ b/scripts/whack-ripe-asns.py
@@ -0,0 +1,83 @@
+"""
+Fix problems in asns.csv generated from RIPE's database.
+
+RIPE's database contains inconsistancies, overlaps, and format errors
+that make it impossible to feed the output of ripe-to-csv.awk directly
+into testbed-rootcert.py without OpenSSL rejecting the resulting
+root.conf.  This script takes a brute force approach to fixing this:
+it converts all ASNs to range form, runs the resulting file through
+the unix sort program to put the data into canonical order, then reads
+it back, merging overlaps, and writing the result in a form acceptable
+to testbed-rootcert.py.
+
+Since we're doing all this anyway, the script also merges adjacent
+blocks.
+
+Ordinarily, it would be dangerous to have the same program act as both
+the source and sink of a pipeline, particularly for such a large data
+set, as the chance of deadlock would approach 100%, but in this case
+we know that the sort program must consume and buffer (somehow) all of
+its input before writing a single line of output, so a single script
+can safely act as a filter both before and after sort.
+
+$Id$
+
+Copyright (C) 2010  Internet Systems Consortium ("ISC")
+
+Permission to use, copy, modify, and distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+"""
+
+import sys, subprocess
+
+sorter = subprocess.Popen(("sort", "-T.", "-n"),
+                          stdin = subprocess.PIPE,
+                          stdout = subprocess.PIPE)
+
+for line in sys.stdin:
+  handle, asn = line.split()
+
+  if "-" in asn:
+    range_min, range_max = asn.split("-")
+  else:
+    range_min, range_max = asn, asn
+
+  sorter.stdin.write("%d %d\n" % (long(range_min), long(range_max)))
+
+sorter.stdin.close()
+
+prev_min = None
+prev_max = None
+
+def show():
+  if prev_min and prev_max:
+    sys.stdout.write("x\t%s-%s\n" % (prev_min, prev_max))
+
+for line in sorter.stdout:
+  this_min, this_max = line.split()
+  this_min = long(this_min)
+  this_max = long(this_max)
+
+  if prev_min and prev_max and prev_max + 1 >= this_min:
+    prev_min = min(prev_min, this_min)
+    prev_max = max(prev_max, this_max)
+
+  else:
+    show()
+    prev_min = this_min
+    prev_max = this_max
+
+show()
+
+sorter.stdout.close()
+
+sys.exit(sorter.wait())