diff options
| -rw-r--r-- | myrpki.rototill/apnic-to-csv.py | 7 | ||||
| -rw-r--r-- | myrpki.rototill/arin-rootcert.py | 4 | ||||
| -rw-r--r-- | myrpki.rototill/arin-to-csv.py | 10 | ||||
| -rw-r--r-- | myrpki.rototill/convert-from-csv-to-entitydb.py | 6 | ||||
| -rw-r--r-- | myrpki.rototill/myrpki.py | 65 | ||||
| -rw-r--r-- | myrpki.rototill/ripe-asns-to-csv.py | 5 | ||||
| -rw-r--r-- | myrpki.rototill/yamltest.py | 2 |
7 files changed, 62 insertions, 37 deletions
diff --git a/myrpki.rototill/apnic-to-csv.py b/myrpki.rototill/apnic-to-csv.py index af801f23..65b71a36 100644 --- a/myrpki.rototill/apnic-to-csv.py +++ b/myrpki.rototill/apnic-to-csv.py @@ -1,10 +1,7 @@ import csv, myrpki, rpki.ipaddrs -def csvout(fn): - return csv.writer(open(fn, "w"), dialect = myrpki.csv_dialect) - -asns = csvout("asns.csv") -prefixes = csvout("prefixes.csv") +asns = myrpki.csv_writer("asns.csv") +prefixes = myrpki.csv_writer("prefixes.csv") for line in open("delegated-apnic-extended-latest"): diff --git a/myrpki.rototill/arin-rootcert.py b/myrpki.rototill/arin-rootcert.py index 7fdd5e3e..41369056 100644 --- a/myrpki.rototill/arin-rootcert.py +++ b/myrpki.rototill/arin-rootcert.py @@ -55,7 +55,7 @@ sbgp-ipAddrBlock = critical,@rfc3997_addrs ''' % { "holder" : holder.lower(), "HOLDER" : holder.upper() } -for i, asn in enumerate(asn for handle, asn in myrpki.csv_open("asns.csv")): +for i, asn in enumerate(asn for handle, asn in myrpki.csv_reader("asns.csv", columns = 2)): print "AS.%d = %s" % (i, asn) print '''\ @@ -64,6 +64,6 @@ print '''\ ''' -for i, prefix in enumerate(prefix for handle, prefix in myrpki.csv_open("prefixes.csv")): +for i, prefix in enumerate(prefix for handle, prefix in myrpki.csv_reader("prefixes.csv", columns = 2)): v = 6 if ":" in prefix else 4 print "IPv%d.%d = %s" % (v, i, prefix) diff --git a/myrpki.rototill/arin-to-csv.py b/myrpki.rototill/arin-to-csv.py index 5b4caae1..55e5762a 100644 --- a/myrpki.rototill/arin-to-csv.py +++ b/myrpki.rototill/arin-to-csv.py @@ -94,15 +94,11 @@ class main(object): assert sep, "Couldn't find separator in %r" % line return tag.strip(), val.strip() - @staticmethod - def csvout(fn): - return csv.writer(open(fn, "w"), dialect = myrpki.csv_dialect) - def __init__(self): - self.asns = self.csvout("asns.csv") - self.prefixes = self.csvout("prefixes.csv") + self.asns = myrpki.csv_writer("asns.csv") + self.prefixes = myrpki.csv_writer("prefixes.csv") try: - self.translations = dict((src, dst) for src, dst in myrpki.csv_open("translations.csv")) + self.translations = dict((src, dst) for src, dst in myrpki.csv_reader("translations.csv", columns = 2)) except IOError: pass f = gzip.open("arin_db.txt.gz") diff --git a/myrpki.rototill/convert-from-csv-to-entitydb.py b/myrpki.rototill/convert-from-csv-to-entitydb.py index 92c85180..282d2e75 100644 --- a/myrpki.rototill/convert-from-csv-to-entitydb.py +++ b/myrpki.rototill/convert-from-csv-to-entitydb.py @@ -165,7 +165,7 @@ for d in map(entitydb, ("children", "parents", "repositories", "pubclients")): one_year_from_now = str(rpki.sundial.now() + rpki.sundial.timedelta(days = 365)) if os.path.exists(children_csv): - for child_handle, valid_until, child_resource_pemfile in myrpki.csv_open(children_csv): + for child_handle, valid_until, child_resource_pemfile in myrpki.csv_reader(children_csv, columns = 3): try: e = Element("parent", @@ -182,7 +182,7 @@ if os.path.exists(children_csv): pass if os.path.exists(parents_csv): - for parent_handle, parent_service_uri, parent_cms_pemfile, parent_https_pemfile, parent_myhandle, parent_sia_base in myrpki.csv_open(parents_csv): + for parent_handle, parent_service_uri, parent_cms_pemfile, parent_https_pemfile, parent_myhandle, parent_sia_base in myrpki.csv_reader(parents_csv, columns = 6): try: e = Element("parent", @@ -213,7 +213,7 @@ if os.path.exists(parents_csv): pass if os.path.exists(pubclients_csv): - for client_handle, client_resource_pemfile, client_sia_base in myrpki.csv_open(pubclients_csv): + for client_handle, client_resource_pemfile, client_sia_base in myrpki.csv_reader(pubclients_csv, columns = 3): try: parent_handle = client_handle.split("/")[-2] if "/" in client_handle else handle diff --git a/myrpki.rototill/myrpki.py b/myrpki.rototill/myrpki.py index 1a45370f..a618577d 100644 --- a/myrpki.rototill/myrpki.py +++ b/myrpki.rototill/myrpki.py @@ -71,13 +71,6 @@ namespace = "http://www.hactrn.net/uris/rpki/myrpki/" version = "2" namespaceQName = "{" + namespace + "}" -# Dialect for our use of CSV files, here to make it easy to change if -# your site needs to do something different. See doc for the csv -# module in the Python standard libraries for details if you need to -# customize this. - -csv_dialect = csv.get_dialect("excel-tab") - # Whether to include incomplete entries when rendering to XML. allow_incomplete = False @@ -180,7 +173,7 @@ class roa_requests(dict): """ self = cls() # format: p/n-m asn group - for pnm, asn, group in csv_open(roa_csv_file): + for pnm, asn, group in csv_reader(roa_csv_file, columns = 3): self.add(asn = asn, group = group, prefix = pnm) return self @@ -283,10 +276,10 @@ class children(dict): validity = c.get("valid_until"), bpki_certificate = fxcert(c.findtext("bpki_child_ta"))) # childname p/n - for handle, pn in csv_open(prefix_csv_file): + for handle, pn in csv_reader(prefix_csv_file, columns = 2): self.add(handle = handle, prefix = pn) # childname asn - for handle, asn in csv_open(asn_csv_file): + for handle, asn in csv_reader(asn_csv_file, columns = 2): self.add(handle = handle, asn = asn) return self @@ -478,13 +471,55 @@ class repositories(dict): bpki_certificate = fxcert(r.findtext("bpki_server_ta"))) return self -def csv_open(filename): +class csv_reader(object): """ - Open a CSV file, with settings that make it a tab-delimited file. - You may need to tweak this function for your environment, see the - csv module in the Python standard libraries for details. + Reader for tab-delimited text that's (slightly) friendlier than the + stock Python csv module (which isn't intended for direct use by + humans anyway, and neither was this package originally, but that + seems to be the way that it has evolved...). + + Columns parameter specifies how many columns users of the reader + expect to see; lines with fewer columns will be padded with None + values. + + Original API design for this class courtesy of Warren Kumari, but + don't blame him if you don't like what I did with his ideas. """ - return csv.reader(open(filename, "rb"), dialect = csv_dialect) + + def __init__(self, filename, columns = None, min_columns = None, comment_characters = "#;"): + assert columns is None or isinstance(columns, int) + assert min_columns is None or isinstance(min_columns, int) + if columns is not None and min_columns is None: + min_columns = columns + self.filename = filename + self.columns = columns + self.min_columns = min_columns + self.comment_characters = comment_characters + self.file = open(filename, "r") + + def __iter__(self): + line_number = 0 + for line in self.file: + line_number += 1 + line = line.strip() + if not line or line[0] in self.comment_characters: + continue + fields = line.split() + if self.min_columns is not None and len(fields) < self.min_columns: + raise RuntimeError, "%s:%d: Not enough columns in line %r" % (self.filename, line_number, line) + if self.columns is not None and len(fields) > self.columns: + raise RuntimeError, "%s:%d: Too many columns in line %r" % (self.filename, line_number, line) + if self.columns is not None and len(fields) < self.columns: + fields += tuple(None for i in xrange(self.columns - len(fields))) + yield fields + +def csv_writer(filename): + """ + Writer object for tab delimited text. We just use the stock CSV + module in excel-tab mode for this. + """ + return csv.writer(open(filename, "w"), dialect = csv.get_dialect("excel-tab")) + def PEMElement(e, tag, filename, **kwargs): """ diff --git a/myrpki.rototill/ripe-asns-to-csv.py b/myrpki.rototill/ripe-asns-to-csv.py index 01c0f82a..04a92627 100644 --- a/myrpki.rototill/ripe-asns-to-csv.py +++ b/myrpki.rototill/ripe-asns-to-csv.py @@ -70,9 +70,6 @@ class main(object): types = dict((x.want_tags[0], x) for x in (aut_num,)) - @staticmethod - def csvout(fn): - return csv.writer(open(fn, "w"), dialect = myrpki.csv_dialect) def finish_statement(self, done): if self.statement: @@ -91,7 +88,7 @@ class main(object): filenames = ("ripe.db.aut-num.gz",) def __init__(self): - self.asns = self.csvout("asns.csv") + self.asns = myrpki.csv_writer("asns.csv") for fn in self.filenames: f = gzip.open(fn) self.statement = "" diff --git a/myrpki.rototill/yamltest.py b/myrpki.rototill/yamltest.py index 1e34abb5..360f6ad7 100644 --- a/myrpki.rototill/yamltest.py +++ b/myrpki.rototill/yamltest.py @@ -293,7 +293,7 @@ class allocation(object): """ path = self.path(fn) print "Writing", path - return csv.writer(open(path, "w"), dialect = myrpki.csv_dialect) + return myrpki.csv_writer(path) def up_down_url(self): """ |