Something whacky going on either with gzip module or with our code to

generate children.csv.  Try disabling the latter first; if that
doesn't work, rewrite as filter taking unzipped data on stdin.

svn path=/myrpki/arin-to-csv.py; revision=2675

1 files changed, 17 insertions, 6 deletions

diff --git a/myrpki/arin-to-csv.py b/myrpki/arin-to-csv.py
index ab2819b4..27f7c5b9 100644
--- a/myrpki/arin-to-csv.py
+++ b/myrpki/arin-to-csv.py
@@ -54,7 +54,7 @@ class ASHandle(Handle):
     def finish(self, ctx):
         if self.check():
             ctx.asns.writerow((self.OrgID, self.ASNumber))
-            ctx.orgids.add(self.OrgID)
+            ctx.orgid(self.OrgID)
 
 class NetHandle(Handle):
 
@@ -76,7 +76,7 @@ class NetHandle(Handle):
         if self.NetType in self.useful_types and self.check():
             if str(self.Prefix).find("/") >= 0:
                 ctx.prefixes.writerow((self.OrgID, self.Prefix))
-                ctx.orgids.add(self.OrgID)
+                ctx.orgid(self.OrgID)
             else:
                 print "Not a prefix: %r" % self
 
@@ -108,21 +108,30 @@ def parseline(line):
 
 class gctx(object):
 
+    make_children = False
+
     def csvout(self, fn):
         return csv.writer(open(fn, "w"), dialect = myrpki.csv_dialect)
 
     def __init__(self):
         self.asns = self.csvout("asns.csv")
         self.prefixes = self.csvout("prefixes.csv")
-        self.children = self.csvout("children.csv")
-        self.orgids = set()
+        if self.make_children:
+            self.orgids = set()
+
+    def orgid(self, o):
+        if self.make_children:
+            self.orgids.add(o)
 
     def finish(self):
         expires = rpki.sundial.now() + rpki.sundial.timedelta(days = 365)
-        self.children.writerows((orgid, expires, "children/%s.ta.cer" % orgid)
-                                for orgid in self.orgids)
+        if self.make_children:
+            self.csvout("children.csv").writerows(
+                (orgid, expires, "children/%s.ta.cer" % orgid)
+                for orgid in self.orgids)
 
 def main():
+    print "Starting at %s" % rpki.sundial.now()
     f = gzip.open("arin_db.txt.gz")
     cur = None
     ctx = gctx()
@@ -140,6 +149,8 @@ def main():
                 cur.set(tag, val)
     if cur:
         cur.finish(ctx)
+    print "Finished main loop at %s" % rpki.sundial.now()
     ctx.finish()
+    print "All done at %s" % rpki.sundial.now()
 
 main()