From ca8584eab7e1d8752fe68d5c404e73e3d1333918 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Wed, 7 Dec 2011 02:37:13 +0000 Subject: Checkpoint svn path=/branches/tk100/; revision=4101 --- scripts/convert-from-entitydb-to-sql.py | 93 +++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 scripts/convert-from-entitydb-to-sql.py (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py new file mode 100644 index 00000000..2bbe0eac --- /dev/null +++ b/scripts/convert-from-entitydb-to-sql.py @@ -0,0 +1,93 @@ +""" +Merge XML entitydb and OpenSSL command-line BPKI into SQL IRDB. + +This is a work in progress, don't use it unless you really know what +you're doing. + +Copyright (C) 2011 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import sys, os, time, getopt +import rpki.config +from rpki.mysql_import import MySQLdb +from lxml.etree import ElementTree + +if os.getlogin() != "sra": + sys.exit("I //said// this was a work in progress") + +cfg_file = "rpki.conf" +entitydb = "entitydb" + +opts, argv = getopt.getopt(sys.argv[1:], "c:e:h?", ["config=", "entitydb=", "help"]) +for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + if o in ("-c", "--config"): + cfg_file = a + elif o in ("-e", "--entitydb"): + entitydb = a +if argv: + sys.exit("Unexpected arguments %s" % argv) + +cfg = rpki.config.parser(cfg_file, "irdbd") + +sql_database = cfg.get("sql-database") +sql_username = cfg.get("sql-username") +sql_password = cfg.get("sql-password") + +db = MySQLdb.connect(user = sql_username, db = sql_database, passwd = sql_password) +cur = db.cursor() + +cur.execute("SHOW TABLES") + +tables = [r[0] for r in cur.fetchall()] + +for table in tables: + if "old_" + table not in tables and table in ("registrant", + "registrant_asn", + "registrant_net", + "roa_request", + "roa_request_prefix", + "ghostbuster_request"): + print "Renaming %s to old_%s" % (table, table) + cur.execute("ALTER TABLE %s RENAME TO old_%s" % (table, table)) + +from django.conf import settings + +settings.configure(DATABASES = { "default" : { + "ENGINE" : "django.db.backends.mysql", + "NAME" : sql_database, + "USER" : sql_username, + "PASSWORD" : sql_password, + "HOST" : "", + "PORT" : "" }}) + +import rpki.irdb + +def ns(tag): + return "{http://www.hactrn.net/uris/rpki/myrpki/}" + tag + +e = ElementTree(file = os.path.join(entitydb, "identity.xml")).getroot() + +t = ns("identity") + +if e.tag == t: + print "Found", t, "handle", e.get("handle") +else: + print "Didn't find", t, "found", e.tag, "instead, oops" + +cur.close() +db.close() -- cgit v1.2.3 From 027d5d72b41ba16ea78482d0ff6c76ba6bcac28c Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Wed, 7 Dec 2011 02:39:14 +0000 Subject: Id svn path=/branches/tk100/; revision=4102 --- scripts/convert-from-entitydb-to-sql.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 2bbe0eac..43a51cf7 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -4,6 +4,8 @@ Merge XML entitydb and OpenSSL command-line BPKI into SQL IRDB. This is a work in progress, don't use it unless you really know what you're doing. +$Id$ + Copyright (C) 2011 Internet Systems Consortium ("ISC") Permission to use, copy, modify, and distribute this software for any -- cgit v1.2.3 From 0e73356dee60228a97475d9fa106f475e41a16fb Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Wed, 7 Dec 2011 18:49:08 +0000 Subject: Sort out how to run syncdb svn path=/branches/tk100/; revision=4103 --- scripts/convert-from-entitydb-to-sql.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 43a51cf7..4e82783c 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -69,16 +69,23 @@ for table in tables: from django.conf import settings -settings.configure(DATABASES = { "default" : { - "ENGINE" : "django.db.backends.mysql", - "NAME" : sql_database, - "USER" : sql_username, - "PASSWORD" : sql_password, - "HOST" : "", - "PORT" : "" }}) +settings.configure( + DATABASES = { "default" : { + "ENGINE" : "django.db.backends.mysql", + "NAME" : sql_database, + "USER" : sql_username, + "PASSWORD" : sql_password, + "HOST" : "", + "PORT" : "" }}, + INSTALLED_APPS = ("rpki.irdb",), +) import rpki.irdb +import django.core.management + +django.core.management.call_command("syncdb", verbosity = 4) + def ns(tag): return "{http://www.hactrn.net/uris/rpki/myrpki/}" + tag -- cgit v1.2.3 From ce1e9d0cdaa5c189497de6aec51483e3c85e2f21 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Wed, 7 Dec 2011 22:28:24 +0000 Subject: Checkpoint. Conversion of some SQL tables works now. svn path=/branches/tk100/; revision=4104 --- scripts/convert-from-entitydb-to-sql.py | 72 ++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 9 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 4e82783c..09c5e973 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -50,6 +50,8 @@ sql_database = cfg.get("sql-database") sql_username = cfg.get("sql-username") sql_password = cfg.get("sql-password") +# Rename the old SQL tables, if they exist + db = MySQLdb.connect(user = sql_username, db = sql_database, passwd = sql_password) cur = db.cursor() @@ -67,6 +69,8 @@ for table in tables: print "Renaming %s to old_%s" % (table, table) cur.execute("ALTER TABLE %s RENAME TO old_%s" % (table, table)) +# Configure the Django model system + from django.conf import settings settings.configure( @@ -82,21 +86,71 @@ settings.configure( import rpki.irdb +# Create the model-based tables if they don't already exist + import django.core.management -django.core.management.call_command("syncdb", verbosity = 4) +django.core.management.call_command("syncdb", verbosity = 4, load_initial_data = False) -def ns(tag): - return "{http://www.hactrn.net/uris/rpki/myrpki/}" + tag +# From here down will be an awful lot of messing about with XML and +# X.509 data, extracting stuff from the old database and whacking it +# into the new. Still working out these bits. -e = ElementTree(file = os.path.join(entitydb, "identity.xml")).getroot() +xmlns = "{http://www.hactrn.net/uris/rpki/myrpki/}" -t = ns("identity") +tag_authorization = xmlns + "authorization" +tag_bpki_child_ta = xmlns + "bpki_child_ta" +tag_bpki_client_ta = xmlns + "bpki_client_ta" +tag_bpki_resource_ta = xmlns + "bpki_resource_ta" +tag_bpki_server_ta = xmlns + "bpki_server_ta" +tag_bpki_ta = xmlns + "bpki_ta" +tag_contact_info = xmlns + "contact_info" +tag_identity = xmlns + "identity" +tag_parent = xmlns + "parent" +tag_repository = xmlns + "repository" -if e.tag == t: - print "Found", t, "handle", e.get("handle") -else: - print "Didn't find", t, "found", e.tag, "instead, oops" +e = ElementTree(file = os.path.join(entitydb, "identity.xml")).getroot() +assert e.tag == tag_identity + +handle = e.get("handle") + +# Check handle against what's in rpki.conf? + +# Create identity if we haven't already + +identity = rpki.irdb.Identity.objects.get_or_create(handle = handle)[0] + +# Copy over any ROA requests + +cur.execute(""" + SELECT roa_request_id, asn FROM old_roa_request + WHERE roa_request_handle = %s + """, (handle,)) +for roa_request_id, asn in cur.fetchall(): + roa_request = rpki.irdb.ROARequest.objects.get_or_create(identity = identity, asn = asn)[0] + cur.execute(""" + SELECT prefix, prefixlen, max_prefixlen, version FROM old_roa_request_prefix + WHERE roa_request_id = %s + """, (roa_request_id,)) + for prefix, prefixlen, max_prefixlen, version in cur.fetchall(): + rpki.irdb.ROARequestPrefix.objects.get_or_create( + roa_request = roa_request, + version = version, + prefix = prefix, + prefixlen = prefixlen, + max_prefixlen = max_prefixlen) + +# Copy over any Ghostbuster requests. This doesn't handle +# Ghostbusters bound to specific parents yet, because I haven't yet +# written the code to copy parent objects from entitydb. + +cur.execute(""" + SELECT vcard FROM old_ghostbuster_request + WHERE self_handle = %s AND parent_handle IS NULL + """, (handle,)) +for row in cur.fetchall(): + rpki.irdb.GhostbusterRequest.objects.get_or_create(identity = identity, vcard = row[0], + defaults = { "parent" : None }) cur.close() db.close() -- cgit v1.2.3 From 61a8790f819698d446fad92678af6b73027f6bbf Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Fri, 9 Dec 2011 01:33:38 +0000 Subject: Rework models slightly. Now parsing most of the BPKI stuff. svn path=/branches/tk100/; revision=4109 --- scripts/convert-from-entitydb-to-sql.py | 145 ++++++++++++++++++++++++++++---- 1 file changed, 129 insertions(+), 16 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 09c5e973..b69e1ec2 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -21,8 +21,8 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -import sys, os, time, getopt -import rpki.config +import sys, os, time, getopt, glob, subprocess +import rpki.config, rpki.x509, rpki.relaxng from rpki.mysql_import import MySQLdb from lxml.etree import ElementTree @@ -31,24 +31,23 @@ if os.getlogin() != "sra": cfg_file = "rpki.conf" entitydb = "entitydb" +bpki = "bpki" -opts, argv = getopt.getopt(sys.argv[1:], "c:e:h?", ["config=", "entitydb=", "help"]) +opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"]) for o, a in opts: if o in ("-h", "--help", "-?"): print __doc__ sys.exit(0) if o in ("-c", "--config"): cfg_file = a - elif o in ("-e", "--entitydb"): - entitydb = a if argv: sys.exit("Unexpected arguments %s" % argv) -cfg = rpki.config.parser(cfg_file, "irdbd") +cfg = rpki.config.parser(cfg_file) -sql_database = cfg.get("sql-database") -sql_username = cfg.get("sql-username") -sql_password = cfg.get("sql-password") +sql_database = cfg.get("sql-database", section = "irdbd") +sql_username = cfg.get("sql-username", section = "irdbd") +sql_password = cfg.get("sql-password", section = "irdbd") # Rename the old SQL tables, if they exist @@ -80,7 +79,8 @@ settings.configure( "USER" : sql_username, "PASSWORD" : sql_password, "HOST" : "", - "PORT" : "" }}, + "PORT" : "", + "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }}}, INSTALLED_APPS = ("rpki.irdb",), ) @@ -93,8 +93,8 @@ import django.core.management django.core.management.call_command("syncdb", verbosity = 4, load_initial_data = False) # From here down will be an awful lot of messing about with XML and -# X.509 data, extracting stuff from the old database and whacking it -# into the new. Still working out these bits. +# X.509 data, extracting stuff from the old SQL database and whacking +# it into the new. Still working out these bits. xmlns = "{http://www.hactrn.net/uris/rpki/myrpki/}" @@ -110,16 +110,129 @@ tag_parent = xmlns + "parent" tag_repository = xmlns + "repository" e = ElementTree(file = os.path.join(entitydb, "identity.xml")).getroot() +rpki.relaxng.myrpki.assertValid(e) assert e.tag == tag_identity handle = e.get("handle") - -# Check handle against what's in rpki.conf? +assert handle == cfg.get("handle", section = "myrpki") # Create identity if we haven't already identity = rpki.irdb.Identity.objects.get_or_create(handle = handle)[0] +# Some BPKI utillity routines + +def read_openssl_serial(filename): + f = open(filename, "r") + text = f.read() + f.close() + return int(text.strip(), 16) + +def get_or_create_CA(purpose): + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, purpose, "ca.cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, purpose, "ca.key")) + crl = rpki.x509.CRL(Auto_file = os.path.join(bpki, purpose, "ca.crl")) + serial = read_openssl_serial(os.path.join(bpki, purpose, "serial")) + crl_number = read_openssl_serial(os.path.join(bpki, purpose, "crl_number")) + + return rpki.irdb.CA.objects.get_or_create(identity = identity, + purpose = rpki.irdb.CA.purpose_map[purpose], + certificate = cer.get_DER(), + private_key = key.get_DER(), + next_serial = serial, + next_crl_number = crl_number, + last_crl_update = crl.getThisUpdate().to_sql(), + next_crl_update = crl.getNextUpdate().to_sql())[0] + +def get_or_create_EECertificate(issuer, purpose): + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", purpose + ".cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", purpose + ".key")) + rpki.irdb.EECertificate.objects.get_or_create( + issuer = issuer, + purpose = rpki.irdb.EECertificate.purpose_map[purpose], + certificate = cer.get_DER(), + private_key = key.get_DER()) + +# Load BPKI CA data + +resource_ca = get_or_create_CA("resources") + +# Load BPKI server EE certificates and keys + +run_flags = dict((i, cfg.getboolean(i, section = "myrpki")) + for i in ("run_rpkid", "run_pubd", "run_rootd")) + +if any(run_flags.itervalues()): + server_ca = get_or_create_CA("servers") + get_or_create_EECertificate(server_ca, "irbe") + if run_flags["run_rpkid"]: + get_or_create_EECertificate(server_ca, "rpkid") + get_or_create_EECertificate(server_ca, "irdbd") + if run_flags["run_pubd"]: + get_or_create_EECertificate(server_ca, "pubd") + if run_flags["run_rootd"]: + get_or_create_EECertificate(server_ca, "rootd") +else: + server_ca = None + +# Load BSC certificates and requests + +for fn in glob.iglob(os.path.join(bpki, "resources", "bsc.*.cer")): + cer = rpki.x509.X509(Auto_file = fn) + req = rpki.x509.X509(Auto_file = fn[:-4] + ".req") + rpki.irdb.BSC.objects.get_or_create( + issuer = resource_ca, + certificate = cer.get_DER(), + pkcs10 = req.get_DER()) + + +def xcert_hash(cert): + """ + Generate the filename hash that myrpki would have generated for a + cross-certification. This is nasty, don't look. + """ + + cmd1 = ("openssl", "x509", "-noout", "-pubkey", "-subject") + cmd2 = ("openssl", "dgst", "-md5") + + env = { "PATH" : os.environ["PATH"], "OPENSSL_CONF" : "/dev/null" } + p1 = subprocess.Popen(cmd1, env = env, stdin = subprocess.PIPE, stdout = subprocess.PIPE) + p2 = subprocess.Popen(cmd2, env = env, stdin = p1.stdout, stdout = subprocess.PIPE) + p1.stdin.write(cert.get_PEM()) + p1.stdin.close() + hash = p2.stdout.read() + if p1.wait() != 0: + raise subprocess.CalledProcessError(returncode = p1.returncode, cmd = cmd1) + if p2.wait() != 0: + raise subprocess.CalledProcessError(returncode = p2.returncode, cmd = cmd2) + + hash = "".join(hash.split()) + if hash.startswith("(stdin)="): + hash = hash[len("(stdin)="):] + return hash + +# Build a table of all the cross-certified BPKI certificates. + +xcerts = {} + +for filename in glob.iglob(os.path.join("bpki", "*", "xcert.*.cer")): + h = filename.split(".")[-2] + + if not h in xcerts: + xcerts[h] = [] + xcerts[h].append(filename) + + # While we're at this, check to make sure that our reproduction of + # the hash algorithm is working correctly. + # + assert xcert_hash(rpki.x509.X509(Auto_file = filename)) == h + + + +# Somewhere around here I'm going to run out of things to do other +# than scraping through the horrible entitydb XML. Bother. + + # Copy over any ROA requests cur.execute(""" @@ -149,8 +262,8 @@ cur.execute(""" WHERE self_handle = %s AND parent_handle IS NULL """, (handle,)) for row in cur.fetchall(): - rpki.irdb.GhostbusterRequest.objects.get_or_create(identity = identity, vcard = row[0], - defaults = { "parent" : None }) + rpki.irdb.GhostbusterRequest.objects.get_or_create(identity = identity, parent = None, + vcard = row[0]) cur.close() db.close() -- cgit v1.2.3 From 0ea3bf33e59556949013ab9b7edc7e82dafd01cc Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Fri, 9 Dec 2011 05:33:11 +0000 Subject: First sort-of-complete version of conversion script. Only handles self-hosted case so far, needs minor tweaks to deal with hosting. Result looks vaguely sane but there's no code to test it yet. svn path=/branches/tk100/; revision=4111 --- scripts/convert-from-entitydb-to-sql.py | 216 +++++++++++++++++++++++++++----- 1 file changed, 187 insertions(+), 29 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index b69e1ec2..11fc4b98 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -21,8 +21,8 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -import sys, os, time, getopt, glob, subprocess -import rpki.config, rpki.x509, rpki.relaxng +import sys, os, time, getopt, glob, subprocess, base64 +import rpki.config, rpki.x509, rpki.relaxng, rpki.sundial from rpki.mysql_import import MySQLdb from lxml.etree import ElementTree @@ -113,12 +113,12 @@ e = ElementTree(file = os.path.join(entitydb, "identity.xml")).getroot() rpki.relaxng.myrpki.assertValid(e) assert e.tag == tag_identity -handle = e.get("handle") -assert handle == cfg.get("handle", section = "myrpki") +self_handle = e.get("handle") +assert self_handle == cfg.get("handle", section = "myrpki") # Create identity if we haven't already -identity = rpki.irdb.Identity.objects.get_or_create(handle = handle)[0] +identity = rpki.irdb.Identity.objects.get_or_create(handle = self_handle)[0] # Some BPKI utillity routines @@ -211,34 +211,184 @@ def xcert_hash(cert): hash = hash[len("(stdin)="):] return hash -# Build a table of all the cross-certified BPKI certificates. - -xcerts = {} - -for filename in glob.iglob(os.path.join("bpki", "*", "xcert.*.cer")): - h = filename.split(".")[-2] - - if not h in xcerts: - xcerts[h] = [] - xcerts[h].append(filename) - - # While we're at this, check to make sure that our reproduction of - # the hash algorithm is working correctly. - # - assert xcert_hash(rpki.x509.X509(Auto_file = filename)) == h +# OK, all this wretched cross-certification looks complicated, but +# that's partly because of the way we've been doing it on disk. The +# new SQL/object based approach should make it much clearer: +# +# Child cross certifies parent's resource TA in child's resource CA. +# +# Parent cross certifies child's resource TA in parent's resource +# CA. +# +# Repository cross certifies client's resource TA in repository's +# server CA. +# +# Client cross certifies repository's server TA in client's resource +# CA. +# +# The remaining xcert files look to be TLS relics which no longer +# serve any real purpose; in theory, those can just go away. + +# Let's try keeping track of all the xcert filenames we use, so we can +# list the ones we didn't. + +xcert_filenames = set(glob.iglob(os.path.join(bpki, "*", "xcert.*.cer"))) + +# Scrape child data out of the entitydb. + +for filename in glob.iglob(os.path.join(entitydb, "children", "*.xml")): + child_handle = os.path.splitext(os.path.split(filename)[1])[0] + + e = ElementTree(file = filename).getroot() + rpki.relaxng.myrpki.assertValid(e) + assert e.tag == tag_parent + + ta = rpki.x509.X509(Base64 = e.findtext(tag_bpki_child_ta)) + xcfn = os.path.join(bpki, "resources", "xcert.%s.cer" % xcert_hash(ta)) + xcert_filenames.discard(xcfn) + xcert = rpki.x509.X509(Auto_file = xcfn) + cur.execute(""" + SELECT registrant_id, valid_until FROM old_registrant + WHERE registry_handle = %s AND registrant_handle = %s + """, (self_handle, child_handle)) + assert cur.rowcount == 1 + registrant_id, valid_until = cur.fetchone() + + valid_until = rpki.sundial.datetime.fromdatetime(valid_until) + assert valid_until == rpki.sundial.datetime.fromXMLtime(e.get("valid_until")) + + child = rpki.irdb.Child.objects.get_or_create( + handle = child_handle, + valid_until = valid_until.to_sql(), + ta = ta.get_DER(), + certificate = xcert.get_DER(), + issuer = resource_ca)[0] + cur.execute(""" + SELECT start_as, end_as FROM old_registrant_asn WHERE registrant_id = %s + """, (registrant_id,)) + for start_as, end_as in cur.fetchall(): + rpki.irdb.ChildASN.objects.get_or_create( + start_as = start_as, + end_as = end_as, + child = child) -# Somewhere around here I'm going to run out of things to do other -# than scraping through the horrible entitydb XML. Bother. + cur.execute(""" + SELECT start_ip, end_ip, version FROM old_registrant_net WHERE registrant_id = %s + """, (registrant_id,)) + for start_ip, end_ip, version in cur.fetchall(): + rpki.irdb.ChildNet.objects.get_or_create( + start_ip = start_ip, + end_ip = end_ip, + version = version, + child = child) + +# Scrape parent data out of the entitydb. + +for filename in glob.iglob(os.path.join(entitydb, "parents", "*.xml")): + parent_handle = os.path.splitext(os.path.split(filename)[1])[0] + + e = ElementTree(file = filename).getroot() + rpki.relaxng.myrpki.assertValid(e) + assert e.tag == tag_parent + + ta = rpki.x509.X509(Base64 = e.findtext(tag_bpki_resource_ta)) + xcfn = os.path.join(bpki, "resources", "xcert.%s.cer" % xcert_hash(ta)) + xcert_filenames.discard(xcfn) + xcert = rpki.x509.X509(Auto_file = xcfn) + + r = e.find(tag_repository) + repository_type = r.get("type") + if repository_type == "referral": + a = r.find(tag_authorization) + referrer = a.get("referrer") + referral_authorization = base64.b64decode(a.text) + else: + referrer = None + referral_authorization = None + + parent = rpki.irdb.Parent.objects.get_or_create( + handle = parent_handle, + parent_handle = e.get("parent_handle"), + child_handle = e.get("child_handle"), + ta = ta.get_DER(), + certificate = xcert.get_DER(), + repository_type = rpki.irdb.Parent.repository_type_map[repository_type], + referrer = referrer, + referral_authorization = referral_authorization, + issuer = resource_ca)[0] + + # While we have the parent object in hand, load any Ghostbuster + # entries specific to this parent. + cur.execute(""" + SELECT vcard FROM old_ghostbuster_request + WHERE self_handle = %s AND parent_handle = %s + """, (self_handle, parent_handle)) + for row in cur.fetchall(): + rpki.irdb.GhostbusterRequest.objects.get_or_create( + identity = identity, + parent = parent, + vcard = row[0]) + +# Scrape repository data out of the entitydb. + +for filename in glob.iglob(os.path.join(entitydb, "repositories", "*.xml")): + repository_handle = os.path.splitext(os.path.split(filename)[1])[0] + + e = ElementTree(file = filename).getroot() + rpki.relaxng.myrpki.assertValid(e) + assert e.tag == tag_repository + + if e.get("type") != "confirmed": + continue + + ta = rpki.x509.X509(Base64 = e.findtext(tag_bpki_server_ta)) + xcfn = os.path.join(bpki, "resources", "xcert.%s.cer" % xcert_hash(ta)) + xcert_filenames.discard(xcfn) + xcert = rpki.x509.X509(Auto_file = xcfn) + + parent = rpki.irdb.Parent.objects.get(handle = e.get("parent_handle")) + + rpki.irdb.Repository.objects.get_or_create( + handle = repository_handle, + client_handle = e.get("client_handle"), + ta = ta.get_DER(), + certificate = xcert.get_DER(), + service_uri = e.get("service_uri"), + sia_base = e.get("sia_base"), + parent = parent, + issuer = resource_ca) + +# Scrape client data out of the entitydb. + +for filename in glob.iglob(os.path.join(entitydb, "pubclients", "*.xml")): + client_handle = os.path.splitext(os.path.split(filename)[1])[0] + + e = ElementTree(file = filename).getroot() + rpki.relaxng.myrpki.assertValid(e) + assert e.tag == tag_repository + + assert e.get("type") == "confirmed" + + ta = rpki.x509.X509(Base64 = e.findtext(tag_bpki_client_ta)) + xcfn = os.path.join(bpki, "servers", "xcert.%s.cer" % xcert_hash(ta)) + xcert_filenames.discard(xcfn) + xcert = rpki.x509.X509(Auto_file = xcfn) + + rpki.irdb.Repository.objects.get_or_create( + handle = client_handle, + ta = ta.get_DER(), + certificate = xcert.get_DER(), + issuer = server_ca) # Copy over any ROA requests cur.execute(""" SELECT roa_request_id, asn FROM old_roa_request WHERE roa_request_handle = %s - """, (handle,)) + """, (self_handle,)) for roa_request_id, asn in cur.fetchall(): roa_request = rpki.irdb.ROARequest.objects.get_or_create(identity = identity, asn = asn)[0] cur.execute(""" @@ -253,17 +403,25 @@ for roa_request_id, asn in cur.fetchall(): prefixlen = prefixlen, max_prefixlen = max_prefixlen) -# Copy over any Ghostbuster requests. This doesn't handle -# Ghostbusters bound to specific parents yet, because I haven't yet -# written the code to copy parent objects from entitydb. +# Copy over any non-parent-specific Ghostbuster requests. cur.execute(""" SELECT vcard FROM old_ghostbuster_request WHERE self_handle = %s AND parent_handle IS NULL - """, (handle,)) + """, (self_handle,)) for row in cur.fetchall(): - rpki.irdb.GhostbusterRequest.objects.get_or_create(identity = identity, parent = None, - vcard = row[0]) + rpki.irdb.GhostbusterRequest.objects.get_or_create( + identity = identity, + parent = None, + vcard = row[0]) + +# List cross certifications we didn't use. + +for filename in sorted(xcert_filenames): + cer = rpki.x509.X509(Auto_file = filename) + print "Unused cross-certificate:", filename, cer.getSubject() + +# Done! cur.close() db.close() -- cgit v1.2.3 From 67a6ef6a1a7e741c5ab2c5b53ad119cb1c5880d3 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sat, 10 Dec 2011 05:31:35 +0000 Subject: Tighten uniqueness constraints on IRDB models. svn path=/branches/tk100/; revision=4114 --- scripts/convert-from-entitydb-to-sql.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 11fc4b98..9de2edf2 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -49,25 +49,9 @@ sql_database = cfg.get("sql-database", section = "irdbd") sql_username = cfg.get("sql-username", section = "irdbd") sql_password = cfg.get("sql-password", section = "irdbd") -# Rename the old SQL tables, if they exist - db = MySQLdb.connect(user = sql_username, db = sql_database, passwd = sql_password) cur = db.cursor() -cur.execute("SHOW TABLES") - -tables = [r[0] for r in cur.fetchall()] - -for table in tables: - if "old_" + table not in tables and table in ("registrant", - "registrant_asn", - "registrant_net", - "roa_request", - "roa_request_prefix", - "ghostbuster_request"): - print "Renaming %s to old_%s" % (table, table) - cur.execute("ALTER TABLE %s RENAME TO old_%s" % (table, table)) - # Configure the Django model system from django.conf import settings @@ -249,7 +233,7 @@ for filename in glob.iglob(os.path.join(entitydb, "children", "*.xml")): xcert = rpki.x509.X509(Auto_file = xcfn) cur.execute(""" - SELECT registrant_id, valid_until FROM old_registrant + SELECT registrant_id, valid_until FROM registrant WHERE registry_handle = %s AND registrant_handle = %s """, (self_handle, child_handle)) assert cur.rowcount == 1 @@ -266,7 +250,7 @@ for filename in glob.iglob(os.path.join(entitydb, "children", "*.xml")): issuer = resource_ca)[0] cur.execute(""" - SELECT start_as, end_as FROM old_registrant_asn WHERE registrant_id = %s + SELECT start_as, end_as FROM registrant_asn WHERE registrant_id = %s """, (registrant_id,)) for start_as, end_as in cur.fetchall(): rpki.irdb.ChildASN.objects.get_or_create( @@ -275,7 +259,7 @@ for filename in glob.iglob(os.path.join(entitydb, "children", "*.xml")): child = child) cur.execute(""" - SELECT start_ip, end_ip, version FROM old_registrant_net WHERE registrant_id = %s + SELECT start_ip, end_ip, version FROM registrant_net WHERE registrant_id = %s """, (registrant_id,)) for start_ip, end_ip, version in cur.fetchall(): rpki.irdb.ChildNet.objects.get_or_create( @@ -323,7 +307,7 @@ for filename in glob.iglob(os.path.join(entitydb, "parents", "*.xml")): # entries specific to this parent. cur.execute(""" - SELECT vcard FROM old_ghostbuster_request + SELECT vcard FROM ghostbuster_request WHERE self_handle = %s AND parent_handle = %s """, (self_handle, parent_handle)) for row in cur.fetchall(): @@ -386,13 +370,13 @@ for filename in glob.iglob(os.path.join(entitydb, "pubclients", "*.xml")): # Copy over any ROA requests cur.execute(""" - SELECT roa_request_id, asn FROM old_roa_request + SELECT roa_request_id, asn FROM roa_request WHERE roa_request_handle = %s """, (self_handle,)) for roa_request_id, asn in cur.fetchall(): roa_request = rpki.irdb.ROARequest.objects.get_or_create(identity = identity, asn = asn)[0] cur.execute(""" - SELECT prefix, prefixlen, max_prefixlen, version FROM old_roa_request_prefix + SELECT prefix, prefixlen, max_prefixlen, version FROM roa_request_prefix WHERE roa_request_id = %s """, (roa_request_id,)) for prefix, prefixlen, max_prefixlen, version in cur.fetchall(): @@ -406,7 +390,7 @@ for roa_request_id, asn in cur.fetchall(): # Copy over any non-parent-specific Ghostbuster requests. cur.execute(""" - SELECT vcard FROM old_ghostbuster_request + SELECT vcard FROM ghostbuster_request WHERE self_handle = %s AND parent_handle IS NULL """, (self_handle,)) for row in cur.fetchall(): -- cgit v1.2.3 From 9151b67621f017e63a046b872dee1008dff6da5a Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Mon, 12 Dec 2011 07:16:26 +0000 Subject: Checkpoint. Add X501DN abstraction, start sorting out BPKI generation code. svn path=/branches/tk100/; revision=4116 --- scripts/convert-from-entitydb-to-sql.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 9de2edf2..c8a85620 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -113,33 +113,30 @@ def read_openssl_serial(filename): return int(text.strip(), 16) def get_or_create_CA(purpose): - cer = rpki.x509.X509(Auto_file = os.path.join(bpki, purpose, "ca.cer")) - key = rpki.x509.RSA(Auto_file = os.path.join(bpki, purpose, "ca.key")) crl = rpki.x509.CRL(Auto_file = os.path.join(bpki, purpose, "ca.crl")) serial = read_openssl_serial(os.path.join(bpki, purpose, "serial")) crl_number = read_openssl_serial(os.path.join(bpki, purpose, "crl_number")) return rpki.irdb.CA.objects.get_or_create(identity = identity, purpose = rpki.irdb.CA.purpose_map[purpose], - certificate = cer.get_DER(), - private_key = key.get_DER(), next_serial = serial, next_crl_number = crl_number, last_crl_update = crl.getThisUpdate().to_sql(), next_crl_update = crl.getNextUpdate().to_sql())[0] -def get_or_create_EECertificate(issuer, purpose): +def get_or_create_KeyedCertificate(issuer, purpose): cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", purpose + ".cer")) key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", purpose + ".key")) - rpki.irdb.EECertificate.objects.get_or_create( + rpki.irdb.KeyedCertificate.objects.get_or_create( issuer = issuer, - purpose = rpki.irdb.EECertificate.purpose_map[purpose], + purpose = rpki.irdb.KeyedCertificate.purpose_map[purpose], certificate = cer.get_DER(), private_key = key.get_DER()) # Load BPKI CA data resource_ca = get_or_create_CA("resources") +get_or_create_KeyedCertificate(resource_ca, "ca") # Load BPKI server EE certificates and keys @@ -148,14 +145,15 @@ run_flags = dict((i, cfg.getboolean(i, section = "myrpki")) if any(run_flags.itervalues()): server_ca = get_or_create_CA("servers") - get_or_create_EECertificate(server_ca, "irbe") + get_or_create_KeyedCertificate(server_ca, "ca") + get_or_create_KeyedCertificate(server_ca, "irbe") if run_flags["run_rpkid"]: - get_or_create_EECertificate(server_ca, "rpkid") - get_or_create_EECertificate(server_ca, "irdbd") + get_or_create_KeyedCertificate(server_ca, "rpkid") + get_or_create_KeyedCertificate(server_ca, "irdbd") if run_flags["run_pubd"]: - get_or_create_EECertificate(server_ca, "pubd") + get_or_create_KeyedCertificate(server_ca, "pubd") if run_flags["run_rootd"]: - get_or_create_EECertificate(server_ca, "rootd") + get_or_create_KeyedCertificate(server_ca, "rootd") else: server_ca = None -- cgit v1.2.3 From 63676e02d7e58487cb0794659de6602168e36e90 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Mon, 12 Dec 2011 21:07:56 +0000 Subject: Checkpoint. Custom IRDB model fields to handle automatic type conversion. Debug last night's rewrite of BPKI certificate issuance. svn path=/branches/tk100/; revision=4117 --- scripts/convert-from-entitydb-to-sql.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index c8a85620..8885893b 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -130,8 +130,8 @@ def get_or_create_KeyedCertificate(issuer, purpose): rpki.irdb.KeyedCertificate.objects.get_or_create( issuer = issuer, purpose = rpki.irdb.KeyedCertificate.purpose_map[purpose], - certificate = cer.get_DER(), - private_key = key.get_DER()) + certificate = cer, + private_key = key) # Load BPKI CA data @@ -157,16 +157,15 @@ if any(run_flags.itervalues()): else: server_ca = None -# Load BSC certificates and requests +# Load BSC certificates and requests. Yes, this currently wires in +# exactly one BSC handle, "bsc". So does the old myrpki code. Ick. for fn in glob.iglob(os.path.join(bpki, "resources", "bsc.*.cer")): - cer = rpki.x509.X509(Auto_file = fn) - req = rpki.x509.X509(Auto_file = fn[:-4] + ".req") rpki.irdb.BSC.objects.get_or_create( issuer = resource_ca, - certificate = cer.get_DER(), - pkcs10 = req.get_DER()) - + handle = "bsc", + certificate = rpki.x509.X509(Auto_file = fn), + pkcs10 = rpki.x509.PKCS10(Auto_file = fn[:-4] + ".req")) def xcert_hash(cert): """ @@ -243,8 +242,8 @@ for filename in glob.iglob(os.path.join(entitydb, "children", "*.xml")): child = rpki.irdb.Child.objects.get_or_create( handle = child_handle, valid_until = valid_until.to_sql(), - ta = ta.get_DER(), - certificate = xcert.get_DER(), + ta = ta, + certificate = xcert, issuer = resource_ca)[0] cur.execute(""" @@ -294,8 +293,8 @@ for filename in glob.iglob(os.path.join(entitydb, "parents", "*.xml")): handle = parent_handle, parent_handle = e.get("parent_handle"), child_handle = e.get("child_handle"), - ta = ta.get_DER(), - certificate = xcert.get_DER(), + ta = ta, + certificate = xcert, repository_type = rpki.irdb.Parent.repository_type_map[repository_type], referrer = referrer, referral_authorization = referral_authorization, @@ -336,8 +335,8 @@ for filename in glob.iglob(os.path.join(entitydb, "repositories", "*.xml")): rpki.irdb.Repository.objects.get_or_create( handle = repository_handle, client_handle = e.get("client_handle"), - ta = ta.get_DER(), - certificate = xcert.get_DER(), + ta = ta, + certificate = xcert, service_uri = e.get("service_uri"), sia_base = e.get("sia_base"), parent = parent, @@ -361,8 +360,8 @@ for filename in glob.iglob(os.path.join(entitydb, "pubclients", "*.xml")): rpki.irdb.Repository.objects.get_or_create( handle = client_handle, - ta = ta.get_DER(), - certificate = xcert.get_DER(), + ta = ta, + certificate = xcert, issuer = server_ca) # Copy over any ROA requests -- cgit v1.2.3 From 266a24c2d42c6e064561e39aef3bd40a674e61cc Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Tue, 13 Dec 2011 00:17:41 +0000 Subject: Flesh out BPKI methods svn path=/branches/tk100/; revision=4118 --- scripts/convert-from-entitydb-to-sql.py | 38 ++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 17 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 8885893b..1ab5201d 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -113,30 +113,35 @@ def read_openssl_serial(filename): return int(text.strip(), 16) def get_or_create_CA(purpose): + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, purpose, "ca.cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, purpose, "ca.key")) crl = rpki.x509.CRL(Auto_file = os.path.join(bpki, purpose, "ca.crl")) serial = read_openssl_serial(os.path.join(bpki, purpose, "serial")) crl_number = read_openssl_serial(os.path.join(bpki, purpose, "crl_number")) - return rpki.irdb.CA.objects.get_or_create(identity = identity, - purpose = rpki.irdb.CA.purpose_map[purpose], - next_serial = serial, - next_crl_number = crl_number, - last_crl_update = crl.getThisUpdate().to_sql(), - next_crl_update = crl.getNextUpdate().to_sql())[0] - -def get_or_create_KeyedCertificate(issuer, purpose): + return rpki.irdb.CA.objects.get_or_create( + identity = identity, + purpose = rpki.irdb.CA.purpose_map[purpose], + certificate = cer, + private_key = key, + latest_crl = crl, + next_serial = serial, + next_crl_number = crl_number, + last_crl_update = crl.getThisUpdate().to_sql(), + next_crl_update = crl.getNextUpdate().to_sql())[0] + +def get_or_create_EECertificate(issuer, purpose): cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", purpose + ".cer")) key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", purpose + ".key")) - rpki.irdb.KeyedCertificate.objects.get_or_create( + rpki.irdb.EECertificate.objects.get_or_create( issuer = issuer, - purpose = rpki.irdb.KeyedCertificate.purpose_map[purpose], + purpose = rpki.irdb.EECertificate.purpose_map[purpose], certificate = cer, private_key = key) # Load BPKI CA data resource_ca = get_or_create_CA("resources") -get_or_create_KeyedCertificate(resource_ca, "ca") # Load BPKI server EE certificates and keys @@ -145,15 +150,14 @@ run_flags = dict((i, cfg.getboolean(i, section = "myrpki")) if any(run_flags.itervalues()): server_ca = get_or_create_CA("servers") - get_or_create_KeyedCertificate(server_ca, "ca") - get_or_create_KeyedCertificate(server_ca, "irbe") + get_or_create_EECertificate(server_ca, "irbe") if run_flags["run_rpkid"]: - get_or_create_KeyedCertificate(server_ca, "rpkid") - get_or_create_KeyedCertificate(server_ca, "irdbd") + get_or_create_EECertificate(server_ca, "rpkid") + get_or_create_EECertificate(server_ca, "irdbd") if run_flags["run_pubd"]: - get_or_create_KeyedCertificate(server_ca, "pubd") + get_or_create_EECertificate(server_ca, "pubd") if run_flags["run_rootd"]: - get_or_create_KeyedCertificate(server_ca, "rootd") + get_or_create_EECertificate(server_ca, "rootd") else: server_ca = None -- cgit v1.2.3 From fc2941d92286ca1e01400c3a46abbd88b411f6f8 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Wed, 14 Dec 2011 04:01:52 +0000 Subject: Start hacking replacement for myrpki comamnd (rpkic) which uses the new Django-model-based entitydb and IRDB. svn path=/branches/tk100/; revision=4121 --- scripts/convert-from-entitydb-to-sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 1ab5201d..1fb1bbea 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -404,7 +404,8 @@ for row in cur.fetchall(): for filename in sorted(xcert_filenames): cer = rpki.x509.X509(Auto_file = filename) - print "Unused cross-certificate:", filename, cer.getSubject() + #print "Unused cross-certificate:", filename, cer.getSubject() + print "Unused cross-certificate:", filename, cer.get_POW().pprint() # Done! -- cgit v1.2.3 From 30923f4cb2ae26aca367c01fab6ead0b59e59db9 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Thu, 15 Dec 2011 15:19:13 +0000 Subject: Checkpoint. Add rpki.irdb.models.CertificateManager() to consolidate BPKI object creation logic. Move CSV code out of rpkic. svn path=/branches/tk100/; revision=4122 --- scripts/convert-from-entitydb-to-sql.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 1fb1bbea..bea12e84 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -130,18 +130,20 @@ def get_or_create_CA(purpose): last_crl_update = crl.getThisUpdate().to_sql(), next_crl_update = crl.getNextUpdate().to_sql())[0] -def get_or_create_EECertificate(issuer, purpose): - cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", purpose + ".cer")) - key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", purpose + ".key")) +def get_or_create_EECertificate(issuer, capurpose, eepurpose): + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, capurpose, eepurpose + ".cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, capurpose, eepurpose + ".key")) rpki.irdb.EECertificate.objects.get_or_create( issuer = issuer, - purpose = rpki.irdb.EECertificate.purpose_map[purpose], + purpose = rpki.irdb.EECertificate.purpose_map[eepurpose], certificate = cer, private_key = key) # Load BPKI CA data resource_ca = get_or_create_CA("resources") +if os.path.exists(os.path.join(bpki, "resources", "referral.cer")): + get_or_create_EECertificate(resource_ca, "resources", "referral") # Load BPKI server EE certificates and keys @@ -150,14 +152,14 @@ run_flags = dict((i, cfg.getboolean(i, section = "myrpki")) if any(run_flags.itervalues()): server_ca = get_or_create_CA("servers") - get_or_create_EECertificate(server_ca, "irbe") + get_or_create_EECertificate(server_ca, "servers", "irbe") if run_flags["run_rpkid"]: - get_or_create_EECertificate(server_ca, "rpkid") - get_or_create_EECertificate(server_ca, "irdbd") + get_or_create_EECertificate(server_ca, "servers", "rpkid") + get_or_create_EECertificate(server_ca, "servers", "irdbd") if run_flags["run_pubd"]: - get_or_create_EECertificate(server_ca, "pubd") + get_or_create_EECertificate(server_ca, "servers", "pubd") if run_flags["run_rootd"]: - get_or_create_EECertificate(server_ca, "rootd") + get_or_create_EECertificate(server_ca, "servers", "rootd") else: server_ca = None -- cgit v1.2.3 From bbde00990b0aae93d4b3d9fac8d163f66eca0c43 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Fri, 16 Dec 2011 02:44:20 +0000 Subject: Checkpoint. Add EnumField. Debug CertificateManager. svn path=/branches/tk100/; revision=4124 --- scripts/convert-from-entitydb-to-sql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index bea12e84..d0a080e9 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -121,7 +121,7 @@ def get_or_create_CA(purpose): return rpki.irdb.CA.objects.get_or_create( identity = identity, - purpose = rpki.irdb.CA.purpose_map[purpose], + purpose = purpose, certificate = cer, private_key = key, latest_crl = crl, @@ -135,7 +135,7 @@ def get_or_create_EECertificate(issuer, capurpose, eepurpose): key = rpki.x509.RSA(Auto_file = os.path.join(bpki, capurpose, eepurpose + ".key")) rpki.irdb.EECertificate.objects.get_or_create( issuer = issuer, - purpose = rpki.irdb.EECertificate.purpose_map[eepurpose], + purpose = eepurpose, certificate = cer, private_key = key) @@ -301,7 +301,7 @@ for filename in glob.iglob(os.path.join(entitydb, "parents", "*.xml")): child_handle = e.get("child_handle"), ta = ta, certificate = xcert, - repository_type = rpki.irdb.Parent.repository_type_map[repository_type], + repository_type = repository_type, referrer = referrer, referral_authorization = referral_authorization, issuer = resource_ca)[0] -- cgit v1.2.3 From eccdce39b0332f002344ba9c4dc2db3b05c3a4cd Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Fri, 16 Dec 2011 22:43:07 +0000 Subject: Checkpoint. Add synchronize_prefixes and synchronize_asns commands. svn path=/branches/tk100/; revision=4125 --- scripts/convert-from-entitydb-to-sql.py | 124 +++++++++++++++++--------------- 1 file changed, 65 insertions(+), 59 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index d0a080e9..1fae02c4 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -29,9 +29,10 @@ from lxml.etree import ElementTree if os.getlogin() != "sra": sys.exit("I //said// this was a work in progress") -cfg_file = "rpki.conf" -entitydb = "entitydb" -bpki = "bpki" +cfg_file = "rpki.conf" +entitydb = "entitydb" +bpki = "bpki" +copy_csv_data = True opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"]) for o, a in opts: @@ -252,24 +253,26 @@ for filename in glob.iglob(os.path.join(entitydb, "children", "*.xml")): certificate = xcert, issuer = resource_ca)[0] - cur.execute(""" - SELECT start_as, end_as FROM registrant_asn WHERE registrant_id = %s - """, (registrant_id,)) - for start_as, end_as in cur.fetchall(): - rpki.irdb.ChildASN.objects.get_or_create( - start_as = start_as, - end_as = end_as, - child = child) - - cur.execute(""" - SELECT start_ip, end_ip, version FROM registrant_net WHERE registrant_id = %s - """, (registrant_id,)) - for start_ip, end_ip, version in cur.fetchall(): - rpki.irdb.ChildNet.objects.get_or_create( - start_ip = start_ip, - end_ip = end_ip, - version = version, - child = child) + if copy_csv_data: + + cur.execute(""" + SELECT start_as, end_as FROM registrant_asn WHERE registrant_id = %s + """, (registrant_id,)) + for start_as, end_as in cur.fetchall(): + rpki.irdb.ChildASN.objects.get_or_create( + start_as = start_as, + end_as = end_as, + child = child) + + cur.execute(""" + SELECT start_ip, end_ip, version FROM registrant_net WHERE registrant_id = %s + """, (registrant_id,)) + for start_ip, end_ip, version in cur.fetchall(): + rpki.irdb.ChildNet.objects.get_or_create( + start_ip = start_ip, + end_ip = end_ip, + version = version, + child = child) # Scrape parent data out of the entitydb. @@ -309,15 +312,16 @@ for filename in glob.iglob(os.path.join(entitydb, "parents", "*.xml")): # While we have the parent object in hand, load any Ghostbuster # entries specific to this parent. - cur.execute(""" - SELECT vcard FROM ghostbuster_request - WHERE self_handle = %s AND parent_handle = %s - """, (self_handle, parent_handle)) - for row in cur.fetchall(): - rpki.irdb.GhostbusterRequest.objects.get_or_create( - identity = identity, - parent = parent, - vcard = row[0]) + if copy_csv_data: + cur.execute(""" + SELECT vcard FROM ghostbuster_request + WHERE self_handle = %s AND parent_handle = %s + """, (self_handle, parent_handle)) + for row in cur.fetchall(): + rpki.irdb.GhostbusterRequest.objects.get_or_create( + identity = identity, + parent = parent, + vcard = row[0]) # Scrape repository data out of the entitydb. @@ -370,37 +374,39 @@ for filename in glob.iglob(os.path.join(entitydb, "pubclients", "*.xml")): certificate = xcert, issuer = server_ca) -# Copy over any ROA requests +if copy_csv_data: + + # Copy over any ROA requests -cur.execute(""" - SELECT roa_request_id, asn FROM roa_request - WHERE roa_request_handle = %s - """, (self_handle,)) -for roa_request_id, asn in cur.fetchall(): - roa_request = rpki.irdb.ROARequest.objects.get_or_create(identity = identity, asn = asn)[0] cur.execute(""" - SELECT prefix, prefixlen, max_prefixlen, version FROM roa_request_prefix - WHERE roa_request_id = %s - """, (roa_request_id,)) - for prefix, prefixlen, max_prefixlen, version in cur.fetchall(): - rpki.irdb.ROARequestPrefix.objects.get_or_create( - roa_request = roa_request, - version = version, - prefix = prefix, - prefixlen = prefixlen, - max_prefixlen = max_prefixlen) - -# Copy over any non-parent-specific Ghostbuster requests. - -cur.execute(""" - SELECT vcard FROM ghostbuster_request - WHERE self_handle = %s AND parent_handle IS NULL - """, (self_handle,)) -for row in cur.fetchall(): - rpki.irdb.GhostbusterRequest.objects.get_or_create( - identity = identity, - parent = None, - vcard = row[0]) + SELECT roa_request_id, asn FROM roa_request + WHERE roa_request_handle = %s + """, (self_handle,)) + for roa_request_id, asn in cur.fetchall(): + roa_request = rpki.irdb.ROARequest.objects.get_or_create(identity = identity, asn = asn)[0] + cur.execute(""" + SELECT prefix, prefixlen, max_prefixlen, version FROM roa_request_prefix + WHERE roa_request_id = %s + """, (roa_request_id,)) + for prefix, prefixlen, max_prefixlen, version in cur.fetchall(): + rpki.irdb.ROARequestPrefix.objects.get_or_create( + roa_request = roa_request, + version = version, + prefix = prefix, + prefixlen = prefixlen, + max_prefixlen = max_prefixlen) + + # Copy over any non-parent-specific Ghostbuster requests. + + cur.execute(""" + SELECT vcard FROM ghostbuster_request + WHERE self_handle = %s AND parent_handle IS NULL + """, (self_handle,)) + for row in cur.fetchall(): + rpki.irdb.GhostbusterRequest.objects.get_or_create( + identity = identity, + parent = None, + vcard = row[0]) # List cross certifications we didn't use. -- cgit v1.2.3 From b87cc14f975ed5cf1e0b34d3a8e30d49ca1a4632 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Tue, 20 Dec 2011 19:00:07 +0000 Subject: Checkpoint. More schema tweaks (HostedCA model). svn path=/branches/tk100/; revision=4129 --- scripts/convert-from-entitydb-to-sql.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 1fae02c4..aa15b461 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -101,10 +101,6 @@ assert e.tag == tag_identity self_handle = e.get("handle") assert self_handle == cfg.get("handle", section = "myrpki") -# Create identity if we haven't already - -identity = rpki.irdb.Identity.objects.get_or_create(handle = self_handle)[0] - # Some BPKI utillity routines def read_openssl_serial(filename): @@ -121,8 +117,7 @@ def get_or_create_CA(purpose): crl_number = read_openssl_serial(os.path.join(bpki, purpose, "crl_number")) return rpki.irdb.CA.objects.get_or_create( - identity = identity, - purpose = purpose, + handle = self_handle if purpose == "resources" else "", certificate = cer, private_key = key, latest_crl = crl, @@ -319,7 +314,7 @@ for filename in glob.iglob(os.path.join(entitydb, "parents", "*.xml")): """, (self_handle, parent_handle)) for row in cur.fetchall(): rpki.irdb.GhostbusterRequest.objects.get_or_create( - identity = identity, + issuer = resource_ca, parent = parent, vcard = row[0]) @@ -383,7 +378,7 @@ if copy_csv_data: WHERE roa_request_handle = %s """, (self_handle,)) for roa_request_id, asn in cur.fetchall(): - roa_request = rpki.irdb.ROARequest.objects.get_or_create(identity = identity, asn = asn)[0] + roa_request = rpki.irdb.ROARequest.objects.get_or_create(issuer = resource_ca, asn = asn)[0] cur.execute(""" SELECT prefix, prefixlen, max_prefixlen, version FROM roa_request_prefix WHERE roa_request_id = %s @@ -404,7 +399,7 @@ if copy_csv_data: """, (self_handle,)) for row in cur.fetchall(): rpki.irdb.GhostbusterRequest.objects.get_or_create( - identity = identity, + issuer = resource_ca, parent = None, vcard = row[0]) -- cgit v1.2.3 From 92eea1a9220231354236bc1c323060b40a3708aa Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Wed, 21 Dec 2011 07:22:38 +0000 Subject: Checkpoint. First pass at rewriting daemon synchronization code for new model. svn path=/branches/tk100/; revision=4130 --- scripts/convert-from-entitydb-to-sql.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index aa15b461..8c1e6bbb 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -363,11 +363,12 @@ for filename in glob.iglob(os.path.join(entitydb, "pubclients", "*.xml")): xcert_filenames.discard(xcfn) xcert = rpki.x509.X509(Auto_file = xcfn) - rpki.irdb.Repository.objects.get_or_create( + rpki.irdb.Client.objects.get_or_create( handle = client_handle, ta = ta, certificate = xcert, - issuer = server_ca) + issuer = server_ca, + sia_base = e.get("sia_base")) if copy_csv_data: -- cgit v1.2.3 From 081d4284a989485236514ff80c9cce6676f35102 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Fri, 23 Dec 2011 01:48:08 +0000 Subject: Checkpoint. Temporary hack around CA model database collision problem; needs b etter fix, but want this fix in version control. svn path=/branches/tk100/; revision=4132 --- scripts/convert-from-entitydb-to-sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 8c1e6bbb..dbdde34c 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -117,7 +117,7 @@ def get_or_create_CA(purpose): crl_number = read_openssl_serial(os.path.join(bpki, purpose, "crl_number")) return rpki.irdb.CA.objects.get_or_create( - handle = self_handle if purpose == "resources" else "", + handle = self_handle if purpose == "resources" else "*", certificate = cer, private_key = key, latest_crl = crl, -- cgit v1.2.3 From 75c16c86b64dc47bc8559946d4e133586b9a2919 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Fri, 23 Dec 2011 06:20:41 +0000 Subject: Rework schema using abstract models rather than stuffing everything with the same syntax into the same SQL table. svn path=/branches/tk100/; revision=4133 --- scripts/convert-from-entitydb-to-sql.py | 95 ++++++++++++++++----------------- 1 file changed, 47 insertions(+), 48 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index dbdde34c..64f0d31a 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -109,37 +109,40 @@ def read_openssl_serial(filename): f.close() return int(text.strip(), 16) -def get_or_create_CA(purpose): - cer = rpki.x509.X509(Auto_file = os.path.join(bpki, purpose, "ca.cer")) - key = rpki.x509.RSA(Auto_file = os.path.join(bpki, purpose, "ca.key")) - crl = rpki.x509.CRL(Auto_file = os.path.join(bpki, purpose, "ca.crl")) - serial = read_openssl_serial(os.path.join(bpki, purpose, "serial")) - crl_number = read_openssl_serial(os.path.join(bpki, purpose, "crl_number")) - - return rpki.irdb.CA.objects.get_or_create( - handle = self_handle if purpose == "resources" else "*", - certificate = cer, - private_key = key, - latest_crl = crl, - next_serial = serial, - next_crl_number = crl_number, - last_crl_update = crl.getThisUpdate().to_sql(), - next_crl_update = crl.getNextUpdate().to_sql())[0] - -def get_or_create_EECertificate(issuer, capurpose, eepurpose): - cer = rpki.x509.X509(Auto_file = os.path.join(bpki, capurpose, eepurpose + ".cer")) - key = rpki.x509.RSA(Auto_file = os.path.join(bpki, capurpose, eepurpose + ".key")) - rpki.irdb.EECertificate.objects.get_or_create( +def get_or_create_ServerCertificate(issuer, purpose): + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", purpose + ".cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", purpose + ".key")) + rpki.irdb.ServerCertificate.objects.get_or_create( issuer = issuer, - purpose = eepurpose, + purpose = purpose, certificate = cer, private_key = key) # Load BPKI CA data -resource_ca = get_or_create_CA("resources") +cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "resources", "ca.cer")) +key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "resources", "ca.key")) +crl = rpki.x509.CRL(Auto_file = os.path.join(bpki, "resources", "ca.crl")) +serial = read_openssl_serial(os.path.join(bpki, "resources", "serial")) +crl_number = read_openssl_serial(os.path.join(bpki, "resources", "crl_number")) + +resource_ca = rpki.irdb.ResourceHolderCA.objects.get_or_create( + handle = self_handle, + certificate = cer, + private_key = key, + latest_crl = crl, + next_serial = serial, + next_crl_number = crl_number, + last_crl_update = crl.getThisUpdate().to_sql(), + next_crl_update = crl.getNextUpdate().to_sql())[0] + if os.path.exists(os.path.join(bpki, "resources", "referral.cer")): - get_or_create_EECertificate(resource_ca, "resources", "referral") + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "resources", "referral.cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "resources", "referral.key")) + rpki.irdb.ReferralCertificate.objects.get_or_create( + issuer = resource_ca, + certificate = cer, + private_key = key) # Load BPKI server EE certificates and keys @@ -147,15 +150,29 @@ run_flags = dict((i, cfg.getboolean(i, section = "myrpki")) for i in ("run_rpkid", "run_pubd", "run_rootd")) if any(run_flags.itervalues()): - server_ca = get_or_create_CA("servers") - get_or_create_EECertificate(server_ca, "servers", "irbe") + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", "ca.cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", "ca.key")) + crl = rpki.x509.CRL(Auto_file = os.path.join(bpki, "servers", "ca.crl")) + serial = read_openssl_serial(os.path.join(bpki, "servers", "serial")) + crl_number = read_openssl_serial(os.path.join(bpki, "servers", "crl_number")) + + server_ca = rpki.irdb.ServerCA.objects.get_or_create( + certificate = cer, + private_key = key, + latest_crl = crl, + next_serial = serial, + next_crl_number = crl_number, + last_crl_update = crl.getThisUpdate().to_sql(), + next_crl_update = crl.getNextUpdate().to_sql())[0] + + get_or_create_ServerCertificate(server_ca, "irbe") if run_flags["run_rpkid"]: - get_or_create_EECertificate(server_ca, "servers", "rpkid") - get_or_create_EECertificate(server_ca, "servers", "irdbd") + get_or_create_ServerCertificate(server_ca, "rpkid") + get_or_create_ServerCertificate(server_ca, "irdbd") if run_flags["run_pubd"]: - get_or_create_EECertificate(server_ca, "servers", "pubd") + get_or_create_ServerCertificate(server_ca, "pubd") if run_flags["run_rootd"]: - get_or_create_EECertificate(server_ca, "servers", "rootd") + get_or_create_ServerCertificate(server_ca, "rootd") else: server_ca = None @@ -194,24 +211,6 @@ def xcert_hash(cert): hash = hash[len("(stdin)="):] return hash -# OK, all this wretched cross-certification looks complicated, but -# that's partly because of the way we've been doing it on disk. The -# new SQL/object based approach should make it much clearer: -# -# Child cross certifies parent's resource TA in child's resource CA. -# -# Parent cross certifies child's resource TA in parent's resource -# CA. -# -# Repository cross certifies client's resource TA in repository's -# server CA. -# -# Client cross certifies repository's server TA in client's resource -# CA. -# -# The remaining xcert files look to be TLS relics which no longer -# serve any real purpose; in theory, those can just go away. - # Let's try keeping track of all the xcert filenames we use, so we can # list the ones we didn't. -- cgit v1.2.3 From 523a1f269dc1c19e3537fc0d1dc9b96a1e7fb8dc Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sun, 25 Dec 2011 18:36:49 +0000 Subject: Clean up rootd cross-certification nastiness (another TLS relic). svn path=/branches/tk100/; revision=4134 --- scripts/convert-from-entitydb-to-sql.py | 59 ++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 20 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 64f0d31a..3ba5241a 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -109,16 +109,16 @@ def read_openssl_serial(filename): f.close() return int(text.strip(), 16) -def get_or_create_ServerCertificate(issuer, purpose): +def get_or_create_ServerEE(issuer, purpose): cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", purpose + ".cer")) key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", purpose + ".key")) - rpki.irdb.ServerCertificate.objects.get_or_create( + rpki.irdb.ServerEE.objects.get_or_create( issuer = issuer, purpose = purpose, certificate = cer, private_key = key) -# Load BPKI CA data +# Load BPKI CAs and directly certified EEs cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "resources", "ca.cer")) key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "resources", "ca.key")) @@ -139,23 +139,21 @@ resource_ca = rpki.irdb.ResourceHolderCA.objects.get_or_create( if os.path.exists(os.path.join(bpki, "resources", "referral.cer")): cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "resources", "referral.cer")) key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "resources", "referral.key")) - rpki.irdb.ReferralCertificate.objects.get_or_create( + rpki.irdb.Referral.objects.get_or_create( issuer = resource_ca, certificate = cer, private_key = key) -# Load BPKI server EE certificates and keys +run_rpkid = cfg.getboolean("run_rpkid", section = "myrpki") +run_pubd = cfg.getboolean("run_pubd", section = "myrpki") +run_rootd = cfg.getboolean("run_rootd", section = "myrpki") -run_flags = dict((i, cfg.getboolean(i, section = "myrpki")) - for i in ("run_rpkid", "run_pubd", "run_rootd")) - -if any(run_flags.itervalues()): +if run_rpkid or run_pubd: cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", "ca.cer")) key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", "ca.key")) crl = rpki.x509.CRL(Auto_file = os.path.join(bpki, "servers", "ca.crl")) serial = read_openssl_serial(os.path.join(bpki, "servers", "serial")) crl_number = read_openssl_serial(os.path.join(bpki, "servers", "crl_number")) - server_ca = rpki.irdb.ServerCA.objects.get_or_create( certificate = cer, private_key = key, @@ -164,18 +162,29 @@ if any(run_flags.itervalues()): next_crl_number = crl_number, last_crl_update = crl.getThisUpdate().to_sql(), next_crl_update = crl.getNextUpdate().to_sql())[0] + get_or_create_ServerEE(server_ca, "irbe") - get_or_create_ServerCertificate(server_ca, "irbe") - if run_flags["run_rpkid"]: - get_or_create_ServerCertificate(server_ca, "rpkid") - get_or_create_ServerCertificate(server_ca, "irdbd") - if run_flags["run_pubd"]: - get_or_create_ServerCertificate(server_ca, "pubd") - if run_flags["run_rootd"]: - get_or_create_ServerCertificate(server_ca, "rootd") else: server_ca = None +if run_rpkid: + get_or_create_ServerEE(server_ca, "rpkid") + get_or_create_ServerEE(server_ca, "irdbd") + +if run_pubd: + get_or_create_ServerEE(server_ca, "pubd") + +# Certification model for rootd has changed. We can reuse the old +# key, but we have to recertify under a different CA than previously. +# Yes, we're pulling a key from the servers BPKI tree and certifying +# it under the resource holder CA, that's part of the change. + +if run_rootd: + rpki.irdb.Rootd.objects.get_or_certify( + issuer = resource_ca, + service_uri = "http://localhost:%s/" % cfg.get("rootd_server_port", section = "myrpki"), + private_key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", "rootd.key"))) + # Load BSC certificates and requests. Yes, this currently wires in # exactly one BSC handle, "bsc". So does the old myrpki code. Ick. @@ -277,6 +286,11 @@ for filename in glob.iglob(os.path.join(entitydb, "parents", "*.xml")): rpki.relaxng.myrpki.assertValid(e) assert e.tag == tag_parent + if parent_handle == self_handle: + assert run_rootd + assert e.get("service_uri") == "http://localhost:%s/" % cfg.get("rootd_server_port", section = "myrpki") + continue + ta = rpki.x509.X509(Base64 = e.findtext(tag_bpki_resource_ta)) xcfn = os.path.join(bpki, "resources", "xcert.%s.cer" % xcert_hash(ta)) xcert_filenames.discard(xcfn) @@ -298,6 +312,7 @@ for filename in glob.iglob(os.path.join(entitydb, "parents", "*.xml")): child_handle = e.get("child_handle"), ta = ta, certificate = xcert, + service_uri = e.get("service_uri"), repository_type = repository_type, referrer = referrer, referral_authorization = referral_authorization, @@ -334,7 +349,11 @@ for filename in glob.iglob(os.path.join(entitydb, "repositories", "*.xml")): xcert_filenames.discard(xcfn) xcert = rpki.x509.X509(Auto_file = xcfn) - parent = rpki.irdb.Parent.objects.get(handle = e.get("parent_handle")) + parent_handle = e.get("parent_handle") + if parent_handle == self_handle: + turtle = resource_ca.rootd + else: + turtle = rpki.irdb.Parent.objects.get(handle = parent_handle) rpki.irdb.Repository.objects.get_or_create( handle = repository_handle, @@ -343,7 +362,7 @@ for filename in glob.iglob(os.path.join(entitydb, "repositories", "*.xml")): certificate = xcert, service_uri = e.get("service_uri"), sia_base = e.get("sia_base"), - parent = parent, + turtle = turtle, issuer = resource_ca) # Scrape client data out of the entitydb. -- cgit v1.2.3 From ec05982af002fd02c50c977f8807ab6870b54118 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Mon, 26 Dec 2011 05:26:58 +0000 Subject: Checkpoint. Start debugging synchronization code. svn path=/branches/tk100/; revision=4135 --- scripts/convert-from-entitydb-to-sql.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 3ba5241a..d96dd62d 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -368,7 +368,7 @@ for filename in glob.iglob(os.path.join(entitydb, "repositories", "*.xml")): # Scrape client data out of the entitydb. for filename in glob.iglob(os.path.join(entitydb, "pubclients", "*.xml")): - client_handle = os.path.splitext(os.path.split(filename)[1])[0] + client_handle = os.path.splitext(os.path.split(filename)[1])[0].replace(".", "/") e = ElementTree(file = filename).getroot() rpki.relaxng.myrpki.assertValid(e) @@ -424,10 +424,11 @@ if copy_csv_data: # List cross certifications we didn't use. -for filename in sorted(xcert_filenames): - cer = rpki.x509.X509(Auto_file = filename) - #print "Unused cross-certificate:", filename, cer.getSubject() - print "Unused cross-certificate:", filename, cer.get_POW().pprint() +if False: + for filename in sorted(xcert_filenames): + cer = rpki.x509.X509(Auto_file = filename) + #print "Unused cross-certificate:", filename, cer.getSubject() + print "Unused cross-certificate:", filename, cer.get_POW().pprint() # Done! -- cgit v1.2.3 From 39676701fc6c5dedde2248178b16b394e90d48b9 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sat, 7 Jan 2012 09:19:03 +0000 Subject: Checkpoint. rpkic and yamltest mostly work, but irdbd is still acting weird. svn path=/branches/tk100/; revision=4146 --- scripts/convert-from-entitydb-to-sql.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'scripts/convert-from-entitydb-to-sql.py') diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index d96dd62d..1b469261 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -247,7 +247,11 @@ for filename in glob.iglob(os.path.join(entitydb, "children", "*.xml")): registrant_id, valid_until = cur.fetchone() valid_until = rpki.sundial.datetime.fromdatetime(valid_until) - assert valid_until == rpki.sundial.datetime.fromXMLtime(e.get("valid_until")) + if valid_until != rpki.sundial.datetime.fromXMLtime(e.get("valid_until")): + print "WARNING: valid_until dates in XML and SQL do not match for child", child_handle + print " SQL:", str(valid_until) + print " XML:", str(rpki.sundial.datetime.fromXMLtime(e.get("valid_until"))) + print "Blundering onwards" child = rpki.irdb.Child.objects.get_or_create( handle = child_handle, -- cgit v1.2.3