Merge branches/tk161 to trunk.

svn path=/trunk/; revision=4415

1 files changed, 106 insertions, 137 deletions

diff --git a/rpkid/rpki/irdbd.py b/rpkid/rpki/irdbd.py
index c2e01287..28e26b07 100644
--- a/rpkid/rpki/irdbd.py
+++ b/rpkid/rpki/irdbd.py
@@ -5,7 +5,7 @@ Usage: python irdbd.py [ { -c | --config } configfile ] [ { -h | --help } ]
 
 $Id$
 
-Copyright (C) 2009--2011  Internet Systems Consortium ("ISC")
+Copyright (C) 2009--2012  Internet Systems Consortium ("ISC")
 
 Permission to use, copy, modify, and distribute this software for any
 purpose with or without fee is hereby granted, provided that the above
@@ -38,160 +38,90 @@ import sys, os, time, getopt, urlparse, warnings
 import rpki.http, rpki.config, rpki.resource_set, rpki.relaxng
 import rpki.exceptions, rpki.left_right, rpki.log, rpki.x509
 
-from rpki.mysql_import import MySQLdb
-
 class main(object):
 
-
   def handle_list_resources(self, q_pdu, r_msg):
-
+    child  = rpki.irdb.Child.objects.get(issuer__handle__exact = q_pdu.self_handle,
+                                         handle = q_pdu.child_handle)
+    resources = child.resource_bag
     r_pdu = rpki.left_right.list_resources_elt()
     r_pdu.tag = q_pdu.tag
     r_pdu.self_handle = q_pdu.self_handle
     r_pdu.child_handle = q_pdu.child_handle
-
-    self.cur.execute(
-      "SELECT registrant_id, valid_until FROM registrant WHERE registry_handle = %s AND registrant_handle = %s",
-      (q_pdu.self_handle, q_pdu.child_handle))
-
-    if self.cur.rowcount != 1:
-      raise rpki.exceptions.NotInDatabase, \
-            "This query should have produced a single exact match, something's messed up (rowcount = %d, self_handle = %s, child_handle = %s)" \
-            % (self.cur.rowcount, q_pdu.self_handle, q_pdu.child_handle)
-
-    registrant_id, valid_until = self.cur.fetchone()
-
-    r_pdu.valid_until = valid_until.strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    r_pdu.asn  = rpki.resource_set.resource_set_as.from_sql(
-      self.cur,
-      "SELECT start_as, end_as FROM registrant_asn WHERE registrant_id = %s",
-      (registrant_id,))
-
-    r_pdu.ipv4 = rpki.resource_set.resource_set_ipv4.from_sql(
-      self.cur,
-      "SELECT start_ip, end_ip FROM registrant_net WHERE registrant_id = %s AND version = 4",
-      (registrant_id,))
-
-    r_pdu.ipv6 = rpki.resource_set.resource_set_ipv6.from_sql(
-      self.cur,
-      "SELECT start_ip, end_ip FROM registrant_net WHERE registrant_id = %s AND version = 6",
-      (registrant_id,))
-
+    r_pdu.valid_until = child.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ")
+    r_pdu.asn = resources.asn
+    r_pdu.ipv4 = resources.v4
+    r_pdu.ipv6 = resources.v6
     r_msg.append(r_pdu)
 
-
   def handle_list_roa_requests(self, q_pdu, r_msg):
-
-    self.cur.execute(
-      "SELECT roa_request_id, asn FROM roa_request WHERE roa_request_handle = %s",
-      (q_pdu.self_handle,))
-
-    for roa_request_id, asn in self.cur.fetchall():
-
+    for request in rpki.irdb.ROARequest.objects.filter(issuer__handle__exact = q_pdu.self_handle):
+      prefix_bag = request.roa_prefix_bag
       r_pdu = rpki.left_right.list_roa_requests_elt()
       r_pdu.tag = q_pdu.tag
       r_pdu.self_handle = q_pdu.self_handle
-      r_pdu.asn = asn
-
-      r_pdu.ipv4 = rpki.resource_set.roa_prefix_set_ipv4.from_sql(
-        self.cur,
-        "SELECT prefix, prefixlen, max_prefixlen FROM roa_request_prefix WHERE roa_request_id = %s AND version = 4",
-        (roa_request_id,))
-
-      r_pdu.ipv6 = rpki.resource_set.roa_prefix_set_ipv6.from_sql(
-        self.cur,
-        "SELECT prefix, prefixlen, max_prefixlen FROM roa_request_prefix WHERE roa_request_id = %s AND version = 6",
-        (roa_request_id,))
-
+      r_pdu.asn = request.asn
+      r_pdu.ipv4 = prefix_bag.v4
+      r_pdu.ipv6 = prefix_bag.v6
       r_msg.append(r_pdu)
 
-
   def handle_list_ghostbuster_requests(self, q_pdu, r_msg):
-
-    self.cur.execute(
-      "SELECT vcard  FROM ghostbuster_request WHERE self_handle = %s AND parent_handle = %s",
-      (q_pdu.self_handle, q_pdu.parent_handle))
-
-    vcards = [result[0] for result in self.cur.fetchall()]
-
-    if not vcards:
-
-      self.cur.execute(
-        "SELECT vcard  FROM ghostbuster_request WHERE self_handle = %s AND parent_handle IS NULL",
-        (q_pdu.self_handle,))
-
-      vcards = [result[0] for result in self.cur.fetchall()]
-
-    for vcard in vcards:
+    ghostbusters = rpki.irdb.GhostbusterRequest.objects.filter(
+      issuer__handle__exact = q_pdu.self_handle,
+      parent__handle__exact = q_pdu.parent_handle)
+    if ghostbusters.count() == 0:
+      ghostbusters = rpki.irdb.GhostbusterRequest.objects.filter(
+        issuer__handle__exact = q_pdu.self_handle,
+        parent = None)
+    for ghostbuster in ghostbusters:
       r_pdu = rpki.left_right.list_ghostbuster_requests_elt()
       r_pdu.tag = q_pdu.tag
       r_pdu.self_handle = q_pdu.self_handle
       r_pdu.parent_handle = q_pdu.parent_handle
-      r_pdu.vcard = vcard
+      r_pdu.vcard = ghostbuster.vcard
       r_msg.append(r_pdu)
 
-
-  handle_dispatch = {
-    rpki.left_right.list_resources_elt            : handle_list_resources,
-    rpki.left_right.list_roa_requests_elt         : handle_list_roa_requests,
-    rpki.left_right.list_ghostbuster_requests_elt : handle_list_ghostbuster_requests}
-
-
   def handler(self, query, path, cb):
     try:
-
-      self.db.ping(True)
-
+      q_pdu = None
       r_msg = rpki.left_right.msg.reply()
-
+      self.start_new_transaction()
+      serverCA = rpki.irdb.ServerCA.objects.get()
+      rpkid = serverCA.ee_certificates.get(purpose = "rpkid")
       try:
-
-        q_msg = rpki.left_right.cms_msg(DER = query).unwrap((self.bpki_ta, self.rpkid_cert))
-
+        q_msg = rpki.left_right.cms_msg(DER = query).unwrap((serverCA.certificate, rpkid.certificate))
         if not isinstance(q_msg, rpki.left_right.msg) or not q_msg.is_query():
-          raise rpki.exceptions.BadQuery, "Unexpected %r PDU" % q_msg
-
+          raise rpki.exceptions.BadQuery("Unexpected %r PDU" % q_msg)
         for q_pdu in q_msg:
-
-          try:
-
-            try:
-              h = self.handle_dispatch[type(q_pdu)]
-            except KeyError:
-              raise rpki.exceptions.BadQuery, "Unexpected %r PDU" % q_pdu
-            else:
-              h(self, q_pdu, r_msg)
-
-          except (rpki.async.ExitNow, SystemExit):
-            raise
-
-          except Exception, e:
-            rpki.log.traceback()
-            r_msg.append(rpki.left_right.report_error_elt.from_exception(e, q_pdu.self_handle, q_pdu.tag))
-
+          self.dispatch(q_pdu, r_msg)
       except (rpki.async.ExitNow, SystemExit):
         raise
-
       except Exception, e:
         rpki.log.traceback()
-        r_msg.append(rpki.left_right.report_error_elt.from_exception(e))
-
-      cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, self.irdbd_key, self.irdbd_cert))
-
+        if q_pdu is None:
+          r_msg.append(rpki.left_right.report_error_elt.from_exception(e))
+        else:
+          r_msg.append(rpki.left_right.report_error_elt.from_exception(e, q_pdu.self_handle, q_pdu.tag))
+      irdbd = serverCA.ee_certificates.get(purpose = "irdbd")
+      cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, irdbd.private_key, irdbd.certificate))
     except (rpki.async.ExitNow, SystemExit):
       raise
-
     except Exception, e:
       rpki.log.traceback()
-
-      # We only get here in cases where we couldn't or wouldn't generate
-      # <report_error/>, so just return HTTP failure.
-
       cb(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e))
 
+  def dispatch(self, q_pdu, r_msg):
+    try:
+      handler = self.dispatch_vector[type(q_pdu)]
+    except KeyError:
+      raise rpki.exceptions.BadQuery("Unexpected %r PDU" % q_pdu)
+    else:
+      handler(q_pdu, r_msg)
+
+  def __init__(self, **kwargs):
 
-  def __init__(self):
+    global rpki
+    from django.conf import settings
 
     os.environ["TZ"] = "UTC"
     time.tzset()
@@ -208,31 +138,69 @@ class main(object):
       elif o in ("-d", "--debug"):
         rpki.log.use_syslog = False
     if argv:
-      raise rpki.exceptions.CommandParseFailure, "Unexpected arguments %s" % argv
+      raise rpki.exceptions.CommandParseFailure("Unexpected arguments %s" % argv)
 
     rpki.log.init("irdbd")
 
-    self.cfg = rpki.config.parser(cfg_file, "irdbd")
+    cfg = rpki.config.parser(cfg_file, "irdbd")
 
-    startup_msg = self.cfg.get("startup-message", "")
+    startup_msg = cfg.get("startup-message", "")
     if startup_msg:
       rpki.log.info(startup_msg)
 
-    self.cfg.set_global_flags()
-
-    self.db = MySQLdb.connect(user   = self.cfg.get("sql-username"),
-                              db     = self.cfg.get("sql-database"),
-                              passwd = self.cfg.get("sql-password"))
-
-    self.cur = self.db.cursor()
-    self.db.autocommit(True)
-
-    self.bpki_ta         = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta"))
-    self.rpkid_cert      = rpki.x509.X509(Auto_update = self.cfg.get("rpkid-cert"))
-    self.irdbd_cert      = rpki.x509.X509(Auto_update = self.cfg.get("irdbd-cert"))
-    self.irdbd_key       = rpki.x509.RSA( Auto_update = self.cfg.get("irdbd-key"))
-
-    u = urlparse.urlparse(self.cfg.get("http-url"))
+    cfg.set_global_flags()
+
+    # Do -not- turn on DEBUG here except for short-lived tests,
+    # otherwise irdbd will eventually run out of memory and crash.
+    #
+    # If you must enable debugging, use django.db.reset_queries() to
+    # clear the query list manually, but it's probably better just to
+    # run with debugging disabled, since that's the expectation for
+    # production code.
+    #
+    # https://docs.djangoproject.com/en/dev/faq/models/#why-is-django-leaking-memory
+
+    settings.configure(
+      DATABASES = {
+        "default" : {
+          "ENGINE"   : "django.db.backends.mysql",
+          "NAME"     : cfg.get("sql-database"),
+          "USER"     : cfg.get("sql-username"),
+          "PASSWORD" : cfg.get("sql-password"),
+          "HOST"     : "",
+          "PORT"     : "" }},
+      INSTALLED_APPS = ("rpki.irdb",),)
+
+    import rpki.irdb
+
+    # Entirely too much fun with read-only access to transactional databases.
+    # 
+    # http://stackoverflow.com/questions/3346124/how-do-i-force-django-to-ignore-any-caches-and-reload-data
+    # http://devblog.resolversystems.com/?p=439
+    # http://groups.google.com/group/django-users/browse_thread/thread/e25cec400598c06d
+    # http://stackoverflow.com/questions/1028671/python-mysqldb-update-query-fails
+    # http://dev.mysql.com/doc/refman/5.0/en/set-transaction.html
+    #
+    # It turns out that MySQL is doing us a favor with this weird
+    # transactional behavior on read, because without it there's a
+    # race condition if multiple updates are committed to the IRDB
+    # while we're in the middle of processing a query.  Note that
+    # proper transaction management by the committers doesn't protect
+    # us, this is a transactional problem on read.  So we need to use
+    # explicit transaction management.  Since irdbd is a read-only
+    # consumer of IRDB data, this means we need to commit an empty
+    # transaction at the beginning of processing each query, to reset
+    # the transaction isolation snapshot.
+
+    import django.db.transaction
+    self.start_new_transaction = django.db.transaction.commit_manually(django.db.transaction.commit)
+
+    self.dispatch_vector = {
+      rpki.left_right.list_resources_elt            : self.handle_list_resources,
+      rpki.left_right.list_roa_requests_elt         : self.handle_list_roa_requests,
+      rpki.left_right.list_ghostbuster_requests_elt : self.handle_list_ghostbuster_requests }
+
+    u = urlparse.urlparse(cfg.get("http-url"))
 
     assert u.scheme in ("", "http") and \
            u.username is None and \
@@ -241,6 +209,7 @@ class main(object):
            u.query    == "" and \
            u.fragment == ""
 
-    rpki.http.server(host         = u.hostname or "localhost",
-                     port         = u.port or 443,
-                     handlers     = ((u.path, self.handler),))
+    rpki.http.server(
+      host     = u.hostname or "localhost",
+      port     = u.port or 443,
+      handlers = ((u.path, self.handler),))