Using a Tornado yield dictionary is elegant (not to mention cute), but

using a single bulk IRDB query for all the child data that
UpdateChildrenTask needs is a better solution.

svn path=/branches/tk705/; revision=6295

2 files changed, 27 insertions, 19 deletions

diff --git a/rpki/rpkid.py b/rpki/rpkid.py
index b65b638e..4c3c5e7e 100644
--- a/rpki/rpkid.py
+++ b/rpki/rpkid.py
@@ -356,26 +356,36 @@ class main(object):
         raise tornado.gen.Return(r_msg)
 
     @tornado.gen.coroutine
-    def irdb_query_child_resources(self, tenant_handle, child_handle):
+    def irdb_query_children_resources(self, tenant_handle, child_handles):
         """
-        Ask IRDB about a child's resources.
+        Ask IRDB about resources for one or more children.
         """
 
         q_msg = self.compose_left_right_query()
-        SubElement(q_msg, rpki.left_right.tag_list_resources, tenant_handle = tenant_handle, child_handle = child_handle)
+        for child_handle in child_handles:
+            SubElement(q_msg, rpki.left_right.tag_list_resources, tenant_handle = tenant_handle, child_handle = child_handle)
 
         r_msg = yield self.irdb_query(q_msg)
 
-        if len(r_msg) != 1:
-            raise rpki.exceptions.BadIRDBReply("Expected exactly one PDU from IRDB: %s" % r_msg.pretty_print_content())
+        if len(r_msg) != len(q_msg):
+            raise rpki.exceptions.BadIRDBReply("Expected IRDB response to be same length as query: %s" % r_msg.pretty_print_content())
+
+        bags = [rpki.resource_set.resource_bag(asn         = rpki.resource_set.resource_set_as(r_pdu.get("asn")),
+                                               v4          = rpki.resource_set.resource_set_ipv4(r_pdu.get("ipv4")),
+                                               v6          = rpki.resource_set.resource_set_ipv6(r_pdu.get("ipv6")),
+                                               valid_until = rpki.sundial.datetime.fromXMLtime(r_pdu.get("valid_until")))
+                for r_pdu in r_msg]
 
-        bag = rpki.resource_set.resource_bag(
-            asn         = rpki.resource_set.resource_set_as(r_msg[0].get("asn")),
-            v4          = rpki.resource_set.resource_set_ipv4(r_msg[0].get("ipv4")),
-            v6          = rpki.resource_set.resource_set_ipv6(r_msg[0].get("ipv6")),
-            valid_until = rpki.sundial.datetime.fromXMLtime(r_msg[0].get("valid_until")))
+        raise tornado.gen.Return(bags)
+
+    @tornado.gen.coroutine
+    def irdb_query_child_resources(self, tenant_handle, child_handle):
+        """
+        Ask IRDB about a single child's resources.
+        """
 
-        raise tornado.gen.Return(bag)
+        bags = yield self.irdb_query_children_resources(tenant_handle, (child_handle,))
+        raise tornado.gen.Return(bags[0])
 
     @tornado.gen.coroutine
     def irdb_query_roa_requests(self, tenant_handle):
diff --git a/rpki/rpkid_tasks.py b/rpki/rpkid_tasks.py
index 51f7033d..9386f796 100644
--- a/rpki/rpkid_tasks.py
+++ b/rpki/rpkid_tasks.py
@@ -292,15 +292,13 @@ class UpdateChildrenTask(AbstractTask):
         postponing = False
 
         child_certs    = rpki.rpkidb.models.ChildCert.objects.filter(child__tenant = self.tenant, ca_detail__state = "active")
-        child_handles  = set(child_cert.child.child_handle for child_cert in child_certs)
-        irdb_resources = yield dict((child_handle,
-                                     self.rpkid.irdb_query_child_resources(self.tenant.tenant_handle, child_handle))
-                                    for child_handle in child_handles)
+        child_handles  = sorted(set(child_cert.child.child_handle for child_cert in child_certs))
+        irdb_resources = dict(zip(child_handles, (yield self.rpkid.irdb_query_children_resources(self.tenant.tenant_handle, child_handles))))
 
         for child_cert in child_certs:
             try:
                 ca_detail = child_cert.ca_detail
-                child_handle = child_cert.child.handle
+                child_handle = child_cert.child.child_handle
                 old_resources = child_cert.cert.get_3779resources()
                 new_resources = old_resources & irdb_resources[child_handle] & ca_detail.latest_ca_cert.get_3779resources()
                 old_aia = child_cert.cert.get_AIA()[0]
@@ -313,11 +311,11 @@ class UpdateChildrenTask(AbstractTask):
                     child_cert.revoke(publisher = publisher)
                     ca_detail.generate_crl_and_manifest(publisher = publisher)
 
-                elif old_resources != new_resources or old_aia != new_aia or (old_resources.valid_until < rsn and irdb_resources.valid_until > now and old_resources.valid_until != irdb_resources.valid_until):
+                elif old_resources != new_resources or old_aia != new_aia or (old_resources.valid_until < rsn and irdb_resources[child_handle].valid_until > now and old_resources.valid_until != irdb_resources[child_handle].valid_until):
                     logger.debug("Need to reissue child %s certificate g(SKI) %s", child_handle, child_cert.gski)
                     if old_resources != new_resources:
                         logger.debug("Child %s g(SKI) %s resources changed: old %s new %s", child_handle, child_cert.gski, old_resources, new_resources)
-                    if old_resources.valid_until != irdb_resources.valid_until:
+                    if old_resources.valid_until != irdb_resources[child_handle].valid_until:
                         logger.debug("Child %s g(SKI) %s validity changed: old %s new %s", child_handle, child_cert.gski, old_resources.valid_until, irdb_resources.valid_until)
 
                     new_resources.valid_until = irdb_resources.valid_until
@@ -330,7 +328,7 @@ class UpdateChildrenTask(AbstractTask):
                     ca_detail.generate_crl_and_manifest(publisher = publisher)
 
             except:
-                logger.exception("%r: Couldn't update %r, skipping", self, child)
+                logger.exception("%r: Couldn't update %r, skipping", self, child_cert)
 
             finally:
                 if (yield self.overdue()):