Automate SQL pings. Suppress case where ROA regeneration gratuitously

updated CRL and manifest.  More debug logging.

svn path=/branches/tk274/; revision=4632

3 files changed, 27 insertions, 6 deletions

diff --git a/rpkid/rpki/left_right.py b/rpkid/rpki/left_right.py
index b9a2c43f..a4e9db3f 100644
--- a/rpkid/rpki/left_right.py
+++ b/rpkid/rpki/left_right.py
@@ -48,7 +48,7 @@ enforce_strict_up_down_xml_sender = False
 # the goal is to avoid going totally compute bound when somebody
 # throws 50,000 new ROA requests at us in a single batch.
 
-max_new_roas_at_once = 50
+max_new_roas_at_once = 200
 
 class left_right_namespace(object):
   """
@@ -760,12 +760,16 @@ class self_elt(data_elt):
             rpki.log.traceback()
             rpki.log.warn("Could not revoke %r: %s" % (roa, e))
         self.gctx.sql.sweep()
-        for ca_detail in ca_details:
-          ca_detail.generate_crl(publisher = publisher)
-          ca_detail.generate_manifest(publisher = publisher)
-        self.gctx.sql.sweep()
         self.gctx.checkpoint()
-        publisher.call_pubd(cb, publication_failed)
+        if publisher.size > 0:
+          for ca_detail in ca_details:
+            ca_detail.generate_crl(publisher = publisher)
+            ca_detail.generate_manifest(publisher = publisher)
+          self.gctx.sql.sweep()
+          self.gctx.checkpoint()
+          publisher.call_pubd(cb, publication_failed)
+        else:
+          cb()
 
       rpki.async.iterator(updates, loop, done, pop_list = True)
       
diff --git a/rpkid/rpki/rpkid.py b/rpkid/rpki/rpkid.py
index ca27dbdc..f2a10abb 100644
--- a/rpkid/rpki/rpkid.py
+++ b/rpkid/rpki/rpkid.py
@@ -1181,13 +1181,16 @@ class ca_detail_obj(rpki.sql.sql_persistent):
       nextUpdate = now + crl_interval
 
     if self.latest_manifest_cert is None or self.latest_manifest_cert.getNotAfter() < nextUpdate:
+      rpki.log.debug("Generating manifest certificate")
       self.generate_manifest_cert()
 
+    rpki.log.debug("Constructing manifest object list")
     objs = [(self.crl_uri_tail, self.latest_crl)]
     objs.extend((c.uri_tail, c.cert) for c in self.child_certs)
     objs.extend((r.uri_tail, r.roa) for r in self.roas if r.roa is not None)
     objs.extend((g.uri_tail, g.ghostbuster) for g in self.ghostbusters)
 
+    rpki.log.debug("Building manifest object")
     self.latest_manifest = rpki.x509.SignedManifest.build(
       serial         = ca.next_manifest_number(),
       thisUpdate     = now,
@@ -1196,6 +1199,7 @@ class ca_detail_obj(rpki.sql.sql_persistent):
       keypair        = self.manifest_private_key_id,
       certs          = self.latest_manifest_cert)
 
+    rpki.log.debug("Manifest generation took %s" % (rpki.sundial.now() - now))
 
     self.manifest_published = rpki.sundial.now()
     self.sql_mark_dirty()
diff --git a/rpkid/rpki/sql.py b/rpkid/rpki/sql.py
index 9d420c80..e79993d2 100644
--- a/rpkid/rpki/sql.py
+++ b/rpkid/rpki/sql.py
@@ -46,6 +46,13 @@ class session(object):
   SQL session layer.
   """
 
+  ## @var ping_threshold
+  # Timeout after which we should issue a ping command before the real
+  # one.  Intent is to keep the MySQL connection alive without pinging
+  # before every single command.
+
+  ping_threshold = rpki.sundial.timedelta(seconds = 60)
+
   def __init__(self, cfg):
 
     self.username = cfg.get("sql-username")
@@ -61,6 +68,7 @@ class session(object):
     self.db = MySQLdb.connect(user = self.username, db = self.database, passwd = self.password)
     self.cur = self.db.cursor()
     self.db.autocommit(True)
+    self.timestamp = rpki.sundial.now()
 
   def close(self):
     if self.cur:
@@ -71,10 +79,15 @@ class session(object):
     self.db = None
 
   def ping(self):
+    rpki.log.debug("Pinging SQL server")
     return self.db.ping(True)
 
   def _wrap_execute(self, func, query, args):
     try:
+      now = rpki.sundial.now()
+      if now > self.timestamp + self.ping_threshold:
+        self.ping()
+      self.timestamp = now
       return func(query, args)
     except _mysql_exceptions.MySQLError:
       if self.dirty: