diff options
Diffstat (limited to 'rp')
| -rwxr-xr-x | rp/rcynic/rcynicng | 96 |
1 files changed, 48 insertions, 48 deletions
diff --git a/rp/rcynic/rcynicng b/rp/rcynic/rcynicng index 01b6a68f..e22711e7 100755 --- a/rp/rcynic/rcynicng +++ b/rp/rcynic/rcynicng @@ -915,10 +915,9 @@ class Fetcher(object): # "you don't need to retrieve this" result comes back, # probably a benign exception we need to catch. Supporting # this means adding another null-able timestamp field to the - # RRDPSnapshot model (which probably should be named the - # RRDPZone model instead), and storing a datetime there. - # Would also need to pull timestamp from the Last-Modified - # header in the response object. + # RRDPZone model, and storing a datetime there. Would also + # need to pull timestamp from the Last-Modified header in the + # response object. try: ok = False @@ -1041,7 +1040,7 @@ class Fetcher(object): yield tornado.gen.moment try: existing_objs.append(RPKIObject.objects.values_list("pk", flat = True).get(der = new_objs[i].der)) - logger.debug("Object existed in SQL but, apparently, not in prior copy of snapshot: uri %s sha256 %s", + logger.debug("Object existed in SQL but, apparently, not in prior copy of zone: uri %s sha256 %s", new_objs[i].uri, new_objs[i].sha256) except RPKIObject.DoesNotExist: i += 1 @@ -1072,36 +1071,36 @@ class Fetcher(object): session_id = notification.get("session_id") serial = long(notification.get("serial")) - snapshot = RRDPSnapshot.objects.filter( + zone = RRDPZone.objects.filter( session_id = session_id).order_by("-retrieved__started").first() - logger.debug("RRDP notification for %s session_id %s serial %s current snapshot %r", - self.uri, session_id, serial, snapshot) + logger.debug("RRDP notification for %s session_id %s serial %s current zone %r", + self.uri, session_id, serial, zone) - if snapshot is not None and snapshot.serial == serial: + if zone is not None and zone.serial == serial: logger.debug("RRDP data for %s is up-to-date, nothing to do", self.uri) return deltas = dict((long(delta.get("serial")), (delta.get("uri"), delta.get("hash"))) for delta in notification.iterchildren(tag_delta)) - if snapshot is None or snapshot.serial + 1 not in deltas: + if zone is None or zone.serial + 1 not in deltas: existing_rpkiobject_map = dict() - if snapshot is not None: - logger.debug("RRDP %s no deltas available for serial %s", self.uri, snapshot.serial) - existing_rpkiobject_map.update(snapshot.rpkiobject_set.values_list("sha256", "pk")) + if zone is not None: + logger.debug("RRDP %s no deltas available for serial %s", self.uri, zone.serial) + existing_rpkiobject_map.update(zone.rpkiobject_set.values_list("sha256", "pk")) x = notification.find(tag_snapshot) url, hash = x.get("uri"), x.get("hash") - logger.debug("RRDP %s loading from snapshot %s serial %s", self.uri, url, serial) + logger.debug("RRDP %s loading from zone %s serial %s", self.uri, url, serial) retrieval, response, xml_file = yield self._rrdp_fetch_data_file(url, hash) - snapshot = RRDPSnapshot.objects.create(session_id = session_id, serial = serial) + zone = RRDPZone.objects.create(session_id = session_id, serial = serial) # Value of "chunk" here may need to be configurable. Larger numbers batch more objects in # a single bulk addition, which is faster ... unless one or more of them isn't really new, in @@ -1122,7 +1121,7 @@ class Fetcher(object): if root is None or root.tag != tag_snapshot \ or root.get("version") != "1" \ or any(a not in ("version", "session_id", "serial") for a in root.attrib): - raise RRDP_ParseFailure("{} doesn't look like an RRDP snapshot file".format(url)) + raise RRDP_ParseFailure("{} doesn't look like an RRDP zone file".format(url)) if root.get("session_id") != session_id: raise RRDP_ParseFailure("Expected RRDP session_id {} for {}, got {}".format( session_id, url, root.get("session_id"))) @@ -1160,25 +1159,25 @@ class Fetcher(object): if len(new_rpkiobjects) > 0: yield self._rrdp_bulk_create(new_rpkiobjects, existing_rpkiobjects) - RPKIObject.snapshot.through.objects.bulk_create([ - RPKIObject.snapshot.through(rrdpsnapshot_id = snapshot.id, rpkiobject_id = i) + RPKIObject.zone.through.objects.bulk_create([ + RPKIObject.zone.through(rrdpzone_id = zone.id, rpkiobject_id = i) for i in retrieval.rpkiobject_set.values_list("pk", flat = True)]) - RPKIObject.snapshot.through.objects.bulk_create([ - RPKIObject.snapshot.through(rrdpsnapshot_id = snapshot.id, rpkiobject_id = i) + RPKIObject.zone.through.objects.bulk_create([ + RPKIObject.zone.through(rrdpzone_id = zone.id, rpkiobject_id = i) for i in existing_rpkiobjects]) - snapshot.retrieved = retrieval - snapshot.save() + zone.updated = rpki.sundial.datetime.now() + zone.save() xml_file.close() else: logger.debug("RRDP %s %s deltas (%s--%s)", self.uri, - (serial - snapshot.serial), snapshot.serial, serial) + (serial - zone.serial), zone.serial, serial) deltas = [(serial, deltas[serial][0], deltas[serial][1]) - for serial in xrange(snapshot.serial + 1, serial + 1)] + for serial in xrange(zone.serial + 1, serial + 1)] futures = [] while deltas or futures: @@ -1193,9 +1192,10 @@ class Fetcher(object): root = None with transaction.atomic(): - snapshot.serial += 1 - snapshot.save() - logger.debug("RRDP %s serial %s loading", self.uri, snapshot.serial) + zone.serial += 1 + zone.updated = None + zone.save() + logger.debug("RRDP %s serial %s loading", self.uri, zone.serial) for event, node in iterparse(xml_file): if node is root: @@ -1210,9 +1210,9 @@ class Fetcher(object): if root.get("session_id") != session_id: raise RRDP_ParseFailure("Expected RRDP session_id {} for {}, got {}".format( session_id, url, root.get("session_id"))) - if long(root.get("serial")) != snapshot.serial: + if long(root.get("serial")) != zone.serial: raise RRDP_ParseFailure("Expected RRDP serial {} for {}, got {}".format( - snapshot.serial, url, root.get("serial"))) + zone.serial, url, root.get("serial"))) hash = node.get("hash") @@ -1222,7 +1222,7 @@ class Fetcher(object): raise RRDP_ParseFailure("{} doesn't look like an RRDP delta file".format(url)) if node.tag == tag_withdraw or node.get("hash") is not None: - snapshot.rpkiobject_set.remove(snapshot.rpkiobject_set.get(sha256 = node.get("hash").lower())) + zone.rpkiobject_set.remove(zone.rpkiobject_set.get(sha256 = node.get("hash").lower())) if node.tag == tag_publish: uri = node.get("uri") @@ -1230,7 +1230,7 @@ class Fetcher(object): if cls is None: raise RRDP_ParseFailure("Unexpected URI %s" % uri) obj, created = cls.store_if_new(node.text.decode("base64"), uri, retrieval) - obj.snapshot.add(snapshot) + obj.zone.add(zone) node.clear() while node.getprevious() is not None: @@ -1239,6 +1239,8 @@ class Fetcher(object): #yield tornado.gen.moment xml_file.close() + zone.updated = rpki.sundial.datetime.now() + zone.save() logger.debug("RRDP %s done processing deltas", self.uri) @@ -1346,18 +1348,18 @@ def final_cleanup(): from django.db import transaction, models def report(when): - logger.debug("Database %s cleanup: %s Authenticated %s RRDPSnapshot %s RPKIObject %s Retrieval", when, - Authenticated.objects.all().count(), RRDPSnapshot.objects.all().count(), + logger.debug("Database %s cleanup: %s Authenticated %s RRDPZone %s RPKIObject %s Retrieval", when, + Authenticated.objects.all().count(), RRDPZone.objects.all().count(), RPKIObject.objects.all().count(), Retrieval.objects.all().count()) report("before") with transaction.atomic(): - #logger.debug("Flushing incomplete RRDP snapshots") + #logger.debug("Flushing incomplete RRDP zones") - q = RRDPSnapshot.objects - q = q.filter(retrieved__isnull = True) + q = RRDPZone.objects + q = q.filter(updated__isnull = True) q.delete() #logger.debug("Flushing old authenticated sets") @@ -1366,32 +1368,31 @@ def final_cleanup(): q = q.exclude(id = authenticated.id) q.delete() - #logger.debug("Flushing RRDP snapshots which don't contain anything in the (remaining) authenticated set") + #logger.debug("Flushing RRDP zones which don't contain anything in the (remaining) authenticated set") q = RPKIObject.objects q = q.filter(authenticated = authenticated.id) - q = q.exclude(snapshot = None) - q = q.order_by("snapshot__id") - q = q.values_list("snapshot__id", flat = True) + q = q.exclude(zone = None) + q = q.order_by("zone__id") + q = q.values_list("zone__id", flat = True) q = q.distinct() - q = RRDPSnapshot.objects.exclude(id__in = q) + q = RRDPZone.objects.exclude(id__in = q) q.delete() - #logger.debug("Flushing RPKI objects which are in neither current authenticated set nor current RRDP snapshot") + #logger.debug("Flushing RPKI objects which are in neither current authenticated set nor current RRDP zone") q = RPKIObject.objects q = q.filter(authenticated = None) # was: q = q.exclude(authenticated = authenticated.id) - q = q.filter(snapshot = None) + q = q.filter(zone = None) q.delete() - #logger.debug("Flushing retrieval objects which are no longer related to any RPKI objects or RRDP snapshot") + #logger.debug("Flushing retrieval objects which are no longer related to any RPKI objects") q = RPKIObject.objects q = q.order_by("retrieved__id") q = q.values_list("retrieved__id", flat = True) q = q.distinct() q = Retrieval.objects.exclude(id__in = q) - q = q.filter(rrdpsnapshot = None) q.delete() report("after") @@ -1482,14 +1483,13 @@ def main(): import rpki.rcynicdb global Retrieval global Authenticated - global RRDPSnapshot + global RRDPZone global RPKIObject Retrieval = rpki.rcynicdb.models.Retrieval Authenticated = rpki.rcynicdb.models.Authenticated - RRDPSnapshot = rpki.rcynicdb.models.RRDPSnapshot + RRDPZone = rpki.rcynicdb.models.RRDPZone RPKIObject = rpki.rcynicdb.models.RPKIObject - global authenticated authenticated = Authenticated.objects.create(started = rpki.sundial.datetime.now()) |