diff options
| -rwxr-xr-x | rp/rcynic/rcynic-dump | 3 | ||||
| -rwxr-xr-x | rp/rcynic/rcynicng | 39 | ||||
| -rw-r--r-- | rpki/rcynicdb/models.py | 6 |
3 files changed, 40 insertions, 8 deletions
diff --git a/rp/rcynic/rcynic-dump b/rp/rcynic/rcynic-dump index ae685d05..a217d063 100755 --- a/rp/rcynic/rcynic-dump +++ b/rp/rcynic/rcynic-dump @@ -48,8 +48,7 @@ def uri_to_filename(obj, base): return os.path.join(args.output_tree, base, obj.uri[obj.uri.index("://") + 3:]) def sha256_to_filename(obj): - return os.path.join(args.output_tree, "sha256", - obj.sha256[0:8], obj.sha256[8:16], obj.sha256[16:24], obj.sha256[24:]) + return os.path.join(args.output_tree, "sha256", obj.sha256[:2], obj.sha256 + obj.uri[-4:]) def authenticated_to_dirname(authenticated): return "authenticated-{}".format(authenticated.started.strftime("%Y-%m-%dT%H:%M:%SZ")) diff --git a/rp/rcynic/rcynicng b/rp/rcynic/rcynicng index d89fda3a..12395c62 100755 --- a/rp/rcynic/rcynicng +++ b/rp/rcynic/rcynicng @@ -421,7 +421,6 @@ class Manifest(rpki.POW.Manifest, CMS_Mixin): return not any(s.kind == "bad" for s in status) def find_crl_candidate_hashes(self): - diruri = self.uri[:self.uri.rindex("/") + 1] for fn, digest in self.fah: if fn.endswith(".crl"): yield digest.encode("hex") @@ -514,8 +513,11 @@ class WalkFrame(object): @tornado.gen.coroutine def initial(self, wsk): - self.diruri = first_rsync_uri(self.cer.caDirectory) - self.fetcher = Fetcher(self.diruri) + + # Will need some logic to decide whether to prefer RRDP or rsync. + # For the moment, we've only implemented rsync, so this is easy. + + self.fetcher = Fetcher(first_rsync_uri(self.cer.caDirectory)) if not self.fetcher.needed(): self.state = self.ready @@ -602,7 +604,7 @@ class WalkFrame(object): yield tornado.gen.moment - uri = self.diruri + fn + uri = self.mft.uri[:self.mft.uri.rindex("/") + 1] + fn # Need general URI validator here? @@ -952,6 +954,29 @@ def final_report(): ElementTree(doc).write(file = args.xml_file, pretty_print = True) +def final_cleanup(): + + # This will almost certainly become more complex once we add RRDP. + + # http://stackoverflow.com/questions/7635838/django-queryset-excluding-many-to-many-objects + # http://blog.roseman.org.uk/2010/01/11/django-patterns-part-2-efficient-reverse-lookups/ + # http://stackoverflow.com/questions/5437335/django-queryset-with-filtering-on-reverse-foreign-key + # https://docs.djangoproject.com/en/1.8/ref/models/querysets/#in + + # https://docs.djangoproject.com/en/1.8/faq/models/#how-can-i-see-the-raw-sql-queries-django-is-running + + # All RPKI objects other than the most recent authenticated set. + # This will definitely need work once we support RRDP. + rpki.rcynicdb.models.RPKIObject.objects.exclude(authenticated = authenticated.id).delete() + + # Flush all but the most recent authenticated set. + rpki.rcynicdb.models.Authenticated.objects.exclude(id = authenticated.id).delete() + + # Flush retrieval objects which have no related RPKI objects. + rpki.rcynicdb.models.Retrieval.objects.exclude( + id__in = rpki.rcynicdb.models.RPKIObject.objects.values_list("retrieved__id", flat = True)).delete() + + @tornado.gen.coroutine def launcher(): for i in xrange(args.workers): @@ -1020,11 +1045,13 @@ def main(): task_queue = tornado.queues.Queue() tornado.ioloop.IOLoop.current().run_sync(launcher) - final_report() - authenticated.finished = rpki.sundial.datetime.now() authenticated.save() + final_report() + + final_cleanup() + if __name__ == "__main__": main() diff --git a/rpki/rcynicdb/models.py b/rpki/rcynicdb/models.py index e2d69d40..80badcff 100644 --- a/rpki/rcynicdb/models.py +++ b/rpki/rcynicdb/models.py @@ -62,6 +62,12 @@ class RRDPSnapshot(models.Model): retrieved = models.OneToOneField(Retrieval) # RPKI objects. +# +# Might need to add an on_delete argument to the ForeignKey for the +# retrieved field: the default behavior is CASCADE, which is may not +# what we want in this case. +# +# https://docs.djangoproject.com/en/1.9/ref/models/fields/#django.db.models.ForeignKey.on_delete class RPKIObject(models.Model): der = models.BinaryField(unique = True) |