diff options
| -rwxr-xr-x | rp/rcynic/rcynicng | 146 | ||||
| -rw-r--r-- | rpki/POW/__init__.py | 4 | ||||
| -rw-r--r-- | rpki/django_settings/common.py | 5 | ||||
| -rw-r--r-- | rpki/django_settings/rcynic.py | 61 | ||||
| -rw-r--r-- | rpki/rcynicdb/__init__.py | 0 | ||||
| -rw-r--r-- | rpki/rcynicdb/models.py | 78 |
6 files changed, 274 insertions, 20 deletions
diff --git a/rp/rcynic/rcynicng b/rp/rcynic/rcynicng index 096ed467..167a3b56 100755 --- a/rp/rcynic/rcynicng +++ b/rp/rcynic/rcynicng @@ -13,6 +13,7 @@ import shutil import errno import logging import argparse +import datetime import subprocess import tornado.gen @@ -188,6 +189,20 @@ class X509(rpki.POW.X509): return "<X509 at 0x{:x}>".format(id(self)) @classmethod + def store_if_new(cls, der, uri, retrieval): + self = cls.derRead(der) + aki = self.getAKI() + ski = self.getSKI() + return rpki.rcynicdb.models.RPKIObject.objects.get_or_create( + der = der, + defaults = dict( + uri = uri, + aki = "" if aki is None else aki.encode("hex"), + ski = "" if ski is None else ski.encode("hex"), + hash = sha256(der).encode("hex"), + retrieved = retrieval)) + + @classmethod def derReadURI(cls, uri, generation, cms = None): fn = uri_to_filename(uri, generation.tree) if not os.path.exists(fn): @@ -296,6 +311,19 @@ class CRL(rpki.POW.CRL): return "<CRL at 0x{:x}>".format(id(self)) @classmethod + def store_if_new(cls, der, uri, retrieval): + self = cls.derRead(der) + aki = self.getAKI() + return rpki.rcynicdb.models.RPKIObject.objects.get_or_create( + der = der, + defaults = dict( + uri = uri, + aki = "" if aki is None else aki.encode("hex"), + ski = "", + hash = sha256(der).encode("hex"), + retrieved = retrieval)) + + @classmethod def derReadURI(cls, uri, generation): fn = uri_to_filename(uri, generation.tree) if not os.path.exists(fn): @@ -346,7 +374,25 @@ class CRL(rpki.POW.CRL): return not any(s.kind == "bad" for s in status) -class Ghostbuster(rpki.POW.CMS): +class CMS_Mixin(object): + + @classmethod + def store_if_new(cls, der, uri, retrieval): + self = cls.derRead(der) + cert = self.certs()[0] + aki = cert.getAKI() + ski = cert.getSKI() + return rpki.rcynicdb.models.RPKIObject.objects.get_or_create( + der = der, + defaults = dict( + uri = uri, + aki = "" if aki is None else aki.encode("hex"), + ski = "" if ski is None else ski.encode("hex"), + hash = sha256(der).encode("hex"), + retrieved = retrieval)) + + +class Ghostbuster(rpki.POW.CMS, CMS_Mixin): def __repr__(self): try: @@ -384,7 +430,7 @@ class Ghostbuster(rpki.POW.CMS): return not any(s.kind == "bad" for s in status) -class Manifest(rpki.POW.Manifest): +class Manifest(rpki.POW.Manifest, CMS_Mixin): def __repr__(self): try: @@ -406,6 +452,7 @@ class Manifest(rpki.POW.Manifest): self.ee = X509.derReadURI(uri, generation, self) self.fah = None self.generation = generation + self.sha256 = sha256(der) self.thisUpdate = None self.nextUpdate = None self.number = None @@ -443,7 +490,7 @@ class Manifest(rpki.POW.Manifest): yield diruri + fn, digest -class ROA(rpki.POW.ROA): +class ROA(rpki.POW.ROA, CMS_Mixin): def __repr__(self): try: @@ -483,6 +530,19 @@ class ROA(rpki.POW.ROA): return not any(s.kind == "bad" for s in status) +class_dispatch = dict(cer = X509, + crl = CRL, + gbr = Ghostbuster, + mft = Manifest, + roa = ROA) + +def uri_to_class(uri): + cls = class_dispatch.get(uri[-3:]) if len(uri) > 4 and uri[-4] == "." else None + if cls is None: + Status.add(uri, None, codes.UNKNOWN_OBJECT_TYPE_SKIPPED) + return cls + + class WalkFrame(object): """ Certificate tree walk stack frame. This is basically just a @@ -492,10 +552,6 @@ class WalkFrame(object): after an rsync or RRDP fetch completes). """ - fns2 = dict(cer = X509, - gbr = Ghostbuster, - roa = ROA) - def __init__(self, cer): self.cer = cer self.state = self.initial @@ -626,15 +682,19 @@ class WalkFrame(object): yield tornado.gen.moment uri = self.diruri + fn - cls = self.fns2.get(uri[-3:]) # Need general URI validator here? if uri == self.crl.uri: continue - if uri[-4] != "." or cls is None: - Status.add(uri, None, codes.UNKNOWN_OBJECT_TYPE_SKIPPED) + cls = uri_to_class(uri) + + if cls is None: + continue + + if cls in (Manifest, CRL): + Status.add(uri, None, codes.INAPPROPRIATE_OBJECT_TYPE_SKIPPED) continue for generation in (Generation.current, Generation.backup): @@ -706,12 +766,12 @@ class WalkTask(object): def read_tals(): - for root, dirs, files in os.walk(args.tals): + for head, dirs, files in os.walk(args.tals): for fn in files: if fn.endswith(".tal"): - furi = "file://" + os.path.abspath(os.path.join(root, fn)) + furi = "file://" + os.path.abspath(os.path.join(head, fn)) try: - with open(os.path.join(root, fn), "r") as f: + with open(os.path.join(head, fn), "r") as f: lines = f.readlines() uri = lines.pop(0).strip() b64 = "".join(lines[lines.index("\n"):]) @@ -822,14 +882,15 @@ class Fetcher(object): self._rsync_history[path] = self try: + path = uri_to_filename(self.uri, args.unauthenticated) cmd = ["rsync", "--update", "--times", "--copy-links", "--itemize-changes"] if self.uri.endswith("/"): cmd.append("--recursive") cmd.append("--delete") cmd.append(self.uri) - cmd.append(uri_to_filename(self.uri, args.unauthenticated)) + cmd.append(path) - dn = os.path.dirname(cmd[-1]) + dn = os.path.dirname(path) if not os.path.exists(dn): os.makedirs(dn) @@ -862,11 +923,44 @@ class Fetcher(object): # Should do something with rsync result and validation status database here. + # We probably don't want to yield in the middle of a + # transaction, and this doesn't really need to be wrapped + # in a transaction in any case, so leave well enough alone. + # + #from django.db import IntegrityError, transaction + #with transaction.atomic(): + + retrieval = rpki.rcynicdb.models.Retrieval.objects.create( + uri = self.uri, + started = datetime.datetime.fromtimestamp(t0), + finished = datetime.datetime.fromtimestamp(t1), + successful = self.status == 0) + + for fn in self._rsync_walk(path): + yield tornado.gen.moment + uri = "rsync://" + fn[len(args.unauthenticated):].lstrip("/") + cls = uri_to_class(uri) + if cls is not None: + try: + with open(fn, "rb") as f: + cls.store_if_new(f.read(), uri, retrieval) + except: + Status.add(uri, Generation.current, codes.UNREADABLE_OBJECT) + logger.exception("Couldn't read %s from rsync tree", uri) + finally: pending = self.pending self.pending = None pending.notify_all() + def _rsync_walk(self, path): + if self.uri.endswith("/"): + for head, dirs, files in os.walk(path): + for fn in files: + yield os.path.join(head, fn) + elif os.path.exists(path): + yield path + class CheckTALTask(object): @@ -964,7 +1058,8 @@ class posint(int): def main(): - os.putenv("TZ", "UTC") + os.environ.update(TZ = "UTC", + DJANGO_SETTINGS_MODULE = "rpki.django_settings.rcynic") time.tzset() parser = argparse.ArgumentParser(description = __doc__) @@ -975,13 +1070,26 @@ def main(): parser.add_argument("--tals", default = "sample-trust-anchors") - parser.add_argument("--workers", type = posint, default = 10) - parser.add_argument("--no-fetch", action = "store_true") - parser.add_argument("--no-spawn-on-fetch", action = "store_true") + parser.add_argument("--workers", type = posint, default = 10) + + parser.add_argument("--no-fetch", action = "store_true") + parser.add_argument("--no-spawn-on-fetch", action = "store_true") + parser.add_argument("--no-migrate", action = "store_true") global args args = parser.parse_args() + import django + django.setup() + + if not args.no_migrate: + # Not sure we should be doing this on every run, but sure simplifies things. + import django.core.management + django.core.management.call_command("migrate", verbosity = 0, interactive = False) + + global rpki + import rpki.rcynicdb + global new_authenticated, old_authenticated new_authenticated = args.authenticated.rstrip("/") + time.strftime(".%Y-%m-%dT%H:%M:%SZ") old_authenticated = args.authenticated.rstrip("/") diff --git a/rpki/POW/__init__.py b/rpki/POW/__init__.py index e88fae80..b6f15a39 100644 --- a/rpki/POW/__init__.py +++ b/rpki/POW/__init__.py @@ -164,6 +164,7 @@ validation_status = StatusCodeDB( TRUST_ANCHOR_WITH_CRLDP = "Trust anchor can't have CRLDP", UNKNOWN_AFI = "Unknown AFI", UNKNOWN_OPENSSL_VERIFY_ERROR = "Unknown OpenSSL verify error", + UNREADABLE_OBJECT = "Unreadable object", UNREADABLE_TRUST_ANCHOR = "Unreadable trust anchor", UNREADABLE_TRUST_ANCHOR_LOCATOR = "Unreadable trust anchor locator", WRONG_OBJECT_VERSION = "Wrong object version", @@ -181,7 +182,8 @@ validation_status = StatusCodeDB( DIGEST_MISMATCH = "Digest mismatch", EE_CERTIFICATE_WITH_1024_BIT_KEY = "EE certificate with 1024 bit key", GRATUITOUSLY_CRITICAL_EXTENSION = "Gratuitously critical extension", - ISSUER_USES_MULTIPLE_CRLDP_VALUES = "Issuer uses multiple CRLDP values",\ + INAPPROPRIATE_OBJECT_TYPE_SKIPPED = "Inappropriate object type skipped", + ISSUER_USES_MULTIPLE_CRLDP_VALUES = "Issuer uses multiple CRLDP values", MULTIPLE_RSYNC_URIS_IN_EXTENSION = "Multiple rsync URIs in extension", NONCONFORMANT_ISSUER_NAME = "Nonconformant X.509 issuer name", NONCONFORMANT_SUBJECT_NAME = "Nonconformant X.509 subject name", diff --git a/rpki/django_settings/common.py b/rpki/django_settings/common.py index 4aa3e119..2f41fe77 100644 --- a/rpki/django_settings/common.py +++ b/rpki/django_settings/common.py @@ -118,3 +118,8 @@ if cfg.has_option("secret-key", section = "web_portal"): SECRET_KEY = cfg.get("secret-key", section = "web_portal") else: SECRET_KEY = os.urandom(66).encode("hex") + + +# Django defaults to thinking everybody lives in Chicago. + +TIME_ZONE = "UTC" diff --git a/rpki/django_settings/rcynic.py b/rpki/django_settings/rcynic.py new file mode 100644 index 00000000..90491ddc --- /dev/null +++ b/rpki/django_settings/rcynic.py @@ -0,0 +1,61 @@ +# $Id$ + +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +This module contains configuration settings for Django libraries. +At present, rcynicng only uses the Django ORM, not the rest of Django. +Unlike the CA tools rcynicng defaults to using SQLite3 as its database +engine, so we tweak the defaults a little before instantiating the +database configuration here. +""" + +from .common import * # pylint: disable=W0401,W0614 + +__version__ = "$Id$" + + +# Database configuration. + +class DBConfigurator(DatabaseConfigurator): + + default_sql_engine = "sqlite3" + + @property + def sqlite3(self): + return dict( + ENGINE = "django.db.backends.sqlite3", + NAME = cfg.get("sql-database", section = self.section, default = "rcynic.db")) + + +DATABASES = DBConfigurator().configure(cfg, "rcynic") + +del DBConfigurator +del DatabaseConfigurator + + +# Apps. + +INSTALLED_APPS = ["rpki.rcynicdb"] + + +# Allow local site to override any setting above -- but if there's +# anything that local sites routinely need to modify, please consider +# putting that configuration into rpki.conf and just adding code here +# to read that configuration. +try: + from local_settings import * # pylint: disable=W0401,F0401 +except ImportError: + pass diff --git a/rpki/rcynicdb/__init__.py b/rpki/rcynicdb/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/rpki/rcynicdb/__init__.py diff --git a/rpki/rcynicdb/models.py b/rpki/rcynicdb/models.py new file mode 100644 index 00000000..318f87e3 --- /dev/null +++ b/rpki/rcynicdb/models.py @@ -0,0 +1,78 @@ +# First cut at ORM models for rcynicng, assuming for now that we're +# going to go with Django rather than raw SQL. + +from django.db import models + +# HTTP/HTTPS/RSYNC fetch event. +# +# Open issue: for RRDP, are we just recording the notification fetch, +# or the snapshot/delta fetches as well? If the latter, to which +# retrieval event does the RRDPSnapshot 1:1 relationship refer? For +# that matter, should we somehow be recording the relationship between +# the notification and snapshot/delta fetches? Given that, at least +# in the current protocol, we will only do either one snapshot fetch +# or one delta fetch after the notify fetch, we could just use two +# URIs in the retrieval record, if we allow the second to be empty +# (which we would have to do anyway for rsync). +# +# Or we could add some kind of fun SQL self-reference, which, in +# Django, looks like: +# +# models.ForeignKey('self', on_delete = models.CASCADE) +# +# except that it's more like a 1:1 recursive relationship, which isn't +# mentioned in the Django docs, but which supposedly +# (http://stackoverflow.com/questions/18271001/django-recursive-relationship) +# works the same way: +# +# models.OneToOneField('self', null = True) +# +# Unclear whether we still need "on_delete = models.CASCADE", probably. +# Example on StackOverflow has a complex .save() method, but that may +# be specific to the original poster's desired semantics. + +class Retrieval(models.Model): + uri = models.TextField() + started = models.DateTimeField() + finished = models.DateTimeField() + successful = models.BooleanField() + +# Collection of validated objects (like current +# rsync-data/authenticated.yyyy-mm-ddTHH:MM:SS/ tree) + +class Authenticated(models.Model): + timestamp = models.DateTimeField() + +# One instance of an RRDP snapshot. +# +# Deltas are processed by finding the RRDPSnapshot holding the +# starting point, creating a new RRDPSnapshot for the endpoint, and +# applying all necessary deltas (with consistency checks all along the +# way) to get from one to the other; we don't commit the endpoint (or +# anything created during the process) until and unless it all works. +# +# Not sure we want uuid field, drop if not useful. + +class RRDPSnapshot(models.Model): + timestamp = models.DateTimeField() + uuid = models.UUIDField() + serial = models.BigIntegerField() + retrieved = models.OneToOneField(Retrieval) + +# RPKI objects. + +class RPKIObject(models.Model): + der = models.BinaryField(unique = True) + uri = models.TextField() + aki = models.SlugField(max_length = 40) # hex SHA-1 + ski = models.SlugField(max_length = 40) # hex SHA-1 + hash = models.SlugField(max_length = 64) # hex SHA-256 + retrieved = models.ForeignKey(Retrieval) + authenticated = models.ManyToManyField(Authenticated) + snapshot = models.ManyToManyField(RRDPSnapshot) + +# No exact analogue to current unauthenticated tree. Generally, when +# we would have looked in the unauthenticated tree we want the most +# recently retrieved copy of a particular object, but particular +# object gets a little weird in RRDP universe. See Tim's draft, not +# gospel but best worked example available to date. |