diff options
| -rwxr-xr-x | ca/irbe_cli | 2 | ||||
| -rw-r--r-- | ca/rpki-confgen.xml | 32 | ||||
| -rwxr-xr-x | ca/rpki-manage | 17 | ||||
| -rwxr-xr-x | ca/rpki-sql-backup | 2 | ||||
| -rwxr-xr-x | ca/rpki-sql-setup | 4 | ||||
| -rwxr-xr-x | ca/rpki-start-servers | 4 | ||||
| -rw-r--r-- | ca/rpki.wsgi | 3 | ||||
| -rw-r--r-- | ca/tests/smoketest.py | 2 | ||||
| -rw-r--r-- | ca/tests/sql-cleaner.py | 2 | ||||
| -rw-r--r-- | ca/tests/sql-dumper.py | 2 | ||||
| -rw-r--r-- | ca/tests/yamlconf.py | 4 | ||||
| -rw-r--r-- | ca/tests/yamltest.py | 14 | ||||
| -rw-r--r-- | potpourri/upgrade-add-ghostbusters.py | 2 | ||||
| -rw-r--r-- | rpki/config.py | 43 | ||||
| -rw-r--r-- | rpki/db_router.py | 57 | ||||
| -rw-r--r-- | rpki/django_settings.py | 238 | ||||
| -rw-r--r-- | rpki/fields.py | 192 | ||||
| -rw-r--r-- | rpki/gui/default_settings.py | 171 | ||||
| -rw-r--r-- | rpki/irdb/models.py | 120 | ||||
| -rw-r--r-- | rpki/irdbd.py | 29 | ||||
| -rw-r--r-- | rpki/old_irdbd.py | 2 | ||||
| -rw-r--r-- | rpki/pubd.py | 2 | ||||
| -rw-r--r-- | rpki/pubdb/__init__.py | 3 | ||||
| -rw-r--r-- | rpki/rootd.py | 2 | ||||
| -rw-r--r-- | rpki/rpkic.py | 2 | ||||
| -rw-r--r-- | rpki/rpkid.py | 2 | ||||
| -rw-r--r-- | rpki/rpkidb/__init__.py | 3 |
27 files changed, 596 insertions, 360 deletions
diff --git a/ca/irbe_cli b/ca/irbe_cli index 1becd403..c38cf93b 100755 --- a/ca/irbe_cli +++ b/ca/irbe_cli @@ -305,7 +305,7 @@ for o, a in opts: if not argv: usage(1) -cfg = rpki.config.parser(cfg_file, "irbe_cli") +cfg = rpki.config.parser(set_filename = cfg_file, section = "irbe_cli") q_msg_left_right = [] q_msg_publication = [] diff --git a/ca/rpki-confgen.xml b/ca/rpki-confgen.xml index e0ed273a..a29ad8cd 100644 --- a/ca/rpki-confgen.xml +++ b/ca/rpki-confgen.xml @@ -816,30 +816,18 @@ <section name = "web_portal"> <doc> - Glue to allow the Django application to pull user configuration - from this file rather than directly editing settings.py. + Glue to allow Django to pull user configuration from this file + rather than requiring the user to edit settings.py. </doc> - <option name = "sql-database" - value = "${myrpki::irdbd_sql_database}"> - <doc> - SQL database name the web portal should use. - </doc> - </option> - - <option name = "sql-username" - value = "${myrpki::irdbd_sql_username}"> - <doc> - SQL user name the web portal should use. - </doc> - </option> - - <option name = "sql-password" - value = "${myrpki::irdbd_sql_password}"> - <doc> - SQL password the web portal should use. - </doc> - </option> + <!-- + We used to have SQL settings for the GUI here, but since + they're pretty much required to be identical to the ones for + irdbd at this point, the duplicate entries were just another + chance to misconfigure something, so I removed them. Not yet + sure whether this was the right approach. Too much historical + baggage in this file. + --> <option name = "secret-key"> <doc> diff --git a/ca/rpki-manage b/ca/rpki-manage index 0d581ce9..db1e9ce3 100755 --- a/ca/rpki-manage +++ b/ca/rpki-manage @@ -5,9 +5,20 @@ from django.core.management import execute_from_command_line # django-admin seems to have problems creating the superuser account when # $LANG is unset or is set to something totally incompatible with UTF-8. -if os.environ.get('LANG') in (None, "", "C"): - os.environ['LANG'] = 'en_US.UTF-8' -os.environ['DJANGO_SETTINGS_MODULE'] = 'rpki.gui.default_settings' +if os.environ.get("LANG") in (None, "", "C"): + os.environ["LANG"] = "en_US.UTF-8" + +# Where to find the Django settings module + +os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings") + +# We don't know whether we're being used to configure the GUI or not +# (well, not without examining the specific command, which we'd like +# to avoid). Default to enabling the GUI so that such commands will +# work, but allow the user to override via the environment variable. + +if not os.environ.get("RPKI_GUI_ENABLE"): + os.environ["RPKI_GUI_ENABLE"] = "yes" execute_from_command_line() diff --git a/ca/rpki-sql-backup b/ca/rpki-sql-backup index e60f9ae3..02835956 100755 --- a/ca/rpki-sql-backup +++ b/ca/rpki-sql-backup @@ -41,7 +41,7 @@ parser.add_argument("-o", "--output", help = "destination for SQL dump (default: stdout)") args = parser.parse_args() -cfg = rpki.config.parser(args.config, "myrpki") +cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") for name in ("rpkid", "irdbd", "pubd"): if cfg.getboolean("start_" + name, False): diff --git a/ca/rpki-sql-setup b/ca/rpki-sql-setup index edc2c242..848e3d0f 100755 --- a/ca/rpki-sql-setup +++ b/ca/rpki-sql-setup @@ -54,7 +54,7 @@ class RootDB(object): user = "root", passwd = getpass.getpass("Please enter your MySQL root password: ")) else: - mysql_cfg = rpki.config.parser(self.mysql_defaults, "client") + mysql_cfg = rpki.config.parser(set_filename = self.mysql_defaults, section = "client") self.db = MySQLdb.connect(db = "mysql", user = mysql_cfg.get("user"), passwd = mysql_cfg.get("password")) @@ -299,7 +299,7 @@ parser.set_defaults(dispatch = do_create_if_missing) args = parser.parse_args() try: - cfg = rpki.config.parser(args.config, "myrpki") + cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") root = RootDB(args.mysql_defaults) current_version = Version(rpki.version.VERSION) for program_name in ("irdbd", "rpkid", "pubd"): diff --git a/ca/rpki-start-servers b/ca/rpki-start-servers index 8a745896..f1f70aa8 100755 --- a/ca/rpki-start-servers +++ b/ca/rpki-start-servers @@ -64,13 +64,13 @@ group.add_argument("--log-syslog", default = "daemon", nargs = "?", help = "log syslog") args = parser.parse_args() -cfg = rpki.config.parser(args.config, "myrpki") +cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") def run(name, old_flag = None): if cfg.getboolean("start_" + name, cfg.getboolean("run_" + name if old_flag is None else old_flag, False)): # pylint: disable=E1103 log_file = os.path.join(args.log_directory, name + ".log") - cmd = (os.path.join(rpki.autoconf.libexecdir, name), "--config", cfg.filename, "--log-level", args.log_level) + cmd = (os.path.join(rpki.autoconf.libexecdir, name), "--log-level", args.log_level) if args.log_file: cmd += ("--log-file", log_file) elif args.log_rotating_file_kbytes: diff --git a/ca/rpki.wsgi b/ca/rpki.wsgi index 72ba75ac..7fa85d73 100644 --- a/ca/rpki.wsgi +++ b/ca/rpki.wsgi @@ -21,7 +21,8 @@ import sys import os import rpki.autoconf -os.environ['DJANGO_SETTINGS_MODULE'] = 'rpki.gui.default_settings' +os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings", + RPKI_GUI_ENABLE = "yes") # Needed for local_settings.py sys.path.insert(1, rpki.autoconf.sysconfdir + '/rpki') diff --git a/ca/tests/smoketest.py b/ca/tests/smoketest.py index 32f11cc3..d1b15253 100644 --- a/ca/tests/smoketest.py +++ b/ca/tests/smoketest.py @@ -68,7 +68,7 @@ parser.add_argument("yaml_file", type = argparse.FileType("r"), help = "YAML description of test network") args = parser.parse_args() -cfg = rpki.config.parser(args.config, "smoketest", allow_missing = True) +cfg = rpki.config.parser(set_filename = args.config, section = "smoketest", allow_missing = True) # Load the YAML script early, so we can report errors ASAP diff --git a/ca/tests/sql-cleaner.py b/ca/tests/sql-cleaner.py index ca88d456..0f0b55b1 100644 --- a/ca/tests/sql-cleaner.py +++ b/ca/tests/sql-cleaner.py @@ -22,7 +22,7 @@ import rpki.config import rpki.sql_schemas from rpki.mysql_import import MySQLdb -cfg = rpki.config.parser(None, "yamltest", allow_missing = True) +cfg = rpki.config.parser(section = "yamltest", allow_missing = True) for name in ("rpkid", "irdbd", "pubd"): diff --git a/ca/tests/sql-dumper.py b/ca/tests/sql-dumper.py index 19cc1b34..d0fe3489 100644 --- a/ca/tests/sql-dumper.py +++ b/ca/tests/sql-dumper.py @@ -22,7 +22,7 @@ import subprocess import rpki.config from rpki.mysql_import import MySQLdb -cfg = rpki.config.parser(None, "yamltest", allow_missing = True) +cfg = rpki.config.parser(section = "yamltest", allow_missing = True) for name in ("rpkid", "irdbd", "pubd"): diff --git a/ca/tests/yamlconf.py b/ca/tests/yamlconf.py index 1b339a89..0f1467f7 100644 --- a/ca/tests/yamlconf.py +++ b/ca/tests/yamlconf.py @@ -500,7 +500,7 @@ class allocation(object): def hire_zookeeper(self): assert not self.is_hosted self._zoo = rpki.irdb.Zookeeper( - cfg = rpki.config.parser(self.path("rpki.conf")), + cfg = rpki.config.parser(filename = self.path("rpki.conf")), logstream = None if quiet else sys.stdout) @property @@ -681,7 +681,7 @@ def main(): # passwords: this is mostly so that I can show a complete working # example without publishing my own server's passwords. - cfg = rpki.config.parser(args.config, "yamlconf", allow_missing = True) + cfg = rpki.config.parser(set_filename = args.config, section = "yamlconf", allow_missing = True) try: cfg.set_global_flags() except: diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index 2b65dbd2..62b1252b 100644 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -539,14 +539,15 @@ class allocation(object): """ Run rpkic for this entity. """ - cmd = [prog_rpkic, "-i", self.name, "-c", self.path("rpki.conf")] + cmd = [prog_rpkic, "-i", self.name] if args.profile: cmd.append("--profile") cmd.append(self.path("rpkic.%s.prof" % rpki.sundial.now())) cmd.extend(str(a) for a in argv if a is not None) print 'Running "%s"' % " ".join(cmd) env = os.environ.copy() - env["YAMLTEST_RPKIC_COUNTER"] = self.next_rpkic_counter() + env.update(YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(), + RPKI_CONF = self.path("rpki.conf")) subprocess.check_call(cmd, cwd = self.host.path(), env = env) def run_python_daemon(self, prog): @@ -556,12 +557,13 @@ class allocation(object): """ basename = os.path.splitext(os.path.basename(prog))[0] cmd = [prog, "--foreground", "--log-level", "debug", - "--log-file", self.path(basename + ".log"), - "--config", self.path("rpki.conf")] + "--log-file", self.path(basename + ".log")] if args.profile and basename != "rootd": cmd.extend(( "--profile", self.path(basename + ".prof"))) - p = subprocess.Popen(cmd, cwd = self.path()) + env = os.environ.copy() + env.update(RPKI_CONF = self.path("rpki.conf")) + p = subprocess.Popen(cmd, cwd = self.path(), env = env) print 'Running %s for %s: pid %d process %r' % (" ".join(cmd), self.name, p.pid, p) return p @@ -672,7 +674,7 @@ try: # passwords: this is mostly so that I can show a complete working # example without publishing my own server's passwords. - cfg = rpki.config.parser(args.config, "yamltest", allow_missing = True) + cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True) only_one_pubd = cfg.getboolean("only_one_pubd", True) allocation.base_port = cfg.getint("base_port", 4400) diff --git a/potpourri/upgrade-add-ghostbusters.py b/potpourri/upgrade-add-ghostbusters.py index a8c8a92b..2548487c 100644 --- a/potpourri/upgrade-add-ghostbusters.py +++ b/potpourri/upgrade-add-ghostbusters.py @@ -43,7 +43,7 @@ for o, a in opts: if o in ("-c", "--config"): cfg_file = a -cfg = rpki.config.parser(cfg_file, "myrpki") +cfg = rpki.config.parser(filename = cfg_file, section = "myrpki") fix("irdbd", """ CREATE TABLE ghostbuster_request ( diff --git a/rpki/config.py b/rpki/config.py index f38427c4..c73fe837 100644 --- a/rpki/config.py +++ b/rpki/config.py @@ -45,10 +45,10 @@ try: except ImportError: default_dirname = None -## @var default_envname +## @var rpki_conf_envname # Name of environment variable containing config file name. -default_envname = "RPKI_CONF" +rpki_conf_envname = "RPKI_CONF" class parser(object): """ @@ -61,14 +61,35 @@ class parser(object): get-methods with default values and default section name. - If no filename is given to the constructor (filename = None), we - check for an environment variable naming the config file, then we - check for a default filename in the current directory, then finally - we check for a global config file if autoconf provided a directory - name to check. + If no filename is given to the constructor (filename and + set_filename both None), we check for an environment variable naming + the config file, then we check for a default filename in the current + directory, then finally we check for a global config file if + autoconf provided a directory name to check. + + NB: In most cases, programs which accept configuration filenames on + their command lines should pass those filenames to us using + set_filename so that we can set the magic environment variable, + because constraints from some external libraries (principally + Django) sometimes require our own library code to look things up in + the configuration file without the knowledge of the controlling + program. Setting the environment variable insures that everybody's + reading from the same script, as it were. """ - def __init__(self, filename = None, section = None, allow_missing = False): + # Odd keyword-only calling sequence is a defense against old code + # that thinks it knows how __init__() handles positional arguments. + + def __init__(self, **kwargs): + section = kwargs.pop("section", None) + allow_missing = kwargs.pop("allow_missing", False) + set_filename = kwargs.pop("set_filename", None) + filename = kwargs.pop("filename", set_filename) + + assert not kwargs, "Unexpected keyword arguments: " + ", ".join("%s = %r" % kv for kv in kwargs.iteritems()) + + if set_filename is not None: + os.environ[rpki_conf_envname] = set_filename self.cfg = ConfigParser.RawConfigParser() self.default_section = section @@ -77,11 +98,11 @@ class parser(object): if filename is not None: filenames.append(filename) else: - if default_envname in os.environ: - filenames.append(os.environ[default_envname]) + if rpki_conf_envname in os.environ: + filenames.append(os.environ[rpki_conf_envname]) filenames.append(default_filename) if default_dirname is not None: - filenames.append("%s/%s" % (default_dirname, default_filename)) + filenames.append(os.path.join(default_dirname, default_filename)) f = fn = None diff --git a/rpki/db_router.py b/rpki/db_router.py new file mode 100644 index 00000000..89ed6e5d --- /dev/null +++ b/rpki/db_router.py @@ -0,0 +1,57 @@ +# $Id$ + +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +Global Django ORM database router for the RPKI CA code. +""" + +# Reference: +# https://docs.djangoproject.com/en/1.6/topics/db/multi-db/ + +class RPKIDBRouter(object): + """ + Django ORM database router for RPKI code. rpkid and pubd get their + own databases, named "rpkidb" and "pubdb", respectively. Everything + else goes to the "default" database. + """ + + dedicated = ("rpkidb", "pubdb") + + def db_for_read(self, model, **hints): + if model._meta.app_label in self.dedicated: + return model._meta.app_label + else: + return "default" + + def db_for_write(self, model, **hints): + if model._meta.app_label in self.dedicated: + return model._meta.app_label + else: + return "default" + + def allow_relation(self, obj1, obj2, **hints): + if obj1._meta.app_label in self.dedicated and obj1._meta.app_label == obj2._meta.app_label: + return True + elif obj1._meta.app_label not in self.dedicated and obj2._meta.app_label not in self.dedicated: + return True + else: + return None + + def allow_syncdb(self, db, model): + if model._meta.app_label in self.dedicated: + return db == model._meta.app_label + else: + return db not in self.dedicated diff --git a/rpki/django_settings.py b/rpki/django_settings.py new file mode 100644 index 00000000..eb3a184b --- /dev/null +++ b/rpki/django_settings.py @@ -0,0 +1,238 @@ +# $Id$ + +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +This module contains configuration settings for Django libraries. + +Most of our CA code uses at least the Django ORM; the web interface +uses a lot more of Django. We also want to handle all normal user +configuration via rpki.conf, so some of the code here is just pulling +settings from rpki.conf and stuffing them into the form Django wants. +""" + +__version__ = "$Id$" + +import os +import socket + +import rpki.config +import rpki.autoconf + +# Some configuration, including SQL authorization, comes from rpki.conf. +cfg = rpki.config.parser() + + +# Do -not- turn on DEBUG here except for short-lived tests, otherwise +# long-running programs like irdbd will eventually run out of memory +# and crash. +# +# If you must enable debugging, use django.db.reset_queries() to +# clear the query list manually, but it's probably better just to +# run with debugging disabled, since that's the expectation for +# production code. +# +# https://docs.djangoproject.com/en/dev/faq/models/#why-is-django-leaking-memory + + +# Database configuration. This is always enabled, and uses a database +# "router" to handle multiple databases. We may want to add yet +# another database to hold South's migration tables, to avoid the +# silliness of requiring an IRDB on, eg, a pubd-only server. +# +# We used to set an option to force MySQL to create InnnoDB databases, +# and we used to set HOST and PORT to the null string, but all of +# these are the defaults with recent versions of MySQL and Django, so +# in theory none of them should be necessary. + +DATABASES = dict( + default = dict(ENGINE = "django.db.backends.mysql", + NAME = cfg.get("sql-database", section = "irdbd"), + USER = cfg.get("sql-username", section = "irdbd"), + PASSWORD = cfg.get("sql-password", section = "irdbd"))) + +if cfg.getboolean("start_rpkid", section = "myrpki"): + DATABASES.update( + rpkidb = dict(ENGINE = "django.db.backends.mysql", + NAME = cfg.get("sql-database", section = "rpkid"), + USER = cfg.get("sql-username", section = "rpkid"), + PASSWORD = cfg.get("sql-password", section = "rpkid"))) + +if cfg.getboolean("start_pubd", section = "myrpki"): + DATABASES.update( + pubdb = dict(ENGINE = "django.db.backends.mysql", + NAME = cfg.get("sql-database", section = "pubd"), + USER = cfg.get("sql-username", section = "pubd"), + PASSWORD = cfg.get("sql-password", section = "pubd"))) + +# ORM database "router" to sort out which apps use which databases. + +DATABASE_ROUTERS = ["rpki.db_router.RPKIDBRouter"] + +# Figure out which apps we're running -- GUI code below adds many more. + +INSTALLED_APPS = ["south"] + +if cfg.getboolean("start_irdbd", section = "myrpki"): + INSTALLED_APPS.append("rpki.irdb") + +if cfg.getboolean("start_rpkid", section = "myrpki"): + INSTALLED_APPS.append("rpki.rpkidb") + +if cfg.getboolean("start_pubd", section = "myrpki"): + INSTALLED_APPS.append("rpki.pubdb") + +# That's about it if we just need the ORM, but Django throws a hissy +# fit if SECRET_KEY isn't set, whether we use it for anything or not. +# +# Make this unique, and don't share it with anybody. +if cfg.has_option("secret-key", section = "web_portal"): + SECRET_KEY = cfg.get("secret-key", section = "web_portal") +else: + SECRET_KEY = os.urandom(66).encode("hex") + + +# If we're the GUI (or a program like rpki-manage that might be +# configuring the GUI) we need a lot of other stuff, so check for an +# environment variable that rpki.wsgi and rpki-manage set for us. + +if os.getenv("RPKI_GUI_ENABLE") == "yes": + + # Where to put static files. + STATIC_ROOT = rpki.autoconf.datarootdir + "/rpki/media" + + # Must end with a slash! + STATIC_URL = "/media/" + + # Where to email server errors. + ADMINS = (("Administrator", "root@localhost"),) + + LOGGING = { + "version": 1, + "formatters": { + "verbose": { + # see http://docs.python.org/2.7/library/logging.html#logging.LogRecord + "format": "%(levelname)s %(asctime)s %(name)s %(message)s" + }, + }, + "handlers": { + "stderr": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "verbose", + }, + "mail_admins": { + "level": "ERROR", + "class": "django.utils.log.AdminEmailHandler", + }, + }, + "loggers": { + "django": { + "level": "ERROR", + "handlers": ["stderr", "mail_admins"], + }, + "rpki.gui": { + "level": "WARNING", + "handlers": ["stderr"], + }, + }, + } + + def select_tz(): + "Find a supported timezone that looks like UTC" + for tz in ("UTC", "GMT", "Etc/UTC", "Etc/GMT"): + if os.path.exists("/usr/share/zoneinfo/" + tz): + return tz + # Can't determine the proper timezone, fall back to UTC and let Django + # report the error to the user. + return "UTC" + + # Local time zone for this installation. Choices can be found here: + # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name + # although not all choices may be available on all operating systems. + # If running in a Windows environment this must be set to the same as your + # system time zone. + TIME_ZONE = select_tz() + + # See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts + # for details on why you might need this. + def get_allowed_hosts(): + allowed_hosts = set(cfg.multiget("allowed-hosts", section = "web_portal")) + allowed_hosts.add(socket.getfqdn()) + try: + import netifaces + for interface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(interface) + for af in (netifaces.AF_INET, netifaces.AF_INET6): + if af in addresses: + for address in addresses[af]: + if "addr" in address: + allowed_hosts.add(address["addr"]) + except ImportError: + pass + return list(allowed_hosts) + + ALLOWED_HOSTS = get_allowed_hosts() + + # List of callables that know how to import templates from various sources. + TEMPLATE_LOADERS = ( + "django.template.loaders.filesystem.Loader", + "django.template.loaders.app_directories.Loader", + "django.template.loaders.eggs.Loader" + ) + + MIDDLEWARE_CLASSES = ( + "django.middleware.common.CommonMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware" + ) + + ROOT_URLCONF = "rpki.gui.urls" + + INSTALLED_APPS.extend(( + "django.contrib.auth", + #"django.contrib.admin", + #"django.contrib.admindocs", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.staticfiles", + "rpki.gui.app", + "rpki.gui.cacheview", + "rpki.gui.routeview", + )) + + TEMPLATE_CONTEXT_PROCESSORS = ( + "django.contrib.auth.context_processors.auth", + "django.core.context_processors.debug", + "django.core.context_processors.i18n", + "django.core.context_processors.media", + "django.contrib.messages.context_processors.messages", + "django.core.context_processors.request", + "django.core.context_processors.static" + ) + +# End of GUI-specific settings. + + +# Allow local site to override any setting above -- but if there's +# anything that local sites routinely need to modify, please consider +# putting that configuration into rpki.conf and just adding code here +# to read that configuration. +try: + from local_settings import * +except: + pass diff --git a/rpki/fields.py b/rpki/fields.py new file mode 100644 index 00000000..3d859aaa --- /dev/null +++ b/rpki/fields.py @@ -0,0 +1,192 @@ +# $Id$ +# +# Copyright (C) 2013--2014 Dragon Research Labs ("DRL") +# Portions copyright (C) 2011--2012 Internet Systems Consortium ("ISC") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notices and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL +# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR +# ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA +# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +Common Django ORM field classes. + +Many of these are complex ASN.1 DER objects stored as SQL BLOBs, since +the only sane text representation would just be the Base64 encoding of +the DER and thus would add no value. +""" + +from django.db import models +from south.modelsinspector import add_introspection_rules + +import rpki.x509 +import rpki.sundial + + +class EnumField(models.PositiveSmallIntegerField): + """ + An enumeration type that uses strings in Python and small integers + in SQL. + """ + + description = "An enumeration type" + + __metaclass__ = models.SubfieldBase + + def __init__(self, *args, **kwargs): + if isinstance(kwargs.get("choices"), (tuple, list)) and isinstance(kwargs["choices"][0], (str, unicode)): + kwargs["choices"] = tuple(enumerate(kwargs["choices"], 1)) + models.PositiveSmallIntegerField.__init__(self, *args, **kwargs) + self.enum_i2s = dict(self.flatchoices) + self.enum_s2i = dict((v, k) for k, v in self.flatchoices) + + def to_python(self, value): + return self.enum_i2s.get(value, value) + + def get_prep_value(self, value): + return self.enum_s2i.get(value, value) + + +class SundialField(models.DateTimeField): + """ + A field type for our customized datetime objects. + """ + __metaclass__ = models.SubfieldBase + + description = "A datetime type using our customized datetime objects" + + def to_python(self, value): + if isinstance(value, rpki.sundial.pydatetime.datetime): + return rpki.sundial.datetime.from_datetime( + models.DateTimeField.to_python(self, value)) + else: + return value + + def get_prep_value(self, value): + if isinstance(value, rpki.sundial.datetime): + return value.to_datetime() + else: + return value + + +class BlobField(models.Field): + """ + Basic BLOB field, no type conversion, just an opaque byte string. + + "BLOB" = "Binary Large OBject". Most SQL implementations seem to + have such a thing, but support appears to predate standardization, + so they all do it slightly differently and we have to cope. + + In PostgreSQL, BLOBs are called "bytea". + + In MySQL, there are different sizes of BLOBs and one must pick the + right one to avoid data truncation. RPKI manifests and CRLs can be + longer than 65535 octets, so in MySQL the only safe BLOB type for + general use is "LONGBLOB". + + SQLite...is not like the other children: data types are more like + guidelines than actual rules. But "BLOB" works. + + For anything else, we just use "BLOB" and hope for the best. + """ + + __metaclass__ = models.SubfieldBase + description = "Raw BLOB type without ASN.1 encoding/decoding" + + def __init__(self, *args, **kwargs): + self.blob_type = kwargs.pop("blob_type", None) + kwargs["serialize"] = False + kwargs["blank"] = True + kwargs["default"] = None + models.Field.__init__(self, *args, **kwargs) + + def db_type(self, connection): + if self.blob_type is not None: + return self.blob_type + elif connection.settings_dict['ENGINE'] == "django.db.backends.mysql": + return "LONGBLOB" + elif connection.settings_dict['ENGINE'] == "django.db.backends.posgresql": + return "bytea" + else: + return "BLOB" + + +# For reasons which now escape me, I had a few fields in the old +# hand-coded SQL which used MySQL type BINARY(20) to hold SKIs. +# Presumably this was so that I could then use those SKIs in indexes +# and searches, but apparently I never got around to that part. +# +# SKIs probably would be better stored as hex strings anyway, so not +# bothering with a separate binary type model for this. Deal with +# this if and when it ever becomes an issue. + + +class DERField(BlobField): + """ + Field class for DER objects. These are derived from BLOBs, but with + automatic translation between ASN.1 and Python types. + + DERField itself is an abstract class, concrete field classes are + derived from it. + """ + + __metaclass__ = models.SubfieldBase + + def to_python(self, value): + assert value is None or isinstance(value, (self.rpki_type, str)) + if isinstance(value, str): + return self.rpki_type(DER = value) + else: + return value + + def get_prep_value(self, value): + assert value is None or isinstance(value, (self.rpki_type, str)) + if isinstance(value, self.rpki_type): + return value.get_DER() + else: + return value + +class CertificateField(DERField): + description = "X.509 certificate" + rpki_type = rpki.x509.X509 + +class KeyField(DERField): + description = "RSA keypair" + rpki_type = rpki.x509.RSA + +class CRLField(DERField): + description = "Certificate Revocation List" + rpki_type = rpki.x509.CRL + +class PKCS10Field(DERField): + description = "PKCS #10 certificate request" + rpki_type = rpki.x509.PKCS10 + +class ManifestField(DERField): + description = "RPKI Manifest" + rpki_type = rpki.x509.SignedManifest + +class ROAField(DERField): + description = "ROA" + rpki_type = rpki.x509.ROA + +class GhostbusterField(DERField): + description = "Ghostbuster Record" + rpki_type = rpki.x509.Ghostbuster + + +field_classes = (EnumField, SundialField, BlobField, CertificateField, KeyField, + CRLField, PKCS10Field, ManifestField, ROAField, GhostbusterField) + +add_introspection_rules([(field_classes, [], {})], + [r"^rpki\.fields\." + cls.__name__ for cls in field_classes]) + +del field_classes diff --git a/rpki/gui/default_settings.py b/rpki/gui/default_settings.py deleted file mode 100644 index 3859247c..00000000 --- a/rpki/gui/default_settings.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -This module contains static configuration settings for the web portal. -""" - -__version__ = '$Id$' - -import os -import random -import string -import socket - -import rpki.config -import rpki.autoconf - -# Where to put static files. -STATIC_ROOT = rpki.autoconf.datarootdir + '/rpki/media' - -# Must end with a slash! -STATIC_URL = '/media/' - -# Where to email server errors. -ADMINS = (('Administrator', 'root@localhost'),) - -LOGGING = { - 'version': 1, - 'formatters': { - 'verbose': { - # see http://docs.python.org/2.7/library/logging.html#logging.LogRecord - 'format': '%(levelname)s %(asctime)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'stderr': { - 'class': 'logging.StreamHandler', - 'level': 'DEBUG', - 'formatter': 'verbose', - }, - 'mail_admins': { - 'level': 'ERROR', - 'class': 'django.utils.log.AdminEmailHandler', - }, - }, - 'loggers': { - 'django': { - 'level': 'ERROR', - 'handlers': ['stderr', 'mail_admins'], - }, - 'rpki.gui': { - 'level': 'WARNING', - 'handlers': ['stderr'], - }, - }, -} - -# Load the SQL authentication bits from the system rpki.conf. -rpki_config = rpki.config.parser(section='web_portal') - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': rpki_config.get('sql-database'), - 'USER': rpki_config.get('sql-username'), - 'PASSWORD': rpki_config.get('sql-password'), - - # Ensure the default storage engine is InnoDB since we need - # foreign key support. The Django documentation suggests - # removing this after the syncdb is performed as an optimization, - # but there isn't an easy way to do this automatically. - - 'OPTIONS': { - 'init_command': 'SET storage_engine=INNODB', - } - } -} - - -def select_tz(): - "Find a supported timezone that looks like UTC" - for tz in ('UTC', 'GMT', 'Etc/UTC', 'Etc/GMT'): - if os.path.exists('/usr/share/zoneinfo/' + tz): - return tz - # Can't determine the proper timezone, fall back to UTC and let Django - # report the error to the user. - return 'UTC' - -# Local time zone for this installation. Choices can be found here: -# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name -# although not all choices may be available on all operating systems. -# If running in a Windows environment this must be set to the same as your -# system time zone. -TIME_ZONE = select_tz() - -def get_secret_key(): - """Retrieve the secret-key value from rpki.conf or generate a random value - if it is not present.""" - d = string.letters + string.digits - val = ''.join([random.choice(d) for _ in range(50)]) - return rpki_config.get('secret-key', val) - -# Make this unique, and don't share it with anybody. -SECRET_KEY = get_secret_key() - -# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts -# for details on why you might need this. -def get_allowed_hosts(): - allowed_hosts = set(rpki_config.multiget("allowed-hosts")) - allowed_hosts.add(socket.getfqdn()) - try: - import netifaces - for interface in netifaces.interfaces(): - addresses = netifaces.ifaddresses(interface) - for af in (netifaces.AF_INET, netifaces.AF_INET6): - if af in addresses: - for address in addresses[af]: - if "addr" in address: - allowed_hosts.add(address["addr"]) - except ImportError: - pass - return list(allowed_hosts) - -ALLOWED_HOSTS = get_allowed_hosts() - -# List of callables that know how to import templates from various sources. -TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', - 'django.template.loaders.eggs.Loader' -) - -MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware' -) - -ROOT_URLCONF = 'rpki.gui.urls' - -INSTALLED_APPS = ( - 'django.contrib.auth', - #'django.contrib.admin', - #'django.contrib.admindocs', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.staticfiles', - 'rpki.irdb', - 'rpki.gui.app', - 'rpki.gui.cacheview', - 'rpki.gui.routeview', - 'south', -) - -TEMPLATE_CONTEXT_PROCESSORS = ( - "django.contrib.auth.context_processors.auth", - "django.core.context_processors.debug", - "django.core.context_processors.i18n", - "django.core.context_processors.media", - "django.contrib.messages.context_processors.messages", - "django.core.context_processors.request", - "django.core.context_processors.static" -) - -# Allow local site to override any setting above -- but if there's -# anything that local sites routinely need to modify, please consider -# putting that configuration into rpki.conf and just adding code here -# to read that configuration. -try: - from local_settings import * -except: - pass diff --git a/rpki/irdb/models.py b/rpki/irdb/models.py index 6fa48c59..26901a68 100644 --- a/rpki/irdb/models.py +++ b/rpki/irdb/models.py @@ -34,6 +34,8 @@ import socket import rpki.POW from south.modelsinspector import add_introspection_rules +from rpki.fields import EnumField, SundialField, CertificateField, DERField, KeyField, CRLField, PKCS10Field + ## @var ip_version_choices # Choice argument for fields implementing IP version numbers. @@ -61,11 +63,11 @@ ee_certificate_lifetime = rpki.sundial.timedelta(days = 60) ### -# Field types +# Field classes class HandleField(django.db.models.CharField): """ - A handle field type. + A handle field class. Replace this with SlugField? """ description = 'A "handle" in one of the RPKI protocols' @@ -74,103 +76,21 @@ class HandleField(django.db.models.CharField): kwargs["max_length"] = 120 django.db.models.CharField.__init__(self, *args, **kwargs) -class EnumField(django.db.models.PositiveSmallIntegerField): - """ - An enumeration type that uses strings in Python and small integers - in SQL. - """ - - description = "An enumeration type" - - __metaclass__ = django.db.models.SubfieldBase - - def __init__(self, *args, **kwargs): - if isinstance(kwargs.get("choices"), (tuple, list)) and isinstance(kwargs["choices"][0], str): - kwargs["choices"] = tuple(enumerate(kwargs["choices"], 1)) - django.db.models.PositiveSmallIntegerField.__init__(self, *args, **kwargs) - self.enum_i2s = dict(self.flatchoices) - self.enum_s2i = dict((v, k) for k, v in self.flatchoices) - - def to_python(self, value): - return self.enum_i2s.get(value, value) - - def get_prep_value(self, value): - return self.enum_s2i.get(value, value) - -class SundialField(django.db.models.DateTimeField): - """ - A field type for our customized datetime objects. - """ - __metaclass__ = django.db.models.SubfieldBase - - description = "A datetime type using our customized datetime objects" - - def to_python(self, value): - if isinstance(value, rpki.sundial.pydatetime.datetime): - return rpki.sundial.datetime.from_datetime( - django.db.models.DateTimeField.to_python(self, value)) - else: - return value - - def get_prep_value(self, value): - if isinstance(value, rpki.sundial.datetime): - return value.to_datetime() - else: - return value - -class DERField(django.db.models.Field): - """ - Field types for DER objects. - """ - - __metaclass__ = django.db.models.SubfieldBase - - def __init__(self, *args, **kwargs): - kwargs["serialize"] = False - kwargs["blank"] = True - kwargs["default"] = None - django.db.models.Field.__init__(self, *args, **kwargs) - - def db_type(self, connection): - if connection.settings_dict['ENGINE'] == "django.db.backends.posgresql": - return "bytea" - else: - return "BLOB" - - def to_python(self, value): - assert value is None or isinstance(value, (self.rpki_type, str)) - if isinstance(value, str): - return self.rpki_type(DER = value) - else: - return value - - def get_prep_value(self, value): - assert value is None or isinstance(value, (self.rpki_type, str)) - if isinstance(value, self.rpki_type): - return value.get_DER() - else: - return value +class SignedReferralField(DERField): + description = "CMS signed object containing XML" + rpki_type = rpki.x509.SignedReferral -class CertificateField(DERField): - description = "X.509 certificate" - rpki_type = rpki.x509.X509 -class RSAKeyField(DERField): - description = "RSA keypair" - rpki_type = rpki.x509.RSA +# Introspection rules for Django South -class CRLField(DERField): - description = "Certificate Revocation List" - rpki_type = rpki.x509.CRL +field_classes = [SignedReferralField] -class PKCS10Field(DERField): - description = "PKCS #10 certificate request" - rpki_type = rpki.x509.PKCS10 +add_introspection_rules([(field_classes, [], {})], + [r"^rpki\.irdb\.models\." + cls.__name__ + for cls in field_classes]) -class SignedReferralField(DERField): - description = "CMS signed object containing XML" - rpki_type = rpki.x509.SignedReferral +del field_classes # Custom managers @@ -231,7 +151,7 @@ class ResourceHolderEEManager(CertificateManager): class CA(django.db.models.Model): certificate = CertificateField() - private_key = RSAKeyField() + private_key = KeyField() latest_crl = CRLField() # Might want to bring these into line with what rpkid does. Current @@ -391,7 +311,7 @@ class ResourceHolderRevocation(Revocation): issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "revocations") class EECertificate(Certificate): - private_key = RSAKeyField() + private_key = KeyField() class Meta: abstract = True @@ -634,13 +554,3 @@ class Client(CrossCertification): # This shouldn't be necessary class Meta: unique_together = ("issuer", "handle") - -# for Django South -- these are just simple subclasses -add_introspection_rules([], - (r'^rpki\.irdb\.models\.CertificateField', - r'^rpki\.irdb\.models\.CRLField', - r'^rpki\.irdb\.models\.EnumField', - r'^rpki\.irdb\.models\.HandleField', - r'^rpki\.irdb\.models\.RSAKeyField', - r'^rpki\.irdb\.models\.SignedReferralField', - r'^rpki\.irdb\.models\.SundialField')) diff --git a/rpki/irdbd.py b/rpki/irdbd.py index d53ae67c..856f91bf 100644 --- a/rpki/irdbd.py +++ b/rpki/irdbd.py @@ -166,7 +166,7 @@ class main(object): rpki.log.init("irdbd", args) - self.cfg = rpki.config.parser(args.config, "irdbd") + self.cfg = rpki.config.parser(set_filename = args.config, section = "irdbd") self.cfg.set_global_flags() if not args.foreground: @@ -185,34 +185,15 @@ class main(object): def main(self): - global rpki # pylint: disable=W0602 - from django.conf import settings - startup_msg = self.cfg.get("startup-message", "") if startup_msg: logger.info(startup_msg) - # Do -not- turn on DEBUG here except for short-lived tests, - # otherwise irdbd will eventually run out of memory and crash. - # - # If you must enable debugging, use django.db.reset_queries() to - # clear the query list manually, but it's probably better just to - # run with debugging disabled, since that's the expectation for - # production code. - # - # https://docs.djangoproject.com/en/dev/faq/models/#why-is-django-leaking-memory - - settings.configure( - DATABASES = { - "default" : { - "ENGINE" : "django.db.backends.mysql", - "NAME" : self.cfg.get("sql-database"), - "USER" : self.cfg.get("sql-username"), - "PASSWORD" : self.cfg.get("sql-password"), - "HOST" : "", - "PORT" : "" }}, - INSTALLED_APPS = ("rpki.irdb",),) + # Now that we know which configuration file to use, it's OK to + # load modules that require Django's settings module. + os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings") + global rpki # pylint: disable=W0602 import rpki.irdb # pylint: disable=W0621 # Entirely too much fun with read-only access to transactional databases. diff --git a/rpki/old_irdbd.py b/rpki/old_irdbd.py index 6c026a31..bb75ae5b 100644 --- a/rpki/old_irdbd.py +++ b/rpki/old_irdbd.py @@ -290,7 +290,7 @@ class main(object): rpki.log.init("irdbd", args) - self.cfg = rpki.config.parser(args.config, "irdbd") + self.cfg = rpki.config.parser(set_filename = args.config, section = "irdbd") startup_msg = self.cfg.get("startup-message", "") if startup_msg: diff --git a/rpki/pubd.py b/rpki/pubd.py index 79315a78..cf5b1b6a 100644 --- a/rpki/pubd.py +++ b/rpki/pubd.py @@ -68,7 +68,7 @@ class main(object): rpki.log.init("pubd", args) - self.cfg = rpki.config.parser(args.config, "pubd") + self.cfg = rpki.config.parser(set_filename = args.config, section = "pubd") self.cfg.set_global_flags() if not args.foreground: diff --git a/rpki/pubdb/__init__.py b/rpki/pubdb/__init__.py new file mode 100644 index 00000000..5e25c7e3 --- /dev/null +++ b/rpki/pubdb/__init__.py @@ -0,0 +1,3 @@ +# $Id$ +# +# Placeholder for pubdb Django models not yet written. diff --git a/rpki/rootd.py b/rpki/rootd.py index fb445213..41c9e656 100644 --- a/rpki/rootd.py +++ b/rpki/rootd.py @@ -349,7 +349,7 @@ class main(object): rpki.log.init("rootd", args) - self.cfg = rpki.config.parser(args.config, "rootd") + self.cfg = rpki.config.parser(set_filename = args.config, section = "rootd") self.cfg.set_global_flags() if not args.foreground: diff --git a/rpki/rpkic.py b/rpki/rpkic.py index d7b76c51..62921308 100644 --- a/rpki/rpkic.py +++ b/rpki/rpkic.py @@ -124,7 +124,7 @@ class main(Cmd): global rpki # pylint: disable=W0602 try: - cfg = rpki.config.parser(self.cfg_file, "myrpki") + cfg = rpki.config.parser(set_filename = self.cfg_file, section = "myrpki") cfg.set_global_flags() except IOError, e: sys.exit("%s: %s" % (e.strerror, e.filename)) diff --git a/rpki/rpkid.py b/rpki/rpkid.py index 36ee2ea9..db81d56f 100644 --- a/rpki/rpkid.py +++ b/rpki/rpkid.py @@ -75,7 +75,7 @@ class main(object): rpki.log.init("rpkid", args) - self.cfg = rpki.config.parser(args.config, "rpkid") + self.cfg = rpki.config.parser(set_filename = args.config, section = "rpkid") self.cfg.set_global_flags() if not args.foreground: diff --git a/rpki/rpkidb/__init__.py b/rpki/rpkidb/__init__.py new file mode 100644 index 00000000..7764913c --- /dev/null +++ b/rpki/rpkidb/__init__.py @@ -0,0 +1,3 @@ +# $Id$ +# +# Placeholder for rpkidb Django models not yet written. |