From 2c749a18db7886b7c9931f2b98eac6f099d304d2 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sun, 25 Oct 2015 22:25:40 +0000 Subject: Tweak binary field types to get SQLite3 working as a Django ORM backend. Switch yamltest's default database configuration to sqlite3. MySQL still has character set issues, which are almost certainly to do with the communication channel rather than the database tables. It's possible that one of the newer DB API drivers for MySQL fixes this, might be worth trying one of them at some point (see the "MySQL notes" discussion of MySQL DB API drivers in the Django documentation). svn path=/branches/tk705/; revision=6149 --- ca/tests/Makefile.in | 1 - ca/tests/yamltest.py | 27 +++++++++++++++++------- rpki/django_settings/common.py | 26 +++++++++++++++-------- rpki/fields.py | 48 +++++++++++++++++++----------------------- rpki/pubd.py | 4 ++-- rpki/pubdb/models.py | 4 ++-- rpki/rpkidb/models.py | 6 +++--- 7 files changed, 65 insertions(+), 51 deletions(-) diff --git a/ca/tests/Makefile.in b/ca/tests/Makefile.in index 618a741e..eb4357b1 100644 --- a/ca/tests/Makefile.in +++ b/ca/tests/Makefile.in @@ -90,7 +90,6 @@ YAMLTEST_CONFIG = smoketest.1.yaml yamltest: rm -rf yamltest.dir rcynic-data - ${PYTHON} sql-cleaner.py ${PYTHON} yamltest.py ${YAMLTEST_CONFIG} YAMLCONF_CONFIG = ${YAMLTEST_CONFIG} diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index 04a2634a..a26fefb9 100644 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -225,7 +225,7 @@ class allocation(object): @classmethod def allocate_engine(cls): """ - Allocate an engine number, mostly used to construct MySQL database + Allocate an engine number, mostly used to construct SQL database names. """ @@ -490,17 +490,11 @@ class allocation(object): run_rpkid = str(not self.is_hosted), run_pubd = str(self.runs_pubd), run_rootd = str(self.is_root), - irdbd_sql_database = "irdb%d" % self.engine, - irdbd_sql_username = "irdb", - rpkid_sql_database = "rpki%d" % self.engine, - rpkid_sql_username = "rpki", rpkid_server_host = "localhost", rpkid_server_port = str(self.rpkid_port), irdbd_server_host = "localhost", irdbd_server_port = str(self.irdbd_port), rootd_server_port = str(self.rootd_port), - pubd_sql_database = "pubd%d" % self.engine, - pubd_sql_username = "pubd", pubd_server_host = "localhost", pubd_server_port = str(self.pubd.pubd_port), publication_rsync_server = "localhost:%s" % self.pubd.rsync_port, @@ -508,7 +502,22 @@ class allocation(object): bpki_servers_directory = self.path(), publication_base_directory = self.path("publication"), rrdp_publication_base_directory = self.path("rrdp-publication"), - shared_sql_password = "fnord") + shared_sql_engine = args.sql_engine, + shared_sql_password = "fnord", + irdbd_sql_username = "irdb", + rpkid_sql_username = "rpki", + pubd_sql_username = "pubd") + + if args.sql_engine == "sqlite3": + r.update( + irdbd_sql_database = self.path("irdb.sqlite3"), + rpkid_sql_database = self.path("rpkidb.sqlite3"), + pubd_sql_database = self.path("pubdb.sqlite3")) + else: + r.update( + irdbd_sql_database = "irdb%d" % self.engine, + rpkid_sql_database = "rpki%d" % self.engine, + pubd_sql_database = "pubd%d" % self.engine) r.update(config_overrides) @@ -755,6 +764,8 @@ parser.add_argument("--notify-when-startup-complete", type = int, help = "send SIGUSR1 to this process when startup is complete") parser.add_argument("--store-router-private-keys", action = "store_true", help = "write generate router private keys to disk") +parser.add_argument("--sql-engine", choices = ("mysql", "sqlite3", "postgresql"), default = "sqlite3", + help = "select SQL engine to use") parser.add_argument("yaml_file", type = argparse.FileType("r"), help = "YAML description of test network") args = parser.parse_args() diff --git a/rpki/django_settings/common.py b/rpki/django_settings/common.py index ef386cf0..d2fe496c 100644 --- a/rpki/django_settings/common.py +++ b/rpki/django_settings/common.py @@ -56,11 +56,17 @@ if os.getenv("RPKI_DJANGO_DEBUG") == "yes": class DatabaseConfigurator(object): + default_sql_engine = "mysql" + def configure(self, cfg, section): self.cfg = cfg self.section = section - return dict(default = getattr(self, cfg.get("sql-engine", section = section, default = "mysql"))()) + engine = cfg.get("sql-engine", section = section, + default = self.default_sql_engine) + return dict( + default = getattr(self, engine)) + @property def mysql(self): return dict( ENGINE = "django.db.backends.mysql", @@ -70,21 +76,23 @@ class DatabaseConfigurator(object): # # Using "latin1" here is totally evil and wrong, but # without it MySQL 5.6 (and, probably, later versions) - # whine incessantly about bad UTF-8 characters when one - # stores ASN.1 DER in BLOB columns. Which makes no - # freaking sense at all, but this is MySQL, which has a - # character set management interface from hell, so good - # luck with that. If anybody really understands how to - # fix this, tell me; for now, we force MySQL to revert to - # the default behavior in MySQL 5.5. + # whine incessantly about bad UTF-8 characters in BLOB + # columns. Which makes no freaking sense at all, but this + # is MySQL, which has the character set management interface + # from hell, so good luck with that. If anybody really + # understands how to fix this, tell me; for now, we force + # MySQL to revert to the default behavior in MySQL 5.5. # - OPTIONS = dict(charset = "latin1")) + #OPTIONS = dict(charset = "latin1") + ) + @property def sqlite3(self): return dict( ENGINE = "django.db.backends.sqlite3", NAME = cfg.get("sql-database", section = self.section)) + @property def postgresql(self): return dict( ENGINE = "django.db.backends.postgresql_psycopg2", diff --git a/rpki/fields.py b/rpki/fields.py index b0252840..a470e272 100644 --- a/rpki/fields.py +++ b/rpki/fields.py @@ -123,43 +123,39 @@ class BlobField(models.Field): # a subclass of BinaryField instead, leave BlobField (for now) for # backwards compatability during migrations, - class DERField(models.BinaryField): """ - Field class for DER objects. These are derived from BinaryField, - but with automatic translation between ASN.1 and Python types. - - DERField itself is an abstract class, concrete field classes are - derived from it. + Field class for DER objects, with automatic translation between + ASN.1 and Python types. This is an abstract class, concrete field + classes are derived from it. """ def __init__(self, *args, **kwargs): - kwargs["serialize"] = False kwargs["blank"] = True kwargs["default"] = None super(DERField, self).__init__(*args, **kwargs) - if False: - def to_python(self, value): - assert value is None or isinstance(value, (self.rpki_type, str)) - if isinstance(value, str): - return self.rpki_type(DER = value) - else: - return value - - def get_prep_value(self, value): - assert value is None or isinstance(value, (self.rpki_type, str)) - if isinstance(value, self.rpki_type): - return value.get_DER() - else: - return value + def deconstruct(self): + name, path, args, kwargs = super(DERField, self).deconstruct() + del kwargs["blank"] + del kwargs["default"] + return name, path, args, kwargs def from_db_value(self, value, expression, connection, context): - assert value is None or isinstance(value, (self.rpki_type, str)) - if isinstance(value, str): - return self.rpki_type(DER = value) - else: - return value + if value is not None: + value = self.rpki_type(DER = str(value)) + return value + + def to_python(self, value): + value = super(DERField, self).to_python(value) + if value is not None and not isinstance(value, self.rpki_type): + value = self.rpki_type(DER = str(value)) + return value + + def get_prep_value(self, value): + if value is not None: + value = value.get_DER() + return super(DERField, self).get_prep_value(value) class CertificateField(DERField): diff --git a/rpki/pubd.py b/rpki/pubd.py index b5d36199..f917c18d 100644 --- a/rpki/pubd.py +++ b/rpki/pubd.py @@ -184,10 +184,10 @@ class main(object): client.base_uri = q_pdu.get("base_uri") bpki_cert = q_pdu.find(rpki.publication_control.tag_bpki_cert) if bpki_cert is not None: - client.bpki_cert = bpki_cert.text.decode("base64") + client.bpki_cert = rpki.x509.X509(Base64 = bpki_cert.text) bpki_glue = q_pdu.find(rpki.publication_control.tag_bpki_glue) if bpki_glue is not None: - client.bpki_glue = bpki_glue.text.decode("base64") + client.bpki_glue = rpki.x509.X509(Base64 = bpki_glue.text) if q_pdu.get("clear_replay_protection") == "yes": client.last_cms_timestamp = None client.save() diff --git a/rpki/pubdb/models.py b/rpki/pubdb/models.py index 446867b8..2b6d67e4 100644 --- a/rpki/pubdb/models.py +++ b/rpki/pubdb/models.py @@ -20,7 +20,7 @@ Django ORM models for pubd. from __future__ import unicode_literals from django.db import models -from rpki.fields import BlobField, CertificateField, SundialField +from rpki.fields import CertificateField, SundialField from lxml.etree import Element, SubElement, tostring as ElementToString import os @@ -303,7 +303,7 @@ class Delta(models.Model): class PublishedObject(models.Model): uri = models.CharField(max_length = 255) - der = BlobField() + der = models.BinaryField() hash = models.CharField(max_length = 64) client = models.ForeignKey(Client) session = models.ForeignKey(Session) diff --git a/rpki/rpkidb/models.py b/rpki/rpkidb/models.py index 2d99df7c..32028a1f 100644 --- a/rpki/rpkidb/models.py +++ b/rpki/rpkidb/models.py @@ -18,7 +18,7 @@ from django.db import models import rpki.left_right -from rpki.fields import (EnumField, SundialField, BlobField, +from rpki.fields import (EnumField, SundialField, CertificateField, RSAPrivateKeyField, PublicKeyField, CRLField, PKCS10Field, ManifestField, ROAField, GhostbusterField) @@ -1705,7 +1705,7 @@ class Child(models.Model): class ChildCert(models.Model): cert = CertificateField() published = SundialField(null = True) - ski = BlobField() + ski = models.BinaryField() child = models.ForeignKey(Child, related_name = "child_certs") ca_detail = models.ForeignKey(CADetail, related_name = "child_certs") @@ -1820,7 +1820,7 @@ class ChildCert(models.Model): class EECertificate(models.Model): - ski = BlobField() + ski = models.BinaryField() cert = CertificateField() published = SundialField(null = True) tenant = models.ForeignKey(Tenant, related_name = "ee_certificates") -- cgit v1.2.3