Switched from BlobField to BinaryField, but doesn't seem to have

helped the MySQL UTF-8 whining.  Same Python code running with MySQL
5.5 doesn't do this, so it's some kind of upgrade trainwreck.
BinaryField uses BLOBs too, just as one would expect, so in theory
this can't be happening.  So it's an undocumented feature.  Yum.

But we wanted to move to BinaryField anyway, and doing so doesn't seem
to have made the problem worse, so committing the changes.

svn path=/branches/tk705/; revision=6145

4 files changed, 344 insertions, 36 deletions

diff --git a/rpki/fields.py b/rpki/fields.py
index 9fc57e51..b0252840 100644
--- a/rpki/fields.py
+++ b/rpki/fields.py
@@ -19,9 +19,9 @@
 """
 Common Django ORM field classes.
 
-Many of these are complex ASN.1 DER objects stored as SQL BLOBs, since
-the only sane text representation would just be the Base64 encoding of
-the DER and thus would add no value.
+Many of these are complex ASN.1 DER objects stored as binaray data,
+since the only sane text representation would just be the Base64
+encoding of the DER and thus would add no value.
 """
 
 import logging
@@ -83,27 +83,9 @@ class SundialField(models.DateTimeField):
 
 class BlobField(models.Field):
   """
-  Basic BLOB field, no type conversion, just an opaque byte string.
+  Old BLOB field type, predating Django's BinaryField type.
 
-  "BLOB" = "Binary Large OBject".  Most SQL implementations seem to
-  have such a thing, but support appears to predate standardization,
-  so they all do it slightly differently and we have to cope.
-
-  In PostgreSQL, BLOBs are called "bytea".
-
-  In MySQL, there are different sizes of BLOBs and one must pick the
-  right one to avoid data truncation.  RPKI manifests and CRLs can be
-  longer than 65535 octets, so in MySQL the only safe BLOB type for
-  general use is "LONGBLOB".
-
-  SQLite...is not like the other children: data types are more like
-  guidelines than actual rules.  But "BLOB" works.
-
-  For anything else, we just use "BLOB" and hope for the best.
-
-  NB: This field type predates Django 1.6's BinaryField.  Probably
-  this should be retired in favor of BinaryField, but I'd have to
-  figure out what that does to field types that derive from this one.
+  Do not use, this is only here for backwards compatabilty during migrations.
   """
 
   __metaclass__ = models.SubfieldBase
@@ -137,26 +119,33 @@ class BlobField(models.Field):
 # this if and when it ever becomes an issue.
 
 
-class DERField(BlobField):
+# DERField used to be a subclass of BlobField.  Try changing it to be
+# a subclass of BinaryField instead, leave BlobField (for now) for
+# backwards compatability during migrations,
+
+
+class DERField(models.BinaryField):
   """
-  Field class for DER objects.  These are derived from BLOBs, but with
-  automatic translation between ASN.1 and Python types.
+  Field class for DER objects.  These are derived from BinaryField,
+  but with automatic translation between ASN.1 and Python types.
 
   DERField itself is an abstract class, concrete field classes are
   derived from it.
   """
 
-  __metaclass__ = models.SubfieldBase
+  def __init__(self, *args, **kwargs):
+    kwargs["serialize"] = False
+    kwargs["blank"] = True
+    kwargs["default"] = None
+    super(DERField, self).__init__(*args, **kwargs)
 
-  def to_python(self, value):
-    if value is not None and not isinstance(value, (self.rpki_type, str)):
-      logger.warning("Why am I now seeing a %r instead of str or %r in the %r rpki.fields.DERField.to_python() method?",
-                     type(value), self.rpki_type, type(self))
-    assert value is None or isinstance(value, (self.rpki_type, str))
-    if isinstance(value, str):
-      return self.rpki_type(DER = value)
-    else:
-      return value
+  if False:
+    def to_python(self, value):
+      assert value is None or isinstance(value, (self.rpki_type, str))
+      if isinstance(value, str):
+        return self.rpki_type(DER = value)
+      else:
+        return value
 
   def get_prep_value(self, value):
     assert value is None or isinstance(value, (self.rpki_type, str))
@@ -165,6 +154,14 @@ class DERField(BlobField):
     else:
       return value
 
+  def from_db_value(self, value, expression, connection, context):
+    assert value is None or isinstance(value, (self.rpki_type, str))
+    if isinstance(value, str):
+      return self.rpki_type(DER = value)
+    else:
+      return value
+
+
 class CertificateField(DERField):
   description = "X.509 certificate"
   rpki_type   = rpki.x509.X509
diff --git a/rpki/irdb/migrations/0005_auto_20151023_2151.py b/rpki/irdb/migrations/0005_auto_20151023_2151.py
new file mode 100644
index 00000000..7d16e7a0
--- /dev/null
+++ b/rpki/irdb/migrations/0005_auto_20151023_2151.py
@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import rpki.irdb.models
+import rpki.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('irdb', '0004_auto_20151018_1603'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='bsc',
+            name='certificate',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='bsc',
+            name='pkcs10',
+            field=rpki.fields.PKCS10Field(),
+        ),
+        migrations.AlterField(
+            model_name='child',
+            name='certificate',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='child',
+            name='ta',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='client',
+            name='certificate',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='client',
+            name='ta',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='eecertificaterequest',
+            name='pkcs10',
+            field=rpki.fields.PKCS10Field(),
+        ),
+        migrations.AlterField(
+            model_name='hostedca',
+            name='certificate',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='parent',
+            name='certificate',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='parent',
+            name='referral_authorization',
+            field=rpki.irdb.models.SignedReferralField(null=True, blank=True),
+        ),
+        migrations.AlterField(
+            model_name='parent',
+            name='ta',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='referral',
+            name='certificate',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='referral',
+            name='private_key',
+            field=rpki.fields.RSAPrivateKeyField(),
+        ),
+        migrations.AlterField(
+            model_name='repository',
+            name='certificate',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='repository',
+            name='ta',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='resourceholderca',
+            name='certificate',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='resourceholderca',
+            name='latest_crl',
+            field=rpki.fields.CRLField(),
+        ),
+        migrations.AlterField(
+            model_name='resourceholderca',
+            name='private_key',
+            field=rpki.fields.RSAPrivateKeyField(),
+        ),
+        migrations.AlterField(
+            model_name='rootd',
+            name='certificate',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='rootd',
+            name='private_key',
+            field=rpki.fields.RSAPrivateKeyField(),
+        ),
+        migrations.AlterField(
+            model_name='serverca',
+            name='certificate',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='serverca',
+            name='latest_crl',
+            field=rpki.fields.CRLField(),
+        ),
+        migrations.AlterField(
+            model_name='serverca',
+            name='private_key',
+            field=rpki.fields.RSAPrivateKeyField(),
+        ),
+        migrations.AlterField(
+            model_name='serveree',
+            name='certificate',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='serveree',
+            name='private_key',
+            field=rpki.fields.RSAPrivateKeyField(),
+        ),
+    ]
diff --git a/rpki/pubdb/migrations/0002_auto_20151023_2151.py b/rpki/pubdb/migrations/0002_auto_20151023_2151.py
new file mode 100644
index 00000000..ae354a9b
--- /dev/null
+++ b/rpki/pubdb/migrations/0002_auto_20151023_2151.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import rpki.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pubdb', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='client',
+            name='bpki_cert',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='client',
+            name='bpki_glue',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+    ]
diff --git a/rpki/rpkidb/migrations/0008_auto_20151023_2151.py b/rpki/rpkidb/migrations/0008_auto_20151023_2151.py
new file mode 100644
index 00000000..415cda5f
--- /dev/null
+++ b/rpki/rpkidb/migrations/0008_auto_20151023_2151.py
@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import rpki.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rpkidb', '0007_auto_20151020_1345'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='bsc',
+            name='pkcs10_request',
+            field=rpki.fields.PKCS10Field(),
+        ),
+        migrations.AlterField(
+            model_name='bsc',
+            name='private_key_id',
+            field=rpki.fields.RSAPrivateKeyField(),
+        ),
+        migrations.AlterField(
+            model_name='bsc',
+            name='signing_cert',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='bsc',
+            name='signing_cert_crl',
+            field=rpki.fields.CRLField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='cadetail',
+            name='latest_ca_cert',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='cadetail',
+            name='latest_crl',
+            field=rpki.fields.CRLField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='cadetail',
+            name='latest_manifest',
+            field=rpki.fields.ManifestField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='cadetail',
+            name='latest_manifest_cert',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='cadetail',
+            name='manifest_private_key_id',
+            field=rpki.fields.RSAPrivateKeyField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='cadetail',
+            name='manifest_public_key',
+            field=rpki.fields.PublicKeyField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='cadetail',
+            name='private_key_id',
+            field=rpki.fields.RSAPrivateKeyField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='cadetail',
+            name='public_key',
+            field=rpki.fields.PublicKeyField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='child',
+            name='bpki_cert',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='child',
+            name='bpki_glue',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='childcert',
+            name='cert',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='eecertificate',
+            name='cert',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='ghostbuster',
+            name='cert',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='ghostbuster',
+            name='ghostbuster',
+            field=rpki.fields.GhostbusterField(),
+        ),
+        migrations.AlterField(
+            model_name='parent',
+            name='bpki_cert',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='parent',
+            name='bpki_glue',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='repository',
+            name='bpki_cert',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='repository',
+            name='bpki_glue',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='roa',
+            name='cert',
+            field=rpki.fields.CertificateField(),
+        ),
+        migrations.AlterField(
+            model_name='roa',
+            name='roa',
+            field=rpki.fields.ROAField(),
+        ),
+        migrations.AlterField(
+            model_name='tenant',
+            name='bpki_cert',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='tenant',
+            name='bpki_glue',
+            field=rpki.fields.CertificateField(null=True),
+        ),
+    ]