JSONField

“A field for storing JSON-encoded data.”

JSONField

“A field for storing JSON-encoded data.”

In Python, represented as:

• dict
• list
• str
• int
• float
• bool
• None

JSON-encoded data

{
  "name": "Sage",
  "active": true,
  "age": 22,
  "height": 170.0,
  "interests": [
    {"hobbies": ["reading", "coding"]},
    {"others": ["cats", 42]}
  ]
}
{
  "name": "Sage",
  "active": true,
  "age": 22,
  "height": 170.0,
  "interests": [
    {"hobbies": ["reading", "coding"]},
    {"others": ["cats", 42]}
  ]
}
{
  "name": "Sage",
  "active": true,
  "age": 22,
  "height": 170.0,
  "interests": [
    {"hobbies": ["reading", "coding"]},
    {"others": ["cats", 42]}
  ]
}
{
  "name": "Sage",
  "active": true,
  "age": 22,
  "height": 170.0,
  "interests": [
    {"hobbies": ["reading", "coding"]},
    {"others": ["cats", 42]}
  ]
}
{
  "name": "Sage",
  "active": true,
  "age": 22,
  "height": 170.0,
  "interests": [
    {"hobbies": ["reading", "coding"]},
    {"others": ["cats", 42]}
  ]
}
{
  "name": "Sage",
  "active": true,
  "age": 22,
  "height": 170.0,
  "interests": [
    {"hobbies": ["reading", "coding"]},
    {"others": ["cats", 42]}
  ]
}
{
  "name": "Sage",
  "active": true,
  "age": 22,
  "height": 170.0,
  "interests": [
    {"hobbies": ["reading", "coding"]},
    {"others": ["cats", 42]}
  ]
}

JSON-encoded data

{
  "name": "Sage",
  "active": true,
  "age": 22,
  "height": 170.0,
  "interests": [
    {"hobbies": ["reading", "coding"]},
    {"others": ["cats", 42]}
  ],
  "partner": null
}
{
  "name": "Sage",
  "active": true,
  "age": 22,
  "height": 170.0,
  "interests": [
    {"hobbies": ["reading", "coding"]},
    {"others": ["cats", 42]}
  ],
  "partner": null
}

JSON-encoded data

# This is in Python
data = '''{
  "name": "Sage",
  "active": true,
  "age": 22,
  "height": 170.0,
  "interests": [
    {"hobbies": ["reading", "coding"]},
    {"others": ["cats", 42]}
  ],
  "partner": null
}'''

Databases

class Profile(models.Model):
    user = models.OneToOneField(User, on_delete=models.CASCADE)
    status = models.CharField(max_length=255)
    last_sync = models.DateTimeField(auto_now=True)
    config = models.JSONField()

Databases

{
  "dark_mode": true,
  "font_size": 1,
  "color_scheme": "blue"
}

{
  "dark_mode": false,
  "font_size": 3,
  "color_scheme": "red"
}

Databases

JSONField

How does it work?

JSONField

How does it work?

class Profile(models.Model):
    ...
    config = models.JSONField()

JSONField

How does it work?

class Profile(models.Model):
    ...
    config = models.JSONField()

>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile.objects.create(config=config)
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config == config
True
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile.objects.create(config=config)
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config == config
True
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile.objects.create(config=config)
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config == config
True
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile.objects.create(config=config)
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config == config
True
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile.objects.create(config=config)
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config == config
True

JSONField

How does it work... in the background?

>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile.objects.create(config=config)
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config == config
True
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> saved_profile.config['font_size'] = 3
>>> saved_profile.save()
>>> Profile.objects.get(id=saved_profile.id).config['font_size']
3
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile.objects.create(config=config)
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config == config
True
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> saved_profile.config['font_size'] = 3
>>> saved_profile.save()
>>> Profile.objects.get(id=saved_profile.id).config['font_size']
3
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile.objects.create(config=config)
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config == config
True
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> saved_profile.config['font_size'] = 3
>>> saved_profile.save()
>>> Profile.objects.get(id=saved_profile.id).config['font_size']
3
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile.objects.create(config=config)
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config == config
True
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> saved_profile.config['font_size'] = 3
>>> saved_profile.save()
>>> Profile.objects.get(id=saved_profile.id).config['font_size']
3

JSONField

How does it work... in the background?

>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile.objects.create(config=config)
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile.objects.create(config=config)
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}

JSONField

How does it work... in the background?

>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile(config=config)
>>> profile.save()
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile(config=config)
>>> profile.save()
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile(config=config)
>>> profile.save()
>>> # Some time later...
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}

JSONField

How does it work... in the background?

>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile(config=config)
>>> profile.save()
>>> # Turn the config into JSON-encoded data!
>>> '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}

JSONField

How does it work... in the background?

>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile(config=config)
>>> profile.save()
>>> # Turn the config into JSON-encoded data!
>>> '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> # Eventually, it will be:
>>> """
INSERT INTO myapp_profile
VALUES (42, '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}')
"""
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile(config=config)
>>> profile.save()
>>> # Turn the config into JSON-encoded data!
>>> '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> # Eventually, it will be:
>>> """
INSERT INTO myapp_profile
VALUES (42, '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}')
"""
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}

JSONField

How does it work... in the background?

>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile(config=config)
>>> profile.save()
>>> # Turn the config into JSON-encoded data!
>>> '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> # Eventually, it will be:
>>> """
INSERT INTO myapp_profile
VALUES (42, '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}')
"""
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> """SELECT id, config FROM myapp_profile WHERE id = 42"""
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}

JSONField

How does it work... in the background?

>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile(config=config)
>>> profile.save()
>>> # Turn the config into JSON-encoded data!
>>> '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> # Eventually, it will be:
>>> """
INSERT INTO myapp_profile
VALUES (42, '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}')
"""
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> """SELECT id, config FROM myapp_profile WHERE id = 42"""
>>> '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> profile = Profile(config=config)
>>> profile.save()
>>> # Turn the config into JSON-encoded data!
>>> '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> # Eventually, it will be:
>>> """
INSERT INTO myapp_profile
VALUES (42, '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}')
"""
>>> saved_profile = Profile.objects.get(id=profile.id)
>>> """SELECT id, config FROM myapp_profile WHERE id = 42"""
>>> '{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> saved_profile.config
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}

Python's json library

>>> import json
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> encoded = json.dumps(config)
>>> encoded
'{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> decoded = json.loads(encoded)
>>> decoded
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> decoded == config
True
>>> import json
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> encoded = json.dumps(config)
>>> encoded
'{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> decoded = json.loads(encoded)
>>> decoded
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> decoded == config
True
>>> import json
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> encoded = json.dumps(config)
>>> encoded
'{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> decoded = json.loads(encoded)
>>> decoded
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> decoded == config
True
>>> import json
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> encoded = json.dumps(config)
>>> encoded
'{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> decoded = json.loads(encoded)
>>> decoded
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> decoded == config
True
>>> import json
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> encoded = json.dumps(config)
>>> encoded
'{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> decoded = json.loads(encoded)
>>> decoded
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> decoded == config
True
>>> import json
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> encoded = json.dumps(config)
>>> encoded
'{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> decoded = json.loads(encoded)
>>> decoded
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> decoded == config
True
>>> import json
>>> config = {'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> encoded = json.dumps(config)
>>> encoded
'{"dark_mode": true, "font_size": 2, "color_scheme": "pink"}'
>>> decoded = json.loads(encoded)
>>> decoded
{'dark_mode': True, 'font_size': 2, 'color_scheme': 'pink'}
>>> decoded == config
True

Transforms and Lookups

{
  "name": "Sage",
  "age": 22
}

>>> MyModel.objects.filter(some_json_field__name='Sage')

Transforms and Lookups

{
  "name": "Sage",
  "age": 22
}

{
  "name": "Sage",
  "age": 22,
  "partner": null
}

>>> MyModel.objects.filter(
        some_json_field__partner__isnull=True)

>>> MyModel.objects.filter(
        some_json_field__partner=None)

Transforms and Lookups

Containment

{
  "name": "Sage",
  "age": 22,
  "pets": [
    {"name": "Bagol", "species": "cat"},
    {"name": "Foxy", "species": "fox"}
  ]
}

>>> MyModel.objects.filter(
        some_json_field__contains={
            "age": 22,
            "pets": [{"species": "cat"}]
        }
    )

Transforms and Lookups

Containment

{
  "name": "Sage",
  "age": 22
}

>>> MyModel.objects.filter(
        some_json_field__contained_by={
            "age": 22,
            "name": "Sage",
            "pets": [
              {"name": "Bagol", "species": "cat"},
              {"name": "Foxy", "species": "fox"}
            ]
        }
    )

Transforms and Lookups

Key existence

>>> MyModel.objects.filter(some_json_field__has_key='pets')

>>> MyModel.objects.filter(
        some_json_field__has_keys=['pets', 'age'])

>>> MyModel.objects.filter(
        some_json_field__has_any_keys=['pets', 'age'])

PageRevision

PageRevision

class PageRevision(models.Model):
    ...
    content_json = models.TextField(
        verbose_name=_("content JSON")
    )
    ...
class PageRevision(models.Model):
    ...
    content_json = models.TextField(
        verbose_name=_("content JSON")
    )
    ...

PageRevision

class PageRevision(models.Model):
    ...
    content_json = models.JSONField(
        verbose_name=_("content JSON")
    )
    ...

PageRevision

class PageRevision(models.Model):
    ...
    content = models.JSONField(
        verbose_name=_("content JSON")
    )
    ...

PageRevision

class PageRevision(models.Model):
    ...
    content = models.JSONField(
        verbose_name=_("content JSON"),
        encoder=DjangoJSONEncoder
    )
    ...
class PageRevision(models.Model):
    ...
    content = models.JSONField(
        verbose_name=_("content JSON"),
        encoder=DjangoJSONEncoder
    )
    ...

PageRevision

BaseLogEntry

BaseLogEntry

class BaseLogEntry(models.Model):
    ...
    data_json = models.TextField(blank=True)
    ...
class BaseLogEntry(models.Model):
    ...
    data_json = models.TextField(blank=True)
    ...

BaseLogEntry

class BaseLogEntry(models.Model):
    ...
    data = models.JSONField(blank=True)
    ...

BaseLogEntry

class BaseLogEntry(models.Model):
    ...
    data = models.JSONField(blank=True, default=dict)
    ...

BaseLogEntry

BaseLogEntry

StreamField

StreamField

StreamField

class StreamField(models.Field):
    ...
    def get_internal_type(self):
        return "TextField"

StreamField

class StreamField(models.Field):
    ...
    def get_internal_type(self):
        return "JSONField"

StreamField

class StreamField(models.Field):
    ...
    def get_internal_type(self):
        return "JSONField"  # Doesn't work! 😢

The problem

The problem

The solution

The solution

JSONStreamField? 🤔

The solution

JSONStreamField? ❎

use_json_field ✅

The solution

class StreamField(models.Field):
    def __init__(self, block_types, use_json_field=None, **kwargs):
        ...
        self.use_json_field = use_json_field
        ...
 
    def deconstruct(self):
        name, path, _, kwargs = super().deconstruct()
        ...
        kwargs["use_json_field"] = self.use_json_field
        return name, path, args, kwargs
class StreamField(models.Field):
    def __init__(self, block_types, use_json_field=None, **kwargs):
        ...
        self.use_json_field = use_json_field
        ...
 
    def deconstruct(self):
        name, path, _, kwargs = super().deconstruct()
        ...
        kwargs["use_json_field"] = self.use_json_field
        return name, path, args, kwargs
class StreamField(models.Field):
    def __init__(self, block_types, use_json_field=None, **kwargs):
        ...
        self.use_json_field = use_json_field
        ...
 
    def deconstruct(self):
        name, path, _, kwargs = super().deconstruct()
        ...
        kwargs["use_json_field"] = self.use_json_field
        return name, path, args, kwargs
class StreamField(models.Field):
    def __init__(self, block_types, use_json_field=None, **kwargs):
        ...
        self.use_json_field = use_json_field
        ...
 
    def deconstruct(self):
        name, path, _, kwargs = super().deconstruct()
        ...
        kwargs["use_json_field"] = self.use_json_field
        return name, path, args, kwargs

The solution

class StreamField(models.Field):
    ...
    def get_internal_type(self):
        return "JSONField" if self.use_json_field else "TextField"
class StreamField(models.Field):
    ...
    def get_internal_type(self):
        return "JSONField" if self.use_json_field else "TextField"

The solution

class StreamField(models.Field):
    ...
    def get_lookup(self, lookup_name):
        if self.use_json_field:
            return models.JSONField().get_lookup(lookup_name)
        return super().get_lookup(lookup_name)
 
    def get_transform(self, lookup_name):
        if self.use_json_field:
            return models.JSONField().get_transform(lookup_name)
        return super().get_transform(lookup_name)
class StreamField(models.Field):
    ...
    def get_lookup(self, lookup_name):
        if self.use_json_field:
            return models.JSONField().get_lookup(lookup_name)
        return super().get_lookup(lookup_name)
 
    def get_transform(self, lookup_name):
        if self.use_json_field:
            return models.JSONField().get_transform(lookup_name)
        return super().get_transform(lookup_name)
class StreamField(models.Field):
    ...
    def get_lookup(self, lookup_name):
        if self.use_json_field:
            return models.JSONField().get_lookup(lookup_name)
        return super().get_lookup(lookup_name)
 
    def get_transform(self, lookup_name):
        if self.use_json_field:
            return models.JSONField().get_transform(lookup_name)
        return super().get_transform(lookup_name)

The solution

class StreamField(models.Field):
    ...
    @property
    def json_field(self):
        return models.JSONField(encoder=DjangoJSONEncoder)
 
    def get_lookup(self, lookup_name):
        if self.use_json_field:
            return self.json_field.get_lookup(lookup_name)
        return super().get_lookup(lookup_name)
 
    def get_transform(self, lookup_name):
        if self.use_json_field:
            return self.json_field.get_transform(lookup_name)
        return super().get_transform(lookup_name)
class StreamField(models.Field):
    ...
    @property
    def json_field(self):
        return models.JSONField(encoder=DjangoJSONEncoder)
 
    def get_lookup(self, lookup_name):
        if self.use_json_field:
            return self.json_field.get_lookup(lookup_name)
        return super().get_lookup(lookup_name)
 
    def get_transform(self, lookup_name):
        if self.use_json_field:
            return self.json_field.get_transform(lookup_name)
        return super().get_transform(lookup_name)

The solution

class StreamField(models.Field):
    def __init__(self, block_types, use_json_field=None, **kwargs):
        ...
        self.use_json_field = use_json_field
        self._check_json_field()
        ...
 
    def _check_json_field(self):
        if type(self.use_json_field) is not bool:
            warnings.warn(
                "StreamField must explicitly set use_json_field "
                "argument to True/False instead of "
                f"{self.use_json_field}.",
                RemovedInWagtail219Warning,
                stacklevel=3,
            )
class StreamField(models.Field):
    def __init__(self, block_types, use_json_field=None, **kwargs):
        ...
        self.use_json_field = use_json_field
        self._check_json_field()
        ...
 
    def _check_json_field(self):
        if type(self.use_json_field) is not bool:
            warnings.warn(
                "StreamField must explicitly set use_json_field "
                "argument to True/False instead of "
                f"{self.use_json_field}.",
                RemovedInWagtail219Warning,
                stacklevel=3,
            )

Where we're at

Where we're at

What could this bring?

• Improved read performance*
• Smarter search with lookups and transforms
• New features for StreamFields?
• You tell me 😉

Thank you!

{
  "name": "Sage Abdullah",
  "username": "laymonage",
  "slides": {
    "hosted": "https://slides.laymonage.com/wagtail-jsonfield",
    "source": "https://github.com/laymonage/slides-wagtail-jsonfield"
  },
  "prs": [
    "https://github.com/wagtail/wagtail/pull/8021",
    "https://github.com/wagtail/wagtail/pull/8024",
    "https://github.com/wagtail/wagtail/pull/8039"
  ]
}