diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..67361ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +*.swp +*.pyc +.DS_Store +/.coverage +/dist/ +/build/ +/MANIFEST +/wagtail_airtable.egg-info/ +/docs/_build/ +/.tox/ +/venv +/node_modules/ +npm-debug.log* +*.idea/ +/*.egg/ +/.cache/ +/.pytest_cache/ +*.sqlite3 +__pycache__ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..820a941 --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2019, Torchbox +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..251f75d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include LICENSE *.rst *.txt *.md +graft wagtail_airtable +global-exclude __pycache__ +global-exclude *.py[co] +global-exclude .DS_Store +global-exclude examples/ diff --git a/README.md b/README.md index 05216fa..7b2ccce 100644 --- a/README.md +++ b/README.md @@ -1 +1,129 @@ -# Future home of wagtail-airtable +# Wagtail/Airtable + +An extension for Wagtail allowing content to be transferred between Airtable sheets and your Wagtail/Django models. + +Developed by [Torchbox](https://torchbox.com/) and sponsored by [The Motley Fool](https://www.fool.com/). + +![Wagtail Airtable demo](examples/preview.gif) + +### How it works + +When you setup a model to "map" to an Airtable sheet, every time you save the model it will attempt to update the row in Airtable. If a row is not found, it will create a new row in your Airtable. + +When you want to sync your Airtable data to your Wagtail website, you can go to `Settings -> Airtable Import`. You can then import entire tables into your Wagtail instance with the click of a button. If you see "_{Your Model}_ is not setup with the correct Airtable settings" you will need to double check your settings. By default the import page can be found at http://yourwebsite.com/admin/airtable-import/, or if you use a custom /admin/ url it'll be http://yourwebsite.com/{custom_admin_url}/airtable-import/. + +##### Behind the scenes... +This package will attempt to match a model object against row in Airtable using a `record_id`. If a model does not have a record_id value, it will look for a match using the `AIRTABLE_UNIQUE_IDENTIFIER` to try and match a unique value in the Airtable to the unique value in your model. Should that succeed your model object will be "paired" with the row in Airtable. But should the record-search fail, a new row in Airtable will be created when you save your model, or a new model object will attempt to be created when you import a model from Airtable. + +> **Note**: Object creation _can_ fail when importing from Airtable. This is expected behaviour as an Airtable might not have all the data a model requires. For instance, a Wagtail Page uses django-treebeard, with path as a required field. If the page model import settings do not include the path field, or a path column isn't present in Airtable, the page cannot be created. This same rule applies to other required fields on any Django model including other required fields on a Wagtail Page. + +### Installation & Configuration + +* Install the package with `pip install wagtail-airtable` +* Add `'wagtail_airtable'` to your project's `INSTALLED_APPS` +* In your settings you will need to map models to Airtable settings. Every model you want to map to an Airtable sheet will need: + * An `AIRTABLE_BASE_KEY`. You can find the base key in your Airtable docs when you're signed in to Airtable.com + * An `AIRTABLE_TABLE_NAME` to determine which table to connect to. + * An `AIRTABLE_UNIQUE_IDENTIFIER`. This can either be a string or a dictionary mapping the Airtable column name to your unique field in your model. + * ie. `AIRTABLE_UNIQUE_IDENTIFIER: 'slug',` this will match the `slug` field on your model with the `slug` column name in Airtable. Use this option if your model field and your Airtable column name are identical. + * ie. `AIRTABLE_UNIQUE_IDENTIFIER: {'Airtable Column Name': 'model_field_name'},` this will map the `Airtable Column Name` to a model field called `model_field_name`. Use this option if your Airtable column name and your model field name are different. + * An `AIRTABLE_SERIALIZER` that takes a string path to your serializer. This helps map incoming data from Airtable to your model fields. Django Rest Framework is required for this. See the [examples/](examples/) directory for serializer examples. + +* Lastly make sure you enable wagtail-airtable with `WAGTAIL_AIRTABLE_ENABLED = True`. By default this is disabled so data in your Wagtail site and your Airtable sheets aren't accidentally overwritten. Data is hard to recover, this option helps prevent accidental data loss. + +### Example Base Configuration + +Below is a base configuration or `ModelName` and `OtherModelName` (both are registered Wagtail snippets), along with `HomePage`. + +```python +# your settings.py +AIRTABLE_API_KEY = 'yourSuperSecretKey' +WAGTAIL_AIRTABLE_ENABLED = True +AIRTABLE_IMPORT_SETTINGS = { + 'appname.ModelName': { + 'AIRTABLE_BASE_KEY': 'app3ds912jFam032S', + 'AIRTABLE_TABLE_NAME': 'Your Airtable Table Name', + 'AIRTABLE_UNIQUE_IDENTIFIER': 'slug', # Must match the Airtable Column name + 'AIRTABLE_SERIALIZER': 'path.to.your.model.serializer.CustomModelSerializer' + }, + 'appname.OtherModelName': { + 'AIRTABLE_BASE_KEY': 'app4ds902jFam035S', + 'AIRTABLE_TABLE_NAME': 'Your Airtable Table Name', + 'AIRTABLE_UNIQUE_IDENTIFIER': + 'Page Slug': 'slug', # 'Page Slug' column name in Airtable, 'slug' field name in Wagtail. + }, + 'AIRTABLE_SERIALIZER': 'path.to.your.model.serializer.OtherCustomModelSerializer' + }, + 'pages.HomePage': { + 'AIRTABLE_BASE_KEY': 'app2ds123jP23035Z', + 'AIRTABLE_TABLE_NAME': 'Wagtail Page Tracking Table', + 'AIRTABLE_UNIQUE_IDENTIFIER': + 'Wagtail Page ID': 'pk', + }, + 'AIRTABLE_SERIALIZER': 'path.to.your.pages.serializer.PageSerializer', + # Below is an OPTIONAL setting. + # By disabling `AIRTABLE_IMPORT_ALLOWED` you can prevent Airtable imports + # Use cases may be: + # - disabling page imports since they are difficult to setup and maintain, + # - one-way sync to Airtable only (ie. when a model/Page is saved) + # Default is True + 'AIRTABLE_IMPORT_ALLOWED': False, + }, + # ... +} +``` + +##### Have multiple models with the same Airtable settings? +The most common approach will likely be to support a handful of models, in which case using the below example would be faster and cleaner. Write a config dictionary once to prevent config bloat. + +```python +AIRTABLE_API_KEY = 'yourSuperSecretKey' +WAGTAIL_AIRTABLE_ENABLED = True +CUSTOM_PAGE_SETTINGS = { + 'AIRTABLE_BASE_KEY': 'app3ds912jFam032S', + 'AIRTABLE_TABLE_NAME': 'Your Airtable Table Name', + 'AIRTABLE_UNIQUE_IDENTIFIER': 'slug', # Must match the Airtable Column name + 'AIRTABLE_SERIALIZER': 'path.to.your.model.serializer.CustomModelSerializer' +}, +AIRTABLE_IMPORT_SETTINGS = { + 'home.HomePage': CUSTOM_PAGE_SETTINGS, + 'blog.BlogPage': CUSTOM_PAGE_SETTINGS, + 'appname.YourModel': CUSTOM_PAGE_SETTINGS, +} +``` + +### Management Commands + +```bash +python manage.py import_airtable appname.ModelName secondapp.SecondModel +``` + +Optionally you can turn up the verbosity for better debugging with the `--verbosity=2` flag. + +##### import_airtable command +This command will look for any `appname.ModelName`s you provide it and use the mapping settings to find data in the Airtable. See the "Behind the scenes" section for more details on how importing works. + +### Local Testing Advice + +> **Note:** Be careful not to use the production settings as you could overwrite Wagtail or Airtable data. + +Because Airtable doesn't provide a testing environment, you'll need to test against a live table. The best way to do this is to copy your live table to a new table (renaming it will help avoid naming confusion), and update your local settings. With this method, you can test to everything safely against a throw-away Airtable. Should something become broken beyond repair, delete the testing table and re-copy the original one. + +### Local debugging +Due to the complexity and fragility of connecting Wagtail and Airtable (because an Airtable column can be almost any value) you may need some help debugging your setup. To turn on higher verbosity output, you can enable the Aritable debug setting `WAGTAIL_AIRTABLE_DEBUG = True`. All this does is increase the default verbosity when running the management command. In a standard Django management command you could run `python manage.py import_airtable appname.ModelName --verbosty=2` however when you import from Airtable using the Wagtail admin import page you won't have access to this verbosity argument. But enabling `WAGTAIL_AIRTABLE_DEBUG` you can manually increase the verbosity. + +> **Note**: This only only work while `DEBUG = True` in your settings as to not accidentally flood your production logs. + +### Airtable Best Practice +Airtable columns can be one of numerous "types", very much like a Python data type or Django field. You can have email columns, url columns, single line of text, checkbox, etc. + +To help maintain proper data synchronisation between your Django/Wagtail instance and your Airtable Base's, you _should_ set the column types to be as similar to your Django fields as possible. + +For example, if you have a BooleanField in a Django model (or Wagtail Page) and you want to support pushing that data to Airtable amd support importing that same data from Airtable, you should set the column type in Airtable to be a Checkbox (because it can only be on/off, much like how a BooleanField can only be True/False). + +In other cases such as Airtables Phone Number column type: if you are using a 3rd party package to handle phone numbers and phone number validation, you'll want to write a custom serializer to handle the incoming value from Airtable (when you import from Airtable). The data will likely come through to Wagtail as a string and you'll want to adjust the string value to be a proper phone number format for internal Wagtail/Django storage. (You may also need to convert the phone number to a standard string when exporting to Airtable as well) + +### Running Tests +Clone the project and cd into the `wagtail-airtable/tests/` directory. Then run `python runtests.py tests`. This project is using standard Django unit tests. + +To target a specific test you can run `python runtests.py tests.test_file.TheTestClass.test_specific_model` diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..7f00a1e --- /dev/null +++ b/examples/README.md @@ -0,0 +1,8 @@ +# Wagtail Airtable Examples + +> The purpose of these examples are to help clarify how to use this package. Most Python packages are fairly self-contained however wagtail-airtable requires more involvement that may or may not be completely intuitive based on your skill level. + +See the following code examples: +- [x] [Django Model Support](model_example.py) +- [x] [Wagtail Page Support](page_example.py) +- [x] [Wagtail Multi-Page Support](multi_page_example.py) diff --git a/examples/model_example.py b/examples/model_example.py new file mode 100644 index 0000000..32af43e --- /dev/null +++ b/examples/model_example.py @@ -0,0 +1,44 @@ +from django.db import models + +from wagtail_airtable.mixins import AirtableMixin + + +class YourModel(AirtableMixin, models.Model): + + name = models.CharField(max_length=200, blank=False) + slug = models.SlugField(max_length=200, unique=True, editable=True) + + @classmethod + def map_import_fields(cls): + """ + Maps your Airtable columns to your Django Model Fields. + + Always provide explicit field names as to not accidentally overwrite sensitive information + such as model pk's. + + Return a dictionary such as: {'Airtable column name': 'model_field_name', ...} + """ + mappings = { + # "Name" is the column name in Airtable. "name" (lowercase) is the field name on line 8. + "Name": "name", + # "Slug" is the column name in Airtable. "slug" (lowercase) is the field name on line 8. + "Slug": "slug", + } + return mappings + + def get_export_fields(self): + """ + Export fields are the fields you want to map when saving a model object. + + Everytime a model is saved, it will take the Airtable Column Name and fill the appropriate cell + with the data you tell it. Most often this will be from self.your_field_name. + + Always provide explicit field names as to not accidentally share sensitive information such as + hashed passwords. + + Return a dictionary such as: {"Airtable Column Name": "update_value", ...} + """ + return { + "Name": self.name, + "Slug": self.slug, + } diff --git a/examples/multi_page_example.py b/examples/multi_page_example.py new file mode 100644 index 0000000..a8796c8 --- /dev/null +++ b/examples/multi_page_example.py @@ -0,0 +1,79 @@ +from django.db import models +from django.utils import timezone + +from wagtail.core.models import Page +from wagtail_airtable.mixins import AirtableMixin + + +class BasePage(AirtableMixin, Page): + """ + This is using the AirtableMixin and the Wagtail Page model to create a new + "BasePage" model. Then new Pages are created using the "BasePage model + they will automatically inherit the import/export field mapping you see + below. + + Note: You'll most likely want BasePage to be an Abstract Model. + """ + + @classmethod + def map_import_fields(cls): + """ + Fields to update when importing a specific page. + These are just updating the seo_title, title, and search_description + fields that come with wagtailcore.Page. + + NOTE: Unless you store required data like the page depth or tree value + in Airtable, when you import a new page it won't be automatically created. + Wagtail doesn't know where you'd like new pages to be created but requires + tree-structure data. + + Example: + {'Airtable column name': 'model_field_name', ...} + """ + + return { + "SEO Title": "seo_title", + "Title": "title", + "Meta Description": "search_description", + } + + def get_export_fields(self): + """ + Map Airtable columns to values from Wagtail or Django. + + Example: + {'Airtable Column Name': updated_value, ...} + """ + return { + "SEO Title": self.seo_title, + "Title": self.title, + "URL": self.full_url, + "Last Published": self.last_published_at.isoformat() + if self.last_published_at + else "", + "Meta Description": self.search_description, + "Type": self.__class__.__name__, + "Live": self.live, + "Unpublished Changes": self.has_unpublished_changes, + "Wagtail Page ID": self.id, + "Slug": self.slug, + } + + class Meta: + abstract = True + + +class HomePage2(BasePage): + pass + + +class ContactPage(BasePage): + pass + + +class BlogPage(BasePage): + pass + + +class MiscPage(BasePage): + pass diff --git a/examples/page_example.py b/examples/page_example.py new file mode 100644 index 0000000..24dcf13 --- /dev/null +++ b/examples/page_example.py @@ -0,0 +1,82 @@ +from django.db import models +from django.utils import timezone + +from wagtail.core.models import Page +from wagtail_airtable.mixins import AirtableMixin + + +class HomePage(AirtableMixin, Page): + + # Wagtail stuff.. + parent_page_types = ["wagtailcore.page"] + template = ["templates/home_page.html"] + + # Custom fields + name = models.CharField(max_length=200, blank=False) + total_awesomeness = models.DecimalField( + blank=True, null=True, decimal_places=1, max_digits=2 + ) + + # Custom property or methods allowed when exporting + # There is no custom property/method support when importing from Airtable because it won't know + # where to map the data to since custom properties and methods are not stored fields. + @property + def top_rated_page(self): + if self.total_awesomeness >= 80: + return True + return False + + @classmethod + def map_import_fields(cls): + """ + Maps your Airtable columns to your Django Model Fields. + + Always provide explicit field names as to not accidentally overwrite sensitive information + such as model pk's. + + Return a dictionary such as: + { + 'Name': 'title', + 'Awesomeness Rating': 'total_awesomeness', + 'Other Airtable Column Name': 'your_django_or_wagtail_field_name', + } + """ + mappings = { + # "Name" is the column name in Airtable. "title" (lowercase) is the field name on line 26. + "Name": "title", + # "Slug" is the column name in Airtable. "slug" (lowercase) comes from Page.slug. + # I've kept "slug" commented out so Airtable cannot overwrite the Page slug as that could cause a lot of trouble with URLs and SEO. But it's possible to do this assuming there aren't two pages with the same slug. + # "Slug": "slug", + "Awesomeness Rating": "total_awesomeness", + "Last Updated": "last_published_at", + } + return mappings + + def get_export_fields(self): + """ + Export fields are the fields you want to map when saving a model object. + + Everytime a model is saved, it will take the Airtable Column Name and fill the appropriate cell + with the data you tell it. Most often this will be from self.your_field_name. + + Always provide explicit field names as to not accidentally share sensitive information such as + hashed passwords. + + Return a dictionary such as: {"Airtable Column Name": "update_value", ...} + """ + return { + "Name": self.name, + "Slug": self.slug, # `slug` is a field found on Page that comes with Wagtail + "Awesomeness Rating": str(self.total_awesomeness) + if self.total_awesomeness + else None, # Send the Decimal as a string. + "Top Rated Awesomeness": self.top_rated_page, # Must be a checkbox column in Airtable. + # If a cell in Airtable should always be filled, but the data might be optional at some point + # You can use a function, method, custom property or ternary operator to set the defaults. + "Last Updated": self.last_published_at + if self.last_published_at + else timezone.now().isoformat(), + } + + class Meta: + verbose_name = "The Best HomePage Ever" diff --git a/examples/preview.gif b/examples/preview.gif new file mode 100644 index 0000000..12af0df Binary files /dev/null and b/examples/preview.gif differ diff --git a/examples/serializers.py b/examples/serializers.py new file mode 100644 index 0000000..4eb80c7 --- /dev/null +++ b/examples/serializers.py @@ -0,0 +1,108 @@ +# from django.utils.dateparse import parse_datetime +from rest_framework import serializers +from taggit.models import Tag +from wagtail_airtable.serializers import AirtableSerializer + + +class TagSerializer(serializers.RelatedField): + """ + A tag serializer to convert a string of tags (ie. `Tag1, Tag2`) into a list of Tag objects (ie. `[Tag], [Tag]`). + + If a tag in Airtable doesn't exist in Wagtail, this snippet will create a new Tag. + + Usage: + class YourModelSerializer(AirtableSerializer): + ... + tags = TagSerializer(required=False) + ... + """ + + def to_internal_value(self, data): + if type(data) == str: + tags = [] + for tag in data.split(","): + tag, _ = Tag.objects.get_or_create(name=tag.strip()) + tags.append(tag) + return tags + elif type(data) == list: + for tag in data: + tag, _ = Tag.objects.get_or_create(name=tag.strip()) + tags.append(tag) + return tags + return data + + def get_queryset(self): + pass + + +class BankNameSerializer(serializers.RelatedField): + """ + Let's assume there's a "bank_name" column in Airtable but it stores a string. + + When importing from Airtable you'll need to find a model object based on that name. + That's what this serializer is doing. + + Usage: + class YourModelSerializer(AirtableSerializer): + ... + bank_name = BankNameSerializer(required=False) + ... + """ + + def to_internal_value(self, data): + from .models import BankOrganisation + + if data: + try: + bank = BankOrganisation.objects.get(name=data) + except BankOrganisation.DoesNotExist: + return None + else: + return bank + return data + + def get_queryset(self): + pass + + +class DateSerializer(serializers.DateTimeField): + def to_internal_value(self, date): + if type(date) == str and len(date): + date = parse_datetime(date).isoformat() + return date + + +class YourModelSerializer(AirtableSerializer): + """ + YourModel serializer used when importing Airtable records. + + This serializer will help validate data coming in from Airtable and help prevent + malicious intentions. + + This model assumes there is a "name" mapping in YourModel.map_import_fields() + """ + + name = serializers.CharField(max_length=200, required=True) + slug = serializers.CharField(max_length=200, required=True) + + +class YourPageSerializer(AirtableSerializer): + """ + YourModel serializer used when importing Airtable records. + + This serializer will help validate data coming in from Airtable and help prevent + malicious intentions. + + This model assumes there is a "name" mapping in YourModel.map_import_fields() + """ + + # Page.title from wagtailcore.page. Airtable can update this value. + title = serializers.CharField(max_length=200, required=True) + # Allow Airtable to overwrite the last_published_at date using a custom serializer. + # This is probably a bad idea to allow this field to be imported, but it's a good code example. + last_published_at = DateSerializer(required=False) + # Custom field we created on `class YourPage`. + # We want Airtable to import and validate this data before updating the value. + name = serializers.CharField(max_length=200, required=True) + # Not supported because we don't want a slug to be overwritten. + # slug = serializers.CharField(max_length=200, required=True) diff --git a/examples/settings.py b/examples/settings.py new file mode 100644 index 0000000..5b3859f --- /dev/null +++ b/examples/settings.py @@ -0,0 +1,35 @@ +# AIRTABLE SETTINGS +COMMON_AIRTABLE_SETTINGS = { + "AIRTABLE_BASE_KEY": "", + "AIRTABLE_TABLE_NAME": "Your Table Name", + "AIRTABLE_UNIQUE_IDENTIFIER": {"Wagtail Page ID": "pk",}, + "AIRTABLE_SERIALIZER": "yourapp.serializers.YourPageSerializer", +}, +WAGTAIL_AIRTABLE_ENABLED = True +WAGTAIL_AIRTABLE_DEBUG = True +AIRTABLE_IMPORT_SETTINGS = { + # Applies to model_example.py + "yourapp.YourModel": { + "AIRTABLE_BASE_KEY": "", # The Airtable Base Code + "AIRTABLE_TABLE_NAME": "Your Table Name", # Airtable Bases can have multiple tables. Tell it which one to use. + "AIRTABLE_UNIQUE_IDENTIFIER": "slug", # Must match the Airtable Column name + "AIRTABLE_SERIALIZER": "yourapp.serializers.YourModelSerializer", # A custom serializer for validating imported data. + }, + # Applies to page_example.py + "yourapp.HomePage": { + "AIRTABLE_BASE_KEY": "", # The Airtable Base Code + "AIRTABLE_TABLE_NAME": "Your Table Name", # Airtable Bases can have multiple tables. Tell it which one to use. + "AIRTABLE_UNIQUE_IDENTIFIER": { # Takes an {'Airtable Column Name': 'django_field_name'} mapping + "Wagtail Page ID": "pk", + }, + "AIRTABLE_SERIALIZER": "yourapp.serializers.YourPageSerializer", # A custom serializer for validating imported data. + }, + # Applies to multi_page_example.py + "yourapp.HomePage2": COMMON_AIRTABLE_SETTINGS, + "yourapp.ContactPage": COMMON_AIRTABLE_SETTINGS, + "yourapp.BlogPage": COMMON_AIRTABLE_SETTINGS, + "yourapp.MiscPage": COMMON_AIRTABLE_SETTINGS, + # { + # ... More settings + # } +} diff --git a/known-issues.md b/known-issues.md new file mode 100644 index 0000000..a4e71e9 --- /dev/null +++ b/known-issues.md @@ -0,0 +1,21 @@ +## Known Issues and work arounds +> All of these would make amazing PR's ;) + +#### Decimal Fields +In `get_export_fields()` if you map a decimal field you'll need to convert it to a string for it to be JSONified and for Airtable to accept it. ie: +```python +rating = models.DecimalField(...) + +def get_export_fields(..): + return { + ... + "rating": str(self.rating) if self.rating else None, + } +``` + +#### Duplicate records by unique column value +If any operation needs to find a new Airtable record by its unique column name (and value) such as a `slug` or `id`, and multiple records are returned at the same time, wagtail-airtable will use the first available option that Airtable returns. + +The problem with this lies in editing Airtable records. Because of this someone may edit the wrong record and the `import` function may not import the correct data. + +Also Airtable does not need to return the records in order from first to last. For example, if you saved a model and it had 4 matched records in your Airtable because there were 4 cells with the slug of "testing-record-slug", you may not get the first record in the list of returned records. In several test cases there were random cases of the first, middle and last records being selected. This is more of an issue with Airtable not giving us all the records in proper order. Whichever record is found first is the record your Django object will be tied to moving forward. diff --git a/runtests.py b/runtests.py new file mode 100644 index 0000000..3374921 --- /dev/null +++ b/runtests.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +import sys +import os + +from django.core.management import execute_from_command_line + +os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.settings' +execute_from_command_line([sys.argv[0], 'test'] + sys.argv[1:]) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..725f958 --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +from setuptools import setup, find_packages + +install_requires = [ + "airtable-python-wrapper>=0.13.0", + "djangorestframework>=3.11.0", +] + +setup( + name='wagtail-airtable', + version='0.1', + description="Sync data between Wagtail and Airtable", + author='Kalob Taulien', + author_email='kalob.taulien@torchbox.com', + url='https://github.com/wagtail/wagtail-airtable', + packages=find_packages(exclude=('tests',)), + include_package_data=True, + license='BSD', + long_description="An extension for Wagtail allowing content to be transferred between Airtable sheets and your Wagtail/Django models", + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Web Environment', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Framework :: Django', + 'Framework :: Wagtail', + 'Framework :: Wagtail :: 2', + ], + install_requires=install_requires, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/test.json b/tests/fixtures/test.json new file mode 100644 index 0000000..9ee9473 --- /dev/null +++ b/tests/fixtures/test.json @@ -0,0 +1,84 @@ +[ +{ + "pk": 1, + "model": "wagtailcore.page", + "fields": { + "title": "Root", + "numchild": 1, + "show_in_menus": false, + "live": true, + "depth": 1, + "content_type": [ + "wagtailcore", + "page" + ], + "path": "0001", + "url_path": "/", + "slug": "root" + } +}, +{ + "model": "tests.advert", + "pk": 1, + "fields": { + "title": "Red! It's the new blue!", + "description": "Red is a scientifically proven color that moves faster than all other colors.", + "external_link": "https://example.com/", + "is_active": true, + "rating": "1.5", + "long_description": "

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.

", + "points": 95, + "slug": "red-its-new-blue" + } +}, +{ + "pk": 2, + "model": "wagtailcore.page", + "fields": { + "title": "Home", + "numchild": 3, + "show_in_menus": false, + "live": true, + "depth": 2, + "content_type": ["tests", "simplepage"], + "path": "00010001", + "url_path": "/home/", + "slug": "home" + } +}, +{ + "pk": 2, + "model": "tests.simplepage", + "fields": { + "intro": "This is the homepage" + } +}, +{ + "pk": 1, + "model": "wagtailcore.site", + "fields": { + "root_page": 2, + "hostname": "localhost", + "port": 80, + "is_default_site": true + } +}, + +{ + "pk": 1, + "model": "auth.user", + "fields": { + "username": "admin", + "first_name": "admin", + "last_name": "admin", + "is_active": true, + "is_superuser": true, + "is_staff": true, + "groups": [ + ], + "user_permissions": [], + "password": "md5$seasalt$1e9bf2bf5606aa5c39852cc30f0f6f22", + "email": "admin@example.com" + } +} +] diff --git a/tests/migrations/0001_initial.py b/tests/migrations/0001_initial.py new file mode 100644 index 0000000..b6fa0e4 --- /dev/null +++ b/tests/migrations/0001_initial.py @@ -0,0 +1,47 @@ +# Generated by Django 3.0.6 on 2020-05-14 19:00 + +from django.db import migrations, models +import django.db.models.deletion +import wagtail.core.fields + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('wagtailcore', '0045_assign_unlock_grouppagepermission'), + ] + + operations = [ + migrations.CreateModel( + name='Advert', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('airtable_record_id', models.CharField(blank=True, db_index=True, max_length=35)), + ('title', models.CharField(max_length=255)), + ('description', models.TextField(blank=True)), + ('external_link', models.URLField(blank=True, max_length=500)), + ('is_active', models.BooleanField(default=False)), + ('rating', models.DecimalField(choices=[(1.0, '1'), (1.5, '1.5'), (2.0, '2'), (2.5, '2.5'), (3.0, '3'), (3.5, '3.5'), (4.0, '4'), (4.5, '4.5'), (5.0, '5')], decimal_places=1, max_digits=2, null=True)), + ('long_description', wagtail.core.fields.RichTextField(blank=True, null=True)), + ('points', models.IntegerField(blank=True, null=True)), + ('slug', models.SlugField(max_length=100, unique=True)), + ], + options={ + 'verbose_name': 'Advertisement', + 'verbose_name_plural': 'Advertisements', + }, + ), + migrations.CreateModel( + name='SimplePage', + fields=[ + ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.Page')), + ('intro', models.TextField()), + ], + options={ + 'abstract': False, + }, + bases=('wagtailcore.page',), + ), + ] diff --git a/tests/migrations/0002_modelnotused.py b/tests/migrations/0002_modelnotused.py new file mode 100644 index 0000000..b51518e --- /dev/null +++ b/tests/migrations/0002_modelnotused.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.6 on 2020-05-15 18:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tests', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ModelNotUsed', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('airtable_record_id', models.CharField(blank=True, db_index=True, max_length=35)), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/tests/migrations/0003_similartoadvert.py b/tests/migrations/0003_similartoadvert.py new file mode 100644 index 0000000..bb2ee21 --- /dev/null +++ b/tests/migrations/0003_similartoadvert.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.6 on 2020-05-15 18:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('tests', '0002_modelnotused'), + ] + + operations = [ + migrations.CreateModel( + name='SimilarToAdvert', + fields=[ + ('advert_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='tests.Advert')), + ], + options={ + 'abstract': False, + }, + bases=('tests.advert',), + ), + ] diff --git a/tests/migrations/__init__.py b/tests/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/models.py b/tests/models.py new file mode 100644 index 0000000..a79b110 --- /dev/null +++ b/tests/models.py @@ -0,0 +1,82 @@ +from django.db import models +from wagtail.core.fields import RichTextField +from wagtail.core.models import Page +from wagtail.snippets.models import register_snippet + +from taggit.managers import TaggableManager +from wagtail_airtable.mixins import AirtableMixin + + +class SimplePage(Page): + intro = models.TextField() + +@register_snippet +class Advert(AirtableMixin, models.Model): + + STAR_RATINGS = ( + (1.0, "1"), + (1.5, "1.5"), + (2.0, "2"), + (2.5, "2.5"), + (3.0, "3"), + (3.5, "3.5"), + (4.0, "4"), + (4.5, "4.5"), + (5.0, "5"), + ) + + title = models.CharField(max_length=255) + description = models.TextField(blank=True) + external_link = models.URLField(blank=True, max_length=500) + is_active = models.BooleanField(default=False) + rating = models.DecimalField( + null=True, choices=STAR_RATINGS, decimal_places=1, max_digits=2 + ) + long_description = RichTextField(blank=True, null=True) + points = models.IntegerField(null=True, blank=True) + slug = models.SlugField(max_length=100, unique=True, editable=True) + + @classmethod + def map_import_fields(cls): + """{'Airtable column name': 'model_field_name', ...}""" + mappings = { + "title": "title", + "description": "description", + "external_link": "external_link", + "is_active": "is_active", + "rating": "rating", + "long_description": "long_description", + "points": "points", + "slug": "slug", + } + return mappings + + def get_export_fields(self): + + return { + "title": self.title, + "description": self.description, + "external_link": self.external_link, + "is_active": self.is_active, + "rating": self.rating, + "long_description": self.long_description, + "points": self.points, + "slug": self.slug, + } + + class Meta: + verbose_name = "Advertisement" + verbose_name_plural = "Advertisements" + + def __str__(self): + return self.title + + +@register_snippet +class SimilarToAdvert(Advert): + pass + + +@register_snippet +class ModelNotUsed(AirtableMixin, models.Model): + pass diff --git a/tests/serializers.py b/tests/serializers.py new file mode 100644 index 0000000..ebfbbf0 --- /dev/null +++ b/tests/serializers.py @@ -0,0 +1,10 @@ +# from django.utils.dateparse import parse_datetime +from rest_framework import serializers +from wagtail_airtable.serializers import AirtableSerializer + + +class AdvertSerializer(AirtableSerializer): + + slug = serializers.CharField(max_length=100, required=True) + title = serializers.CharField(max_length=255) + external_link = serializers.URLField(required=False) diff --git a/tests/settings.py b/tests/settings.py new file mode 100644 index 0000000..9cc14b0 --- /dev/null +++ b/tests/settings.py @@ -0,0 +1,145 @@ +import os + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/checklist/ + + +# Application definition +INSTALLED_APPS = [ + 'tests', + 'wagtail_airtable', + + 'wagtail.contrib.forms', + 'wagtail.contrib.redirects', + 'wagtail.embeds', + 'wagtail.sites', + 'wagtail.users', + 'wagtail.snippets', + 'wagtail.documents', + 'wagtail.images', + 'wagtail.search', + 'wagtail.admin', + 'wagtail.core', + + 'modelcluster', + 'taggit', + + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', +] + +MIDDLEWARE = [ + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.security.SecurityMiddleware', + + 'wagtail.core.middleware.SiteMiddleware', + 'wagtail.contrib.redirects.middleware.RedirectMiddleware', +] + +ROOT_URLCONF = 'tests.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'tests.wsgi.application' + + +# Database +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'test-db.sqlite3'), + } +} + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +PASSWORD_HASHERS = ( + 'django.contrib.auth.hashers.MD5PasswordHasher', # don't use the intentionally slow default password hasher +) + +LANGUAGE_CODE = 'en-us' +TIME_ZONE = 'UTC' +USE_I18N = True +USE_L10N = True +USE_TZ = True + + +STATICFILES_FINDERS = [ + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', +] + +STATIC_ROOT = os.path.join(BASE_DIR, 'static') +STATIC_URL = '/static/' + +MEDIA_ROOT = os.path.join(BASE_DIR, 'test-media') +MEDIA_URL = 'http://media.example.com/media/' + +SECRET_KEY = 'not needed' + +# Wagtail settings + +WAGTAIL_SITE_NAME = "wagtail-airtable" +BASE_URL = 'http://example.com' + +AIRTABLE_API_KEY = 'keyWoWoWoWoW' +WAGTAIL_AIRTABLE_ENABLED = True +WAGTAIL_AIRTABLE_DEBUG = False +AIRTABLE_IMPORT_SETTINGS = { + 'tests.SimplePage': { + 'AIRTABLE_BASE_KEY': 'xxx', + 'AIRTABLE_TABLE_NAME': 'xxx', + 'AIRTABLE_UNIQUE_IDENTIFIER': 'slug', + 'AIRTABLE_SERIALIZER': 'tests.serializers.SimplePageSerializer', + }, + 'tests.Advert': { + 'AIRTABLE_BASE_KEY': 'app_airtable_advert_base_key', + 'AIRTABLE_TABLE_NAME': 'Advert Table Name', + 'AIRTABLE_UNIQUE_IDENTIFIER': 'slug', + 'AIRTABLE_SERIALIZER': 'tests.serializers.AdvertSerializer', + }, + 'tests.SimilarToAdvert': { # Exact same as 'tests.Advert' + 'AIRTABLE_BASE_KEY': 'app_airtable_advert_base_key', + 'AIRTABLE_TABLE_NAME': 'Advert Table Name', + 'AIRTABLE_UNIQUE_IDENTIFIER': 'slug', + 'AIRTABLE_SERIALIZER': 'tests.serializers.AdvertSerializer', + }, +} diff --git a/tests/test_import.py b/tests/test_import.py new file mode 100644 index 0000000..016d9d0 --- /dev/null +++ b/tests/test_import.py @@ -0,0 +1,389 @@ +from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import BaseCommand +from django.conf import settings +from django.test import TestCase, TransactionTestCase + +from wagtail_airtable.management.commands.import_airtable import Importer +from wagtail_airtable.tests import MockAirtable +from tests.models import Advert, ModelNotUsed, SimilarToAdvert, SimplePage +from tests.serializers import AdvertSerializer + + +class TestImportClass(TestCase): + fixtures = ['test.json'] + + def setUp(self): + self.options = { + 'verbosity': 2 + } + + def get_valid_record_fields(self): + """Common used method for standard valid airtable records.""" + return { + "Page Title": "Red! It's the new blue!", + "SEO Description": "Red is a scientifically proven...", + "External Link": "https://example.com/", + "Is Active": True, + "rating": "1.5", + "long_description": "

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.

", + "points": 95, + "slug": "red-its-new-blue", + } + + def get_valid_mapped_fields(self): + """Common used method for standard valid airtable mapped fields.""" + return { + "Page Title": "title", + "SEO Description": "description", + "External Link": "external_link", + "Is Active": "is_active", + "slug": "slug", + } + + def get_invalid_record_fields(self): + """Common used method for standard invalid airtable records.""" + return { + "SEO Description": "Red is a scientifically proven...", + "External Link": "https://example.com/", + "slug": "red-its-new-blue", + } + + def get_invalid_mapped_fields(self): + """Common used method for standard invalid airtable mapped fields.""" + return { + "SEO Description": "description", + "External Link": "external_link", + "slug": "slug", + } + + def test_debug_message(self): + models = ["fake.ModelName"] + text = "Testing debug message with high verbosity" + + importer = Importer(models=models, options=self.options) + debug_message = importer.debug_message(text) + self.assertEqual(debug_message, text) + + # Lower verbosity + importer = Importer(models=models, options={'verbosity': 1}) + debug_message = importer.debug_message(text) + self.assertEqual(debug_message, None) + + def test_get_model_serializer(self): + importer = Importer(models=["tests.Advert"]) + advert_serializer = importer.get_model_serializer("tests.serializers.AdvertSerializer") + self.assertEqual(advert_serializer, AdvertSerializer) + + def test_get_incorrect_model_serializer(self): + importer = Importer(models=["tests.Advert"]) + with self.assertRaises(AttributeError) as context: + advert_serializer = importer.get_model_serializer("tests.serializers.MissingSerializer") + self.assertEqual("module 'tests.serializers' has no attribute 'MissingSerializer'", str(context.exception)) + + def test_get_model_settings(self): + importer = Importer() + # Finds config settings + advert_settings = importer.get_model_settings(Advert) + self.assertEqual(type(advert_settings), dict) + self.assertDictEqual(settings.AIRTABLE_IMPORT_SETTINGS['tests.Advert'], advert_settings) + + # Does not find config settings + unused_model_settings = importer.get_model_settings(ModelNotUsed) + self.assertEqual(type(unused_model_settings), dict) + self.assertDictEqual(unused_model_settings, {}) + + # Finds adjacent model settings + advert_settings = importer.get_model_settings(SimilarToAdvert) + self.assertEqual(type(advert_settings), dict) + self.assertDictEqual(settings.AIRTABLE_IMPORT_SETTINGS['tests.Advert'], advert_settings) + + def test_get_column_to_field_names(self): + importer = Importer() + # The airtable column is the same name as the django field name + # ie: slug and slug + column, field = importer.get_column_to_field_names('slug') + self.assertEqual(column, 'slug') + self.assertEqual(field, 'slug') + # The airtable column is different from the django field name + # ie: "Page Title" and "title" + column, field = importer.get_column_to_field_names({"Page Title": "title"}) + self.assertEqual(column, 'Page Title') + self.assertEqual(field, 'title') + # Different settings were specified and arent currently handled + # Returns empty values + column, field = importer.get_column_to_field_names(None) + self.assertEqual(column, None) + self.assertEqual(field, None) + + def test_get_or_set_cached_records(self): + importer = Importer() + self.assertEqual(importer.cached_records, {}) + + # Make one API call. Update the cached_records. + client1 = MockAirtable() + all_records = importer.get_or_set_cached_records(client1) + client1.get_all.assert_called() + self.assertEqual(len(all_records), 4) + + cached_records = { + client1.table_name: all_records + } + self.assertEqual(importer.cached_records, cached_records) + self.assertEqual(len(importer.cached_records), 1) + + # Second API call is the same. Use the pre-existing cached records. + client2 = MockAirtable() + all_records = importer.get_or_set_cached_records(client2) + self.assertEqual(importer.cached_records, cached_records) + self.assertEqual(len(importer.cached_records), 1) + + # Third API call will create a new cached record in the cached_records dict + client3 = MockAirtable() + client3.table_name = "second_cached_entry" + all_records = importer.get_or_set_cached_records(client3) + self.assertNotEqual(importer.cached_records, cached_records) + self.assertEqual(len(importer.cached_records), 2) + # Ensure the internal dictionary "cache" has been updated + cached_records["second_cached_entry"] = all_records + self.assertEqual(importer.cached_records, cached_records) + + def test_convert_mapped_fields(self): + importer = Importer() + record_fields_dict = self.get_valid_record_fields() + record_fields_dict['extra_field_from_airtable'] = "Not mapped" + mapped_fields = importer.convert_mapped_fields( + record_fields_dict, + self.get_valid_mapped_fields(), + ) + # Ensure the new mapped fields have the proper django field keys + # And that each value is the value from the airtable record. + self.assertEqual( + mapped_fields['title'], + "Red! It's the new blue!", + ) + self.assertEqual( + mapped_fields['description'], + "Red is a scientifically proven...", + ) + # Ensure a field from Airtable that's not mapped to a model does not get + # passed into the newly mapped fields + self.assertFalse(hasattr(mapped_fields, 'extra_field_from_airtable')) + + def test_update_object(self): + importer = Importer(models=["tests.Advert"]) + advert_serializer = importer.get_model_serializer("tests.serializers.AdvertSerializer") + record_fields_dict = self.get_valid_record_fields() + record_fields_dict["SEO Description"] = "Red is a scientifically proven..." + mapped_fields_dict = { + "Page Title": "title", + "SEO Description": "description", + "External Link": "external_link", + "Is Active": "is_active", + "slug": "slug", + } + mapped_fields = importer.convert_mapped_fields( + record_fields_dict, + self.get_valid_mapped_fields(), + ) + # Ensure mapped_fields are mapped properly + self.assertEqual( + mapped_fields['description'], + "Red is a scientifically proven...", + ) + # Check serialized data is valid + serialized_data = advert_serializer(data=mapped_fields) + is_valid = serialized_data.is_valid() + self.assertTrue(is_valid) + # Get the advert object. + instance = Advert.objects.first() + self.assertEqual(instance.airtable_record_id, '') + # Importer should have zero updates objects. + self.assertEqual(importer.updated, 0) + + saved = importer.update_object(instance, 'recNewRecordId', serialized_data) + + self.assertTrue(saved) + self.assertEqual(importer.updated, 1) + self.assertEqual(importer.records_used, ['recNewRecordId']) + # Re-fetch the Advert instance and check its airtable_record_id + instance = Advert.objects.first() + self.assertEqual(instance.airtable_record_id, 'recNewRecordId') + + def test_update_object_with_invalid_serialized_data(self): + instance = Advert.objects.first() + importer = Importer(models=["tests.Advert"]) + advert_serializer = importer.get_model_serializer("tests.serializers.AdvertSerializer") + record_fields_dict = { + "SEO Description": "Red is a scientifically proven...", + "External Link": "https://example.com/", + "slug": "red-its-new-blue", + "Rating": "2.5", + } + mapped_fields_dict = { + "SEO Description": "description", + "External Link": "external_link", + "slug": "slug", + "Rating": "rating", + } + mapped_fields = importer.convert_mapped_fields( + record_fields_dict, + mapped_fields_dict, + ) + serialized_data = advert_serializer(data=mapped_fields) + is_valid = serialized_data.is_valid() + self.assertFalse(is_valid) + saved = importer.update_object(instance, 'recNewRecordId', serialized_data) + self.assertFalse(saved) + + def test_update_object_by_uniq_col_name_missing_uniq_id(self): + importer = Importer() + updated = importer.update_object_by_uniq_col_name( + field_mapping={'slug': ''}, + model=Advert, + serialized_data=object, + record_id='', + ) + + self.assertFalse(updated) + self.assertEqual(importer.skipped, 1) + + def test_update_object_by_uniq_col_name_object_found(self): + importer = Importer(models=["tests.Advert"]) + advert_serializer = importer.get_model_serializer("tests.serializers.AdvertSerializer") + record_fields_dict = { + "Page Title": "Red! It's the new blue!", + "SEO Description": "Red is a scientifically proven...", + "External Link": "https://example.com/UPDATED", + "Is Active": True, + "rating": "1.5", + "long_description": "

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.

", + "points": 95, + "slug": "red-its-new-blue", + } + mapped_fields_dict = { + "Page Title": "title", + "SEO Description": "description", + "External Link": "external_link", + "Is Active": "is_active", + "slug": "slug", + } + mapped_fields = importer.convert_mapped_fields( + record_fields_dict, + mapped_fields_dict, + ) + # Check serialized data is valid + serialized_data = advert_serializer(data=mapped_fields) + self.assertTrue(serialized_data.is_valid()) + + # with self.assertRaises(AttributeError) + updated = importer.update_object_by_uniq_col_name( + field_mapping={'slug': 'red-its-new-blue'}, + model=Advert, + serialized_data=serialized_data, + record_id='recNewRecordId', + ) + self.assertTrue(updated) + self.assertEqual(importer.skipped, 0) + self.assertEqual(importer.updated, 1) + + advert = Advert.objects.get(slug="red-its-new-blue") + self.assertEqual(advert.external_link, "https://example.com/UPDATED") + + def test_update_object_by_uniq_col_name_object_not_found(self): + importer = Importer(models=["tests.Advert"]) + advert_serializer = importer.get_model_serializer("tests.serializers.AdvertSerializer") + record_fields_dict = { + "Page Title": "Red! It's the new blue!", + "SEO Description": "Red is a scientifically proven...", + "External Link": "https://example.com/", + "Is Active": True, + "rating": "1.5", + "long_description": "

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.

", + "points": 95, + "slug": "red-its-new-blue", + } + mapped_fields_dict = { + "Page Title": "title", + "SEO Description": "description", + "External Link": "external_link", + "Is Active": "is_active", + "slug": "slug", + } + mapped_fields = importer.convert_mapped_fields( + record_fields_dict, + mapped_fields_dict, + ) + # Check serialized data is valid + serialized_data = advert_serializer(data=mapped_fields) + self.assertTrue(serialized_data.is_valid()) + + # with self.assertRaises(AttributeError) + updated = importer.update_object_by_uniq_col_name( + field_mapping={'slug': 'MISSING-OBJECT'}, + model=Advert, + serialized_data=serialized_data, + record_id='recNewRecordId', + ) + self.assertFalse(updated) + self.assertEqual(importer.updated, 0) + + def test_is_wagtail_page(self): + importer = Importer() + self.assertTrue(importer.is_wagtail_page(SimplePage)) + self.assertFalse(importer.is_wagtail_page(Advert)) + + def test_get_data_for_new_model_with_valid_serialized_data(self): + importer = Importer(models=["tests.Advert"]) + advert_serializer = importer.get_model_serializer("tests.serializers.AdvertSerializer") + mapped_fields = importer.convert_mapped_fields( + self.get_valid_record_fields(), + self.get_valid_mapped_fields(), + ) + # Check serialized data is valid + serialized_data = advert_serializer(data=mapped_fields) + is_valid = serialized_data.is_valid() + self.assertTrue(is_valid) + + data_for_new_model = importer.get_data_for_new_model(serialized_data, mapped_fields, 'recSomeRecordId') + self.assertTrue(data_for_new_model.get('airtable_record_id')) + self.assertEqual(data_for_new_model['airtable_record_id'], 'recSomeRecordId') + self.assertIsNone(data_for_new_model.get('id')) + self.assertIsNone(data_for_new_model.get('pk')) + + new_dict = dict(serialized_data.validated_data) + new_dict['airtable_record_id'] = 'recSomeRecordId' + self.assertDictEqual(new_dict, data_for_new_model) + + def test_get_data_for_new_model_with_invalid_serialized_data(self): + importer = Importer(models=["tests.Advert"]) + advert_serializer = importer.get_model_serializer("tests.serializers.AdvertSerializer") + + mapped_fields = importer.convert_mapped_fields( + self.get_invalid_record_fields(), + self.get_invalid_mapped_fields(), + ) + # Check serialized data is valid + serialized_data = advert_serializer(data=mapped_fields) + is_valid = serialized_data.is_valid() + self.assertFalse(is_valid) + + data_for_new_model = importer.get_data_for_new_model(serialized_data, mapped_fields, 'recSomeRecordId') + self.assertTrue(data_for_new_model.get('airtable_record_id')) + self.assertEqual(data_for_new_model['airtable_record_id'], 'recSomeRecordId') + self.assertIsNone(data_for_new_model.get('id')) + self.assertIsNone(data_for_new_model.get('pk')) + + new_dict = mapped_fields.copy() + new_dict['airtable_record_id'] = 'recSomeRecordId' + self.assertDictEqual(new_dict, data_for_new_model) + + +class TestImportCommand(TransactionTestCase): + fixtures = ['test.json'] + + def setUp(self): + pass + + def test_import_command(self): + from django.core.management import call_command + message = call_command("import_airtable", "tests.Advert", verbosity=1) diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..5e193b8 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,241 @@ +from copy import copy +from unittest import mock + +from airtable import Airtable +from django.test import TestCase + +from tests.models import Advert +from wagtail_airtable.mixins import AirtableMixin + + +class TestAirtableModel(TestCase): + fixtures = ['test.json'] + + def setUp(self): + self.client.login(username='admin', password='password') + self.advert = Advert.objects.first() + + def test_model_connection_settings(self): + # Make sure the settings are being passed into the model after .setup_airtable() is called + advert = copy(self.advert) + advert.setup_airtable() + self.assertEqual(advert.AIRTABLE_BASE_KEY, 'app_airtable_advert_base_key') + self.assertEqual(advert.AIRTABLE_TABLE_NAME, 'Advert Table Name') + self.assertEqual(advert.AIRTABLE_UNIQUE_IDENTIFIER, 'slug') + self.assertEqual(advert.AIRTABLE_SERIALIZER, 'tests.serializers.AdvertSerializer') + + def test_model_connection_settings_before_setup(self): + # Make sure instances are not instantiated with Airtable settings. + # By preventing automatic Airtable API instantiation we can avoid + # holding an Airtable API object on every model instance in Wagtail List views. + advert = copy(self.advert) + self.assertEqual(advert.AIRTABLE_BASE_KEY, None) + self.assertEqual(advert.AIRTABLE_TABLE_NAME, None) + self.assertEqual(advert.AIRTABLE_UNIQUE_IDENTIFIER, None) + # Object don't need to store the AIRTABLE_SERIALIZER property on them. + # Thus they should not have the property at all. + self.assertFalse(hasattr(advert, 'AIRTABLE_SERIALIZER')) + + def test_get_export_fields(self): + self.assertTrue(hasattr(self.advert, 'get_export_fields')) + export_fields = self.advert.get_export_fields() + self.assertEqual(type(export_fields), dict) + + def test_get_import_fields(self): + self.assertTrue(hasattr(self.advert, 'map_import_fields')) + mapped_import_fields = self.advert.map_import_fields() + self.assertEqual(type(mapped_import_fields), dict) + + def test_create_object_from_url(self): + response = self.client.post('/admin/snippets/tests/advert/add/', { + 'title': 'Second advert', + 'description': 'Lorem ipsum dolor sit amet, consectetur adipisicing elit.', + 'rating': "1.5", + 'slug': 'second-advert', + }) + advert = Advert.objects.last() + self.assertEqual(advert.airtable_record_id, 'recNewRecordId') + self.assertEqual(advert.title, 'Second advert') + self.assertFalse(advert._ran_airtable_setup) + self.assertFalse(advert._is_enabled) + self.assertFalse(advert._push_to_airtable) + self.assertFalse(hasattr(advert, 'client')) + + advert.setup_airtable() + + self.assertEqual(advert.AIRTABLE_BASE_KEY, 'app_airtable_advert_base_key') + self.assertEqual(advert.AIRTABLE_TABLE_NAME, 'Advert Table Name') + self.assertEqual(advert.AIRTABLE_UNIQUE_IDENTIFIER, 'slug') + self.assertTrue(advert._ran_airtable_setup) + self.assertTrue(advert._is_enabled) + self.assertTrue(advert._push_to_airtable) + self.assertTrue(hasattr(advert, 'client')) + + def test_create_object_from_orm(self): + advert = Advert.objects.create( + title='Throw away advert', + description='Lorem ipsum dolor sit amet, consectetur adipisicing elit.', + rating="2.5", + slug='disposable-advert', + ) + self.assertEqual(advert.airtable_record_id, 'recNewRecordId') + self.assertEqual(advert.title, 'Throw away advert') + + def test_edit_object(self): + advert = Advert.objects.create( + title='Throw away advert', + description='Lorem ipsum dolor sit amet, consectetur adipisicing elit.', + rating="2.5", + slug='edit-advert', + airtable_record_id='recCustomEditId', + ) + # Get the new advert without the instantiated airtable properties and api client + advert = Advert.objects.get(slug='edit-advert') + self.assertFalse(advert._ran_airtable_setup) + self.assertFalse(advert._is_enabled) + self.assertFalse(advert._push_to_airtable) + advert.title = "Edited title" + advert.description = "Edited description" + advert.save() + advert.client.update.assert_called() + self.assertTrue(advert._ran_airtable_setup) + self.assertTrue(advert._is_enabled) + self.assertTrue(advert._push_to_airtable) + self.assertEqual(advert.title, "Edited title") + + def test_delete_object(self): + advert = Advert.objects.create( + title='Throw away advert', + description='Lorem ipsum dolor sit amet, consectetur adipisicing elit.', + rating="2.5", + slug='delete-me', + ) + # If we werent mocking the Airtable.update() method, we'd assert advert.client.insert + advert.client.update.assert_called() + self.assertEqual(advert.airtable_record_id, 'recNewRecordId') + + advert.delete() + advert.client.delete.assert_called() + find_deleted_advert = Advert.objects.filter(slug='delete-me').count() + self.assertEqual(find_deleted_advert, 0) + + +class TestAirtableMixin(TestCase): + fixtures = ['test.json'] + + def setUp(self): + self.advert = Advert.objects.first() + + def test_setup_airtable(self): + advert = copy(self.advert) + self.assertEqual(advert._ran_airtable_setup, False) + self.assertEqual(advert._is_enabled, False) + self.assertEqual(advert._push_to_airtable, False) + self.assertFalse(hasattr(advert, 'client')) + + advert.setup_airtable() + + self.assertEqual(advert.AIRTABLE_BASE_KEY, 'app_airtable_advert_base_key') + self.assertEqual(advert.AIRTABLE_TABLE_NAME, 'Advert Table Name') + self.assertEqual(advert.AIRTABLE_UNIQUE_IDENTIFIER, 'slug') + self.assertEqual(advert._ran_airtable_setup, True) + self.assertEqual(advert._is_enabled, True) + self.assertEqual(advert._push_to_airtable, True) + self.assertTrue(hasattr(advert, 'client')) + + def test_create_and_attach_airtable_record(self): + advert = copy(self.advert) + self.assertEqual(advert.airtable_record_id, '') + + advert.setup_airtable() + advert.save() + + advert.client.update.assert_called() + advert.client.insert.assert_not_called() + self.assertEqual(advert.airtable_record_id, 'recNewRecordId') + + def test_update_record(self): + advert = Advert.objects.first() + advert.setup_airtable() + self.assertEqual(advert.airtable_record_id, '') + record = advert.update_record('fake record id') + advert.client.update.assert_called() + advert.client.insert.assert_not_called() + self.assertEqual(record['id'], 'recNewRecordId') + self.assertEqual(advert.airtable_record_id, 'recNewRecordId') + + def test_delete_record(self): + advert = copy(self.advert) + advert.setup_airtable() + deleted = advert.delete_record() + self.assertTrue(deleted) + advert.client.delete.assert_called() + + def test_parse_request_error(self): + error_401 = "401 Client Error: Unauthorized for url: https://api.airtable.com/v0/appYourAppId/Your%20Table?filterByFormula=.... [Error: {'type': 'AUTHENTICATION_REQUIRED', 'message': 'Authentication required'}]" + parsed_error = AirtableMixin.parse_request_error(error_401) + self.assertEqual(parsed_error['status_code'], 401) + self.assertEqual(parsed_error['type'], 'AUTHENTICATION_REQUIRED') + self.assertEqual(parsed_error['message'], 'Authentication required') + + error_404 = "404 Client Error: Not Found for url: https://api.airtable.com/v0/app3dozZtsCotiIpf/Brokerages/nope [Error: NOT_FOUND]" + parsed_error = AirtableMixin.parse_request_error(error_404) + self.assertEqual(parsed_error['status_code'], 404) + self.assertEqual(parsed_error['type'], 'NOT_FOUND') + self.assertEqual(parsed_error['message'], 'Record not found') + + error_404 = "404 Client Error: Not Found for url: https://api.airtable.com/v0/app3dozZtsCotiIpf/Brokerages%2022 [Error: {'type': 'TABLE_NOT_FOUND', 'message': 'Could not find table table_name in appxxxxx'}]" + parsed_error = AirtableMixin.parse_request_error(error_404) + self.assertEqual(parsed_error['status_code'], 404) + self.assertEqual(parsed_error['type'], 'TABLE_NOT_FOUND') + self.assertEqual(parsed_error['message'], 'Could not find table table_name in appxxxxx') + + def test_match_record(self): + advert = copy(self.advert) + advert.setup_airtable() + record_id = advert.match_record() + self.assertEqual(record_id, 'recNewRecordId') + advert.client.search.assert_called() + + def test_check_record_exists(self): + advert = copy(self.advert) + advert.setup_airtable() + record_exists = advert.check_record_exists('recNewRecordId') + self.assertTrue(record_exists) + advert.client.get.assert_called() + + def test_is_airtable_enabled(self): + advert = copy(self.advert) + self.assertFalse(advert._ran_airtable_setup) + self.assertFalse(advert._is_enabled) + advert.setup_airtable() + self.assertTrue(advert._ran_airtable_setup) + self.assertTrue(advert._is_enabled) + + def test_save(self): + advert = copy(self.advert) + self.assertFalse(advert._ran_airtable_setup) + self.assertFalse(advert._push_to_airtable) + self.assertFalse(advert._is_enabled) + self.assertEqual(advert.airtable_record_id, '') + + advert.setup_airtable() + self.assertTrue(advert._ran_airtable_setup) + self.assertTrue(advert._push_to_airtable) + self.assertTrue(advert._is_enabled) + + self.assertEqual(advert.airtable_record_id, '') + + advert.save() + self.assertEqual(advert.airtable_record_id, 'recNewRecordId') + advert.client.update.assert_called() + advert.client.insert.assert_not_called() + + def test_delete(self): + advert = copy(self.advert) + advert.setup_airtable() + + deleted = advert.delete_record() + self.assertTrue(deleted) + advert.client.delete.assert_called() + diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..68aa57f --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,38 @@ +from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist +from django.test import TestCase + +from wagtail_airtable.utils import get_model_for_path, get_all_models, get_validated_models + +from tests.models import Advert, ModelNotUsed, SimilarToAdvert, SimplePage +from tests.serializers import AdvertSerializer + + +class TestUtilFunctions(TestCase): + + def test_get_model_for_path(self): + advert_model = get_model_for_path("tests.Advert") + self.assertEqual(advert_model, Advert) + simple_page = get_model_for_path("tests.SimplePage") + self.assertEqual(simple_page, SimplePage) + bad_model_path = get_model_for_path("tests.BadModelPathName") + self.assertFalse(bad_model_path) + + def test_get_validated_models_with_single_valid_model(self): + models = ["tests.Advert"] + models = get_validated_models(models=models) + self.assertListEqual(models, [Advert]) + + def test_get_validated_models_with_multiple_valid_models(self): + models = ["tests.Advert", "tests.SimplePage", "tests.SimilarToAdvert"] + models = get_validated_models(models=models) + self.assertListEqual(models, [Advert, SimplePage, SimilarToAdvert]) + + def test_get_validated_models_with_invalid_model(self): + models = ["fake.ModelName"] + with self.assertRaises(ImproperlyConfigured) as context: + get_validated_models(models=models) + self.assertEqual("'fake.ModelName' is not recognised as a model name.", str(context.exception)) + + def test_get_all_models(self): + available_models = get_all_models() + self.assertListEqual(available_models, [SimplePage, Advert, SimilarToAdvert]) diff --git a/tests/test_views.py b/tests/test_views.py new file mode 100644 index 0000000..a3f71bc --- /dev/null +++ b/tests/test_views.py @@ -0,0 +1,27 @@ +from django.test import TestCase + + +class TestAdminViews(TestCase): + fixtures = ['test.json'] + + def setUp(self): + self.client.login(username='admin', password='password') + + def test_get(self): + response = self.client.get('/admin/airtable-import/') + self.assertEqual(response.status_code, 200) + self.assertContains(response, 'Models you can import from Airtable') + self.assertContains(response, 'Advert') + self.assertNotContains(response, 'Simple Page') + + def test_list_snippets(self): + response = self.client.get('/admin/snippets/tests/advert/') + self.assertEqual(response.status_code, 200) + + def test_snippet_detail(self): + response = self.client.get('/admin/snippets/tests/advert/1/') + self.assertEqual(response.status_code, 200) + # Ensure the default Advert does not have an Airtable Record ID + instance = response.context_data['instance'] + self.assertEqual(instance.airtable_record_id, '') + diff --git a/tests/urls.py b/tests/urls.py new file mode 100644 index 0000000..a2b9ce2 --- /dev/null +++ b/tests/urls.py @@ -0,0 +1,10 @@ +from django.conf.urls import include, url + +from wagtail.admin import urls as wagtailadmin_urls +from wagtail.core import urls as wagtail_urls + + +urlpatterns = [ + url(r'^admin/', include(wagtailadmin_urls)), + url(r'', include(wagtail_urls)), +] diff --git a/wagtail_airtable/__init__.py b/wagtail_airtable/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wagtail_airtable/apps.py b/wagtail_airtable/apps.py new file mode 100644 index 0000000..8ce35f3 --- /dev/null +++ b/wagtail_airtable/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class WagtailAirtableConfig(AppConfig): + name = "wagtail_airtable" diff --git a/wagtail_airtable/forms.py b/wagtail_airtable/forms.py new file mode 100644 index 0000000..615ff3a --- /dev/null +++ b/wagtail_airtable/forms.py @@ -0,0 +1,24 @@ +from django import forms +from django.conf import settings + + +class AirtableImportModelForm(forms.Form): + + model = forms.CharField() + + def clean_model(self): + """Make sure this model is in the AIRTABLE_IMPORT_SETTINGS config.""" + + model_label = self.cleaned_data["model"].lower() + airtable_settings = getattr(settings, "AIRTABLE_IMPORT_SETTINGS", {}) + is_valid_model = False + + for label, model_settings in airtable_settings.items(): + if model_label == label.lower(): + is_valid_model = True + break + + if not is_valid_model: + raise forms.ValidationError("You are importing an unsupported model") + + return model_label diff --git a/wagtail_airtable/management/commands/import_airtable.py b/wagtail_airtable/management/commands/import_airtable.py new file mode 100644 index 0000000..496e5fc --- /dev/null +++ b/wagtail_airtable/management/commands/import_airtable.py @@ -0,0 +1,582 @@ +import sys +from importlib import import_module + +from airtable import Airtable +from django.db import models, IntegrityError +from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.core.management.base import BaseCommand +from logging import getLogger +from modelcluster.contrib.taggit import ClusterTaggableManager +from taggit.managers import TaggableManager +from wagtail.core.models import Page + +from wagtail_airtable.tests import MockAirtable +from wagtail_airtable.utils import get_model_for_path, get_validated_models + +logger = getLogger(__name__) + + +DEFAULT_OPTIONS = { + "verbosity": 1, +} + +TESTING = any(x in ["test", "runtests.py"] for x in sys.argv) + + +class Importer: + def __init__(self, models=[], options=DEFAULT_OPTIONS): + self.models = models + self.options = options + self.records_used = [] + self.cached_records = {} + self.created = 0 + self.updated = 0 + self.skipped = 0 + + def debug_message(self, message): + """ + Local function. Print debug messages if `verbosity` is 2 or higher. + """ + if self.options["verbosity"] >= 2: + if not TESTING: + print(message) + return message + + def get_model_serializer(self, serializer_string): + location, serializer_name = serializer_string.rsplit(".", 1) + module = import_module(location) + serializer_class = getattr(module, serializer_name) + return serializer_class + + def get_model_settings(self, model) -> dict: + return settings.AIRTABLE_IMPORT_SETTINGS.get(model._meta.label, {}) + + def get_column_to_field_names(self, airtable_unique_identifier) -> tuple: + uniq_id_type = type(airtable_unique_identifier) + airtable_unique_identifier_column_name = None + airtable_unique_identifier_field_name = None + if uniq_id_type == str: + # The unique identifier is a string. + # Use it as the Airtable Column name and the Django field name + airtable_unique_identifier_column_name = airtable_unique_identifier + airtable_unique_identifier_field_name = airtable_unique_identifier + elif uniq_id_type == dict: + # Unique identifier is a dictionary. + # Use the key as the Airtable Column name and the value as the Django Field name. + ( + airtable_unique_identifier_column_name, + airtable_unique_identifier_field_name, + ) = list(airtable_unique_identifier.items())[0] + + return ( + airtable_unique_identifier_column_name, + airtable_unique_identifier_field_name, + ) + + def get_or_set_cached_records(self, airtable_client): + # Memoize results from Airtable so we don't hit the same API multiple times + # This is largely used to support additional Wagtail/Airatble settings + # that are identical to each other, as in they would use the + # same Airtable Base as the key in the dictionary. + # ie. + # 'yourapp.YourPage': { + # ... + # 'AIRTABLE_TABLE_NAME': 'Your Table', + # }, + # 'different_app.DifferentPage': { + # ... + # 'AIRTABLE_TABLE_NAME': 'Your Table', # Same Airtable Table name + # } + # All of the above settings will use the 'Your Table' results + # instead of hitting the Airtable API for each model and getting the same + # results every time. This is designed to help with API efficiency, reduce + # load/import times, and to reduce how much memory is required to save all + # the records from Airtable. + if self.cached_records.get(airtable_client.table_name): + all_records = self.cached_records.get(airtable_client.table_name) + else: + # Get all the airtable records for the specified table. + all_records = airtable_client.get_all() + self.cached_records[airtable_client.table_name] = all_records + return all_records + + def convert_mapped_fields(self, record_fields_dict, mapped_fields_dict) -> dict: + # Create a dictionary of newly mapped key:value pairs based on the `mappings` dict above. + # This wil convert "airtable column name" to "django_field_name" + mapped_fields_dict = { + mapped_fields_dict[key]: value + for (key, value) in record_fields_dict.items() + if key in mapped_fields_dict + } + return mapped_fields_dict + + def update_object( + self, instance, record_id, serialized_data, is_wagtail_model=False + ) -> bool: + """ + Attempts to update an object. + + Returns a bool that determines if the object was updated or not. + """ + + if serialized_data.is_valid(): + self.debug_message( + "\t\t Serializer data was valid. Setting attrs on model..." + ) + model = type(instance) + for field_name, value in serialized_data.validated_data.items(): + field_type = type( + model._meta.get_field(field_name) + ) # ie. django.db.models.fields.CharField + # If this field type is a subclass of a known Wagtail Tag, or a Django m2m field + # We need to loop through all the values and add them to the m2m-style field. + if issubclass( + field_type, + (TaggableManager, ClusterTaggableManager, models.ManyToManyField,), + ): + m2m_field = getattr(instance, field_name) + for m2m_value in value: + m2m_field.add(m2m_value) + else: + setattr(instance, field_name, value) + # When an object is saved it should NOT push its newly saved data back to Airtable. + # This could theoretically cause a loop. By default this setting is True. But the + # below line confirms it's false, just to be safe. + instance.airtable_record_id = record_id + instance.push_to_airtable = False + try: + if is_wagtail_model: + self.debug_message("\t\t This is a Wagtail Page model") + # Wagtail page. Requires a .save_revision() + if not instance.locked: + self.debug_message( + "\t\t\t Page is not locked. Saving page and creating a new revision." + ) + # Only save the page if the page is not locked + instance.save() + instance.save_revision() + self.updated = self.updated + 1 + else: + self.debug_message("\t\t\t Page IS locked. Skipping Page save.") + self.skipped = self.skipped + 1 + else: + # Django model. Save normally. + self.debug_message("\t\t Saving Django model") + instance.save() + self.updated = self.updated + 1 + + # New record being processed. Save it to the list of records. + self.records_used.append(record_id) + # Object updated (and record was used) + return True + except ValidationError as error: + self.skipped = self.skipped + 1 + error_message = "; ".join(error.messages) + logger.error( + f"Unable to save {instance._meta.label} -> '{instance}'. Error(s): {error_message}" + ) + self.debug_message( + f"\t\t Could not save Wagtail/Django model. Error: {error_message}" + ) + else: + logger.info(f"Invalid data for record {record_id}") + self.debug_message( + f"\t\t Serializer was invalid for record: {record_id}, model id: {instance.pk}" + ) + self.debug_message( + "\t\t Continuing to look for object by its unique identifier" + ) + # Not updated. + return False + + def update_object_by_uniq_col_name( + self, + field_mapping=None, + model=None, + serialized_data=None, + record_id=None, + is_wagtail_model=False, + ): + k, v = zip(*field_mapping.items()) + airtable_unique_identifier_field_name = k[0] + unique_identifier = v[0] + + if unique_identifier: + self.debug_message( + f"\t\t An Airtable record based on the unique identifier was found: {airtable_unique_identifier_field_name}" + ) + try: + instance = model.objects.get( + **{airtable_unique_identifier_field_name: unique_identifier} + ) + self.debug_message( + f"\t\t Local object found by Airtable unique column name: {airtable_unique_identifier_field_name}" + ) + except model.DoesNotExist: + instance = None + self.debug_message( + "\t\t No object was found based on the Airtable column name" + ) + + if instance: + # A local model object was found by a unique identifier. + if serialized_data.is_valid(): + for field_name, value in serialized_data.validated_data.items(): + field_type = type( + model._meta.get_field(field_name) + ) # ie. django.db.models.fields.CharField + # If this field type is a subclass of a known Wagtail Tag, or a Django m2m field + # We need to loop through all the values and add them to the m2m-style field. + if issubclass( + field_type, + ( + TaggableManager, + ClusterTaggableManager, + models.ManyToManyField, + ), + ): + m2m_field = getattr(instance, field_name) + for m2m_value in value: + m2m_field.add(m2m_value) + else: + setattr(instance, field_name, value) + # When an object is saved it should NOT push its newly saved data back to Airtable. + # This could theoretically cause a loop. By default this setting is False. But the + # below line confirms it's false, just to be safe. + instance.airtable_record_id = record_id + instance.push_to_airtable = False + try: + if is_wagtail_model: + # Wagtail page. Requires a .save_revision() + if not instance.locked: + # Only save the page if the page is not locked + instance.save() + instance.save_revision() + self.updated = self.updated + 1 + else: + self.debug_message( + "\t\t\t Page IS locked. Skipping Page save." + ) + self.skipped = self.skipped + 1 + else: + # Django model. Save normally. + instance.save() + self.debug_message("\t\t\t Saved!") + self.updated = self.updated + 1 + + # Record this record as "used" + self.records_used.append(record_id) + # New record being processed. Save it to the list of records. + return True + except ValidationError as error: + error_message = "; ".join(error.messages) + logger.error( + f"Unable to save {instance}. Error(s): {error_message}" + ) + self.debug_message( + f"\t\t Unable to save {instance} (ID: {instance.pk}; Airtable Record ID: {record_id}). Reason: {error_message}" + ) + self.skipped = self.skipped + 1 + else: + logger.info(f"Invalid data for record {record_id}") + self.debug_message("\t\t Serializer data was invalid.") + self.skipped = self.skipped + 1 + else: + # No object was found by this unique ID. + # Do nothing. The next step will be to create this object in Django + logger.info( + f"{model._meta.verbose_name} with field {airtable_unique_identifier_field_name}={unique_identifier} was not found" + ) + self.debug_message( + f"\t\t {model._meta.verbose_name} with field {airtable_unique_identifier_field_name}={unique_identifier} was not found" + ) + else: + # There was no unique identifier set for this model. + # Nothing can be done about that right now. + logger.info(f"{model._meta.verbose_name} does not have a unique identifier") + self.debug_message( + f"\t\t {model._meta.verbose_name} does not have a unique identifier" + ) + self.skipped = self.skipped + 1 + return False + + def is_wagtail_page(self, model): + if issubclass(model, Page): + logger.info(f"{model._meta.verbose_name} cannot be created from an import.") + self.debug_message( + f"\t\t {model._meta.verbose_name} is a Wagtail Page and cannot be created from an import." + ) + # New record being processed. Save it to the list of records. + return True + return False + + def get_data_for_new_model(self, serialized_data, mapped_import_fields, record_id): + + # Check if we can use the serialized data to create a new model. + # If we can, great! If not, fall back to the original mapped_import_fields + # If this has to fall back to the original mapped_import_fields: failure + # to create a model will be higher than normal. + if serialized_data.is_valid(): + data_for_new_model = dict(serialized_data.validated_data) + else: + data_for_new_model = mapped_import_fields + data_for_new_model["airtable_record_id"] = record_id + + # First things first, remove any "pk" or "id" items from the mapped_import_fields + # This will let Django and Wagtail handle the PK on its own, as it should. + # When the model is saved it'll trigger a push to Airtable and automatically update + # the necessary column with the new PK so it's always accurate. + for key in ( + "pk", + "id", + ): + try: + del data_for_new_model[key] + except KeyError: + pass + + return data_for_new_model + + def run(self): + + models = get_validated_models(models=self.models) + self.debug_message(f"Validated models: {models}") + + # Used for re-using API data instead of making several of API request and waiting/spamming the Airtable API + # Maintain a list of record Ids that were used already. Every record is a unique ID so processing the + # Same record more than once will just be hitting the DB over and over and over again. No bueno. + + for model in models: + self.debug_message(f"IMPORTING MODEL: {model}") + # Wagtail models require a .save_revision() call when being saved. + is_wagtail_model = issubclass(model, Page) + # Airtable global settings. + airtable_settings = self.get_model_settings(model) + + # Set the unique identifier and serializer. + model_serializer = self.get_model_serializer( + airtable_settings.get("AIRTABLE_SERIALIZER") + ) + # Get the unique column name and field name. + # The CAN be the same value if a string is provided in the settings. + ( + airtable_unique_identifier_column_name, + airtable_unique_identifier_field_name, + ) = self.get_column_to_field_names( + airtable_settings.get("AIRTABLE_UNIQUE_IDENTIFIER") + ) + + if ( + not airtable_unique_identifier_field_name + and not airtable_unique_identifier_column_name + ): + logger.error("No unique columns are set in your Airtable configuration") + continue + + # Set the Airtable API client on a per-model basis + if not TESTING: + airtable = Airtable( + airtable_settings.get("AIRTABLE_BASE_KEY"), + airtable_settings.get("AIRTABLE_TABLE_NAME"), + api_key=settings.AIRTABLE_API_KEY, + ) + else: + airtable = MockAirtable( + airtable_settings.get("AIRTABLE_BASE_KEY"), + airtable_settings.get("AIRTABLE_TABLE_NAME"), + api_key=settings.AIRTABLE_API_KEY, + ) + + all_records = self.get_or_set_cached_records(airtable) + + self.debug_message( + f"\t Airtable base key: {airtable_settings.get('AIRTABLE_BASE_KEY')}" + ) + self.debug_message( + f"\t Airtable table name: {airtable_settings.get('AIRTABLE_TABLE_NAME')}" + ) + self.debug_message("\t Airtable unique identifier settings:") + self.debug_message(f"\t Airtable records: {len(all_records)}") + self.debug_message( + f"\t\t Airtable column: {airtable_unique_identifier_column_name}" + ) + self.debug_message( + f"\t\t Django field name: {airtable_unique_identifier_field_name}" + ) + + # Loop through every record in the Airtable. + for record in all_records: + # If a record was used already, skip this iteration. + if record["id"] in self.records_used: + continue + + record_id = record["id"] + record_fields = record["fields"] + mapped_import_fields = self.convert_mapped_fields( + record_fields, model.map_import_fields() + ) + serialized_data = model_serializer(data=mapped_import_fields) + serialized_data.is_valid() + + # Look for a record by its airtable_record_id. + # If it exists, update the data. + self.debug_message( + f"\n\t Looking for existing object with record: {record_id}" + ) + try: + obj = model.objects.get(airtable_record_id=record_id) + self.debug_message(f"\t\t Local object found {obj}") + except model.DoesNotExist: + obj = None + self.debug_message("\t\t Local object was NOT found") + + if obj: + # Model object was found by it's airtable_record_id + was_updated = self.update_object( + instance=obj, + record_id=record_id, + serialized_data=serialized_data, + is_wagtail_model=is_wagtail_model, + ) + # Object was updated. No need to continue through the rest of this function + if was_updated: + continue + + # This `unique_identifier` is the value of an Airtable record. + # ie. + # fields = { + # 'Slug': 'your-model' + # } + # This will return 'your-model' and can now be searched for as model.objects.get(slug='your-model') + unique_identifier = record_fields.get( + airtable_unique_identifier_column_name, None + ) + was_updated = self.update_object_by_uniq_col_name( + field_mapping={ + airtable_unique_identifier_field_name: unique_identifier + }, + model=model, + serialized_data=serialized_data, + record_id=record_id, + is_wagtail_model=is_wagtail_model, + ) + if was_updated: + continue + + # Cannot bulk-create Wagtail pages from Airtable because we don't know where the pages + # Are supposed to live, what their tree depth should be, and a few other factors. + # For this scenario, log information and skip the loop iteration. + if self.is_wagtail_page(model): + self.records_used.append(record_id) + continue + + # Attempt to format valid data to create a new model form either the + # validated data in the serializer, or the mapped_field data. + data_for_new_model = self.get_data_for_new_model( + serialized_data, mapped_import_fields, record_id + ) + + # If there is no match whatsoever, try to create a new `model` instance. + # Note: this may fail if there isn't enough data in the Airtable record. + try: + self.debug_message("\t\t Attempting to create a new object...") + new_model = model(**data_for_new_model) + new_model.save() + self.debug_message("\t\t Object created") + self.created = self.created + 1 + except ValueError as value_error: + logger.info( + f"Could not create new model object. Value Error: {value_error}" + ) + self.debug_message( + f"\t\t Could not create new model object. Value Error: {value_error}" + ) + except IntegrityError as e: + logger.info( + f"Could not create new model object. Integrity Error: {e}" + ) + self.debug_message( + f"\t\t Could not create new model object. Integrity Error: {e}" + ) + except AttributeError as e: + logger.info( + f"Could not create new model object. AttributeError. Error: {e}" + ) + self.debug_message( + f"\t\t Could not create new model object. AttributeError. Error: {e}" + ) + except Exception as e: + logger.error( + f"Unhandled error. Could not create a new object for {model._meta.verbose_name}. Error: {e}" + ) + self.debug_message( + f"\t\t Unhandled error. Could not create a new object for {model._meta.verbose_name}. Error: {e}" + ) + + return self.created, self.skipped, self.updated + + +class Command(BaseCommand): + help = "Import data from an Airtable and overwrite model or page information" + + def add_arguments(self, parser): + parser.add_argument( + "labels", + metavar="model_name", + nargs="+", + help="Model (as app_label.model_name) or app name to populate table entries for, e.g. creditcards.CreditCard", + ) + + def handle(self, *args, **options): + """ + Runs the management command with the app_name.ModelName as parameters. + ie. python manage.py import_airtable pages.HomePage creditcards.CreditCard + + This will handle Wagtail Pages (with page revisions) along with standard + Django models. + + Models passed in to the command go through a quick validation to ensure they + exist. + + Every model is then looped through, and a `settings.AIRTABLE_IMPORT_SETTINGS` + is searched for based on the model's label (ie. pages.HomePage). These + settings are used to connect to a certain Airtable Base (a set of spreadsheets), + the name of the table to use, a unique identifier (used for connecting previously + unrelated Airtable records to Django objects), and a serializer for validating + incoming data from Airtable to make it work with the Django field types. + + Each model loop contains an `airtable.get_all()` command which + will get all the data from the Airtable Table and load it into a + list of dictionaries. This uses memoization to reduce memory hogging and + wasted API calls to Airtable. + + Then every record is iterated over and 3 actions are taken: + 1. Look for an existing model object by its airtable_record_id. + Update if found. Skip to step 2 if not found. + 2. Search for a model object by its unique identifier. + If found, update the object. Skip to step 3 if not found. + 3. Create a new object. This step is very susceptible to fail based on the + model type and fields in the serializer. Wagtail pages cannot be created + from Airtable records as there's missing data such as depth and parent + pages. And so Wagtail pages are skipped entirely in step 3. + """ + # Overwrite verbosity if WAGTAIL_AIRTABLE_DEBUG is enabled. + if settings.DEBUG: + # AIRTABLE_DEBUG can only be enabled if standard Django DEBUG is enabled. + # The idea is to protect logs and output in production from the noise of these imports. + AIRTABLE_DEBUG = getattr(settings, "WAGTAIL_AIRTABLE_DEBUG", False) + if AIRTABLE_DEBUG: + options["verbosity"] = 2 + + importer = Importer(models=options["labels"], options=options) + created, skipped, updated = importer.run() + + if options["verbosity"] >= 1: + self.stdout.write( + f"{created} object created. {updated} object updated. {skipped} object skipped." + ) + + # Use friendlier message. Slightly differs from the stdout.write() string. + return f"{created} items created. {updated} items updated. {skipped} items skipped." diff --git a/wagtail_airtable/management/commands/reset_local_airtable_records.py b/wagtail_airtable/management/commands/reset_local_airtable_records.py new file mode 100644 index 0000000..bef08c8 --- /dev/null +++ b/wagtail_airtable/management/commands/reset_local_airtable_records.py @@ -0,0 +1,22 @@ +from django.conf import settings +from django.core.management.base import BaseCommand + +from wagtail_airtable.utils import get_all_models + + +class Command(BaseCommand): + help = "Looks through every available model in the AIRTABLE_IMPORT_SETTINGS and unsets the `airtable_record_id`" + + def handle(self, *args, **options): + """ + Gets all the models set in AIRTABLE_IMPORT_SETTINGS, loops through them, and set `airtable_record_id=''` to every one. + """ + records_updated = 0 + models = get_all_models() + for model in models: + if hasattr(model, "airtable_record_id"): + total_updated = model.objects.update(airtable_record_id="") + records_updated = records_updated + total_updated + + if options["verbosity"] >= 1: + self.stdout.write(f"Set {records_updated} objects to airtable_record_id=''") diff --git a/wagtail_airtable/migrations/__init__.py b/wagtail_airtable/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wagtail_airtable/mixins.py b/wagtail_airtable/mixins.py new file mode 100644 index 0000000..60ef22c --- /dev/null +++ b/wagtail_airtable/mixins.py @@ -0,0 +1,327 @@ +import sys +from ast import literal_eval +from logging import getLogger + +from airtable import Airtable +from django.conf import settings +from django.db import models +from requests import HTTPError + +from django.utils.functional import cached_property + +from .tests import MockAirtable + +logger = getLogger(__name__) + + +TESTING = any(x in ["test", "runtests.py"] for x in sys.argv) + + +class AirtableMixin(models.Model): + """A mixin to update an Airtable when a model object is saved or deleted.""" + + AIRTABLE_BASE_KEY = None + AIRTABLE_TABLE_NAME = None + AIRTABLE_UNIQUE_IDENTIFIER = None + + # If the Airtable integration for this model is enabled. Used for sending data to Airtable. + _is_enabled = False + # If the Airtable api setup is complete in this model. Used for singleton-like setup_airtable() method. + _ran_airtable_setup = False + # Upon save, should this model's data be sent to Airtable? + # This is an internal variable. Both _push_to_airtable and push_to_airtable needs to be True + # before a push to Airtable will happen. + # _push_to_airtable is for internal use only + _push_to_airtable = False + # Case for disabling this: when importing data from Airtable as to not + # ... import data, save the model, and push the same data back to Airtable. + # push_to_airtable can be set from outside the model + push_to_airtable = True + + airtable_record_id = models.CharField(max_length=35, db_index=True, blank=True) + + def setup_airtable(self) -> None: + """ + This method is used in place of __init__() as to not check global settings and + set the Airtable api client over and over again. + + self._ran_airtable_setup is used to ensure this method is only ever run once. + """ + if not self._ran_airtable_setup: + # Don't run this more than once on a model. + self._ran_airtable_setup = True + + if not hasattr(settings, "AIRTABLE_IMPORT_SETTINGS") or not getattr( + settings, "WAGTAIL_AIRTABLE_ENABLED", False + ): + # No AIRTABLE_IMPORT_SETTINGS were found. Skip checking for settings. + return None + + # Look for airtable settings. Default to an empty dict. + AIRTABLE_SETTINGS = settings.AIRTABLE_IMPORT_SETTINGS.get( + self._meta.label, {} + ) + + # Set the airtable settings. + self.AIRTABLE_BASE_KEY = AIRTABLE_SETTINGS.get("AIRTABLE_BASE_KEY") + self.AIRTABLE_TABLE_NAME = AIRTABLE_SETTINGS.get("AIRTABLE_TABLE_NAME") + self.AIRTABLE_UNIQUE_IDENTIFIER = AIRTABLE_SETTINGS.get( + "AIRTABLE_UNIQUE_IDENTIFIER" + ) + self.AIRTABLE_SERIALIZER = AIRTABLE_SETTINGS.get("AIRTABLE_SERIALIZER") + if ( + AIRTABLE_SETTINGS + and settings.AIRTABLE_API_KEY + and self.AIRTABLE_BASE_KEY + and self.AIRTABLE_TABLE_NAME + and self.AIRTABLE_UNIQUE_IDENTIFIER + ): + if not TESTING: + self.client = Airtable( + self.AIRTABLE_BASE_KEY, + self.AIRTABLE_TABLE_NAME, + api_key=settings.AIRTABLE_API_KEY, + ) + else: + self.client = MockAirtable( + self.AIRTABLE_BASE_KEY, + self.AIRTABLE_TABLE_NAME, + api_key=settings.AIRTABLE_API_KEY, + ) + + self._push_to_airtable = True + self._is_enabled = True + else: + logger.warning( + f"Airtable settings are not enabled for the {self._meta.verbose_name} " + f"({self._meta.model_name}) model" + ) + + @property + def is_airtable_enabled(self): + """ + Used in the template to determine if a model can or cannot be imported from Airtable. + """ + if not self._ran_airtable_setup: + self.setup_airtable() + return self._is_enabled + + def get_import_fields(self): + """ + When implemented, should return a dictionary of the mapped fields from Airtable to the model. + ie. + { + "Airtable Column Name": "model_field_name", + ... + } + """ + raise NotImplementedError + + def get_export_fields(self): + """ + When implemented, should return a dictionary of the mapped fields from Airtable to the model. + ie. + { + "airtable_column": self.airtable_column, + "annual_fee": self.annual_fee, + } + """ + raise NotImplementedError + + @cached_property + def mapped_export_fields(self): + return self.get_export_fields() + + def create_record(self) -> dict: + """ + Create or update a record. + + The create_record() method will look for an Airtable match before trying + to create a new Airtable record (that comes with a new airtable_record_id). + + This function needs to check for a matched record in Airtable first just in case + some data became out of sync, or one person worked in Airtable and one worked in + Wagtail. The idea here is to marry those records whenever possible instead of + duplicating Airtable records. + + If a record in Airtable exists, update this object with the found record_id. (Prevent record duplication) + But if a record is NOT found in Airtable, create a new record. + """ + matched_record = self.match_record() + if matched_record: + record = self.update_record(matched_record) + else: + record = self.client.insert(self.mapped_export_fields) + + self.airtable_record_id = record["id"] + return record + + def check_record_exists(self, airtable_record_id) -> bool: + """ + Check if a record exists in an Airtable by its exact Airtable Record ID. + + This will trigger an Airtable API request. + Returns a True/False response. + """ + try: + record = self.client.get(airtable_record_id) + except HTTPError: + record = {} + return bool(record) + + def update_record(self, airtable_record_id=None): + """ + Update a record. + + Before updating a record this will check to see if a record even exists + in Airtable. If a record is not found using its Airtable record_id it cannot + be updated and may throw an unexpected error. + + If a record DOES exist based on its Airtable record_id, we can update that particular row. + If a record does NOT exist in Airtable, a new record will need to be created. + """ + airtable_record_id = airtable_record_id or self.airtable_record_id + if self.check_record_exists(airtable_record_id): + # Record exists in Airtable + record = self.client.update(airtable_record_id, self.mapped_export_fields) + else: + # No record exists in Airtable. Create a new record now. + record = self.create_record() + + self.airtable_record_id = record["id"] + return record + + def delete_record(self) -> bool: + """ + Deletes a record from Airtable, but does not delete the object from Django. + + Returns True if the record is successfully deleted, otherwise False. + """ + try: + response = self.client.delete(self.airtable_record_id) + deleted = response["deleted"] + except HTTPError: + deleted = False + return deleted + + def match_record(self) -> str: + """ + Look for a record in an Airtable. Search by the AIRTABLE_UNIQUE_IDENTIFIER. + + Instead of looking for an Airtable record by it's exact Record ID, it will + search through the specified Airtable column for a specific value. + + WARNING: If more than one record is found, the first one in the returned + list of records (a list of dicts) will be used. + + This differs from check_record_exists() as this will return the record string + (or an empty string if a record is not found), whereas check_record_exists() + will return a True/False boolean to let you know if a record simply exists, + or doesn't exist. + """ + if type(self.AIRTABLE_UNIQUE_IDENTIFIER) == dict: + keys = list(self.AIRTABLE_UNIQUE_IDENTIFIER.keys()) + values = list(self.AIRTABLE_UNIQUE_IDENTIFIER.values()) + # TODO: Edge case handling: + # - Handle multiple dictionary keys + # - Handle empty dictionary + airtable_column_name = keys[0] + model_field_name = values[0] + value = getattr(self, model_field_name) + else: + _airtable_unique_identifier = self.AIRTABLE_UNIQUE_IDENTIFIER + value = getattr(self, _airtable_unique_identifier) + airtable_column_name = self.AIRTABLE_UNIQUE_IDENTIFIER + records = self.client.search(airtable_column_name, value) + total_records = len(records) + if total_records: + # If more than 1 record was returned log a warning. + if total_records > 1: + logger.info( + f"Found {total_records} Airtable records for {airtable_column_name}={value}. " + f"Using first available record ({records[0]['id']}) and ignoring the others." + ) + # Always return the first record + return records[0]["id"] + + return "" + + def refresh_mapped_export_fields(self) -> None: + """Delete the @cached_property caching on self.mapped_export_fields.""" + try: + del self.mapped_export_fields + except Exception: + # Doesn't matter what the error is. + pass + + @classmethod + def parse_request_error(cls, error): + """ + Parse an Airtable/requests HTTPError string. + + Example: 401 Client Error: Unauthorized for url: https://api.airtable.com/v0/appYourAppId/Your%20Table?filterByFormula=.... [Error: {'type': 'AUTHENTICATION_REQUIRED', 'message': 'Authentication required'}] + """ + + code = int(error.split(":", 1)[0].split(" ")[0]) + error_json = error.split("[Error: ")[1].rstrip("]") + if error_json == "NOT_FOUND": # 404's act different + return { + "status_code": code, + "type": "NOT_FOUND", + "message": "Record not found", + } + else: + error_info = literal_eval(error_json) + return { + "status_code": code, + "type": error_info["type"], + "message": error_info["message"], + } + + def save(self, *args, **kwargs): + """ + If there's an existing airtable record id, update the row. + Otherwise attempt to create a new record. + """ + self.setup_airtable() + if self._push_to_airtable and self.push_to_airtable: + # Every airtable model needs mapped fields. + # mapped_export_fields is a cached property. Delete the cached prop and get new values upon save. + self.refresh_mapped_export_fields() + if self.airtable_record_id: + # If this model has an airtable_record_id, attempt to update the record. + try: + self.update_record() + except HTTPError as e: + error = self.parse_request_error(e.args[0]) + message = ( + f"Could not update Airtable record. Reason: {error['message']}" + ) + logger.warning(message) + # Used in the `after_edit_page` hook. If it exists, an error message will be displayed. + self._airtable_update_error = message + else: + # Creating a record will also search for an existing field match + # ie. Looks for a matching `slug` in Airtable and Wagtail/Django + try: + self.create_record() + except HTTPError as e: + error = self.parse_request_error(e.args[0]) + message = ( + f"Could not create Airtable record. Reason: {error['message']}" + ) + logger.warning(message) + # Used in the `after_edit_page` hook. If it exists, an error message will be displayed. + self._airtable_update_error = message + + return super().save(*args, **kwargs) + + def delete(self, *args, **kwargs): + self.setup_airtable() + if self.airtable_record_id: + # Try to delete the record from the Airtable. + self.delete_record() + return super().delete(*args, **kwargs) + + class Meta: + abstract = True diff --git a/wagtail_airtable/serializers.py b/wagtail_airtable/serializers.py new file mode 100644 index 0000000..eb7f158 --- /dev/null +++ b/wagtail_airtable/serializers.py @@ -0,0 +1,20 @@ +from rest_framework import serializers + + +class AirtableSerializer(serializers.Serializer): + """ + Generic Airtable serializer for parsing Airtable API JSON data into proper model data. + """ + + def validate(self, data): + """ + Loop through all the values, and if anything comes back as 'None' return an empty string. + + Not all fields will have cleaned data. Some fields could be stored as 'None' in Airtable, + so we need to loop through every value and converting 'None' to '' + """ + for key, value in data.items(): + # If any fields pass validation with the string 'None', return a blank string + if value == "None": + data[key] = "" + return data diff --git a/wagtail_airtable/templates/wagtail_airtable/airtable_import_listing.html b/wagtail_airtable/templates/wagtail_airtable/airtable_import_listing.html new file mode 100644 index 0000000..b1df93e --- /dev/null +++ b/wagtail_airtable/templates/wagtail_airtable/airtable_import_listing.html @@ -0,0 +1,31 @@ +{% extends "wagtailadmin/base.html" %} +{% block titletag %}Import Airtable Sheets{% endblock %} +{% block content %} + {% include "wagtailadmin/shared/header.html" with title="Airtable Import" %} + +
+

Models you can import from Airtable

+ {% for model_name, model_path, is_airtable_enabled, grouped_models in models %} +

{{ model_name }}

+ {% if grouped_models %} +
+

When you import {{ model_name }} you'll also be importing these ({{ grouped_models|length }}) as well: {% for model_name in grouped_models %}{{ model_name }}{% if not forloop.last %}, {% endif %} {% endfor %}

+
+ {% endif %} + {% if is_airtable_enabled %} +
+ {% csrf_token %} + + +
+ {% else %} + {{ model_name }} is not setup with the correct Airtable settings + {% endif %} + {% empty %} + There are no models configured yet + {% endfor %} +
+{% endblock %} diff --git a/wagtail_airtable/tests.py b/wagtail_airtable/tests.py new file mode 100644 index 0000000..05d34ba --- /dev/null +++ b/wagtail_airtable/tests.py @@ -0,0 +1,143 @@ +"""A mocked Airtable API wrapper.""" +from unittest import mock + + +class MockAirtable(mock.Mock): + pass + + +MockAirtable.table_name = "app_airtable_advert_base_key" + +MockAirtable.get = mock.MagicMock("get") +MockAirtable.get.return_value = { + "id": "recNewRecordId", + "fields": { + "title": "Red! It's the new blue!", + "description": "Red is a scientifically proven color that moves faster than all other colors.", + "external_link": "https://example.com/", + "is_active": True, + "rating": "1.5", + "long_description": "

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.

", + "points": 95, + "slug": "red-its-new-blue", + }, +} + +MockAirtable.insert = mock.MagicMock("insert") +MockAirtable.insert.return_value = { + "id": "recNewRecordId", + "fields": { + "title": "Red! It's the new blue!", + "description": "Red is a scientifically proven color that moves faster than all other colors.", + "external_link": "https://example.com/", + "is_active": True, + "rating": "1.5", + "long_description": "

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.

", + "points": 95, + "slug": "red-its-new-blue", + }, +} + +MockAirtable.update = mock.MagicMock("update") +MockAirtable.update.return_value = { + "id": "recNewRecordId", + "fields": { + "title": "Red! It's the new blue!", + "description": "Red is a scientifically proven color that moves faster than all other colors.", + "external_link": "https://example.com/", + "is_active": True, + "rating": "1.5", + "long_description": "

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.

", + "points": 95, + "slug": "red-its-new-blue", + }, +} + +MockAirtable.delete = mock.MagicMock("delete") +MockAirtable.delete.return_value = {"deleted": True, "record": "recNewRecordId"} + +MockAirtable.search = mock.MagicMock("search") +MockAirtable.search.return_value = [ + { + "id": "recNewRecordId", + "fields": { + "title": "Red! It's the new blue!", + "description": "Red is a scientifically proven color that moves faster than all other colors.", + "external_link": "https://example.com/", + "is_active": True, + "rating": "1.5", + "long_description": "

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.

", + "points": 95, + "slug": "red-its-new-blue", + }, + }, + { + "id": "Different record", + "fields": { + "title": "Not the used record.", + "description": "This is only used for multiple responses from MockAirtable", + "external_link": "https://example.com/", + "is_active": False, + "rating": "5.5", + "long_description": "", + "points": 1, + "slug": "not-the-used-record", + }, + }, +] + +MockAirtable.get_all = mock.MagicMock("get_all") +MockAirtable.get_all.return_value = [ + { + "id": "recNewRecordId", + "fields": { + "title": "Red! It's the new blue!", + "description": "Red is a scientifically proven color that moves faster than all other colors.", + "external_link": "https://example.com/", + "is_active": True, + "rating": "1.5", + "long_description": "

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.

", + "points": 95, + "slug": "red-its-new-blue", + }, + }, + { + "id": "Different record", + "fields": { + "title": "Not the used record.", + "description": "This is only used for multiple responses from MockAirtable", + "external_link": "https://example.com/", + "is_active": False, + "rating": "5.5", + "long_description": "", + "points": 1, + "slug": "not-the-used-record", + }, + }, + { + "id": "recRecordThree", + "fields": { + "title": "A third record.", + "description": "This is only used for multiple responses from MockAirtable", + "external_link": "https://example.com/", + "is_active": False, + "rating": "5.5", + "long_description": "", + "points": 1, + "slug": "record-3", + }, + }, + { + "id": "recRecordFour", + "fields": { + "title": "A fourth record.", + "description": "This is only used for multiple responses from MockAirtable", + "external_link": "https://example.com/", + "is_active": False, + "rating": "5.5", + "long_description": "", + "points": 1, + "slug": "record-4", + }, + }, +] diff --git a/wagtail_airtable/utils.py b/wagtail_airtable/utils.py new file mode 100644 index 0000000..88f8a5b --- /dev/null +++ b/wagtail_airtable/utils.py @@ -0,0 +1,73 @@ +""" +Utility functions for wagtail-airtable. +""" +from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist + + +def get_model_for_path(model_path): + """ + Given an 'app_name.model_name' string, return the model class or False + """ + app_label, model_name = model_path.lower().split(".") + try: + return ContentType.objects.get_by_natural_key( + app_label, model_name + ).model_class() + except ObjectDoesNotExist: + return False + + +def get_all_models() -> list: + """ + Gets all models from settings.AIRTABLE_IMPORT_SETTINGS. + + Returns a list of models. + """ + airtable_settings = getattr(settings, "AIRTABLE_IMPORT_SETTINGS", {}) + validated_models = [] + for label, model_settings in airtable_settings.items(): + if model_settings.get("AIRTABLE_IMPORT_ALLOWED", True): + label = label.lower() + if "." in label: + try: + model = get_model_for_path(label) + validated_models.append(model) + except ObjectDoesNotExist: + raise ImproperlyConfigured( + "%r is not recognised as a model name." % label + ) + + return validated_models + + +def get_validated_models(models=[]) -> list: + """ + Accept a list of model paths (ie. ['appname.Model1', 'appname.Model2']). + + Looks for models from a string and checks if the mode actually exists. + Then it'll loop through each model and check if it's allowed to be imported. + + Returns a list of validated models. + """ + validated_models = [] + for label in models: + if "." in label: + # interpret as a model + model = get_model_for_path(label) + if not model: + raise ImproperlyConfigured( + "%r is not recognised as a model name." % label + ) + + validated_models.append(model) + + models = validated_models[:] + for model in validated_models: + airtable_settings = settings.AIRTABLE_IMPORT_SETTINGS.get(model._meta.label, {}) + # Remove this model from the `models` list so it doesn't hit the Airtable API. + if not airtable_settings.get("AIRTABLE_IMPORT_ALLOWED", True): + models.remove(model) + + return models diff --git a/wagtail_airtable/views.py b/wagtail_airtable/views.py new file mode 100644 index 0000000..d7dd0aa --- /dev/null +++ b/wagtail_airtable/views.py @@ -0,0 +1,122 @@ +from django.conf import settings +from django.contrib import messages +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ImproperlyConfigured +from django.shortcuts import redirect +from django.urls import reverse +from django.views.generic import TemplateView +from django.core.exceptions import ObjectDoesNotExist +from logging import getLogger + +from wagtail_airtable.forms import AirtableImportModelForm +from wagtail_airtable.utils import get_model_for_path + +from wagtail_airtable.management.commands.import_airtable import Importer + +logger = getLogger(__name__) + + +class AirtableImportListing(TemplateView): + """ + Loads options for importing Airtable data + """ + + template_name = "wagtail_airtable/airtable_import_listing.html" + http_method_names = ["get", "post"] + + def post(self, request, *args, **kwargs): + form = AirtableImportModelForm(request.POST) + if form.is_valid(): + model_label = form.cleaned_data["model"] + importer = Importer(models=[model_label], options={"verbosity": 1}) + importer.run() + message = f"{importer.created} items created. {importer.updated} items updated. {importer.skipped} items skipped." + messages.add_message( + request, messages.SUCCESS, f"Import succeeded with {message}" + ) + else: + messages.add_message(request, messages.ERROR, "Could not import") + + return redirect(reverse("airtable_import_listing")) + + def _get_base_model(self, model): + """ + For the given model, return the highest concrete model in the inheritance tree - + e.g. for BlogPage, return Page + """ + if model._meta.parents: + model = model._meta.get_parent_list()[0] + return model + + def get_validated_models(self): + """Get models from AIRTABLE_IMPORT_SETTINGS, validate they exist, and return a list of tuples. + + returns: + [ + ('Credit Card', 'creditcards.CreditCard', ), + ('..', '..'), + ] + """ + airtable_settings = getattr(settings, "AIRTABLE_IMPORT_SETTINGS", {}) + + # Loop through all the models in the settings and create a new dict + # of the unique settings for each model label. + # If settings were used more than once the second (3rd, 4th, etc) common settings + # will be bulked into a "grouped_models" list. + tracked_settings = [] + models = {} + for label, model_settings in airtable_settings.items(): + if model_settings not in tracked_settings: + tracked_settings.append(model_settings) + models[label] = model_settings + models[label]["grouped_models"] = [] + else: + for label2, model_settings2 in models.items(): + if model_settings is model_settings2: + models[label2]["grouped_models"].append(label) + + # Validated models are models that actually exist. + # This way fake models can't be added. + validated_models = [] + for label, model_settings in models.items(): + # If this model is allowed to be imported. Default is True. + if model_settings.get("AIRTABLE_IMPORT_ALLOWED", True): + # A temporary variable for holding grouped model names. + # This is added to the validated_models item later. + # This is only used for displaying model names in the import template + _grouped_models = [] + # Loop through the grouped_models list in each setting, validate each model, + # then add it to the larger grouped_models + if model_settings.get("grouped_models"): + for grouped_model_label in model_settings.get("grouped_models"): + if "." in grouped_model_label: + model = get_model_for_path(grouped_model_label) + if model: + _grouped_models.append(model._meta.verbose_name_plural) + + if "." in label: + model = get_model_for_path(label) + if model: + # Append a triple-tuple to the validated_models with the: + # (1. Models verbose name, 2. Model label, 3. is_airtable_enabled from the model, and 4. List of grouped models) + airtable_enabled_for_model = getattr( + model, "is_airtable_enabled", False + ) + validated_models.append( + ( + model._meta.verbose_name_plural, + label, + airtable_enabled_for_model, + _grouped_models, + ) + ) + else: + raise ImproperlyConfigured( + "%r is not recognised as a model name." % label + ) + + return validated_models + + def get_context_data(self, **kwargs): + """Add validated models from the AIRTABLE_IMPORT_SETTINGS to the context.""" + return {"models": self.get_validated_models()} diff --git a/wagtail_airtable/wagtail_hooks.py b/wagtail_airtable/wagtail_hooks.py new file mode 100644 index 0000000..97f9d5e --- /dev/null +++ b/wagtail_airtable/wagtail_hooks.py @@ -0,0 +1,78 @@ +from django.conf import settings +from django.conf.urls import url +from django.contrib import messages +from django.urls import reverse +from wagtail.core import hooks +from wagtail.admin.menu import MenuItem + +from wagtail_airtable.views import AirtableImportListing +from .mixins import AirtableMixin + + +@hooks.register("register_admin_urls") +def register_airtable_url(): + return [ + url( + r"^airtable-import/$", + AirtableImportListing.as_view(), + name="airtable_import_listing", + ), + ] + + +@hooks.register("register_settings_menu_item") +def register_airtable_setting(): + def is_shown(request): + return getattr(settings, "WAGTAIL_AIRTABLE_ENABLED", False) + + menu_item = MenuItem( + "Airtable Import", + reverse("airtable_import_listing"), + classnames="icon icon-cog", + order=1000, + ) + menu_item.is_shown = is_shown + return menu_item + + +@hooks.register("after_edit_page") +def after_page_update(request, page): + # Check if the page is an AirtableMixin Subclass + watail_airtable_enabled = getattr(settings, "WAGTAIL_AIRTABLE_ENABLED", False) + if watail_airtable_enabled and issubclass(page.__class__, AirtableMixin): + # When AirtableMixin.save() is called.. + # Either it'll connect with Airtable and update the row as expected, or + # it will have some type of error. + # If _airtable_update_error exists on the page, use that string as the + # message error. + # Otherwise assume a successful update happened on the Airtable row + if hasattr(page, "is_airtable_enabled") and page.is_airtable_enabled: + if hasattr(page, "_airtable_update_error"): + messages.add_message( + request, messages.ERROR, page._airtable_update_error + ) + else: + messages.add_message( + request, messages.SUCCESS, "Airtable record updated" + ) + + +@hooks.register("after_edit_snippet") +def after_snippet_update(request, instance): + watail_airtable_enabled = getattr(settings, "WAGTAIL_AIRTABLE_ENABLED", False) + if watail_airtable_enabled and issubclass(instance.__class__, AirtableMixin): + # When AirtableMixin.save() is called.. + # Either it'll connect with Airtable and update the row as expected, or + # it will have some type of error. + # If _airtable_update_error exists on the page, use that string as the + # message error. + # Otherwise assume a successful update happened on the Airtable row + if hasattr(instance, "is_airtable_enabled") and instance.is_airtable_enabled: + if hasattr(instance, "_airtable_update_error"): + messages.add_message( + request, messages.ERROR, instance._airtable_update_error + ) + else: + messages.add_message( + request, messages.SUCCESS, "Airtable record updated" + )