diff --git a/docs/admin/_assets/img/objecttype_export_import.png b/docs/admin/_assets/img/objecttype_export_import.png new file mode 100644 index 00000000..39fe9cea Binary files /dev/null and b/docs/admin/_assets/img/objecttype_export_import.png differ diff --git a/docs/admin/objecttype.rst b/docs/admin/objecttype.rst index 3fa414b9..38582d4b 100644 --- a/docs/admin/objecttype.rst +++ b/docs/admin/objecttype.rst @@ -128,3 +128,31 @@ if you click on the "history" button in the top right corner of the object type :alt: Show the history of changes. You can see all the versions, their statuses, the creation dates and the related JSON shemas. + + +Export and import object types +------------------------------ + +To export object types including their history, select the desired object types in the Objecttypes changelist +dashboard, then use the "Export selected objecttypes as a file" action from the Action dropdown menu and click +"Go". This will download an archive containing the selected object types with all versions. + +To import object types, navigate to the Objecttypes changelist and select the "Import from file" action. +Upload the archive previously exported, optionally keeping the original UUIDs by checking the +corresponding checkbox. After successful import, you will see a confirmation message. + +.. image:: _assets/img/objecttype_export_import.png + :alt: Export and import + + +.. note:: + If UUIDs are kept during import, existing object types with matching UUIDs should **not** exist. + You cannot overwrite existing objects types. If no objects exist for the types, you should be able to + delete the object type and proceed with importing. + + If UUIDs are not kept, new UUIDs will be generated and existing object types with the same name + will not be overwritten. + +.. note:: + When importing multiple object types, either the whole import succeeds, or the whole import fails. + There are no partial imports. diff --git a/requirements/base.txt b/requirements/base.txt index 5ac132fb..d3521748 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -17,7 +17,7 @@ asgiref==3.11.0 # django-structlog asn1crypto==1.5.1 # via webauthn -attrs==20.3.0 +attrs==25.4.0 # via # glom # jsonschema diff --git a/requirements/ci.txt b/requirements/ci.txt index d908cdfc..ddba9216 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -32,12 +32,13 @@ asn1crypto==1.5.1 # -c requirements/base.txt # -r requirements/base.txt # webauthn -attrs==20.3.0 +attrs==25.4.0 # via # -c requirements/base.txt # -r requirements/base.txt # glom # jsonschema + # referencing babel==2.16.0 # via sphinx beautifulsoup4==4.9.3 @@ -418,6 +419,12 @@ humanize==4.9.0 # -c requirements/base.txt # -r requirements/base.txt # flower +hypothesis==6.151.9 + # via + # -r requirements/test-tools.in + # hypothesis-jsonschema +hypothesis-jsonschema==0.22.1 + # via -r requirements/test-tools.in idna==3.7 # via # -c requirements/base.txt @@ -455,6 +462,9 @@ jsonschema==4.17.3 # -c requirements/base.txt # -r requirements/base.txt # drf-spectacular + # hypothesis-jsonschema +jsonschema-specifications==2025.9.1 + # via -r requirements/test-tools.in kombu==5.5.4 # via # -c requirements/base.txt @@ -733,6 +743,8 @@ redis==6.4.0 # -c requirements/base.txt # -r requirements/base.txt # django-redis +referencing==0.37.0 + # via jsonschema-specifications requests==2.32.4 # via # -c requirements/base.txt @@ -755,6 +767,8 @@ requests-mock==1.12.1 # -r requirements/test-tools.in roman-numerals==4.1.0 # via sphinx +rpds-py==0.30.0 + # via referencing ruamel-yaml==0.18.10 # via # -c requirements/base.txt @@ -790,6 +804,8 @@ six==1.16.0 # webtest snowballstemmer==2.2.0 # via sphinx +sortedcontainers==2.4.0 + # via hypothesis soupsieve==2.2.1 # via beautifulsoup4 sphinx==9.1.0 @@ -853,6 +869,7 @@ typing-extensions==4.9.0 # pydantic # pydantic-core # pyopenssl + # referencing # zgw-consumers tzdata==2025.2 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index f4f647ec..63a300b6 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -37,12 +37,13 @@ asn1crypto==1.5.1 # -c requirements/ci.txt # -r requirements/ci.txt # webauthn -attrs==20.3.0 +attrs==25.4.0 # via # -c requirements/ci.txt # -r requirements/ci.txt # glom # jsonschema + # referencing autopep8==2.3.2 # via django-silk babel==2.16.0 @@ -482,6 +483,15 @@ humanize==4.9.0 # -c requirements/ci.txt # -r requirements/ci.txt # flower +hypothesis==6.151.9 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # hypothesis-jsonschema +hypothesis-jsonschema==0.22.1 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt identify==2.6.10 # via pre-commit idna==3.7 @@ -532,6 +542,11 @@ jsonschema==4.17.3 # -c requirements/ci.txt # -r requirements/ci.txt # drf-spectacular + # hypothesis-jsonschema +jsonschema-specifications==2025.9.1 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt kombu==5.5.4 # via # -c requirements/ci.txt @@ -865,6 +880,11 @@ redis==6.4.0 # -c requirements/ci.txt # -r requirements/ci.txt # django-redis +referencing==0.37.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # jsonschema-specifications requests==2.32.4 # via # -c requirements/ci.txt @@ -895,6 +915,11 @@ roman-numerals==4.1.0 # -c requirements/ci.txt # -r requirements/ci.txt # sphinx +rpds-py==0.30.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # referencing ruamel-yaml==0.18.10 # via # -c requirements/ci.txt @@ -939,6 +964,11 @@ snowballstemmer==2.2.0 # -c requirements/ci.txt # -r requirements/ci.txt # sphinx +sortedcontainers==2.4.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # hypothesis soupsieve==2.2.1 # via # -c requirements/ci.txt @@ -1041,6 +1071,7 @@ typing-extensions==4.9.0 # pydantic # pydantic-core # pyopenssl + # referencing # rich-click # zgw-consumers tzdata==2025.2 diff --git a/requirements/test-tools.in b/requirements/test-tools.in index cd401c28..50f6a95b 100644 --- a/requirements/test-tools.in +++ b/requirements/test-tools.in @@ -13,6 +13,9 @@ pyquery # integrates with webtest requests-mock tblib vcrpy +hypothesis +hypothesis-jsonschema +jsonschema-specifications # Code formatting ruff diff --git a/src/objects/api/serializers.py b/src/objects/api/serializers.py index 332cdac2..d462ff9b 100644 --- a/src/objects/api/serializers.py +++ b/src/objects/api/serializers.py @@ -95,7 +95,7 @@ class LabelsField(serializers.JSONField): pass -class ObjectTypeSerializer(serializers.HyperlinkedModelSerializer): +class ObjectTypeSerializer(serializers.HyperlinkedModelSerializer[ObjectType]): labels = LabelsField( required=False, help_text=get_help_text("core.ObjectType", "labels"), diff --git a/src/objects/core/admin.py b/src/objects/core/admin.py index 95c7b35c..512ab150 100644 --- a/src/objects/core/admin.py +++ b/src/objects/core/admin.py @@ -1,4 +1,8 @@ +from __future__ import annotations + +import io import json +from functools import partial from typing import Sequence from django import forms @@ -6,7 +10,7 @@ from django.contrib import admin, messages from django.contrib.gis.db.models import GeometryField from django.db import models -from django.http import HttpRequest, HttpResponseRedirect +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.shortcuts import redirect, render from django.urls import path, reverse from django.utils.html import format_html @@ -19,7 +23,8 @@ from objects.api.v2.filters import filter_queryset_by_data_attr from .constants import ObjectTypeVersionStatus -from .forms import ObjectTypeVersionForm, UrlImportForm +from .forms import FileImportForm, ObjectTypeVersionForm, UrlImportForm +from .import_export import export_data, import_upload from .models import Object, ObjectRecord, ObjectType, ObjectTypeVersion from .widgets import JSONSuit @@ -109,6 +114,8 @@ class ObjectTypeAdmin(admin.ModelAdmin): change_list_template = "admin/core/objecttype/object_list.html" + actions = ["export_objecttypes_action"] + def get_urls(self): urls = super().get_urls() my_urls = [ @@ -117,6 +124,11 @@ def get_urls(self): self.admin_site.admin_view(self.import_from_url_view), name="import_from_url", ), + path( + "import-from-file/", + self.admin_site.admin_view(self.import_from_file_view), + name="import_from_file", + ), ] return my_urls + urls @@ -141,10 +153,11 @@ def publish(self, request, obj): return HttpResponseRedirect(request.path) - def add_new_version(self, request, obj): + def add_new_version(self, request, obj: ObjectType): new_version = obj.last_version + assert new_version new_version.pk = None - new_version.version = new_version.version + 1 + new_version.version = None new_version.status = ObjectTypeVersionStatus.draft new_version.save() @@ -183,10 +196,53 @@ def import_from_url_view(self, request): request, "admin/core/objecttype/object_import_form.html", {"form": form} ) + def import_from_file_view(self, request: HttpRequest) -> HttpResponse: + if request.method == "POST": + form = FileImportForm(request.POST, files=request.FILES) + if form.is_valid(): + imported_types = import_upload( + form.files["export_file"], + form.cleaned_data["keep_uuid"], + partial(form.add_error, "export_file"), + ) + if not form.errors and not imported_types: + form.add_error( + "export_file", _("Found nothing importable in that file") + ) + if not form.errors: + self.message_user( + request, + _("{resource_types} imported successfully.").format( + resource_types=",".join(imported_types) + ), + ) + return redirect(reverse("admin:core_objecttype_changelist")) + else: + form = FileImportForm() + + return render( + request, + "admin/core/objecttype/object_import_form.html", + {"form": form, "source": "file"}, + ) + + @admin.action(description=_("Export selected objecttypes as a file")) + def export_objecttypes_action( + self, request: HttpRequest, queryset: models.QuerySet[ObjectType] + ) -> HttpResponse: + output = io.BytesIO() + export_data(output, objecttypes=queryset) + + response = HttpResponse(output.getvalue(), content_type="application/zip") + response["Content-Disposition"] = ( + 'attachment; filename="objecttypes-export.zip"' + ) + return response + class ObjectRecordForm(forms.ModelForm): class Meta: - model: ObjectRecord + model = ObjectRecord help_texts = { "geometry": get_help_text("core.ObjectRecord", "geometry") + "\n\n format: SRID=4326;POINT|LINESTRING|POLYGON (LAT LONG, ...)" diff --git a/src/objects/core/forms.py b/src/objects/core/forms.py index a8b05fdb..a2feb8b1 100644 --- a/src/objects/core/forms.py +++ b/src/objects/core/forms.py @@ -58,6 +58,20 @@ def clean_objecttype_url(self): self.cleaned_data["json"] = response_json +class FileImportForm(forms.Form): + export_file = forms.FileField( + label=_("Object type export file"), + required=True, + help_text=_("The file exported with the Export option from the Action menu."), + ) + keep_uuid = forms.BooleanField( + label=_("Keep the UUIDs the same"), + help_text=_("Import keeping the same UUIDs as in the export."), + initial=False, + required=False, + ) + + class ObjectTypeVersionForm(forms.ModelForm): class Meta: model = ObjectTypeVersion diff --git a/src/objects/core/import_export.py b/src/objects/core/import_export.py new file mode 100644 index 00000000..7a69c595 --- /dev/null +++ b/src/objects/core/import_export.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import json +import zipfile +from datetime import datetime, timezone +from typing import ( + BinaryIO, + Callable, + Iterable, + Mapping, + NoReturn, + Sequence, +) + +from django.core.exceptions import SuspiciousFileOperation +from django.core.files import File +from django.core.files.uploadedfile import UploadedFile +from django.core.serializers.json import DjangoJSONEncoder +from django.db import transaction +from django.db.models import Manager, QuerySet +from django.utils.translation import gettext_lazy as _ + +import structlog +from rest_framework.exceptions import ValidationError +from rest_framework.serializers import BaseSerializer, ModelSerializer + +from objects import __version__ +from objects.api.serializers import ( + ObjectTypeSerializer as _ObjectTypeSerializer, + ObjectTypeVersionSerializer as _ObjectTypeVersionSerializer, +) +from objects.core.models import ObjectType, ObjectTypeVersion + +logger = structlog.stdlib.get_logger(__name__) + + +class ObjectTypeVersionSerializer(ModelSerializer[ObjectTypeVersion]): + class Meta(_ObjectTypeVersionSerializer.Meta): # type: ignore + fields = [ + field + for field in _ObjectTypeVersionSerializer.Meta.fields + if field not in ["url", "objectType"] # exclude + ] + extra_kwargs = { + f: kwargs | {"read_only": False} + for f, kwargs in _ObjectTypeVersionSerializer.Meta.extra_kwargs.items() + } + + +class ObjectTypeSerializer(_ObjectTypeSerializer): + """ObjectTypeSerializer adapted for exporting/importing into a different instance. + + No hyperlinks + """ + + versions = ObjectTypeVersionSerializer(many=True) + + class Meta(_ObjectTypeSerializer.Meta): + fields = [ + field for field in _ObjectTypeSerializer.Meta.fields if field != "url" + ] + extra_kwargs = { + f: kwargs | {"read_only": False} + for f, kwargs in _ObjectTypeSerializer.Meta.extra_kwargs.items() + } + + def create(self, validated_data) -> ObjectType: + versions = validated_data.pop("versions") + object_type = super().create(validated_data | {"is_imported": True}) + for data in versions: + ObjectTypeVersion.objects.create(object_type=object_type, **data) + + return object_type + + +IMPORT_ORDER: Mapping[str, type[BaseSerializer]] = { + "objectTypes": ObjectTypeSerializer, +} +EXPORT_MAP = {v: k for k, v in IMPORT_ORDER.items()} + + +def export_data( + output: BinaryIO, + /, + *, + objecttypes: Sequence[ObjectType] | QuerySet[ObjectType] | Manager[ObjectType], +) -> None: + """Export + + The zip will be written to output. + """ + + if isinstance(objecttypes, (QuerySet, Manager)): # pyright: ignore[reportArgumentType] + objecttypes = objecttypes.prefetch_related("versions") + + with zipfile.ZipFile(output, "w") as zf: + zf.writestr( + _filename(EXPORT_MAP[ObjectTypeSerializer]), + _encode( + ObjectTypeSerializer( + instance=objecttypes, many=True, context={"request": None} + ).data + ), + ) + zf.writestr( + "meta.json", + _encode(_metadata()), + ) + + +def _strip_uuid(data: object): + match data: + case [*members]: + return [_strip_uuid(m) for m in members] + case {"uuid": _, **rest}: + return rest + case _: # pragma: no cover + raise Exception( + "If you want to reuse this for whatever you're doing, make it recursive" + ) + + +@transaction.atomic +def import_data( + export_file: BinaryIO | File, keep_uuid: bool = True +) -> set[str] | NoReturn: + """Import the data from export_file return the set of resources imported. + + raises + DRF ValidationError if contents malformed + zipfile.BadZipFile if file is malformed + """ + imported_resources: set[str] = set() + with zipfile.ZipFile(export_file, "r") as zf: + present = zf.namelist().__contains__ + + for resource, Serializer in IMPORT_ORDER.items(): + if not present(_filename(resource)): + continue + + data = _decode(zf.read(_filename(resource))) + + if not keep_uuid: + data = _strip_uuid(data) + + serializer = Serializer(data=data, many=True) + if serializer.is_valid(raise_exception=True): + serializer.save() + imported_resources.add(resource) + return imported_resources + + +def _flatten_drf_error(error: ValidationError) -> Iterable[str]: + return ( + f"Row {index}, {field}: {err}" + for index, item in enumerate(error.detail, 1) + if isinstance(item, dict) + for field, errors in item.items() + for err in errors + ) + + +def import_upload( + file: UploadedFile | object | list[object], + keep_uuid: bool, + report_error_to_user: Callable[[str], None], +) -> set[str]: + """Import data from Uploaded fiLe and return the set of type names that were imported. + + The object | list[object] annotation doesn't mean you can throw anything at it; + just anything from request.FILES / form.files, so anything suspicious will raise + SuspiciousFileOperation + """ + match file: + case UploadedFile(): + with file.open() as fd: + try: + return import_data(fd, keep_uuid=keep_uuid) + except ValidationError as e: + for error_msg in _flatten_drf_error(e): + report_error_to_user(error_msg) + except zipfile.BadZipfile: + report_error_to_user( + _("{file} is not a supported file type").format(file=file.name), + ) + except Exception: # pragma: no cover + report_error_to_user( + _( + "Something unexpected happened during import.\n" + "Please try again. If it fails again and you think " + "the file should be correct, please include the " + "file in your bug report.\n" + ).format() + ) + logger.exception("unhandled import exception") + return set() + case [*fs]: + return { + resource + for f in fs + for resource in import_upload(f, keep_uuid, report_error_to_user) + } + case _: # pragma: no cover + logger.exception("unexected type in upload", contents=file) + raise SuspiciousFileOperation("Unexpected type in upload") + + +def _encode(obj: object) -> str: + return json.dumps(obj, cls=DjangoJSONEncoder) + + +def _decode(s: bytes | str) -> object: + return json.loads(s) + + +def _filename(model_name: str) -> str: + return f"{model_name}.json" + + +def _metadata() -> dict[str, str]: + "Metadata for an export" + return { + "app_version": __version__, + "exported_at": datetime.now(timezone.utc).isoformat() + "Z", + } diff --git a/src/objects/core/models.py b/src/objects/core/models.py index 168e44a7..80a522eb 100644 --- a/src/objects/core/models.py +++ b/src/objects/core/models.py @@ -6,6 +6,7 @@ from django.contrib.gis.db.models import GeometryField from django.contrib.postgres.indexes import GinIndex +from django.core.exceptions import ValidationError from django.core.serializers.json import DjangoJSONEncoder from django.db import models from django.utils.translation import gettext_lazy as _ @@ -139,6 +140,7 @@ class ObjectType(models.Model): ) objects = ObjectTypeQuerySet.as_manager() + versions: models.QuerySet[ObjectTypeVersion] def __str__(self): return f"{self.name}" @@ -217,6 +219,7 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) def generate_version_number(self) -> int: + # XXX: this is racing other queries! existed_versions = ObjectTypeVersion.objects.filter( object_type=self.object_type ) @@ -228,6 +231,10 @@ def generate_version_number(self) -> int: ] version_number = max_version + 1 + if version_number > 32767: # Postgrest has no USMALLINT, only SMALLINT + raise ValidationError( + _("Maximum version number 32767 has been reached for this objecttype") + ) return version_number diff --git a/src/objects/core/tests/test_admin.py b/src/objects/core/tests/test_admin.py index 3b198aae..0984df39 100644 --- a/src/objects/core/tests/test_admin.py +++ b/src/objects/core/tests/test_admin.py @@ -33,7 +33,16 @@ def test_object_changelist_filter_by_objecttype(self): object2 = ObjectFactory.create() # Verify that the number of queries doesn't scale with the number of objecttypes - with self.assertNumQueries(22): + class UpperBound(int): + # NumQueries tests for equality + def __eq__(self, value: object, /) -> bool: + match value: + case int(n): + return n < self + case _: + return super().__eq__(value) + + with self.assertNumQueries(UpperBound(40)): response = self.app.get( reverse("admin:core_object_changelist"), {"object_type__id__exact": object_type.pk}, diff --git a/src/objects/core/tests/test_export_import.py b/src/objects/core/tests/test_export_import.py new file mode 100644 index 00000000..0a18eec2 --- /dev/null +++ b/src/objects/core/tests/test_export_import.py @@ -0,0 +1,302 @@ +import json +from io import BytesIO + +from django.core.files.uploadedfile import SimpleUploadedFile + +import hypothesis.strategies as st +import jsonschema_specifications +from hypothesis import HealthCheck, given, settings +from hypothesis.extra.django import TestCase, from_model +from hypothesis_jsonschema import from_schema + +from ..import_export import export_data, import_data, import_upload +from ..models import ObjectType, ObjectTypeVersion +from ..utils import check_json_schema + + +def _valid_schema(json_value): + try: + check_json_schema(json_value) + except Exception: + return False + else: + # no escaped null-bytes PG JSONFields can't handle them + return r"\u0000" not in json.dumps(json_value) + + +def jsonschemas(): + "Hypothesesis strategy that generates valid jsonschemata." + # from_schema doesn't resolve $dynamicRef (yet), which the 2020 draft meta schema uses + meta_schema = "https://json-schema.org/draft/2019-09/schema" + return st.one_of( + st.booleans(), + st.just({}), + ( + from_schema( + jsonschema_specifications.REGISTRY[meta_schema].contents, # type: ignore + ) + .map( + lambda schema: schema | {"$schema": meta_schema} + if isinstance(schema, dict) + else schema + ) + .filter(_valid_schema) + ), + ) + + +@st.composite +def objecttypes( + draw: st.DrawFn, *, min_versions: int = 0, max_versions: int | None = None +) -> ObjectType: + # postgres can't store + + object_type = draw( + from_model( + ObjectType, + is_imported=st.just(False), + contact_email=st.just("") | st.emails(), # optional email + ) + ) + # for better scr + schemata = jsonschemas() + schema = draw(schemata) + + # create some versions + draw( + st.lists( + from_model( + ObjectTypeVersion, + object_type=st.just(object_type), + json_schema=st.one_of( + st.just(schema), # re-use same + schemata, # change to a new schema + ), + # hypothesis infers the bounds correctly, but also tries 0 + # and will bump into the auto gen going out of bounds + version=st.integers(min_value=1, max_value=(1 << 15) - 1), + ), + min_size=min_versions, + max_size=max_versions, + ) + ) + return object_type + + +class RoundTripExportImportTests(TestCase): + def setup_example(self): + # empty before each hypothesis example + ObjectType.objects.all().delete() + ObjectTypeVersion.objects.all().delete() + return super().setup_example() + + @given(objecttypes=st.lists(objecttypes(max_versions=3), min_size=1, max_size=3)) + @settings( + suppress_health_check=[HealthCheck.too_slow], + ) + def test_export_import_roundtrip(self, objecttypes: list[ObjectType]) -> None: + output = BytesIO() + + export_data(output, objecttypes=objecttypes) + + output.seek(0) + + original_versions = { + ot.uuid: list(ot.versions.order_by("version")) for ot in objecttypes + } + + # purge after export + ObjectType.objects.all().delete() + ObjectTypeVersion.objects.all().delete() + + import_data(output) + + assert ObjectType.objects.count() == len(objecttypes) + + imported_types = ( + ObjectType.objects.all().prefetch_related("versions").order_by("uuid") + ) + assert imported_types.count() == len(objecttypes) + + for imported_type, original_type in zip( + imported_types, sorted(objecttypes, key=lambda x: x.uuid) + ): + assert imported_type.is_imported is True + + assert imported_type.uuid == original_type.uuid + assert imported_type.name == original_type.name.strip() + assert imported_type.name_plural == original_type.name_plural.strip() + assert imported_type.description == original_type.description.strip() + assert ( + imported_type.data_classification + == original_type.data_classification.strip() + ) + assert ( + imported_type.maintainer_organization + == original_type.maintainer_organization.strip() + ) + assert ( + imported_type.maintainer_department + == original_type.maintainer_department.strip() + ) + assert imported_type.contact_person == original_type.contact_person.strip() + assert imported_type.contact_email == original_type.contact_email.strip() + assert imported_type.source == original_type.source.strip() + assert ( + imported_type.update_frequency == original_type.update_frequency.strip() + ) + assert ( + imported_type.provider_organization + == original_type.provider_organization.strip() + ) + assert ( + imported_type.documentation_url + == original_type.documentation_url.strip() + ) + assert imported_type.labels == original_type.labels + assert imported_type.allow_geometry == original_type.allow_geometry + + assert imported_type.versions.count() == len( + original_versions[original_type.uuid] + ) + + for imported_version, original_version in zip( + imported_type.versions.order_by("version"), + original_versions[original_type.uuid], + ): + assert imported_version.version == original_version.version + assert imported_version.json_schema == original_version.json_schema + assert imported_version.status == original_version.status + assert imported_version.published_at == original_version.published_at + + @given(objecttype=objecttypes(min_versions=1, max_versions=3)) + @settings( + suppress_health_check=[HealthCheck.too_slow], + ) + def test_export_import_roundtrip_from_queryset( + self, objecttype: ObjectType + ) -> None: + "Ensure a QuerySet[ObjectType] doesn't need an explicit prefect" + num_versions = objecttype.versions.count() + output = BytesIO() + + export_data(output, objecttypes=ObjectType.objects.all()) + + output.seek(0) + # purge after export + ObjectType.objects.all().delete() + ObjectTypeVersion.objects.all().delete() + + import_data(output) + output.truncate() + + assert ObjectTypeVersion.objects.count() == num_versions + + @given( + objecttypes=st.lists( + objecttypes(min_versions=1, max_versions=3), min_size=1, max_size=3 + ) + ) + @settings( + suppress_health_check=[HealthCheck.too_slow], + ) + def test_export_import_with_new_uuids(self, objecttypes: list[ObjectType]) -> None: + num_versions = ObjectTypeVersion.objects.count() + output = BytesIO() + + export_data(output, objecttypes=objecttypes) + + output.seek(0) + import_data(output, keep_uuid=False) + output.truncate() + + assert ObjectType.objects.count() == len(objecttypes) * 2 + assert ObjectTypeVersion.objects.count() == num_versions * 2 + + @given(objecttypes(min_versions=1, max_versions=3)) + @settings(suppress_health_check=[HealthCheck.too_slow]) + def test_export_import_with_new_uuid(self, original_type: ObjectType) -> None: + output = BytesIO() + + export_data(output, objecttypes=[original_type]) + + output.seek(0) + # import a new, keeping original + import_data(output, keep_uuid=False) + output.truncate() + + imported_type = ObjectType.objects.exclude(uuid=original_type.uuid).get() + + assert imported_type.is_imported is True + + assert imported_type.uuid != original_type.uuid + assert imported_type.name == original_type.name.strip() + assert imported_type.name_plural == original_type.name_plural.strip() + assert imported_type.description == original_type.description.strip() + assert ( + imported_type.data_classification + == original_type.data_classification.strip() + ) + assert ( + imported_type.maintainer_organization + == original_type.maintainer_organization.strip() + ) + assert ( + imported_type.maintainer_department + == original_type.maintainer_department.strip() + ) + assert imported_type.contact_person == original_type.contact_person.strip() + assert imported_type.contact_email == original_type.contact_email.strip() + assert imported_type.source == original_type.source.strip() + assert imported_type.update_frequency == original_type.update_frequency.strip() + assert ( + imported_type.provider_organization + == original_type.provider_organization.strip() + ) + assert ( + imported_type.documentation_url == original_type.documentation_url.strip() + ) + assert imported_type.labels == original_type.labels + assert imported_type.allow_geometry == original_type.allow_geometry + + assert imported_type.versions.count() == original_type.versions.count() + + for imported_version, original_version in zip( + imported_type.versions.order_by("version"), + original_type.versions.order_by("version"), + ): + assert imported_version.version == original_version.version + assert imported_version.json_schema == original_version.json_schema + assert imported_version.status == original_version.status + assert imported_version.published_at == original_version.published_at + + @given(objecttypes(), objecttypes()) + @settings(suppress_health_check=[HealthCheck.too_slow]) + def test_import_upload_list_of_files(self, ot1, ot2): + output1 = BytesIO() + output2 = BytesIO() + + export_data(output1, objecttypes=[ot1]) + export_data(output2, objecttypes=[ot2]) + + file1 = SimpleUploadedFile( + "file1.zip", output1.getvalue(), content_type="application/zip" + ) + file2 = SimpleUploadedFile( + "file2.zip", output2.getvalue(), content_type="application/zip" + ) + + ObjectType.objects.all().delete() + + errors = [] + + def report_error(msg): + errors.append(msg) + + import_upload([file1, file2], keep_uuid=True, report_error_to_user=report_error) + + assert set(ObjectType.objects.all().values_list("name", flat=True)) == { + ot1.name.strip(), + ot2.name.strip(), + } + assert errors == [] diff --git a/src/objects/core/tests/test_objecttypeversion_generate.py b/src/objects/core/tests/test_objecttypeversion_generate.py index 49a7c52f..0523ee15 100644 --- a/src/objects/core/tests/test_objecttypeversion_generate.py +++ b/src/objects/core/tests/test_objecttypeversion_generate.py @@ -1,3 +1,4 @@ +from django.core.exceptions import ValidationError from django.test import TestCase from objects.core.models import ObjectTypeVersion @@ -31,3 +32,15 @@ def test_generate_version_for_objecttype_with_existed_version(self): ) self.assertEqual(object_version.version, 2) + + def test_version_bounds_check(self): + object_type = ObjectTypeFactory.create() + max_sql_smallint = (1 << 15) - 1 + ObjectTypeVersionFactory.create( + object_type=object_type, version=max_sql_smallint + ) + + with self.assertRaises(ValidationError): + ObjectTypeVersion.objects.create( + json_schema=JSON_SCHEMA, object_type=object_type + ) diff --git a/src/objects/templates/admin/core/objecttype/object_import_form.html b/src/objects/templates/admin/core/objecttype/object_import_form.html index 709b9170..fafc5be2 100644 --- a/src/objects/templates/admin/core/objecttype/object_import_form.html +++ b/src/objects/templates/admin/core/objecttype/object_import_form.html @@ -12,19 +12,24 @@ {% endblock %} {% block content %} -

{% trans 'Import from URL' %}

+

{% blocktrans with source=source|default:'URL' %}Import from {{ source }}{% endblocktrans %}

{% csrf_token %} -
+ {% if form.non_field_errors %} +
+ {{ form.non_field_errors }} +
+ {% endif %} +
{% for field in form %}
{{ field.errors }} {{ field }} {% if field.field.help_text %}  -
{{ field.field.help_text }}
- {% endif %} +
{{ field.field.help_text }}
+ {% endif %}
{% endfor %}
diff --git a/src/objects/templates/admin/core/objecttype/object_list.html b/src/objects/templates/admin/core/objecttype/object_list.html index 86ff2d1b..69df3556 100644 --- a/src/objects/templates/admin/core/objecttype/object_list.html +++ b/src/objects/templates/admin/core/objecttype/object_list.html @@ -7,5 +7,10 @@ {% trans 'Import from URL' %} +
  • + + {% trans 'Import from file' %} + +
    • {{ block.super }} {% endblock %} diff --git a/src/objects/tests/admin/test_objecttype_admin.py b/src/objects/tests/admin/test_objecttype_admin.py index c9bc5ad1..f077ec66 100644 --- a/src/objects/tests/admin/test_objecttype_admin.py +++ b/src/objects/tests/admin/test_objecttype_admin.py @@ -1,5 +1,7 @@ import json +import zipfile from datetime import date +from io import BytesIO from django.urls import reverse, reverse_lazy @@ -7,6 +9,7 @@ from django_webtest import WebTest from freezegun import freeze_time from maykin_2fa.test import disable_admin_mfa +from webtest import Upload from objects.accounts.tests.factories import SuperUserFactory from objects.core.constants import ( @@ -14,6 +17,7 @@ ObjectTypeVersionStatus, UpdateFrequencyChoices, ) +from objects.core.import_export import export_data from objects.core.models import ObjectType from objects.core.tests.factories import ObjectTypeFactory, ObjectTypeVersionFactory @@ -35,6 +39,7 @@ class AdminAddTests(WebTest): url = reverse_lazy("admin:core_objecttype_add") import_from_url = reverse_lazy("admin:import_from_url") + import_from_file = reverse_lazy("admin:import_from_file") @classmethod def setUpTestData(cls): @@ -248,6 +253,123 @@ def test_create_objecttype_from_url_with_nonexistent_url(self): self.assertEqual(response.status_code, 200) self.assertEqual(ObjectType.objects.count(), 0) + def test_create_import_from_file_requires_authentication(self): + self.app.set_user(None) + import_view = self.app.get(self.import_from_file, expect_errors=True) + self.assertEqual(import_view.status_code, 302) + self.assertIn("/admin/login", import_view.location) + + def test_create_import_from_file(self): + # setup an export file for import + objecttype = ObjectTypeFactory.create() + output = BytesIO() + export_data(output, objecttypes=[objecttype]) + export_file_content = output.getvalue() + + # submit the import form with the export file + import_view = self.app.get(self.import_from_file) + form = import_view.form + form["export_file"] = Upload( + "test-export.zip", export_file_content, "application/zip" + ) + response = form.submit() + + # verify redirection after successful import + self.assertEqual(response.status_code, 302) + self.assertEqual(response.location, "/admin/core/objecttype/") + + # check that the objecttype was imported + self.assertEqual(ObjectType.objects.count(), 2) + + def test_create_import_from_file_invalid_zip(self): + objecttype = ObjectTypeFactory.create() + output = BytesIO() + export_data(output, objecttypes=[objecttype]) + # corrupt the zip content by truncating it + export_file_content = output.getvalue()[:10] + + import_view = self.app.get(self.import_from_file) + form = import_view.form + form["export_file"] = Upload( + "test-export.zip", export_file_content, "application/zip" + ) + response = form.submit() + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "is not a supported file type") + + def test_create_import_from_file_empty(self): + import_view = self.app.get(self.import_from_file) + form = import_view.form + # create an empty zip file + empty_zip = BytesIO() + with zipfile.ZipFile(empty_zip, "w") as zf: + zf.writestr("README.md", b"This is the wrong file") + empty_zip.seek(0) + + form["export_file"] = Upload("empty.zip", empty_zip.read(), "application/zip") + response = form.submit() + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Found nothing importable in that file") + + def test_create_import_from_file_validation_error(self): + objecttype = ObjectTypeFactory.create() + + # invalidate some property + objecttype.update_frequency = 1024 * "x" + + output = BytesIO() + export_data(output, objecttypes=[objecttype]) + export_file_content = output.getvalue() + + import_view = self.app.get(self.import_from_file) + form = import_view.form + form["export_file"] = Upload( + "corrupted-export.zip", export_file_content, "application/zip" + ) + response = form.submit() + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Row") + # this error is there if no other errors were found, but nothing was + # imported, because some random zip was uploaded + self.assertNotContains(response, "Found nothing importable in that file") + + def test_create_import_from_file_no_file_selected(self): + import_view = self.app.get(self.import_from_file) + form = import_view.form + response = form.submit() + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "This field is required.") + + def test_export_action(self): + objecttype1 = ObjectTypeFactory.create(name="Tree 1") + objecttype2 = ObjectTypeFactory.create(name="Tree 2") + + response = self.app.get(reverse("admin:core_objecttype_changelist")) + form = response.forms["changelist-form"] + form["action"] = "export_objecttypes_action" + form["_selected_action"] = [objecttype1.pk, objecttype2.pk] + response = form.submit() + + self.assertEqual(response.status_code, 200) + self.assertEqual(response["Content-Type"], "application/zip") + self.assertIn( + 'attachment; filename="objecttypes-export.zip"', + response["Content-Disposition"], + ) + + # verify the exported content + output = BytesIO(response.content) + with zipfile.ZipFile(output, "r") as zf: + self.assertIn("objectTypes.json", zf.namelist()) + + with zf.open("objectTypes.json") as f: + data = json.load(f) + self.assertEqual({d["name"] for d in data}, {"Tree 1", "Tree 2"}) + @disable_admin_mfa() class AdminDetailTests(WebTest):