From ea7375dc308207da3d2c2b5a982f953ad3890f72 Mon Sep 17 00:00:00 2001 From: Maximilian Gutzmann Date: Wed, 5 Nov 2025 10:52:51 +0000 Subject: [PATCH 1/4] added sessions for upload job monitoring --- adit/upload/admin.py | 28 ++++- adit/upload/models.py | 19 ++++ adit/upload/static/upload/upload.js | 170 +++++++++++++++------------- adit/upload/urls.py | 8 +- adit/upload/views.py | 26 ++++- 5 files changed, 172 insertions(+), 79 deletions(-) diff --git a/adit/upload/admin.py b/adit/upload/admin.py index 21693c25f..4e837eda8 100644 --- a/adit/upload/admin.py +++ b/adit/upload/admin.py @@ -1,6 +1,32 @@ from django.contrib import admin # Register your models here. -from .models import UploadSettings +from .models import UploadSession, UploadSettings admin.site.register(UploadSettings, admin.ModelAdmin) + + +class UploadStatisticAdmin(admin.ModelAdmin): + list_display = ( + "get_owner", + "id", + "time_opened", + "get_size_in_mb", + "uploaded_file_count", + ) + list_filter = ("time_opened", "owner") + search_fields = ("owner__username",) + + def get_owner(self, obj): + return obj.owner.username + + get_owner.admin_order_field = "owner__username" + get_owner.short_description = "Owner" + + def get_size_in_mb(self, obj): + return f"{obj.upload_size / (1024 * 1024):.2f} MB" + + get_size_in_mb.short_description = "Upload Size" + + +admin.site.register(UploadSession, UploadStatisticAdmin) diff --git a/adit/upload/models.py b/adit/upload/models.py index 607ea74e2..1a95a0d7c 100644 --- a/adit/upload/models.py +++ b/adit/upload/models.py @@ -1,3 +1,8 @@ +import uuid + +from django.conf import settings +from django.db import models + from adit.core.models import DicomAppSettings @@ -10,3 +15,17 @@ class Meta: ] +class UploadSession(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + time_opened = models.DateTimeField(auto_now_add=True) + upload_size = models.IntegerField(default=0) + uploaded_file_count = models.IntegerField(default=0) + owner_id: int + owner = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + related_name="%(app_label)s_jobs", + ) + + def __str__(self) -> str: + return f"{self.__class__.__name__} [{self.pk}]" diff --git a/adit/upload/static/upload/upload.js b/adit/upload/static/upload/upload.js index 40a8e9b25..70b17c135 100644 --- a/adit/upload/static/upload/upload.js +++ b/adit/upload/static/upload/upload.js @@ -21,9 +21,7 @@ function UploadJobForm(formEl) { uploadCompleteTextVisible: false, initUploadForm: function () { - document.body.addEventListener("chooseFolder", () => { - this.chooseFolder(); - }); + document.body.addEventListener("chooseFolder", this.chooseFolder()); document.body.addEventListener("htmx:afterSwap", (event) => { // @ts-ignore if (event.detail.target.id === "myForm") { @@ -117,95 +115,112 @@ function UploadJobForm(formEl) { if (files.length === 0) { // @ts-ignore - showToast("warning", "Sandbox", `No files selected.${files}`); - } else { - const dataset_length = files.length; - let status = 0; - let loadedFiles = 0; + showToast("warning", "Sandbox", `No files selected.`); + return; + } + //Start session + const startSessionResponse = await fetch("/upload/session/", { + method: "POST", + headers: { "X-CSRFToken": window.public.csrf_token }, + mode: "same-origin", // Do not send CSRF token to another domain. + }); + if (!startSessionResponse.ok) { + // @ts-ignore + showToast("error", "Sandbox", `Failed to start upload session:`); + return; + } + const sessionData = await startSessionResponse.json(); + const SessionId = sessionData["session_id"]; - try { - const checker = await this.isValidSeries(files); + const dataset_length = files.length; + let status = 0; + let loadedFiles = 0; - if (checker) { - const anon = await this.createAnonymizer(); + try { + const checker = await this.isValidSeries(files); - this.buttonVisible = false; - this.stopUploadVar = false; + if (checker) { + const anon = await this.createAnonymizer(); + + this.buttonVisible = false; + this.stopUploadVar = false; - const progBar = formEl.querySelector('[id="pb"]'); - if (!(progBar instanceof HTMLProgressElement)) { + const progBar = formEl.querySelector('[id="pb"]'); + if (!(progBar instanceof HTMLProgressElement)) { + throw new Error( + "progBar must be an instance of HTMLProgressElement" + ); + } + progBar.value = 0; + this.pbVisible = true; + for (const file of files) { + const set = await file.arrayBuffer(); + // Anonymize data and write back to bufferstream + const dicomData = dcmjs.data.DicomMessage.readFile(set, { + ignoreErrors: true, + }); + const pseudonym = formEl.querySelector('[name="pseudonym"]'); + + if (!(pseudonym instanceof HTMLInputElement)) { throw new Error( - "progBar must be an instance of HTMLProgressElement" + "pseudonym must be an instance of HTMLInputElement" ); } - progBar.value = 0; - this.pbVisible = true; - for (const file of files) { - const set = await file.arrayBuffer(); - // Anonymize data and write back to bufferstream - const dicomData = dcmjs.data.DicomMessage.readFile(set, { - ignoreErrors: true, - }); - const pseudonym = formEl.querySelector('[name="pseudonym"]'); - - if (!(pseudonym instanceof HTMLInputElement)) { - throw new Error( - "pseudonym must be an instance of HTMLInputElement" - ); - } - let newPatientID = pseudonym.value; - - await anon.anonymize(dicomData); - dicomData.upsertTag("00100020", "LO", [newPatientID]); // replace PatientID - dicomData.upsertTag("00100010", "PN", [ - { Alphabetic: newPatientID }, - ]); // replace PatientName - const anonymized_set = await dicomData.write(); - - this.stopUploadButtonVisible = true; - if (this.stopUploadVar) { - // Stop uploading if stop button is clicked - break; - } + let newPatientID = pseudonym.value; + + await anon.anonymize(dicomData); + dicomData.upsertTag("00100020", "LO", [newPatientID]); // replace PatientID + dicomData.upsertTag("00100010", "PN", [ + { Alphabetic: newPatientID }, + ]); // replace PatientName + const anonymized_set = await dicomData.write(); + + this.stopUploadButtonVisible = true; + if (this.stopUploadVar) { + // Stop uploading if stop button is clicked + break; + } - // Upload data to server - status = await uploadData({ + // Upload data to server + status = await uploadData( + { dataset: anonymized_set, node_id: nodeId, - }); - - if (status == 200) { - loadedFiles += 1; - let currentPBValue = progBar.value; - let newPBValue = (loadedFiles / dataset_length) * 100; - if (newPBValue > currentPBValue) { - progBar.value = newPBValue; - } - } else { - this.uploadResultText = "Upload Failed"; - this.pbVisible = false; - this.uploadCompleteTextVisible = true; - this.stopUploadButtonVisible = false; - break; + }, + SessionId + ); + + if (status == 200) { + loadedFiles += 1; + let currentPBValue = progBar.value; + let newPBValue = (loadedFiles / dataset_length) * 100; + if (newPBValue > currentPBValue) { + progBar.value = newPBValue; } - } - if (loadedFiles === dataset_length) { - this.finishUploadComplete(); } else { - this.finishUploadIncomplete(); + this.uploadResultText = "Upload Failed"; + this.pbVisible = false; + this.uploadCompleteTextVisible = true; + this.stopUploadButtonVisible = false; + break; } + } + if (loadedFiles === dataset_length) { + this.finishUploadComplete(); } else { - this.uploadResultText = "Upload refused - Incorrect dataset"; - this.buttonVisible = false; - this.uploadCompleteTextVisible = true; + this.finishUploadIncomplete(); } - } catch (e) { - this.uploadResultText = "Upload Failed due to an Error"; + } else { + this.uploadResultText = "Upload refused - Incorrect dataset"; this.buttonVisible = false; this.uploadCompleteTextVisible = true; - this.pbVisible = false; - console.error(e); } + } catch (e) { + this.uploadResultText = "Upload Failed due to an Error"; + this.buttonVisible = false; + this.uploadCompleteTextVisible = true; + this.pbVisible = false; + console.error(e); } }, @@ -354,7 +369,7 @@ function UploadJobForm(formEl) { }; } -const uploadData = async (data) => { +const uploadData = async (data, session) => { const formData = new FormData(); for (const key in data) { const blob = new Blob([data[key]]); @@ -365,7 +380,10 @@ const uploadData = async (data) => { const request = new Request(url, { method: "POST", - headers: { "X-CSRFToken": window.public.csrf_token }, + headers: { + "X-CSRFToken": window.public.csrf_token, + "X-Upload-Session": session, + }, mode: "same-origin", // Do not send CSRF token to another domain. body: formData, }); diff --git a/adit/upload/urls.py b/adit/upload/urls.py index 3598ad697..c91201311 100644 --- a/adit/upload/urls.py +++ b/adit/upload/urls.py @@ -1,6 +1,11 @@ from django.urls import path -from .views import UploadCreateView, UploadUpdatePreferencesView, upload_api_view +from .views import ( + UploadCreateView, + UploadUpdatePreferencesView, + create_upload_session, + upload_api_view, +) urlpatterns = [ path( @@ -13,4 +18,5 @@ name="upload_create", ), path("data-upload//", view=upload_api_view, name="data_upload"), + path("session/", create_upload_session, name="create_upload_session"), ] diff --git a/adit/upload/views.py b/adit/upload/views.py index ee2cb8e56..482c05611 100644 --- a/adit/upload/views.py +++ b/adit/upload/views.py @@ -2,6 +2,7 @@ import logging from io import BytesIO from typing import Any +from uuid import UUID from adit_radis_shared.common.types import AuthenticatedHttpRequest from asgiref.sync import sync_to_async @@ -13,7 +14,7 @@ ) from django.core.exceptions import PermissionDenied, SuspiciousOperation from django.core.files.uploadedfile import UploadedFile -from django.http import HttpResponse +from django.http import HttpResponse, JsonResponse from django.shortcuts import render from django.views.generic.edit import FormView from django_htmx.http import trigger_client_event @@ -24,6 +25,7 @@ from adit.core.views import BaseUpdatePreferencesView from .forms import UploadForm +from .models import UploadSession UPLOAD_SOURCE = "upload_source" UPLOAD_DESTINATION = "upload_destination" @@ -107,6 +109,12 @@ async def upload_api_view(request: AuthenticatedHttpRequest, node_id: str) -> Ht destination_node = await DicomServer.objects.aget(id=node_id) + session_id = request.headers.get("X-Upload-Session") + if session_id is None: + return HttpResponse(status=400, content="No upload session specified") + session_id = UUID(session_id) + session = await UploadSession.objects.aget(id=session_id) + node_accessible = await sync_to_async( lambda: destination_node.is_accessible_by_user(request.user, "destination") )() @@ -120,10 +128,13 @@ async def upload_api_view(request: AuthenticatedHttpRequest, node_id: str) -> Ht if file_data is None: return HttpResponse(status=400, content="No data received") + dataset_size = 0 + if "dataset" in file_data: uploaded_file = file_data.get("dataset") assert isinstance(uploaded_file, UploadedFile) dataset_bytes = BytesIO(uploaded_file.read()) + dataset_size = dataset_bytes.getbuffer().nbytes try: dataset = read_dataset(dataset_bytes) except Exception as e: @@ -139,6 +150,10 @@ async def upload_api_view(request: AuthenticatedHttpRequest, node_id: str) -> Ht if dataset is None or uploaded_file is None: return HttpResponse(status=400, content="No data received") try: + session.upload_size += dataset_size + session.uploaded_file_count += 1 + await sync_to_async(session.save)() + loop = asyncio.get_event_loop() await loop.run_in_executor(None, operator.upload_images, [dataset]) @@ -155,3 +170,12 @@ async def upload_api_view(request: AuthenticatedHttpRequest, node_id: str) -> Ht response["statusText"] = response.reason_phrase return response + + +@login_required +async def create_upload_session(request): + if request.method != "POST": + return JsonResponse({"error": "Method not allowed"}, status=405) + + data = await sync_to_async(UploadSession.objects.create)(owner=request.user) + return JsonResponse({"session_id": str(data.id), "started_at": data.time_opened}) From 69eb9968e2c9ff57a79eb0971429ac5fb7742a7f Mon Sep 17 00:00:00 2001 From: Maximilian Gutzmann Date: Wed, 5 Nov 2025 12:18:13 +0000 Subject: [PATCH 2/4] Added migration & enhanced seed security --- adit/upload/migrations/0002_uploadsession.py | 42 ++++++++++++++++++++ adit/upload/static/upload/upload.js | 18 ++++----- 2 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 adit/upload/migrations/0002_uploadsession.py diff --git a/adit/upload/migrations/0002_uploadsession.py b/adit/upload/migrations/0002_uploadsession.py new file mode 100644 index 000000000..3150b1318 --- /dev/null +++ b/adit/upload/migrations/0002_uploadsession.py @@ -0,0 +1,42 @@ +# Generated by Django 5.2.7 on 2025-11-05 11:03 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("upload", "0001_initial"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="UploadSession", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("time_opened", models.DateTimeField(auto_now_add=True)), + ("upload_size", models.IntegerField(default=0)), + ("uploaded_file_count", models.IntegerField(default=0)), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="%(app_label)s_jobs", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + ] diff --git a/adit/upload/static/upload/upload.js b/adit/upload/static/upload/upload.js index 70b17c135..e44f672ed 100644 --- a/adit/upload/static/upload/upload.js +++ b/adit/upload/static/upload/upload.js @@ -140,7 +140,7 @@ function UploadJobForm(formEl) { const checker = await this.isValidSeries(files); if (checker) { - const anon = await this.createAnonymizer(); + const anon = await this.createAnonymizer(SessionId); this.buttonVisible = false; this.stopUploadVar = false; @@ -259,7 +259,7 @@ function UploadJobForm(formEl) { }, 5000); }, - createAnonymizer: async () => { + createAnonymizer: async (sessionId) => { const seedElement = document.getElementById("anon-seed-json"); if (!seedElement) { throw new Error("anon-seed-json element does not exist"); @@ -268,19 +268,19 @@ function UploadJobForm(formEl) { if (!seedText) { throw new Error("anon-seed-json element is empty"); } - let seed = JSON.parse(seedText); - if (seed == null || Object.keys(seed).length === 0) { + const seedObj = JSON.parse(seedText); + if (seedObj == null || Object.keys(seedObj).length === 0) { throw new Error("Anonymizer seed must not be empty"); } + const encoder = new TextEncoder(); - const seedData = encoder.encode(seed + document.cookie); - const hashBuffer = await crypto.subtle.digest("SHA-256", seedData); + const combined = encoder.encode(JSON.stringify(seedObj) + sessionId); + const hashBuffer = await crypto.subtle.digest("SHA-256", combined); const hashArray = Array.from(new Uint8Array(hashBuffer)); - const hashHex = hashArray + const combinedSeed = hashArray .map((b) => b.toString(16).padStart(2, "0")) .join(""); - - return new Anonymizer({ seed: hashHex }); + return new Anonymizer({ seed: combinedSeed }); }, traverseDirectory: async function (item, files) { From 3ce28bec584d2d692baa6db1d716097940f3dc68 Mon Sep 17 00:00:00 2001 From: Maximilian Gutzmann Date: Wed, 5 Nov 2025 13:36:34 +0000 Subject: [PATCH 3/4] Added frontend in Admin Section --- adit/core/templates/core/admin_section.html | 21 +++++++++++++++++++++ adit/core/views.py | 3 +++ adit/upload/apps.py | 6 ++++++ 3 files changed, 30 insertions(+) diff --git a/adit/core/templates/core/admin_section.html b/adit/core/templates/core/admin_section.html index ec7b4fc8f..e9649688d 100644 --- a/adit/core/templates/core/admin_section.html +++ b/adit/core/templates/core/admin_section.html @@ -28,6 +28,27 @@
Job Overview
{% endfor %} +
Upload Sessions
+ + + + + + + + + + + {% for session in upload_stats %} + + + + + + + {% endfor %} + +
Created byCreated atUpload SizeUploaded Files
{{ session.owner }}{{ session.time_opened }}{{ session.upload_size | filesizeformat }}{{ session.uploaded_file_count }}
Admin Tools
  • diff --git a/adit/core/views.py b/adit/core/views.py index 42b08edc1..efba8e731 100644 --- a/adit/core/views.py +++ b/adit/core/views.py @@ -31,6 +31,7 @@ from procrastinate.contrib.django import app from adit.core.utils.model_utils import reset_tasks +from adit.upload.apps import collect_top_sessions from .models import DicomJob, DicomTask from .site import job_stats_collectors @@ -40,12 +41,14 @@ def admin_section(request: HttpRequest) -> HttpResponse: status_list = DicomJob.Status.choices job_stats = [collector() for collector in job_stats_collectors] + upload_stats = collect_top_sessions() return render( request, "core/admin_section.html", { "status_list": status_list, "job_stats": job_stats, + "upload_stats": upload_stats, }, ) diff --git a/adit/upload/apps.py b/adit/upload/apps.py index 567448641..c14053bc1 100644 --- a/adit/upload/apps.py +++ b/adit/upload/apps.py @@ -22,6 +22,12 @@ def register_app(): ) +def collect_top_sessions(): + from adit.upload.models import UploadSession + + return list(UploadSession.objects.order_by("-time_opened"))[:5] + + def create_app_settings(): from .models import UploadSettings From de06aaba5371d43c7766e35575372aa4f0022038 Mon Sep 17 00:00:00 2001 From: Maximilian Gutzmann Date: Thu, 6 Nov 2025 11:12:00 +0000 Subject: [PATCH 4/4] added Tests --- adit/upload/tests/acceptance/test_upload.py | 36 +++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/adit/upload/tests/acceptance/test_upload.py b/adit/upload/tests/acceptance/test_upload.py index 66f3c4b75..b477a059a 100644 --- a/adit/upload/tests/acceptance/test_upload.py +++ b/adit/upload/tests/acceptance/test_upload.py @@ -11,6 +11,7 @@ from adit.core.utils.dicom_dataset import QueryDataset from adit.core.utils.dicom_operator import DicomOperator from adit.core.utils.testing_helpers import setup_dimse_orthancs +from adit.upload.models import UploadSession from adit.upload.utils.testing_helpers import create_upload_group, get_sample_dicoms_folder fake = Faker() @@ -238,3 +239,38 @@ def test_pseudonym_is_used_as_patientID(live_server: LiveServer, page: Page, pat assert found_patients_list[0].PatientID == test_pseudonym assert found_patients_list[0].NumberOfPatientRelatedStudies == 1 + + +@pytest.mark.acceptance +@pytest.mark.order("last") +@pytest.mark.django_db(transaction=True) +def test_session_statistics(live_server: LiveServer, page: Page): + user = create_and_login_example_user(page, live_server.url) + group = create_upload_group() + add_user_to_group(user, group) + dimse_orthancs = setup_dimse_orthancs() + grant_access(group, dimse_orthancs[1], destination=True) + folder = get_sample_dicoms_folder("1001") + + page.on("console", lambda msg: print(msg.text)) + + page.goto(live_server.url + "/upload/jobs/new") + page.get_by_label("Destination").select_option(label="DICOM Server Orthanc Test Server 2") + page.get_by_label("Pseudonym").fill("Test pseudonym") + + page.get_by_label("Choose a directory").set_input_files(files=[folder]) + + expect(page.locator("button#uploadButton")).to_be_visible() + expect(page.locator("button#clearButton")).to_be_visible() + + page.locator("button#uploadButton").click() + + expect(page.locator("button#stopUploadButton")).to_be_visible() + + page.wait_for_selector("p#uploadCompleteText") + + expect(page.locator("p#uploadCompleteText")).to_contain_text("Upload Successful!") + + session = UploadSession.objects.order_by("-time_opened").first() + assert session is not None + assert session.uploaded_file_count == 11