LocationData for parsers

dojo/db_migrations/0261_remove_url_insert_insert_remove_url_update_update_and_more.py

@@ -0,0 +1,77 @@
+# Generated by Django 5.2.11 on 2026-02-21 14:31
+
+import pgtrigger.compiler
+import pgtrigger.migrations
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dojo', '0260_alter_engagement_status_alter_engagementevent_status'),
+    ]
+
+    operations = [
+        pgtrigger.migrations.RemoveTrigger(
+            model_name='url',
+            name='insert_insert',
+        ),
+        pgtrigger.migrations.RemoveTrigger(
+            model_name='url',
+            name='update_update',
+        ),
+        pgtrigger.migrations.RemoveTrigger(
+            model_name='url',
+            name='delete_delete',
+        ),
+        migrations.RemoveIndex(
+            model_name='url',
+            name='dojo_url_host_25c32d_idx',
+        ),
+        migrations.RenameField(
+            model_name='url',
+            old_name='hash',
+            new_name='identity_hash',
+        ),
+        migrations.RenameField(
+            model_name='urlevent',
+            old_name='hash',
+            new_name='identity_hash',
+        ),
+        migrations.AddField(
+            model_name='locationfindingreference',
+            name='relationship',
+            field=models.CharField(blank=True, choices=[('owned_by', 'is owned by'), ('used_by', 'is used by')], default='', help_text='The relationship between two locations', max_length=16),
+        ),
+        migrations.AddField(
+            model_name='locationfindingreference',
+            name='relationship_data',
+            field=models.JSONField(blank=True, default=dict, help_text='Any extra data about this relationship'),
+        ),
+        migrations.AddField(
+            model_name='locationproductreference',
+            name='relationship',
+            field=models.CharField(blank=True, choices=[('owned_by', 'is owned by'), ('used_by', 'is used by')], default='', help_text='The relationship between two locations', max_length=16),
+        ),
+        migrations.AddField(
+            model_name='locationproductreference',
+            name='relationship_data',
+            field=models.JSONField(blank=True, default=dict, help_text='Any extra data about this relationship'),
+        ),
+        migrations.AddIndex(
+            model_name='url',
+            index=models.Index(fields=['host', 'identity_hash'], name='dojo_url_host_71300b_idx'),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name='url',
+            trigger=pgtrigger.compiler.Trigger(name='insert_insert', sql=pgtrigger.compiler.UpsertTriggerSql(func='INSERT INTO "dojo_urlevent" ("fragment", "host", "host_validation_failure", "id", "identity_hash", "location_id", "path", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "port", "protocol", "query", "user_info") VALUES (NEW."fragment", NEW."host", NEW."host_validation_failure", NEW."id", NEW."identity_hash", NEW."location_id", NEW."path", _pgh_attach_context(), NOW(), \'insert\', NEW."id", NEW."port", NEW."protocol", NEW."query", NEW."user_info"); RETURN NULL;', hash='a2ee432a66ed5941b48145a5186615445416fee7', operation='INSERT', pgid='pgtrigger_insert_insert_9de22', table='dojo_url', when='AFTER')),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name='url',
+            trigger=pgtrigger.compiler.Trigger(name='update_update', sql=pgtrigger.compiler.UpsertTriggerSql(condition='WHEN (OLD.* IS DISTINCT FROM NEW.*)', func='INSERT INTO "dojo_urlevent" ("fragment", "host", "host_validation_failure", "id", "identity_hash", "location_id", "path", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "port", "protocol", "query", "user_info") VALUES (NEW."fragment", NEW."host", NEW."host_validation_failure", NEW."id", NEW."identity_hash", NEW."location_id", NEW."path", _pgh_attach_context(), NOW(), \'update\', NEW."id", NEW."port", NEW."protocol", NEW."query", NEW."user_info"); RETURN NULL;', hash='580aee0302a640be89ca8650645a7abe5c84c465', operation='UPDATE', pgid='pgtrigger_update_update_4785e', table='dojo_url', when='AFTER')),
+        ),
+        pgtrigger.migrations.AddTrigger(
+            model_name='url',
+            trigger=pgtrigger.compiler.Trigger(name='delete_delete', sql=pgtrigger.compiler.UpsertTriggerSql(func='INSERT INTO "dojo_urlevent" ("fragment", "host", "host_validation_failure", "id", "identity_hash", "location_id", "path", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "port", "protocol", "query", "user_info") VALUES (OLD."fragment", OLD."host", OLD."host_validation_failure", OLD."id", OLD."identity_hash", OLD."location_id", OLD."path", _pgh_attach_context(), NOW(), \'delete\', OLD."id", OLD."port", OLD."protocol", OLD."query", OLD."user_info"); RETURN NULL;', hash='e17c960765339d538242b5e60e85f12c4af209ec', operation='DELETE', pgid='pgtrigger_delete_delete_ca7d6', table='dojo_url', when='AFTER')),
+        ),
+    ]

dojo/db_migrations/max_migration.txt

@@ -1 +1 @@
-0260_alter_engagement_status_alter_engagementevent_status
+0261_remove_url_insert_insert_remove_url_update_update_and_more

dojo/fixtures/defect_dojo_sample_data_locations.json

@@ -46113,7 +46113,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "03009c0636425af566fb6b737db82852812fe2969107ef299530a248f78c4761",
+      "identity_hash": "03009c0636425af566fb6b737db82852812fe2969107ef299530a248f78c4761",
       "host": "127.0.0.1",
       "host_validation_failure": false,
       "location": 1,
@@ -46129,7 +46129,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "f1ef169262638cbeabd36e60b021936c5a925beda67b52bd1b52847aa10352e1",
+      "identity_hash": "f1ef169262638cbeabd36e60b021936c5a925beda67b52bd1b52847aa10352e1",
       "host": "127.0.0.1",
       "host_validation_failure": false,
       "location": 2,
@@ -46145,7 +46145,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "025d6b16f8cfba2d8e15e85deb81963a84d5dd3c700614f8e8fda87378cf58aa",
+      "identity_hash": "025d6b16f8cfba2d8e15e85deb81963a84d5dd3c700614f8e8fda87378cf58aa",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 3,
@@ -46161,7 +46161,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "0f93015b731ab1bc5ea1332a90781a2045d46e70ec1090361f68db58265a3271",
+      "identity_hash": "0f93015b731ab1bc5ea1332a90781a2045d46e70ec1090361f68db58265a3271",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 4,
@@ -46177,7 +46177,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "94405be03445dd80d4e542775bf5797252cd907ecfeb381b51f72b5f01c78e4b",
+      "identity_hash": "94405be03445dd80d4e542775bf5797252cd907ecfeb381b51f72b5f01c78e4b",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 5,
@@ -46193,7 +46193,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "96e8eda7ebef554ead75dd22b2b6408b19498648964075d0cd8babf16bcddea9",
+      "identity_hash": "96e8eda7ebef554ead75dd22b2b6408b19498648964075d0cd8babf16bcddea9",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 6,
@@ -46209,7 +46209,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "9eac0b36d89ea41c3ca850519bdf071471991f565dc1f5c0a3385a856a2817af",
+      "identity_hash": "9eac0b36d89ea41c3ca850519bdf071471991f565dc1f5c0a3385a856a2817af",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 7,
@@ -46225,7 +46225,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "b1550d532705cf9e2441cf2d42e143f21d9991b1eb964aa88e52159eb1c5ff31",
+      "identity_hash": "b1550d532705cf9e2441cf2d42e143f21d9991b1eb964aa88e52159eb1c5ff31",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 8,
@@ -46241,7 +46241,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "efaf0a94394a5a2e117454956d67a1c43f95da40b7d7d5c158b049eded4cbe07",
+      "identity_hash": "efaf0a94394a5a2e117454956d67a1c43f95da40b7d7d5c158b049eded4cbe07",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 9,
@@ -46257,7 +46257,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "7c00a6008791144f8e587507469ba1aaf5858a497a5fd2fd4fa5c29e942c4004",
+      "identity_hash": "7c00a6008791144f8e587507469ba1aaf5858a497a5fd2fd4fa5c29e942c4004",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 10,
@@ -46273,7 +46273,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "317895cd12da9908f97ebe8e63cb6395fed39d14925b631d9d4c1362b6458c20",
+      "identity_hash": "317895cd12da9908f97ebe8e63cb6395fed39d14925b631d9d4c1362b6458c20",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 11,
@@ -46289,7 +46289,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "2850f6a08fa0bd443f200525cc170231ce867d810fce483a0e3867f2b8807bfb",
+      "identity_hash": "2850f6a08fa0bd443f200525cc170231ce867d810fce483a0e3867f2b8807bfb",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 12,
@@ -46305,7 +46305,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "c77e801a98a44ae86442c03f8dc2c068a3ce72d745c69aa290cce2c91c5be802",
+      "identity_hash": "c77e801a98a44ae86442c03f8dc2c068a3ce72d745c69aa290cce2c91c5be802",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 13,
@@ -46321,7 +46321,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "0710d27be344e66a9b1554d493f1e14f6b060dacba8ab64b5d2f86f49b85cca2",
+      "identity_hash": "0710d27be344e66a9b1554d493f1e14f6b060dacba8ab64b5d2f86f49b85cca2",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 14,
@@ -46337,7 +46337,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "95f7dc925856be6f19bcfcc4b5c542d8a382815709900d6ab91693e3864bf75e",
+      "identity_hash": "95f7dc925856be6f19bcfcc4b5c542d8a382815709900d6ab91693e3864bf75e",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 15,
@@ -46353,7 +46353,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "fddeec108eaec6a43060f43a44e228740350710cb3c4475ddd7cc7da7f740206",
+      "identity_hash": "fddeec108eaec6a43060f43a44e228740350710cb3c4475ddd7cc7da7f740206",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 16,
@@ -46369,7 +46369,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "312cfc49152f6445cdb53df02fc65c966accd3e90e709fbe63bce73703085708",
+      "identity_hash": "312cfc49152f6445cdb53df02fc65c966accd3e90e709fbe63bce73703085708",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 17,
@@ -46385,7 +46385,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "cac8b9e23292f238ae36c2c98600c1e3c813f48a53b6f8ec4f3accb0dda42886",
+      "identity_hash": "cac8b9e23292f238ae36c2c98600c1e3c813f48a53b6f8ec4f3accb0dda42886",
       "host": "localhost",
       "host_validation_failure": false,
       "location": 18,
@@ -46401,7 +46401,7 @@
   {
     "fields": {
       "fragment": "",
-      "hash": "f28d3752e452cde3e00a3aaf885fe153037ae69d9726e6a0936ee7da3225c1ad",
+      "identity_hash": "f28d3752e452cde3e00a3aaf885fe153037ae69d9726e6a0936ee7da3225c1ad",
       "host": "127.0.0.1",
       "host_validation_failure": false,
       "location": 19,
@@ -93348,4 +93348,4 @@
     "model": "authtoken.token",
     "pk": "6d45bc1d2e5cea8c4559edd68f910cc485f61708"
   }
-]
+]

dojo/fixtures/dojo_testdata_locations.json

@@ -824,7 +824,7 @@
       "query": "",
       "path": "endpoint/420/edit/",
       "port": 80,
-      "hash": "f28d3752e452cde3e00a3aaf885fe153037ae69d9726e6a0936ee7da3225c1ad"
+      "identity_hash": "f28d3752e452cde3e00a3aaf885fe153037ae69d9726e6a0936ee7da3225c1ad"
     }
   },
   {
@@ -838,7 +838,7 @@
       "query": "",
       "path": "",
       "port": 21,
-      "hash": "025d6b16f8cfba2d8e15e85deb81963a84d5dd3c700614f8e8fda87378cf58aa"
+      "identity_hash": "025d6b16f8cfba2d8e15e85deb81963a84d5dd3c700614f8e8fda87378cf58aa"
     }
   },
   {
@@ -852,7 +852,7 @@
       "query": "",
       "path": "",
       "port": 22,
-      "hash": "03009c0636425af566fb6b737db82852812fe2969107ef299530a248f78c4761"
+      "identity_hash": "03009c0636425af566fb6b737db82852812fe2969107ef299530a248f78c4761"
     }
   },
   {
@@ -866,7 +866,7 @@
       "query": "",
       "path": "",
       "port": 21,
-      "hash": "3c82808602507ea78a48dc605d86476f5cc609f99d446d1a9c9539d5c7c10166"
+      "identity_hash": "3c82808602507ea78a48dc605d86476f5cc609f99d446d1a9c9539d5c7c10166"
     }
   },
   {
@@ -880,7 +880,7 @@
       "query": "",
       "path": "",
       "port": 80,
-      "hash": "c1f60206ce7dac6202baec6f6251006e499113356869f605d0168f13e1d593ac"
+      "identity_hash": "c1f60206ce7dac6202baec6f6251006e499113356869f605d0168f13e1d593ac"
     }
   },
   {
@@ -894,7 +894,7 @@
       "query": "",
       "path": "",
       "port": 80,
-      "hash": "99e0dbdf63598bfa7c3310a6785638117134c44ac59a38785e9e81fece0878da"
+      "identity_hash": "99e0dbdf63598bfa7c3310a6785638117134c44ac59a38785e9e81fece0878da"
     }
   },
   {
@@ -908,7 +908,7 @@
       "query": "",
       "path": "",
       "port": 443,
-      "hash": "26259e2d4e77789a856808509ba2d9a50838bdc76d71f596a1bb346bbcb30332"
+      "identity_hash": "26259e2d4e77789a856808509ba2d9a50838bdc76d71f596a1bb346bbcb30332"
     }
   },
   {
@@ -922,7 +922,7 @@
       "query": "",
       "path": "f6",
       "port": 443,
-      "hash": "4e5614106ea6ca9c15c3328731036b06ab32c45fb2ed013601350748e2d3ed85"
+      "identity_hash": "4e5614106ea6ca9c15c3328731036b06ab32c45fb2ed013601350748e2d3ed85"
     }
   },
   {

dojo/importers/base_importer.py

@@ -15,10 +15,10 @@
 import dojo.risk_acceptance.helper as ra_helper
 from dojo.celery_dispatch import dojo_dispatch_task
 from dojo.importers.endpoint_manager import EndpointManager
-from dojo.importers.location_manager import LocationManager
+from dojo.importers.location_manager import LocationManager, UnsavedLocation
 from dojo.importers.options import ImporterOptions
 from dojo.jira_link.helper import is_keep_in_sync_with_jira
-from dojo.location.models import AbstractLocation, Location
+from dojo.location.models import Location
 from dojo.models import (
     # Import History States
     IMPORT_CLOSED_FINDING,
@@ -792,7 +792,7 @@ def process_request_response_pairs(
     def process_locations(
         self,
         finding: Finding,
-        locations_to_add: list[AbstractLocation],
+        locations_to_add: list[UnsavedLocation],
     ) -> None:
         """
         Process any locations to add to the finding. Locations could come from two places