Skip to content

Commit 597a1a8

Browse files
dogboatMaffooch
dogboat
and
Maffooch
authored
LocationData for parsers (#14395)
* wip * cyclonedx test * dependency check update test * wip * wip * wip * wip * wip * wip * replace parser instantiation of URL with locationdata * url model util method updates * wip * wip * wip * locationdata.dependency * wip * move locationdata module * wip * wip * wip * wip * update to use test.unsaved_metadata * test at adding hashes * locationdata entry name update * wip relationship data * assoc rel info in dataclass * updates to include licensing for components * tools updates * comments * remove unused attribute * comments * comments * typing * fixup * deptrack parser update * pass correct reference * update parsers to include file_name for deps if possible * finding hash code computation fix * generic json parser update * updates * fix classmethod/app.task interaction * some tool cleanup * undo mend * tools fixes * wip tools * fixup * trigger GitHub actions * nuclei fix * nuclei fix --------- Co-authored-by: Cody Maffucci <46459665+Maffooch@users.noreply.github.com>
1 parent 46ef61b commit 597a1a8

File tree

130 files changed

+1689
-619
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

130 files changed

+1689
-619
lines changed

‎dojo/db_migrations/0261_remove_url_insert_insert_remove_url_update_update_and_more.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Generated by Django 5.2.11 on 2026-02-21 14:31
2+
3+
import pgtrigger.compiler
4+
import pgtrigger.migrations
5+
from django.db import migrations, models
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
('dojo', '0260_alter_engagement_status_alter_engagementevent_status'),
12+
]
13+
14+
operations = [
15+
pgtrigger.migrations.RemoveTrigger(
16+
model_name='url',
17+
name='insert_insert',
18+
),
19+
pgtrigger.migrations.RemoveTrigger(
20+
model_name='url',
21+
name='update_update',
22+
),
23+
pgtrigger.migrations.RemoveTrigger(
24+
model_name='url',
25+
name='delete_delete',
26+
),
27+
migrations.RemoveIndex(
28+
model_name='url',
29+
name='dojo_url_host_25c32d_idx',
30+
),
31+
migrations.RenameField(
32+
model_name='url',
33+
old_name='hash',
34+
new_name='identity_hash',
35+
),
36+
migrations.RenameField(
37+
model_name='urlevent',
38+
old_name='hash',
39+
new_name='identity_hash',
40+
),
41+
migrations.AddField(
42+
model_name='locationfindingreference',
43+
name='relationship',
44+
field=models.CharField(blank=True, choices=[('owned_by', 'is owned by'), ('used_by', 'is used by')], default='', help_text='The relationship between two locations', max_length=16),
45+
),
46+
migrations.AddField(
47+
model_name='locationfindingreference',
48+
name='relationship_data',
49+
field=models.JSONField(blank=True, default=dict, help_text='Any extra data about this relationship'),
50+
),
51+
migrations.AddField(
52+
model_name='locationproductreference',
53+
name='relationship',
54+
field=models.CharField(blank=True, choices=[('owned_by', 'is owned by'), ('used_by', 'is used by')], default='', help_text='The relationship between two locations', max_length=16),
55+
),
56+
migrations.AddField(
57+
model_name='locationproductreference',
58+
name='relationship_data',
59+
field=models.JSONField(blank=True, default=dict, help_text='Any extra data about this relationship'),
60+
),
61+
migrations.AddIndex(
62+
model_name='url',
63+
index=models.Index(fields=['host', 'identity_hash'], name='dojo_url_host_71300b_idx'),
64+
),
65+
pgtrigger.migrations.AddTrigger(
66+
model_name='url',
67+
trigger=pgtrigger.compiler.Trigger(name='insert_insert', sql=pgtrigger.compiler.UpsertTriggerSql(func='INSERT INTO "dojo_urlevent" ("fragment", "host", "host_validation_failure", "id", "identity_hash", "location_id", "path", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "port", "protocol", "query", "user_info") VALUES (NEW."fragment", NEW."host", NEW."host_validation_failure", NEW."id", NEW."identity_hash", NEW."location_id", NEW."path", _pgh_attach_context(), NOW(), \'insert\', NEW."id", NEW."port", NEW."protocol", NEW."query", NEW."user_info"); RETURN NULL;', hash='a2ee432a66ed5941b48145a5186615445416fee7', operation='INSERT', pgid='pgtrigger_insert_insert_9de22', table='dojo_url', when='AFTER')),
68+
),
69+
pgtrigger.migrations.AddTrigger(
70+
model_name='url',
71+
trigger=pgtrigger.compiler.Trigger(name='update_update', sql=pgtrigger.compiler.UpsertTriggerSql(condition='WHEN (OLD.* IS DISTINCT FROM NEW.*)', func='INSERT INTO "dojo_urlevent" ("fragment", "host", "host_validation_failure", "id", "identity_hash", "location_id", "path", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "port", "protocol", "query", "user_info") VALUES (NEW."fragment", NEW."host", NEW."host_validation_failure", NEW."id", NEW."identity_hash", NEW."location_id", NEW."path", _pgh_attach_context(), NOW(), \'update\', NEW."id", NEW."port", NEW."protocol", NEW."query", NEW."user_info"); RETURN NULL;', hash='580aee0302a640be89ca8650645a7abe5c84c465', operation='UPDATE', pgid='pgtrigger_update_update_4785e', table='dojo_url', when='AFTER')),
72+
),
73+
pgtrigger.migrations.AddTrigger(
74+
model_name='url',
75+
trigger=pgtrigger.compiler.Trigger(name='delete_delete', sql=pgtrigger.compiler.UpsertTriggerSql(func='INSERT INTO "dojo_urlevent" ("fragment", "host", "host_validation_failure", "id", "identity_hash", "location_id", "path", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "port", "protocol", "query", "user_info") VALUES (OLD."fragment", OLD."host", OLD."host_validation_failure", OLD."id", OLD."identity_hash", OLD."location_id", OLD."path", _pgh_attach_context(), NOW(), \'delete\', OLD."id", OLD."port", OLD."protocol", OLD."query", OLD."user_info"); RETURN NULL;', hash='e17c960765339d538242b5e60e85f12c4af209ec', operation='DELETE', pgid='pgtrigger_delete_delete_ca7d6', table='dojo_url', when='AFTER')),
76+
),
77+
]

‎dojo/db_migrations/max_migration.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0260_alter_engagement_status_alter_engagementevent_status
1+
0261_remove_url_insert_insert_remove_url_update_update_and_more

‎dojo/fixtures/defect_dojo_sample_data_locations.json‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -46113,7 +46113,7 @@
4611346113
{
4611446114
"fields": {
4611546115
"fragment": "",
46116-
"hash": "03009c0636425af566fb6b737db82852812fe2969107ef299530a248f78c4761",
46116+
"identity_hash": "03009c0636425af566fb6b737db82852812fe2969107ef299530a248f78c4761",
4611746117
"host": "127.0.0.1",
4611846118
"host_validation_failure": false,
4611946119
"location": 1,
@@ -46129,7 +46129,7 @@
4612946129
{
4613046130
"fields": {
4613146131
"fragment": "",
46132-
"hash": "f1ef169262638cbeabd36e60b021936c5a925beda67b52bd1b52847aa10352e1",
46132+
"identity_hash": "f1ef169262638cbeabd36e60b021936c5a925beda67b52bd1b52847aa10352e1",
4613346133
"host": "127.0.0.1",
4613446134
"host_validation_failure": false,
4613546135
"location": 2,
@@ -46145,7 +46145,7 @@
4614546145
{
4614646146
"fields": {
4614746147
"fragment": "",
46148-
"hash": "025d6b16f8cfba2d8e15e85deb81963a84d5dd3c700614f8e8fda87378cf58aa",
46148+
"identity_hash": "025d6b16f8cfba2d8e15e85deb81963a84d5dd3c700614f8e8fda87378cf58aa",
4614946149
"host": "localhost",
4615046150
"host_validation_failure": false,
4615146151
"location": 3,
@@ -46161,7 +46161,7 @@
4616146161
{
4616246162
"fields": {
4616346163
"fragment": "",
46164-
"hash": "0f93015b731ab1bc5ea1332a90781a2045d46e70ec1090361f68db58265a3271",
46164+
"identity_hash": "0f93015b731ab1bc5ea1332a90781a2045d46e70ec1090361f68db58265a3271",
4616546165
"host": "localhost",
4616646166
"host_validation_failure": false,
4616746167
"location": 4,
@@ -46177,7 +46177,7 @@
4617746177
{
4617846178
"fields": {
4617946179
"fragment": "",
46180-
"hash": "94405be03445dd80d4e542775bf5797252cd907ecfeb381b51f72b5f01c78e4b",
46180+
"identity_hash": "94405be03445dd80d4e542775bf5797252cd907ecfeb381b51f72b5f01c78e4b",
4618146181
"host": "localhost",
4618246182
"host_validation_failure": false,
4618346183
"location": 5,
@@ -46193,7 +46193,7 @@
4619346193
{
4619446194
"fields": {
4619546195
"fragment": "",
46196-
"hash": "96e8eda7ebef554ead75dd22b2b6408b19498648964075d0cd8babf16bcddea9",
46196+
"identity_hash": "96e8eda7ebef554ead75dd22b2b6408b19498648964075d0cd8babf16bcddea9",
4619746197
"host": "localhost",
4619846198
"host_validation_failure": false,
4619946199
"location": 6,
@@ -46209,7 +46209,7 @@
4620946209
{
4621046210
"fields": {
4621146211
"fragment": "",
46212-
"hash": "9eac0b36d89ea41c3ca850519bdf071471991f565dc1f5c0a3385a856a2817af",
46212+
"identity_hash": "9eac0b36d89ea41c3ca850519bdf071471991f565dc1f5c0a3385a856a2817af",
4621346213
"host": "localhost",
4621446214
"host_validation_failure": false,
4621546215
"location": 7,
@@ -46225,7 +46225,7 @@
4622546225
{
4622646226
"fields": {
4622746227
"fragment": "",
46228-
"hash": "b1550d532705cf9e2441cf2d42e143f21d9991b1eb964aa88e52159eb1c5ff31",
46228+
"identity_hash": "b1550d532705cf9e2441cf2d42e143f21d9991b1eb964aa88e52159eb1c5ff31",
4622946229
"host": "localhost",
4623046230
"host_validation_failure": false,
4623146231
"location": 8,
@@ -46241,7 +46241,7 @@
4624146241
{
4624246242
"fields": {
4624346243
"fragment": "",
46244-
"hash": "efaf0a94394a5a2e117454956d67a1c43f95da40b7d7d5c158b049eded4cbe07",
46244+
"identity_hash": "efaf0a94394a5a2e117454956d67a1c43f95da40b7d7d5c158b049eded4cbe07",
4624546245
"host": "localhost",
4624646246
"host_validation_failure": false,
4624746247
"location": 9,
@@ -46257,7 +46257,7 @@
4625746257
{
4625846258
"fields": {
4625946259
"fragment": "",
46260-
"hash": "7c00a6008791144f8e587507469ba1aaf5858a497a5fd2fd4fa5c29e942c4004",
46260+
"identity_hash": "7c00a6008791144f8e587507469ba1aaf5858a497a5fd2fd4fa5c29e942c4004",
4626146261
"host": "localhost",
4626246262
"host_validation_failure": false,
4626346263
"location": 10,
@@ -46273,7 +46273,7 @@
4627346273
{
4627446274
"fields": {
4627546275
"fragment": "",
46276-
"hash": "317895cd12da9908f97ebe8e63cb6395fed39d14925b631d9d4c1362b6458c20",
46276+
"identity_hash": "317895cd12da9908f97ebe8e63cb6395fed39d14925b631d9d4c1362b6458c20",
4627746277
"host": "localhost",
4627846278
"host_validation_failure": false,
4627946279
"location": 11,
@@ -46289,7 +46289,7 @@
4628946289
{
4629046290
"fields": {
4629146291
"fragment": "",
46292-
"hash": "2850f6a08fa0bd443f200525cc170231ce867d810fce483a0e3867f2b8807bfb",
46292+
"identity_hash": "2850f6a08fa0bd443f200525cc170231ce867d810fce483a0e3867f2b8807bfb",
4629346293
"host": "localhost",
4629446294
"host_validation_failure": false,
4629546295
"location": 12,
@@ -46305,7 +46305,7 @@
4630546305
{
4630646306
"fields": {
4630746307
"fragment": "",
46308-
"hash": "c77e801a98a44ae86442c03f8dc2c068a3ce72d745c69aa290cce2c91c5be802",
46308+
"identity_hash": "c77e801a98a44ae86442c03f8dc2c068a3ce72d745c69aa290cce2c91c5be802",
4630946309
"host": "localhost",
4631046310
"host_validation_failure": false,
4631146311
"location": 13,
@@ -46321,7 +46321,7 @@
4632146321
{
4632246322
"fields": {
4632346323
"fragment": "",
46324-
"hash": "0710d27be344e66a9b1554d493f1e14f6b060dacba8ab64b5d2f86f49b85cca2",
46324+
"identity_hash": "0710d27be344e66a9b1554d493f1e14f6b060dacba8ab64b5d2f86f49b85cca2",
4632546325
"host": "localhost",
4632646326
"host_validation_failure": false,
4632746327
"location": 14,
@@ -46337,7 +46337,7 @@
4633746337
{
4633846338
"fields": {
4633946339
"fragment": "",
46340-
"hash": "95f7dc925856be6f19bcfcc4b5c542d8a382815709900d6ab91693e3864bf75e",
46340+
"identity_hash": "95f7dc925856be6f19bcfcc4b5c542d8a382815709900d6ab91693e3864bf75e",
4634146341
"host": "localhost",
4634246342
"host_validation_failure": false,
4634346343
"location": 15,
@@ -46353,7 +46353,7 @@
4635346353
{
4635446354
"fields": {
4635546355
"fragment": "",
46356-
"hash": "fddeec108eaec6a43060f43a44e228740350710cb3c4475ddd7cc7da7f740206",
46356+
"identity_hash": "fddeec108eaec6a43060f43a44e228740350710cb3c4475ddd7cc7da7f740206",
4635746357
"host": "localhost",
4635846358
"host_validation_failure": false,
4635946359
"location": 16,
@@ -46369,7 +46369,7 @@
4636946369
{
4637046370
"fields": {
4637146371
"fragment": "",
46372-
"hash": "312cfc49152f6445cdb53df02fc65c966accd3e90e709fbe63bce73703085708",
46372+
"identity_hash": "312cfc49152f6445cdb53df02fc65c966accd3e90e709fbe63bce73703085708",
4637346373
"host": "localhost",
4637446374
"host_validation_failure": false,
4637546375
"location": 17,
@@ -46385,7 +46385,7 @@
4638546385
{
4638646386
"fields": {
4638746387
"fragment": "",
46388-
"hash": "cac8b9e23292f238ae36c2c98600c1e3c813f48a53b6f8ec4f3accb0dda42886",
46388+
"identity_hash": "cac8b9e23292f238ae36c2c98600c1e3c813f48a53b6f8ec4f3accb0dda42886",
4638946389
"host": "localhost",
4639046390
"host_validation_failure": false,
4639146391
"location": 18,
@@ -46401,7 +46401,7 @@
4640146401
{
4640246402
"fields": {
4640346403
"fragment": "",
46404-
"hash": "f28d3752e452cde3e00a3aaf885fe153037ae69d9726e6a0936ee7da3225c1ad",
46404+
"identity_hash": "f28d3752e452cde3e00a3aaf885fe153037ae69d9726e6a0936ee7da3225c1ad",
4640546405
"host": "127.0.0.1",
4640646406
"host_validation_failure": false,
4640746407
"location": 19,
@@ -93348,4 +93348,4 @@
9334893348
"model": "authtoken.token",
9334993349
"pk": "6d45bc1d2e5cea8c4559edd68f910cc485f61708"
9335093350
}
93351-
]
93351+
]

‎dojo/fixtures/dojo_testdata_locations.json‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -824,7 +824,7 @@
824824
"query": "",
825825
"path": "endpoint/420/edit/",
826826
"port": 80,
827-
"hash": "f28d3752e452cde3e00a3aaf885fe153037ae69d9726e6a0936ee7da3225c1ad"
827+
"identity_hash": "f28d3752e452cde3e00a3aaf885fe153037ae69d9726e6a0936ee7da3225c1ad"
828828
}
829829
},
830830
{
@@ -838,7 +838,7 @@
838838
"query": "",
839839
"path": "",
840840
"port": 21,
841-
"hash": "025d6b16f8cfba2d8e15e85deb81963a84d5dd3c700614f8e8fda87378cf58aa"
841+
"identity_hash": "025d6b16f8cfba2d8e15e85deb81963a84d5dd3c700614f8e8fda87378cf58aa"
842842
}
843843
},
844844
{
@@ -852,7 +852,7 @@
852852
"query": "",
853853
"path": "",
854854
"port": 22,
855-
"hash": "03009c0636425af566fb6b737db82852812fe2969107ef299530a248f78c4761"
855+
"identity_hash": "03009c0636425af566fb6b737db82852812fe2969107ef299530a248f78c4761"
856856
}
857857
},
858858
{
@@ -866,7 +866,7 @@
866866
"query": "",
867867
"path": "",
868868
"port": 21,
869-
"hash": "3c82808602507ea78a48dc605d86476f5cc609f99d446d1a9c9539d5c7c10166"
869+
"identity_hash": "3c82808602507ea78a48dc605d86476f5cc609f99d446d1a9c9539d5c7c10166"
870870
}
871871
},
872872
{
@@ -880,7 +880,7 @@
880880
"query": "",
881881
"path": "",
882882
"port": 80,
883-
"hash": "c1f60206ce7dac6202baec6f6251006e499113356869f605d0168f13e1d593ac"
883+
"identity_hash": "c1f60206ce7dac6202baec6f6251006e499113356869f605d0168f13e1d593ac"
884884
}
885885
},
886886
{
@@ -894,7 +894,7 @@
894894
"query": "",
895895
"path": "",
896896
"port": 80,
897-
"hash": "99e0dbdf63598bfa7c3310a6785638117134c44ac59a38785e9e81fece0878da"
897+
"identity_hash": "99e0dbdf63598bfa7c3310a6785638117134c44ac59a38785e9e81fece0878da"
898898
}
899899
},
900900
{
@@ -908,7 +908,7 @@
908908
"query": "",
909909
"path": "",
910910
"port": 443,
911-
"hash": "26259e2d4e77789a856808509ba2d9a50838bdc76d71f596a1bb346bbcb30332"
911+
"identity_hash": "26259e2d4e77789a856808509ba2d9a50838bdc76d71f596a1bb346bbcb30332"
912912
}
913913
},
914914
{
@@ -922,7 +922,7 @@
922922
"query": "",
923923
"path": "f6",
924924
"port": 443,
925-
"hash": "4e5614106ea6ca9c15c3328731036b06ab32c45fb2ed013601350748e2d3ed85"
925+
"identity_hash": "4e5614106ea6ca9c15c3328731036b06ab32c45fb2ed013601350748e2d3ed85"
926926
}
927927
},
928928
{

‎dojo/importers/base_importer.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@
1515
import dojo.risk_acceptance.helper as ra_helper
1616
from dojo.celery_dispatch import dojo_dispatch_task
1717
from dojo.importers.endpoint_manager import EndpointManager
18-
from dojo.importers.location_manager import LocationManager
18+
from dojo.importers.location_manager import LocationManager, UnsavedLocation
1919
from dojo.importers.options import ImporterOptions
2020
from dojo.jira_link.helper import is_keep_in_sync_with_jira
21-
from dojo.location.models import AbstractLocation, Location
21+
from dojo.location.models import Location
2222
from dojo.models import (
2323
# Import History States
2424
IMPORT_CLOSED_FINDING,
@@ -792,7 +792,7 @@ def process_request_response_pairs(
792792
def process_locations(
793793
self,
794794
finding: Finding,
795-
locations_to_add: list[AbstractLocation],
795+
locations_to_add: list[UnsavedLocation],
796796
) -> None:
797797
"""
798798
Process any locations to add to the finding. Locations could come from two places

0 commit comments

Comments
 (0)