Skip to content

fix: restrict rfc bulkload to those with a Document object #10289

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rjsparks merged 3 commits into from
Jan 21, 2026
Merged

fix: restrict rfc bulkload to those with a Document object #10289

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c310d61
fix: restrict rfc bulkload to those with a Document object
rjsparks Jan 20, 2026
25a2405
fix: only query Document once
rjsparks Jan 20, 2026
7772091
test: update test to match check for existing Document
rjsparks Jan 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ietf/sync/tests_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from django.test import override_settings
from ietf import settings
from ietf.doc.factories import RfcFactory
from ietf.doc.storage_utils import exists_in_storage, retrieve_str
from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper
from ietf.utils.test_utils import TestCase
Expand Down Expand Up @@ -59,6 +60,7 @@ def test_load_rfcs_into_blobdb(self):
rfc_path = Path(faux_rfc_path)
(rfc_path / "prerelease").mkdir()
for num in [12345, 54321]:
RfcFactory(rfc_number=num)
for ext in settings.RFC_FILE_TYPES + ("json",):
with (rfc_path / f"rfc{num}.{ext}").open("w") as f:
f.write(ext)
Expand Down
64 changes: 36 additions & 28 deletions ietf/sync/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
from pathlib import Path

from django.conf import settings

from ietf.utils import log
from ietf.doc.models import Document
from ietf.doc.storage_utils import AlreadyExistsError, store_bytes


Expand All @@ -26,17 +26,42 @@ def build_from_file_content(rfc_numbers: list[int]) -> str:

def load_rfcs_into_blobdb(numbers: list[int]):
types_to_load = settings.RFC_FILE_TYPES + ("json",)
rfc_docs = Document.objects.filter(type="rfc", rfc_number__in=numbers).values_list("rfc_number", flat=True)
for num in numbers:
for ext in types_to_load:
fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}"
if fs_path.is_file():
with fs_path.open("rb") as f:
if num in rfc_docs:
for ext in types_to_load:
fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}"
if fs_path.is_file():
with fs_path.open("rb") as f:
bytes = f.read()
mtime = fs_path.stat().st_mtime
try:
store_bytes(
kind="rfc",
name=f"{ext}/rfc{num}.{ext}",
content=bytes,
allow_overwrite=False, # Intentionally not allowing overwrite.
doc_name=f"rfc{num}",
doc_rev=None,
Comment on lines +44 to +45
Copy link
Member

@jennifer-richards jennifer-richards Jan 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you wanted to be paranoid, you could iterate over Documents instead of numbers and use doc.name and doc.rev here. I think the data are clean in this regard, though.

# Not setting content_type
mtime=datetime.datetime.fromtimestamp(
mtime, tz=datetime.UTC
),
)
except AlreadyExistsError as e:
log.log(str(e))

# store the not-prepped xml
name = f"rfc{num}.notprepped.xml"
source = Path(settings.RFC_PATH) / "prerelease" / name
if source.is_file():
with open(source, "rb") as f:
bytes = f.read()
mtime = fs_path.stat().st_mtime
mtime = source.stat().st_mtime
try:
store_bytes(
kind="rfc",
name=f"{ext}/rfc{num}.{ext}",
name=f"notprepped/{name}",
content=bytes,
allow_overwrite=False, # Intentionally not allowing overwrite.
doc_name=f"rfc{num}",
Expand All @@ -46,24 +71,7 @@ def load_rfcs_into_blobdb(numbers: list[int]):
)
except AlreadyExistsError as e:
log.log(str(e))

# store the not-prepped xml
name = f"rfc{num}.notprepped.xml"
source = Path(settings.RFC_PATH) / "prerelease" / name
if source.is_file():
with open(source, "rb") as f:
bytes = f.read()
mtime = source.stat().st_mtime
try:
store_bytes(
kind="rfc",
name=f"notprepped/{name}",
content=bytes,
allow_overwrite=False, # Intentionally not allowing overwrite.
doc_name=f"rfc{num}",
doc_rev=None,
# Not setting content_type
mtime=datetime.datetime.fromtimestamp(mtime, tz=datetime.UTC),
)
except AlreadyExistsError as e:
log.log(str(e))
else:
log.log(
f"Skipping loading rfc{num} into blobdb as no matching Document exists"
)
Loading