Skip to content

fix: restrict rfc bulkload to those with a Document object #10289

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rjsparks merged 3 commits into from
Jan 21, 2026
+38 −28
Merged

fix: restrict rfc bulkload to those with a Document object #10289

Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c310d61
fix: restrict rfc bulkload to those with a Document object
rjsparks Jan 20, 2026
25a2405
fix: only query Document once
rjsparks Jan 20, 2026
7772091
test: update test to match check for existing Document
rjsparks Jan 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 36 additions & 28 deletions ietf/sync/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
from pathlib import Path

from django.conf import settings

from ietf.utils import log
from ietf.doc.models import Document
from ietf.doc.storage_utils import AlreadyExistsError, store_bytes


Expand All @@ -26,17 +26,42 @@ def build_from_file_content(rfc_numbers: list[int]) -> str:

def load_rfcs_into_blobdb(numbers: list[int]):
types_to_load = settings.RFC_FILE_TYPES + ("json",)
rfc_docs = Document.objects.filter(type="rfc", rfc_number__in=numbers).values_list("rfc_number", flat=True)
for num in numbers:
for ext in types_to_load:
fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}"
if fs_path.is_file():
with fs_path.open("rb") as f:
if num in rfc_docs:
for ext in types_to_load:
fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}"
if fs_path.is_file():
with fs_path.open("rb") as f:
bytes = f.read()
mtime = fs_path.stat().st_mtime
try:
store_bytes(
kind="rfc",
name=f"{ext}/rfc{num}.{ext}",
content=bytes,
allow_overwrite=False, # Intentionally not allowing overwrite.
doc_name=f"rfc{num}",
doc_rev=None,
Comment on lines +44 to +45
Copy link
Member

@jennifer-richards jennifer-richards Jan 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you wanted to be paranoid, you could iterate over Documents instead of numbers and use doc.name and doc.rev here. I think the data are clean in this regard, though.

# Not setting content_type
mtime=datetime.datetime.fromtimestamp(
mtime, tz=datetime.UTC
),
)
except AlreadyExistsError as e:
log.log(str(e))

# store the not-prepped xml
name = f"rfc{num}.notprepped.xml"
source = Path(settings.RFC_PATH) / "prerelease" / name
if source.is_file():
with open(source, "rb") as f:
bytes = f.read()
mtime = fs_path.stat().st_mtime
mtime = source.stat().st_mtime
try:
store_bytes(
kind="rfc",
name=f"{ext}/rfc{num}.{ext}",
name=f"notprepped/{name}",
content=bytes,
allow_overwrite=False, # Intentionally not allowing overwrite.
doc_name=f"rfc{num}",
Expand All @@ -46,24 +71,7 @@ def load_rfcs_into_blobdb(numbers: list[int]):
)
except AlreadyExistsError as e:
log.log(str(e))

# store the not-prepped xml
name = f"rfc{num}.notprepped.xml"
source = Path(settings.RFC_PATH) / "prerelease" / name
if source.is_file():
with open(source, "rb") as f:
bytes = f.read()
mtime = source.stat().st_mtime
try:
store_bytes(
kind="rfc",
name=f"notprepped/{name}",
content=bytes,
allow_overwrite=False, # Intentionally not allowing overwrite.
doc_name=f"rfc{num}",
doc_rev=None,
# Not setting content_type
mtime=datetime.datetime.fromtimestamp(mtime, tz=datetime.UTC),
)
except AlreadyExistsError as e:
log.log(str(e))
else:
log.log(
f"Skipping loading rfc{num} into blobdb as no matching Document exists"
)
Loading