diff --git a/.changeset/rude-chefs-admire.md b/.changeset/rude-chefs-admire.md
new file mode 100644
index 00000000000..c41901da267
--- /dev/null
+++ b/.changeset/rude-chefs-admire.md
@@ -0,0 +1,7 @@
+---
+'@primer/react': patch
+---
+
+Button and IconButton only use aria-labelledby to preserve the button label in a loading state OR if aria-labelledBy was explicitly passed.
+
+<!-- Changed components: Button, IconButton -->
diff --git a/packages/react/src/Button/ButtonBase.tsx b/packages/react/src/Button/ButtonBase.tsx
index 8458ecd8ad5..33a75fbb058 100644
--- a/packages/react/src/Button/ButtonBase.tsx
+++ b/packages/react/src/Button/ButtonBase.tsx
@@ -62,7 +62,6 @@ const ButtonBase = forwardRef(
     }, [baseStyles, sxProp])
     const uuid = useId(id)
     const loadingAnnouncementID = `${uuid}-loading-announcement`
-    const buttonLabelID = ariaLabelledBy || `${uuid}-label`
 
     if (__DEV__) {
       /**
@@ -100,8 +99,10 @@ const ButtonBase = forwardRef(
           aria-describedby={[loadingAnnouncementID, ariaDescribedBy]
             .filter(descriptionID => Boolean(descriptionID))
             .join(' ')}
-          // aria-labelledby is needed because the accessible name becomes unset when the button is in a loading state
-          aria-labelledby={buttonLabelID}
+          // aria-labelledby is needed because the accessible name becomes unset when the button is in a loading state.
+          // We only set it when the button is in a loading state because it will supercede the aria-label when the screen
+          // reader announces the button name.
+          aria-labelledby={loading ? `${uuid}-label` : ariaLabelledBy}
           id={id}
           onClick={loading ? undefined : onClick}
         >
@@ -117,7 +118,7 @@ const ButtonBase = forwardRef(
                 {loading && !LeadingVisual && !TrailingVisual && renderVisual(Spinner, loading, 'loadingSpinner')}
                 {LeadingVisual && renderVisual(LeadingVisual, loading, 'leadingVisual')}
                 {children && (
-                  <span data-component="text" id={buttonLabelID}>
+                  <span data-component="text" id={loading ? `${uuid}-label` : undefined}>
                     {children}
                     {count !== undefined && !TrailingVisual && (
                       <CounterLabel data-component="ButtonCounter" sx={{ml: 2}}>
diff --git a/packages/react/src/Button/__tests__/__snapshots__/Button.test.tsx.snap b/packages/react/src/Button/__tests__/__snapshots__/Button.test.tsx.snap
index d01040717b0..c33ffce6c69 100644
--- a/packages/react/src/Button/__tests__/__snapshots__/Button.test.tsx.snap
+++ b/packages/react/src/Button/__tests__/__snapshots__/Button.test.tsx.snap
@@ -264,7 +264,6 @@ exports[`Button respects block prop 1`] = `
 
 <button
   aria-describedby="test-button-loading-announcement"
-  aria-labelledby="test-button-label"
   class="c0"
   data-block="block"
   data-loading="false"
@@ -278,7 +277,6 @@ exports[`Button respects block prop 1`] = `
   >
     <span
       data-component="text"
-      id="test-button-label"
     >
       Block
     </span>
@@ -550,7 +548,6 @@ exports[`Button respects the alignContent prop 1`] = `
 
 <button
   aria-describedby="test-button-loading-announcement"
-  aria-labelledby="test-button-label"
   class="c0"
   data-loading="false"
   data-no-visuals="true"
@@ -563,7 +560,6 @@ exports[`Button respects the alignContent prop 1`] = `
   >
     <span
       data-component="text"
-      id="test-button-label"
     >
       Align start
     </span>
@@ -835,7 +831,6 @@ exports[`Button respects the large size prop 1`] = `
 
 <button
   aria-describedby="test-button-loading-announcement"
-  aria-labelledby="test-button-label"
   class="c0"
   data-loading="false"
   data-no-visuals="true"
@@ -849,7 +844,6 @@ exports[`Button respects the large size prop 1`] = `
   >
     <span
       data-component="text"
-      id="test-button-label"
     >
       Smol
     </span>
@@ -1121,7 +1115,6 @@ exports[`Button respects the small size prop 1`] = `
 
 <button
   aria-describedby="test-button-loading-announcement"
-  aria-labelledby="test-button-label"
   class="c0"
   data-loading="false"
   data-no-visuals="true"
@@ -1135,7 +1128,6 @@ exports[`Button respects the small size prop 1`] = `
   >
     <span
       data-component="text"
-      id="test-button-label"
     >
       Smol
     </span>
@@ -1410,7 +1402,6 @@ exports[`Button styles danger button appropriately 1`] = `
 
 <button
   aria-describedby="test-button-loading-announcement"
-  aria-labelledby="test-button-label"
   class="c0"
   data-loading="false"
   data-no-visuals="true"
@@ -1423,7 +1414,6 @@ exports[`Button styles danger button appropriately 1`] = `
   >
     <span
       data-component="text"
-      id="test-button-label"
     >
       Danger
     </span>
@@ -1707,7 +1697,6 @@ exports[`Button styles invisible button appropriately 1`] = `
 
 <button
   aria-describedby="test-button-loading-announcement"
-  aria-labelledby="test-button-label"
   class="c0"
   data-loading="false"
   data-no-visuals="true"
@@ -1720,7 +1709,6 @@ exports[`Button styles invisible button appropriately 1`] = `
   >
     <span
       data-component="text"
-      id="test-button-label"
     >
       Invisible
     </span>
@@ -1991,7 +1979,6 @@ exports[`Button styles primary button appropriately 1`] = `
 
 <button
   aria-describedby="test-button-loading-announcement"
-  aria-labelledby="test-button-label"
   class="c0"
   data-loading="false"
   data-no-visuals="true"
@@ -2004,7 +1991,6 @@ exports[`Button styles primary button appropriately 1`] = `
   >
     <span
       data-component="text"
-      id="test-button-label"
     >
       Primary
     </span>
diff --git a/packages/react/src/__tests__/__snapshots__/TextInput.test.tsx.snap b/packages/react/src/__tests__/__snapshots__/TextInput.test.tsx.snap
index 0612f23b7e4..846c0f90855 100644
--- a/packages/react/src/__tests__/__snapshots__/TextInput.test.tsx.snap
+++ b/packages/react/src/__tests__/__snapshots__/TextInput.test.tsx.snap
@@ -2681,7 +2681,6 @@ exports[`TextInput renders trailingAction text button 1`] = `
   >
     <button
       aria-describedby=":r0:-loading-announcement"
-      aria-labelledby=":r0:-label"
       className="c4"
       data-block={null}
       data-loading={false}
@@ -2696,7 +2695,6 @@ exports[`TextInput renders trailingAction text button 1`] = `
       >
         <span
           data-component="text"
-          id=":r0:-label"
         >
           Clear
         </span>
@@ -3249,7 +3247,6 @@ exports[`TextInput renders trailingAction text button with a tooltip 1`] = `
   >
     <button
       aria-describedby=":r2:-loading-announcement :r1:"
-      aria-labelledby=":r2:-label"
       className="c4"
       data-block={null}
       data-loading={false}
@@ -3268,7 +3265,6 @@ exports[`TextInput renders trailingAction text button with a tooltip 1`] = `
       >
         <span
           data-component="text"
-          id=":r2:-label"
         >
           Clear
         </span>