diff --git a/polygerrit-ui/app/elements/shared/gr-account-label/gr-account-label.js b/polygerrit-ui/app/elements/shared/gr-account-label/gr-account-label.js
index aa2edbf..9122a87 100644
--- a/polygerrit-ui/app/elements/shared/gr-account-label/gr-account-label.js
+++ b/polygerrit-ui/app/elements/shared/gr-account-label/gr-account-label.js
@@ -136,7 +136,11 @@
   }
 
   _hasAttention(config, highlight, account, change, force) {
-    if (force) return true;
+    return force || this._hasUnforcedAttention(config, highlight, account,
+        change);
+  }
+
+  _hasUnforcedAttention(config, highlight, account, change) {
     return this._isAttentionSetEnabled(config, highlight, account, change)
         && change.attention_set
         && change.attention_set.hasOwnProperty(account._account_id);
@@ -196,6 +200,12 @@
       targetIsSelf: targetId === selfId,
     };
   }
+
+  _computeAttentionIconTtitle(config, highlight, account, change) {
+    return this._hasUnforcedAttention(config, highlight, account, change)
+      ? 'Click to remove the user from the attention set'
+      : 'Disabled. Use "Modify" to make changes.';
+  }
 }
 
 customElements.define(GrAccountLabel.is, GrAccountLabel);
diff --git a/polygerrit-ui/app/elements/shared/gr-account-label/gr-account-label_html.ts b/polygerrit-ui/app/elements/shared/gr-account-label/gr-account-label_html.ts
index a721734..e3c0e14 100644
--- a/polygerrit-ui/app/elements/shared/gr-account-label/gr-account-label_html.ts
+++ b/polygerrit-ui/app/elements/shared/gr-account-label/gr-account-label_html.ts
@@ -106,8 +106,9 @@
         link=""
         aria-label="Remove user from attention set"
         on-click="_handleRemoveAttentionClick"
-        has-tooltip=""
-        title="Click to remove the user from the attention set"
+        disabled="[[!_hasUnforcedAttention(_config, highlightAttention, account, change)]]"
+        has-tooltip="[[_hasUnforcedAttention(_config, highlightAttention, account, change)]]"
+        title="[[_computeAttentionIconTtitle(_config, highlightAttention, account, change)]]"
         ><iron-icon class="attention" icon="gr-icons:attention"></iron-icon>
       </gr-button>
     </template>