bug#77313: 30.1.50; Regression: flymake indicators are erroneously using margins

[Spencer Baugh]

Here's an updated version of my previous patch which avoids the need for
a new variant in the defcustom.  When flymake-indicator-type is set to
fringes and we're on a text frame, we simply fall back to using margins.
That allows it to be set to fringes by default without losing
functionality for text terminals; this is how this should have been
implemented originally.

Can we make progress on at least installing this on master, so we can
test it before backporting it to Emacs 30.2?

>From d6122447985b9f1f20b86c3ea714cc447eedcf39 Mon Sep 17 00:00:00 2001
From: Spencer Baugh <sbaugh@janestreet.com>
Date: Tue, 8 Apr 2025 08:43:37 -0400
Subject: [PATCH] flymake: fall back to margins on text terminals

Previously, flymake-indicator-type defaulted to either fringes
or margins.  But fringes should be used on graphical frames, and
margins on TTY frames.  So default to fringes instead, and
simply fall back to margins automatically on text frames.

* lisp/progmodes/flymake.el (flymake-indicator-type): Set to
fringes.  (bug#77313)
(flymake--use-fringes-p, flymake--use-margins-p): Add.
(flymake--resize-margins): Check flymake--use-margins-p.
(flymake--highlight-line): Check flymake--use-fringes-p and
flymake--use-margins.
---
 lisp/progmodes/flymake.el | 28 ++++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/lisp/progmodes/flymake.el b/lisp/progmodes/flymake.el
index 6cc7e1f7a79..4b40f9b9c31 100644
--- a/lisp/progmodes/flymake.el
+++ b/lisp/progmodes/flymake.el
@@ -185,22 +185,22 @@ flymake-fringe-indicator-position
                 (const right-fringe)
                 (const :tag "No fringe indicators" nil)))
 
-(defcustom flymake-indicator-type (if (display-graphic-p)
-                                      'fringes
-                                    'margins)
+(defcustom flymake-indicator-type 'fringes
   "Indicate which indicator type to use for display errors.
 
 The value can be nil (don't indicate errors but just highlight them),
-fringes (use fringes) or margins (use margins)
+`fringes' (use fringes) or `margins' (use margins)
 
 Difference between fringes and margin is that fringes support displaying
 bitmaps on graphical displays and margins display text in a blank area
 from current buffer that works in both graphical and text displays.
+Thus, even when `fringes' is selected, margins will still be used on
+text displays.
 
 See Info node `Fringes' and Info node `(elisp)Display Margins'."
   :version "30.1"
   :type '(choice (const :tag "Use Fringes" fringes)
-                 (const :tag "Use Margins "margins)
+                 (const :tag "Use Margins" margins)
                  (const :tag "No indicators" nil)))
 
 (defcustom flymake-margin-indicators-string
@@ -752,9 +752,9 @@ flymake--indicator-overlay-spec
   "Return INDICATOR as propertized string to use in error indicators."
   (let* ((indicator (flymake--lookup-type-property
                      type
-                     (cond ((eq flymake-indicator-type 'fringes)
+                     (cond ((flymake--use-fringes-p)
                             'flymake-bitmap)
-                           ((eq flymake-indicator-type 'margins)
+                           ((flymake--use-margins-p)
                             'flymake-margin-string))
                      (alist-get 'bitmap (alist-get type ; backward compat
                                                    
flymake-diagnostic-types-alist))))
@@ -784,10 +784,22 @@ flymake--indicator-overlay-spec
                                              (format "<%s> <mouse-1>" 
flymake-margin-indicator-position)
                                              
#'flymake-show-buffer-diagnostics-at-event-line))))))))
 
+(defun flymake--use-fringes-p ()
+  (cl-case flymake-indicator-type
+    (fringes (display-graphic-p))
+    ((nil margins) nil)
+    (t t)))
+
+(defun flymake--use-margins-p ()
+  (cl-case flymake-indicator-type
+    (fringes (not (display-graphic-p)))
+    (margins t)
+    (t nil)))
+
 (defun flymake--resize-margins (&optional orig-width)
   "Resize current window margins according to 
`flymake-margin-indicator-position'.
 Return to original margin width if ORIG-WIDTH is non-nil."
-  (when (and (eq flymake-indicator-type 'margins)
+  (when (and (flymake--use-margins-p)
              flymake-autoresize-margins)
     (cond
      ((and orig-width flymake--original-margin-width)
-- 
2.39.3