bug#40808: 27.0.91; inaccuracy in isearch-lazy-count

[Juri Linkov]

> What's the status of this bug?  Apart from the request
> by Juri for changes to be made to Org mode, is the code
> Juri prescribed for this bug fix going to be applied,
> or has it been applied?
>
> Is fixing this bug (e.g. with Juri's code) somehow
> dependent on also fixing some Org-mode code?

Fortunately, the isearch fix doesn't depend on org-mode.
So here is the complete implementation that was made possible
thanks to the suggestions in
https://lists.gnu.org/archive/html/emacs-devel/2022-06/msg01208.html

For example, searching for the word "nongnu" in etc/ORG-NEWS
when all outlines are collapsed with isearch-lazy-count=t
will show such prompts:

search-invisible = t      1/4 I-search:
search-invisible = open   1/2 (invisible 2) I-search:
search-invisible = nil    0/0 (invisible 4) I-search:

where such numbers as "1/2" show the matches that can be visited by isearch,
and "(invisible 2)" are the rest of matches unreachable by isearch.

diff --git a/lisp/isearch.el b/lisp/isearch.el
index db7b53c014..945dce029a 100644
--- a/lisp/isearch.el
+++ b/lisp/isearch.el
@@ -466,6 +466,12 @@ lazy-count-suffix-format
   :group 'lazy-count
   :version "27.1")
 
+(defvar lazy-count-invisible-format " (invisible %s)"
+  "Format of the number of invisible matches for the prompt.
+When invisible matches exist, their number is appended
+after the total number of matches.  Display nothing when
+this variable is nil.")
+
 
 ;; Define isearch help map.
 
@@ -1277,6 +1283,7 @@ isearch-mode
 
        isearch-lazy-count-current nil
        isearch-lazy-count-total nil
+       isearch-lazy-count-invisible nil
 
        ;; Save the original value of `minibuffer-message-timeout', and
        ;; set it to nil so that isearch's messages don't get timed out.
@@ -3529,7 +3536,12 @@ isearch-lazy-count-format
                     (- isearch-lazy-count-total
                        isearch-lazy-count-current
                        -1)))
-                (or isearch-lazy-count-total "?"))
+                (if (and isearch-lazy-count-invisible
+                         lazy-count-invisible-format)
+                    (concat (format "%s" (or isearch-lazy-count-total "?"))
+                            (format lazy-count-invisible-format
+                                    isearch-lazy-count-invisible))
+                  (or isearch-lazy-count-total "?")))
       "")))
 
 
@@ -3780,10 +3792,11 @@ isearch-range-invisible
     (save-excursion
       (goto-char beg)
       (let (;; can-be-opened keeps track if we can open some overlays.
-           (can-be-opened (eq search-invisible 'open))
+           (can-be-opened (memq search-invisible '(open can-be-opened)))
            ;; the list of overlays that could be opened
            (crt-overlays nil))
-       (when (and can-be-opened isearch-hide-immediately)
+       (when (and can-be-opened isearch-hide-immediately
+                   (not (eq search-invisible 'can-be-opened)))
          (isearch-close-unnecessary-overlays beg end))
        ;; If the following character is currently invisible,
        ;; skip all characters with that same `invisible' property value.
@@ -3822,9 +3835,10 @@ isearch-range-invisible
        (if (>= (point) end)
            (if (and can-be-opened (consp crt-overlays))
                (progn
-                 (setq isearch-opened-overlays
-                       (append isearch-opened-overlays crt-overlays))
-                 (mapc 'isearch-open-overlay-temporary crt-overlays)
+                 (unless (eq search-invisible 'can-be-opened)
+                    (setq isearch-opened-overlays
+                         (append isearch-opened-overlays crt-overlays))
+                   (mapc 'isearch-open-overlay-temporary crt-overlays))
                  nil)
              (setq isearch-hidden t)))))))
 
@@ -4008,6 +4022,7 @@ isearch-lazy-highlight-forward
 (defvar isearch-lazy-highlight-error nil)
 (defvar isearch-lazy-count-current nil)
 (defvar isearch-lazy-count-total nil)
+(defvar isearch-lazy-count-invisible nil)
 (defvar isearch-lazy-count-hash (make-hash-table))
 (defvar lazy-count-update-hook nil
   "Hook run after new lazy count results are computed.")
@@ -4086,7 +4101,8 @@ isearch-lazy-highlight-new-loop
         ;; Reset old counter before going to count new numbers
         (clrhash isearch-lazy-count-hash)
         (setq isearch-lazy-count-current nil
-              isearch-lazy-count-total nil)
+              isearch-lazy-count-total nil
+              isearch-lazy-count-invisible nil)
         ;; Delay updating the message if possible, to avoid flicker
         (when (string-equal isearch-string "")
           (when (and isearch-mode (null isearch-message-function))
@@ -4166,11 +4182,13 @@ isearch-lazy-highlight-search
            (isearch-regexp-lax-whitespace
             isearch-lazy-highlight-regexp-lax-whitespace)
            (isearch-forward isearch-lazy-highlight-forward)
-           ;; Don't match invisible text unless it can be opened
-           ;; or when counting matches and user can visit hidden matches
-           (search-invisible (or (eq search-invisible 'open)
-                                 (and isearch-lazy-count search-invisible)))
-           (retry t)
+            ;; Count all invisible matches, but highlight only
+            ;; according to search-invisible without opening overlays.
+           (search-invisible (or isearch-lazy-count
+                                  (if (eq search-invisible 'open)
+                                      'can-be-opened
+                                    search-invisible)))
+            (retry t)
            (success nil))
        ;; Use a loop like in `isearch-search'.
        (while retry
@@ -4186,15 +4204,23 @@ isearch-lazy-highlight-search
     (error nil)))
 
 (defun isearch-lazy-highlight-match (mb me)
-  (let ((ov (make-overlay mb me)))
-    (push ov isearch-lazy-highlight-overlays)
-    ;; 1000 is higher than ediff's 100+,
-    ;; but lower than isearch main overlay's 1001
-    (overlay-put ov 'priority 1000)
-    (overlay-put ov 'face 'lazy-highlight)
-    (unless (or (eq isearch-lazy-highlight 'all-windows)
-                isearch-lazy-highlight-buffer)
-      (overlay-put ov 'window (selected-window)))))
+  (unless (and isearch-lazy-count
+               ;; Recheck the match that possibly was intended
+               ;; for counting only, but not for highlighting
+               (not (let ((search-invisible
+                           (if (eq search-invisible 'open)
+                               'can-be-opened
+                             search-invisible)))
+                      (funcall isearch-filter-predicate mb me))))
+    (let ((ov (make-overlay mb me)))
+      (push ov isearch-lazy-highlight-overlays)
+      ;; 1000 is higher than ediff's 100+,
+      ;; but lower than isearch main overlay's 1001
+      (overlay-put ov 'priority 1000)
+      (overlay-put ov 'face 'lazy-highlight)
+      (unless (or (eq isearch-lazy-highlight 'all-windows)
+                  isearch-lazy-highlight-buffer)
+        (overlay-put ov 'window (selected-window))))))
 
 (defun isearch-lazy-highlight-start ()
   "Start a new lazy-highlight updating loop."
@@ -4328,11 +4354,19 @@ isearch-lazy-highlight-buffer-update
                                (setq found nil)
                              (forward-char -1)))
                        (when isearch-lazy-count
-                         (setq isearch-lazy-count-total
-                               (1+ (or isearch-lazy-count-total 0)))
-                         (puthash (if isearch-lazy-highlight-forward me mb)
-                                  isearch-lazy-count-total
-                                  isearch-lazy-count-hash))
+                          ;; Count as invisible when can't open overlay
+                         (if (not (let ((search-invisible
+                                          (if (eq search-invisible 'open)
+                                              'can-be-opened
+                                            search-invisible)))
+                                     (funcall isearch-filter-predicate mb me)))
+                             (setq isearch-lazy-count-invisible
+                                   (1+ (or isearch-lazy-count-invisible 0)))
+                           (setq isearch-lazy-count-total
+                                 (1+ (or isearch-lazy-count-total 0)))
+                           (puthash (if isearch-lazy-highlight-forward me mb)
+                                    isearch-lazy-count-total
+                                    isearch-lazy-count-hash)))
                        ;; Don't highlight the match when this loop is used
                        ;; only to count matches or when matches were already
                        ;; highlighted within the current window boundaries