feature/tree-sitter 18947103fa 5/5: Use tree-sitter for font-lock's syntactic function (optionally)

[Yuan Fu]

branch: feature/tree-sitter
commit 18947103fabf8070738b3bd9c5a8d02f90988a3d
Author: Yuan Fu <casouri@gmail.com>
Commit: Yuan Fu <casouri@gmail.com>

    Use tree-sitter for font-lock's syntactic function (optionally)
    
    Before this change, we rebind font-lock-fontify-region-function and
    call font-lock's function in tree-sitter's function, and the order of
    fontification is tree-sitter, font-lock syntax, font-lock regexp.
    
    Now we make font-lock's syntax function customizable and replace it
    with tree-sitter's function, and the order of fontification is
    tree-sitter, font-lock regexp.
    
    * doc/lispref/modes.texi (Parser-based Font Lock): Reflect the change
    in manual.
    * lisp/font-lock.el (font-lock-fontify-syntactically-function): New
    varaible.
    (font-lock-default-fontify-region): Call
    font-lock-fontify-syntactically-function rather.
    (font-lock-fontify-syntactically-region): Rename to
    font-lock-default-fontify-syntactically
    (font-lock-default-fontify-syntactically): Rename to this.
    
    * lisp/treesit.el (treesit-font-lock-fontify-region): Don't call
    font-lock functions.
    (treesit-font-lock-enable): Remove this function.  It is not used even
    before this change.
    (treesit-major-mode-setup): Instead of binding
    font-lock-fontify-region-function, now bind to
    font-lock-fontify-syntactically-function.  And we can let font-lock do
    it's thing.
---
 doc/lispref/modes.texi |  3 +--
 lisp/font-lock.el      | 18 +++++++++++++++---
 lisp/treesit.el        | 31 ++++---------------------------
 3 files changed, 20 insertions(+), 32 deletions(-)

diff --git a/doc/lispref/modes.texi b/doc/lispref/modes.texi
index 614bbe1293..d778636d6d 100644
--- a/doc/lispref/modes.texi
+++ b/doc/lispref/modes.texi
@@ -3892,8 +3892,7 @@ Source}) for this purpose.
 
 Parser-based font lock and other font lock mechanisms are not mutually
 exclusive.  By default, if enabled, parser-based font lock runs first,
-then the syntactic font lock (if enabled), then the regexp-based
-font lock.
+replacing syntactic font lock, then the regexp-based font lock.
 
 Although parser-based font lock doesn't share the same customization
 variables with regexp-based font lock, it uses similar customization
diff --git a/lisp/font-lock.el b/lisp/font-lock.el
index b6f4150964..e88dc1d3b7 100644
--- a/lisp/font-lock.el
+++ b/lisp/font-lock.el
@@ -614,6 +614,14 @@ If it fontifies a larger region, it should ideally return 
a list of the form
 \(jit-lock-bounds BEG . END) indicating the bounds of the region actually
 fontified.")
 
+(defvar font-lock-fontify-syntactically-function
+  #'font-lock-default-fontify-syntactically
+  "Function to use for syntactically fontifying a region.
+
+It should take two args, the beginning and end of the region, and
+an optional third arg VERBOSE.  If VERBOSE is non-nil, the
+function should print status messages.")
+
 (defvar font-lock-unfontify-region-function 
#'font-lock-default-unfontify-region
   "Function to use for unfontifying a region.
 It should take two args, the beginning and end of the region.
@@ -942,7 +950,7 @@ The value of this variable is used when Font Lock mode is 
turned on.")
 ;; A further reason to use the fontification indirection feature is when the
 ;; default syntactic fontification, or the default fontification in general,
 ;; is not flexible enough for a particular major mode.  For example, perhaps
-;; comments are just too hairy for `font-lock-fontify-syntactically-region' to
+;; comments are just too hairy for `font-lock-default-fontify-syntactically' to
 ;; cope with.  You need to write your own version of that function, e.g.,
 ;; `hairy-fontify-syntactically-region', and make your own version of
 ;; `hairy-fontify-region' call that function before calling
@@ -952,6 +960,10 @@ The value of this variable is used when Font Lock mode is 
turned on.")
 ;; example, TeX modes could fontify {\foo ...} and \bar{...}  etc. multi-line
 ;; directives correctly and cleanly.  (It is the same problem as fontifying
 ;; multi-line strings and comments; regexps are not appropriate for the job.)
+;; (This comment is written before `font-lock-default-fontify-syntactically'
+;; can be replaced.  Now you can obviously replace
+;; `font-lock-default-fontify-syntactically' with a custom function.)
+
 
 (defvar-local font-lock-extend-after-change-region-function nil
   "A function that determines the region to refontify after a change.
@@ -1181,7 +1193,7 @@ This function is the default 
`font-lock-fontify-region-function'."
            (setq font-lock-syntactically-fontified end))
          (font-lock-fontify-syntactic-keywords-region start end)))
      (unless font-lock-keywords-only
-       (font-lock-fontify-syntactically-region beg end loudly))
+       (funcall font-lock-fontify-syntactically-function beg end loudly))
      (font-lock-fontify-keywords-region beg end loudly)
      `(jit-lock-bounds ,beg . ,end))))
 
@@ -1531,7 +1543,7 @@ START should be at the beginning of a line."
 (defvar font-lock-comment-end-skip nil
   "If non-nil, Font Lock mode uses this instead of `comment-end-skip'.")
 
-(defun font-lock-fontify-syntactically-region (start end &optional loudly)
+(defun font-lock-default-fontify-syntactically (start end &optional loudly)
   "Put proper face on each string and comment between START and END.
 START should be at the beginning of a line."
   (syntax-propertize end)  ; Apply any needed syntax-table properties.
diff --git a/lisp/treesit.el b/lisp/treesit.el
index 4844127509..b1c712fea5 100644
--- a/lisp/treesit.el
+++ b/lisp/treesit.el
@@ -656,24 +656,8 @@ If LOUDLY is non-nil, display some debugging information."
                 (when (or loudly tresit--font-lock-verbose)
                   (message "Fontifying text from %d to %d, Face: %s Language: 
%s"
                            start end face language)))))))))
-  ;; Call regexp font-lock after tree-sitter, as it is usually used
-  ;; for custom fontification.
-  (let ((font-lock-unfontify-region-function #'ignore))
-    (funcall #'font-lock-default-fontify-region start end loudly))
   `(jit-lock-bounds ,start . ,end))
 
-(defun treesit-font-lock-enable ()
-  "Enable tree-sitter font-locking for the current buffer."
-  (treesit-font-lock-recompute-features)
-  (setq-local font-lock-fontify-region-function
-              #'treesit-font-lock-fontify-region)
-  ;; If we don't set `font-lock-defaults' to some non-nil value,
-  ;; font-lock doesn't enable properly (`font-lock-mode-internal'
-  ;; doesn't run).  See `font-lock-specified-p'.
-  (when (null font-lock-defaults)
-    (setq font-lock-defaults '(nil)))
-  (font-lock-mode 1))
-
 ;;; Indent
 
 (defvar treesit--indent-verbose nil
@@ -1306,17 +1290,10 @@ If `treesit-defun-type-regexp' is non-nil, setup
   (when treesit-font-lock-settings
     ;; `font-lock-mode' wouldn't setup properly if
     ;; `font-lock-defaults' is nil, see `font-lock-specified-p'.
-    ;; And we disable syntax-table-based font-lock by setting the
-    ;; KEYWORD-ONLY flag to t, so syntax-table-based font-lock
-    ;; doesn't override tree-sitter's fontification.
-    (setq-local font-lock-defaults '(nil t))
-    (setq-local font-lock-fontify-region-function
-                #'treesit-font-lock-fontify-region)
-    ;; `font-lock-mode' sets this to t when syntactic font-lock is
-    ;; enabled (i.e., `font-lock-keywords-only' is nil).  We disable
-    ;; font-lock's syntactic fontification, and do it ourselves, so we
-    ;; still need `jit-lock-contextually' to be t, set it ourselves.
-    (setq-local jit-lock-contextually t)
+    (setq-local font-lock-defaults
+                '( nil nil nil nil
+                   (font-lock-fontify-syntactically-function
+                    . treesit-font-lock-fontify-region)))
     (font-lock-mode 1)
     (treesit-font-lock-recompute-features))
   ;; Indent.