[elpa] elpa-admin 913bb04: * GNUmakefile: Rewrite the all-in-place code

[Stefan Monnier]

branch: elpa-admin
commit 913bb0425df269e99104c3b2522097577251e5b8
Author: Stefan Monnier <monnier@iro.umontreal.ca>
Commit: Stefan Monnier <monnier@iro.umontreal.ca>

    * GNUmakefile: Rewrite the all-in-place code
    
    (PKG_DESCS_MK, RM): New vars.
    (clean): Use them.  Also remove the *-pkg.el files.
    (clean/%): New target(s).
    (pkgs): Simplify.
    ($(PKG_DESCS_MK)): New target.
    (packages/%-pkg.el): Rely on $(PKG_DESCS_MK) for the dependencies.
    (packages/%): Clean up the output a bit.
    
    * README (Make targets): New node.
    
    * elpa-admin.el:
    (elpaa--make-one-tarball): Handle `:make` before `:doc`.
    (elpaa--make): Allow the argument of `:make` to be a list of targets.
    (elpaa-batch-pkg-spec-make-dependencies): New function.
---
 GNUmakefile   | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------
 README        | 51 +++++++++++++++++++++++++++++++
 elpa-admin.el | 35 +++++++++++++++++++--
 3 files changed, 169 insertions(+), 14 deletions(-)

diff --git a/GNUmakefile b/GNUmakefile
index ca2f120..5eab0b5 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -2,6 +2,9 @@
 #
 
 EMACS=emacs --batch
+RM=rm -f
+
+PKG_DESCS_MK=.pkg-descs.mk
 
 .PHONY: all
 all: all-in-place
@@ -24,8 +27,16 @@ build-all:
 .PHONY: clean
 clean:
 #      rm -rf archive $(ARCHIVE_TMP)
-       rm -f packages/*/*-autoloads.el
-       find packages -name '*.elc' -print0 | xargs -0 rm -f
+       $(RM) $(PKG_DESCS_MK)
+       $(RM) packages/*/*-autoloads.el
+       $(RM) packages/*/*-pkg.el
+       find packages -name '*.elc' -print0 | xargs -0 $(RM)
+
+.PHONY: clean/%
+clean/%:
+       $(RM) packages/$*/*-autoloads.el
+       $(RM) packages/$*/*-pkg.el
+       find packages/$* -name '*.elc' -print0 | xargs -0 $(RM)
 
 .PHONY: readme
 readme:
@@ -34,8 +45,7 @@ readme:
                          (org-export-to-file 'html \"html/readme.html\"))"
 
 ########## Rules for in-place installation ####################################
-pkgs := $(foreach pkg, $(wildcard packages/*), \
-          $(if $(shell [ -d "$(pkg)" ] && echo true), $(pkg)))
+pkgs := $(wildcard packages/*)
 
 define SET-diff
 $(shell $(file > .tmp.setdiff, $(1))  \
@@ -119,20 +129,27 @@ packages/%.elc: packages/%.el
 .PHONY: $(extra_elcs)
 # $(extra_elcs):; rm $@
 
+
+include $(PKG_DESCS_MK)
+$(PKG_DESCS_MK): elpa-packages
+       $(EMACS) -Q -l admin/elpa-admin.el \
+                -f elpaa-batch-pkg-spec-make-dependencies $@
+
 # # Put into single_pkgs the set of -pkg.el files we need to keep up-to-date.
 # # I.e. all the -pkg.el files for the single-file packages.
 pkg_descs:=$(foreach pkg, $(pkgs), $(pkg)/$(notdir $(pkg))-pkg.el)
 #$(foreach al, $(single_pkgs), $(eval $(call RULE-srcdeps, $(al))))
-packages/%-pkg.el: packages/%.el
+packages/%-pkg.el:
        @echo 'Generating description file $@'
        @$(EMACS) -l admin/elpa-admin.el \
                  -f elpaa-batch-generate-description-file "$@"
 
 .PHONY: all-in-place
 # Use order-only prerequisites, so that autoloads are done first.
-all-in-place: | $(extra_elcs) $(autoloads) $(pkg_descs) elcs
+all-in-place: | $(autoloads) $(pkg_descs) $(pkgs) #$(extra_elcs)
 
-define FILE-deps
+# arg1 is the % of packages/%, returns the list of .el and .elc files
+define FILE-files
 $(if $(findstring /, $(1)),                         \
      $(if $(patsubst %.elc,,$(1)),                  \
           $(patsubst %.elc, %.el, $(1))),           \
@@ -143,8 +160,64 @@ $(if $(findstring /, $(1)),                             \
                    --exclude-ignore=.elpaignore      \
                   --exclude='*-pkg.el'              \
                   --exclude='*-autoloads.el'        \
+                  --exclude='.dir-locals.el'        \
                    --exclude-vcs packages/$(1) 2>&1  \
-               | sed -ne 's/\.el$$/.elc/p';}))
+               | sed -ne 's/\(\.elc*\)$$/\1/p';}))
+endef
+
+define FILE-els
+$(filter %.el, $(1))
+endef
+
+define FILE-elcs
+$(filter %.elc, $(1))
+endef
+
+# Takes two args: arg1 is a set of .el and arg2 a set of .elc
+# Return the set of .el files that don't yet have a .elc.
+define FILE-notyetcompiled
+$(call SET-diff, $(1), $(patsubst %.elc, %.el, $(2)))
+endef
+
+# Takes a set of .el files and returns those that can't be byte-compiled.
+define FILE-nobytecompile
+$(foreach el, $(1), \
+          $(if $(shell grep '^;.*no-byte-compile: *t' "$(el)"), $(el)))
+endef
+
+# Takes a set of .el in arg1 and .elc files in arg2
+# and generates the set of .elc files that need to be generated.
+define FILE-computeddeps2
+$(patsubst %.el, %.elc,                        \
+    $(call SET-diff, $(1),                     \
+                     $(call FILE-nobytecompile, \
+                            $(call FILE-notyetcompiled, $(1), $(2)))))
+endef
+
+# Takes a pkgname in arg1 and a set of .el and .elc files in arg2, and
+# generates the set of .elc files that need to be generated.
+define FILE-computeddeps1
+$(call FILE-computeddeps2, $(call FILE-els, $(1)), $(call FILE-elcs, $(1)))
+endef
+
+# Compute the dependencies for a file packages/%.
+# The main case is for the `packages/[PKGNAME]` directory.
+# FIXME: Remove outdated .elc files with no matching .el file!
+define FILE-deps
+$(if $(findstring /, $(1)),                           \
+     $(if $(patsubst %.elc,,$(1)),                    \
+          $(patsubst %.elc, %.el, $(1))),             \
+     packages/$(1)/$(1)-pkg.el                        \
+     packages/$(1)/$(1)-autoloads.el                   \
+     $(call FILE-computeddeps1,                               \
+        $(shell [ -d packages/$(1) ] && {             \
+                  tar -cvhf /dev/null                 \
+                      --exclude-ignore=.elpaignore     \
+                      --exclude='*-pkg.el'            \
+                      --exclude='*-autoloads.el'       \
+                      --exclude='.dir-locals.el'       \
+                      --exclude-vcs packages/$(1) 2>&1 \
+                  | sed -ne 's/\(\.elc*\)$$/\1/p';})))
 endef
 
 # define FILE-cmd
@@ -177,9 +250,11 @@ dummy:
 
 .SECONDEXPANSION:
 packages/% : dummy $$(call FILE-deps,$$*)
-       [ -d packages/$* ] ||               \
-            $(EMACS) -l admin/elpa-admin.el \
-                    -f elpaa-batch-archive-update-worktrees "$(@F)"
+       @[ -d packages/$* ] || {                                      \
+            echo $(EMACS) -l admin/elpa-admin.el                     \
+                    -f elpaa-batch-archive-update-worktrees "$(@F)"; \
+            $(EMACS) -l admin/elpa-admin.el                          \
+                    -f elpaa-batch-archive-update-worktrees "$(@F)"; }
 
 #### Fetching updates from upstream                                        ####
 
diff --git a/README b/README
index 3bb95b9..6ef577a 100644
--- a/README
+++ b/README
@@ -112,6 +112,57 @@ It can be either an Info file, a Texinfo file, or an Org 
file.
 
 ** =:make TARGET=
 Indicates that we should run ~make TARGET~ in order to build some files.
+This is run before processing =:doc=, so it can be used to generate
+the Info and Texinfo file from some other format.
+TARGET can also be a list, in which case each target should be
+an actual file name (rather than a "phony target").
+
+* Make targets
+
+** =all-in-place=
+This is the default target, and it prepares all the directories found in
+the =packages= subdirectory for use by =package.el=.  This assume you will
+want to add =.../packages= to your =package-directory-list= so as to treat
+those packages as installed "in place".
+
+** =packages/[PKGNAME]=
+Prepare the subdirectory =packages/[PKGNAME]= for use by =package.el=.
+If the directory does not exist yet, it checks it out as a Git worktree.
+Else, it generates the auxiliary files like =[PKGNAME]-pkg.el= and
+=[PKGNAME]-autoloads.el= and compiles the Elisp source files.
+
+** =build/[PKGNAME]=
+Build the ELPA tarball(s) for PKGNAME.  The result is placed in
+the =archive= and =archive-devel= subdirectories.
+
+** =build-all=
+Same as before but does it for all the packages listed in the
+=elpa-packages= file.
+
+** =fetch/[PKGNAME]=
+Fetch changes from the upstream repository of package PKGNAME.
+Does not apply them to anywhere, only keeps them in Git and shows
+a concise summary of the new commits compared to the current
+content of the archive.
+
+** =fetch-all=
+Same as before but fetches changes for all the packages listed in the
+=elpa-packages= file.
+
+** =sync/[PKGNAME]=
+Fetch changes from the upstream repository of package PKGNAME
+and push them to the corresponding branch in the main repository (such as
+=elpa.git=).
+
+** =sync-all=
+Same as before but does it for all the packages listed in the
+=elpa-packages= file.
+
+** =clean=
+Delete all the files generated by a plain =make=.
+
+** =clean/[PKGNAME]=
+Delete all the files generated by =make packages/[PKGNAME]=.
 
 * Text below this marker is OUTDATED and still needs to be reviewed/rewritten!!
 
diff --git a/elpa-admin.el b/elpa-admin.el
index 81339e1..f01fff9 100644
--- a/elpa-admin.el
+++ b/elpa-admin.el
@@ -345,9 +345,10 @@ Return non-nil if a new tarball was created."
         (when revision-function
           (elpaa--select-revision dir pkg-spec (funcall revision-function)))
         (elpaa--copyright-check pkg-spec)
-        ;; FIXME: Build Info files and corresponding `dir' file.
-        (elpaa--build-Info pkg-spec dir)
+        ;; Run `make' before building the Info file, so that the `make' rule
+        ;; can be used to build the Info/Texinfo file.
         (elpaa--make pkg-spec dir)
+        (elpaa--build-Info pkg-spec dir)
         (elpaa--write-pkg-file dir pkgname metadata)
         ;; FIXME: Allow renaming files or selecting a subset of the files!
         (cl-assert (not (string-match "[][*\\|?]" pkgname)))
@@ -1440,7 +1441,8 @@ More at " (elpaa--default-url pkgname))
     (when target
       (with-temp-buffer
         (let ((default-directory (elpaa--dirname dir)))
-          (elpaa--call-sandboxed t "make" target)
+          (apply #'elpaa--call-sandboxed t "make"
+                 (if (consp target) target (list target)))
           (elpaa--message "%s" (buffer-string)))))))
 
 ;;; Fetch updates from upstream
@@ -1589,6 +1591,33 @@ More at " (elpaa--default-url pkgname))
 
   (ert-run-tests-batch-and-exit t))
 
+;;; Make dependencies
+
+(defun elpaa-batch-pkg-spec-make-dependencies ()
+  (let ((dst (pop command-line-args-left)))
+    (with-temp-buffer
+      (dolist (pkg-spec (elpaa--get-specs))
+        (let ((pkgname (car pkg-spec)))
+          (insert
+           (format "packages/%s/%s-pkg.el: packages/%s/%s\n"
+                   pkgname pkgname pkgname (elpaa--main-file pkg-spec)))
+          (let ((make-targets (elpaa--spec-get pkg-spec :make)))
+            (when (consp make-targets)
+              (dolist (target make-targets)
+                (insert (format "packages/%s: packages/%s/%s\n"
+                                pkgname pkgname target))
+                (insert (format "packages/%s/%s:
+\tcd packages/%s; $(MAKE) %s\n"
+                                pkgname target pkgname target)))
+              (insert (format "clean-submake/%s:\n\t$(RM) %s\n"
+                              pkgname
+                              (mapconcat (lambda (f)
+                                           (concat "packages/" pkgname "/" f))
+                                         make-targets
+                                         " ")))
+              (insert (format "clean clean/%s: clean-submake/%s\n"
+                              pkgname pkgname))))))
+      (write-region (point-min) (point-max) dst nil 'silent))))
 
 (provide 'elpa-admin)
 ;;; elpa-admin.el ends here