[elpa] elpa-admin a3dcbb3 225/357: Merge branch 'master' into new-website

[Stefan Monnier]

branch: elpa-admin
commit a3dcbb30e7d51e35c5259cdbbabcb5752301c3ac
Merge: 7e4545d 62629ab
Author: Nicolas Petton <nicolas@petton.fr>
Commit: Nicolas Petton <nicolas@petton.fr>

    Merge branch 'master' into new-website
---
 GNUmakefile               |  74 ++++++++++++++++++++-------
 README                    |  63 +++++++++++++++++++----
 admin/archive-contents.el | 128 ++++++++++++++++++++++++++--------------------
 admin/ert-support.el      |  54 +++++++++++++++++++
 4 files changed, 236 insertions(+), 83 deletions(-)

diff --git a/GNUmakefile b/GNUmakefile
index cc5be30..d23d523 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -1,4 +1,5 @@
 # Makefile for GNU Emacs Lisp Package Archive.
+#
 
 EMACS=emacs --batch
 
@@ -13,20 +14,20 @@ CR_EXCEPTIONS=copyright_exceptions
 .PHONY: check_copyrights
 check_copyrights:
        @echo "Compute exceptions >$(CR_EXCEPTIONS)~"
-       @export LC_ALL=C;                                               \
-       (cd packages;                                                   \
-       find . -name '.git' -prune -o                                   \
-              -name 'test' -prune -o                                   \
-              -name '*.el' -print0 |                                   \
-           xargs -0 grep -L 'Free Software Foundation, Inc' |          \
-           grep -v '\(\.dir-locals\|.-\(pkg\|autoloads\)\)\.el$$';     \
-       find . -name '.git' -prune -o -name '*.el' -print |             \
-           while read f; do                                            \
-               fquoted="$$(echo $$f|tr '|' '_')";                      \
-               sed -n -e '/[Cc]opyright.*, *[1-9][-0-9]*,\?$$/N'       \
-                   -e '/Free Software Foundation/d'                    \
-                   -e "s|^\\(.*[Cc]opyright\\)|$$fquoted:\\1|p"        \
-                  "$$f";                                               \
+       @export LC_ALL=C;                                           \
+       (cd packages &&                                             \
+       find . -name '.git' -prune -o                               \
+              -name 'test' -prune -o                               \
+              -name '*.el' -print0 |                               \
+           xargs -0 grep -L 'Free Software Foundation, Inc' |      \
+           grep -v '\(\.dir-locals\|.-\(pkg\|autoloads\)\)\.el$$'; \
+       find . -name '.git' -prune -o -name '*.el' -type f -print | \
+           while read f; do                                        \
+               fquoted="$$(echo $$f|tr '|' '_')";                  \
+               sed -n -e '/[Cc]opyright.*, *[1-9][-0-9]*,\?$$/N'   \
+                   -e '/Free Software Foundation/d'                \
+                   -e "s|^\\(.*[Cc]opyright\\)|$$fquoted:\\1|p"    \
+                  "$$f";                                           \
            done) | sort >$(CR_EXCEPTIONS)~
        diff -u "$(CR_EXCEPTIONS)" "$(CR_EXCEPTIONS)~"
 
@@ -40,14 +41,21 @@ archive-tmp: packages
        mkdir -p $(ARCHIVE_TMP)
        cp -a packages/. $(ARCHIVE_TMP)/packages
 
+# Use && after the cd commands, not ;, to ensure the build fails
+# immediately if the directory $(ARCHIVE_TMP)/packages does not exist.
+# For process-archive this is crucial; otherwise batch-make-archive in
+# archive-contents.el will interpret directories in the current
+# directory as unreleased packages, and recursively delete them,
+# including .git.  Prior to using &&, running "make process-archive"
+# could silently delete all local git history!
 process-archive:
        # FIXME, we could probably speed this up significantly with
        # rules like "%.tar: ../%/ChangeLog" so we only rebuild the packages
        # that have indeed changed.
-       cd $(ARCHIVE_TMP)/packages;                             \
+       cd $(ARCHIVE_TMP)/packages &&                           \
          $(EMACS) -l $(CURDIR)/admin/archive-contents.el       \
                   -f batch-make-archive
-       @cd $(ARCHIVE_TMP)/packages;                            \
+       @cd $(ARCHIVE_TMP)/packages &&                          \
          for pt in *; do                                       \
              if [ -f "$${pt}/.elpaignore" ]; then              \
                  ignore="$${pt}/.elpaignore";                  \
@@ -56,7 +64,7 @@ process-archive:
              fi;                                               \
              if [ -d $$pt ]; then                              \
                  echo "Creating tarball $${pt}.tar" &&         \
-                 tar -cf $${pt}.tar $$pt --exclude-vcs -X "$$ignore";  \
+                 tar -chf $${pt}.tar $$pt --exclude-vcs -X "$$ignore"; \
                  rm -rf $${pt};                                \
              fi;                                               \
          done
@@ -76,7 +84,7 @@ archive-full: archive-tmp org-fetch
 # FIXME: Turn it into an `external', which will require adding the notion of
 # "snapshot" packages.
 org-fetch: archive-tmp
-       cd $(ARCHIVE_TMP)/packages; \
+       cd $(ARCHIVE_TMP)/packages && \
        pkgname=`curl -s http://orgmode.org/elpa/|perl -ne 'push @f, $$1 if 
m/(org-\d{8})\.tar/; END { @f = sort @f; print "$$f[-1]\n"}'`; \
        wget -q http://orgmode.org/elpa/$${pkgname}.tar -O $${pkgname}.tar; \
        if [ -f $${pkgname}.tar ]; then \
@@ -123,7 +131,7 @@ autoloads := $(foreach pkg, $(pkgs), $(pkg)/$(notdir 
$(pkg))-autoloads.el)
 $(foreach al, $(autoloads), $(eval $(call RULE-srcdeps, $(al))))
 %-autoloads.el:
        @echo 'Generating autoloads for $@'
-       @cd $(dir $@); \
+       @cd $(dir $@) && \
          $(EMACS) -l $(CURDIR)/admin/archive-contents.el \
              --eval "(archive--refresh-pkg-file)" \
              --eval "(require 'package)" \
@@ -183,3 +191,31 @@ all-in-place: $(extra_elcs) $(autoloads) $(pkg_descs)
 externals:
        $(EMACS) -l admin/archive-contents.el \
            -f archive-add/remove/update-externals
+
+
+
+
+################### Testing ###############
+
+PACKAGE_DIRS = $(shell find packages -maxdepth 1 -type d)
+PACKAGES=$(subst /,,$(subst packages,,$(PACKAGE_DIRS)))
+
+TOP =$(shell pwd)
+
+define test_template
+$(1)-test:
+       cd packages/$(1);\
+       $(EMACS) -l $(TOP)/admin/ert-support.el \
+               --eval "(ert-support-test-package \"$(TOP)\" '$(1))" \
+
+$(1)-test-log:
+       $(MAKE) $(1)-test > packages/$(1)/$(1).log 2>&1 || { stat=ERROR; }
+endef
+
+$(foreach package,$(PACKAGES),$(eval $(call test_template,$(package))))
+
+PACKAGES_TESTS=$(addsuffix -test-log,$(PACKAGES))
+PACKAGES_LOG=$(foreach package,$(PACKAGES),packages/$(package)/$(package).log)
+
+check: $(PACKAGES_TESTS)
+       $(EMACS) -l ert -f ert-summarize-tests-batch-and-exit $(PACKAGES_LOG)
diff --git a/README b/README
index fe5a4f1..7c5cd36 100644
--- a/README
+++ b/README
@@ -1,4 +1,4 @@
-Copyright (C) 2010-2011, 2014 Free Software Foundation, Inc.
+Copyright (C) 2010-2011, 2014, 2015 Free Software Foundation, Inc.
 See the end of the file for license conditions.
 
 
@@ -34,7 +34,48 @@ safely work on the next version here without worrying about 
the unstable
 code making it to GNU ELPA, and simply update the "version" when you want to
 release the new code.
 
-** To add a package:
+** To add a package: (submission, submit)
+
+Adding a basic package is very simple. There are thorough
+instructional, but the gist is that you:
+
+1. Notify emacs-devel@gnu.org.
+2. Place all files inside `packages/<pkg-name>/'.
+3. `git add', `git commit' and `git push'.
+
+If you don't have push access to the repository, someone will do steps
+2 and 3 for you.
+
+*** Notify emacs-devel@gnu.org
+
+There is no approval process for GNU Elpa packages.  Still,
+you must send an email to emacs-devel for several reasons:
+
+- Notifying other developers;
+- Making sure the package doesn't break FSF rules;
+- Checking if the package is not reinventing the wheel;
+- Ensuring that first-time developers are doing it right.
+
+Before doing anything, please ensure your package follows the
+conventions described in the `** Format' section.  Then, send an email
+to the list with the subject:
+    [ELPA] New package: <pkg-name>
+
+Start your message with an explanation about the package.  A
+copy-paste of the package's Summary and Commentary is perfectly fine
+here, but you can write more or less than that if you'd like.
+
+At the bottom of the message contents include the changes you're going
+to make (the patch).  For a single-file package this can be the
+package file itself instead of the patch.  If you prefer (and if you
+have push access), you can push your changes to a branch called
+`scratch/<pkg-name>', and mention the branch in your message.
+
+After 48h, or once any issues have been addressed, someone will push
+your changes for you.  You should probably also subscribe to
+emacs-devel@gnu.org, since that's where we discuss about GNU Elpa, and
+to bug-gnu-emacs@gnu.org, since that's where people will report bugs
+about your package.
 
 *** Add a simple (1-file) package as packages/<pkg-name>/<pkg-name>.el.
 
@@ -88,12 +129,15 @@ and the web-pages from this source code:
 
 ** External branches
 
-The easiest way to maintain and develop GNU Elpa packages is to just
-edit them right here (in elpa.git).  However, some maintainers may
-prefer to use a dedicated repository or branch for the package.  There
-are two ways to do that: subtrees and externals.
+The above instructions are enough to add regular packages, those that
+are maintained primarily here in the repository.  The instructions
+below are for those maintainers who prefer to use a dedicated
+repository or branch for the package.
+
+There are two ways to do that: subtrees and externals.
 
-Such packages should be listed in the `externals-list' file.
+Either way, such packages should always be listed in the
+`externals-list' file.
 
 In both cases, a copy of the code is kept in the `elpa' repository
 (not necessarily in the master branch) and should be sync'd with the
@@ -101,7 +145,7 @@ upstream every once in a while.  This copy may include 
local changes,
 although these should be kept to a minimum.
 
 If know you don't want a local package, but don't know which of these
-two options you prefere, then use a subtree.
+two options you prefer, then use a subtree.
 
 *** Subtrees
 
@@ -125,7 +169,7 @@ them here, simply do:
 On older git versions "git subtree" might not be available.  You can
 try "git merge -s subtree", or just update git.
 
-- <remote-repo> is the remote's url.  If you've previously used "git
+- <remote-repo> is the remote's URL.  If you've previously used "git
   remote add", then this can be the remote's name.
 - <remote-branch> is the branch you want to pull (probably "master").
 
@@ -193,6 +237,7 @@ packages/ directory.  You can then add that directory, e.g. 
with:
 ** To deploy the package repository as a remotely-accessible archive:
 
    git clone .../elpa
+   (cd elpa; git clone .../emacs)    #If you want to generate :core packages.
    mkdir build
    cd build
    (cd ../elpa; git log --format=%H | tail -n 1) >.changelog-witness
diff --git a/admin/archive-contents.el b/admin/archive-contents.el
index 8862acb..f3b4db8 100755
--- a/admin/archive-contents.el
+++ b/admin/archive-contents.el
@@ -179,7 +179,6 @@ PKG is the name of the package and DIR is the directory 
where it is."
             (error "Can't parse first line of %s" mainfile)
           ;; Grab the other fields, which are not mandatory.
           (let* ((description (match-string 1))
-                 (pv )
                  (version
                   (or (lm-header "package-version")
                       (lm-header "version")
@@ -207,8 +206,9 @@ PKG is the name of the package and DIR is the directory 
where it is."
   "Deploy the contents of DIR into the archive as a simple package.
 Rename DIR/PKG.el to PKG-VERS.el, delete DIR, and return the descriptor."
   ;; Write DIR/foo.el to foo-VERS.el and delete DIR
-  (rename-file (expand-file-name (concat pkg ".el") dir)
-              (concat pkg "-" vers ".el"))
+  (let ((src (expand-file-name (concat pkg ".el") dir)))
+    (funcall (if (file-symlink-p src) #'copy-file #'rename-file)
+            src (concat pkg "-" vers ".el")))
   ;; Add the content of the ChangeLog.
   (let ((cl (expand-file-name "ChangeLog" dir)))
     (with-current-buffer (find-file-noselect (concat pkg "-" vers ".el"))
@@ -436,7 +436,7 @@ Rename DIR/ to PKG-VERS/, and return the descriptor."
   (replace-regexp-in-string "<" "&lt;"
                             (replace-regexp-in-string "&" "&amp;" txt)))
 
-(defun archive--insert-repolinks (name srcdir mainsrcfile url)
+(defun archive--insert-repolinks (name srcdir _mainsrcfile url)
   (when url
     (insert (format "<p>Home page: <a href=%S>%s</a></p>\n"
                     url (archive--quote url)))
@@ -591,41 +591,45 @@ Rename DIR/ to PKG-VERS/, and return the descriptor."
 (defconst archive--emacs-git-url "git://git.sv.gnu.org/emacs.git")
 
 (defun archive--sync-emacs-repo ()
-  "Clone and sync Emacs repository."
-  (let ((reference (expand-file-name
-                    (or (getenv "EMACS_CLONE_REFERENCE") "../emacs/master")))
-        (emacs-repo-root (expand-file-name "emacs")))
-    (when (and (file-exists-p emacs-repo-root)
-               (not (file-exists-p
-                     (expand-file-name "README" emacs-repo-root))))
-      (message "Cleaning stalled Emacs clone: %s" emacs-repo-root)
-      (delete-directory emacs-repo-root t))
-    (cond ((file-exists-p emacs-repo-root)
-           (let ((default-directory emacs-repo-root))
-             (message "Running git pull in %S" default-directory)
-             (call-process "git" nil t nil "pull")))
-          ((file-exists-p reference)
-           (message "Emacs repository reference found: %s" reference)
-           (call-process
-            "git" nil t nil
-            "clone" archive--emacs-git-url
-            "--reference" reference
-            emacs-repo-root))
-          (t
-           (error
-            (concat "Emacs repository not found at: %s\n"
-                    "Point EMACS_CLONE_REFERENCE environment variable to an "
-                    "existing checkout.") reference)))))
-
-(defun archive--cleanup-packages (externals-list)
+  "Sync Emacs repository, if applicable.
+Return non-nil if there's an \"emacs\" repository present."
+  ;; Support for :core packages is important for elpa.gnu.org, but for other
+  ;; cases such as "in-place installation", it's rather secondary since
+  ;; those users can just as well use a development version of Emacs to get
+  ;; those packages.
+  ;; So make the handling of :core packages depend on whether or not the user
+  ;; has setup a clone of Emacs under the "emacs" subdirectory.
+  (let ((emacs-repo-root (expand-file-name "emacs")))
+    (if (not (file-directory-p emacs-repo-root))
+        (progn (message "No \"emacs\" subdir: will skip :core packages")
+               nil)
+      (let ((default-directory emacs-repo-root))
+        (message "Running git pull in %S" default-directory)
+        (call-process "git" nil t nil "pull")
+        t))))
+
+(defun archive--find-non-trivial-file (dir)
+  (catch 'found-important-file
+    (dolist (file (directory-files-recursively dir ".*"))
+      (unless (or (member file '("." ".."))
+                  (string-match "\\.elc\\'" file)
+                  (string-match "-autoloads.el\\'" file)
+                  (string-match "-pkg.el\\'" file)
+                  (file-symlink-p file))
+        (throw 'found-important-file file)))
+    nil))
+
+(defun archive--cleanup-packages (externals-list with-core)
   "Remove subdirectories of `packages/' that do not correspond to known 
packages.
 This is any subdirectory inside `packages/' that's not under
-version control nor listed in EXTERNALS-LIST."
+version control nor listed in EXTERNALS-LIST.
+If WITH-CORE is non-nil, it means we manage :core packages as well."
   (let ((default-directory (expand-file-name "packages/")))
     (dolist (dir (directory-files "."))
       (cond
        ((or (not (file-directory-p dir)) (file-symlink-p dir))
-        ;; We only add/remove plain directories in elpa/packages (not 
symlinks).
+        ;; We only add/remove plain directories in elpa/packages (not
+        ;; symlinks).
         nil)
        ((member dir '("." "..")) nil)
        ((assoc dir externals-list) nil)
@@ -641,10 +645,19 @@ version control nor listed in EXTERNALS-LIST."
                      (message "Deleted all of %s" dir))
             (message "Keeping leftover unclean %s:\n%s" dir status))))
        ;; Check if `dir' is under version control.
-       ((not (zerop (call-process "git" nil nil nil
-                                  "ls-files" "--error-unmatch" dir)))
-        (message "Deleted untracked package %s" dir)
-        (delete-directory dir 'recursive t))))))
+       ((and with-core
+             (not (zerop (call-process "git" nil nil nil
+                                       "ls-files" "--error-unmatch" dir))))
+        ;; Not under version control.  Check if it only contains
+        ;; symlinks and generated files, in which case it is probably
+        ;; a leftover :core package that can safely be deleted.
+        ;; (let ((file (archive--find-non-trivial-file dir)))
+        ;;   (if file
+        ;;       (message "Keeping %s for non-trivial file \"%s\"" dir file)
+        ;;     (progn
+        ;;       (message "Deleted untracked package %s" dir)
+        ;;       (delete-directory dir 'recursive t))))
+        )))))
 
 (defun archive--external-package-sync (name)
   "Sync external package named NAME."
@@ -674,9 +687,9 @@ version control nor listed in EXTERNALS-LIST."
   "Return non-nil if DEST is an empty variant."
   (member dest (list "" "." nil)))
 
-(defun archive--core-package-copy-file
+(defun archive--core-package-link-file
     (source dest emacs-repo-root package-root exclude-regexp)
-  "Copy file from SOURCE to DEST ensuring subdirectories."
+  "Link file from SOURCE to DEST ensuring subdirectories."
   (unless (string-match-p exclude-regexp source)
     (let* ((absolute-package-file-name
             (expand-file-name dest package-root))
@@ -685,14 +698,18 @@ version control nor listed in EXTERNALS-LIST."
            (directory (file-name-directory absolute-package-file-name)))
       (unless (file-directory-p directory)
         (make-directory directory t))
-      (copy-file absolute-core-file-name absolute-package-file-name))
+      (condition-case nil
+         (make-symbolic-link absolute-core-file-name
+                             absolute-package-file-name t)
+       (file-error
+        (copy-file absolute-core-file-name absolute-package-file-name))))
     (message "  %s -> %s" source (if (archive--core-package-empty-dest-p dest)
                                      (file-name-nondirectory source)
                                    dest))))
 
-(defun archive--core-package-copy-directory
+(defun archive--core-package-link-directory
     (source dest emacs-repo-root package-root exclude-regexp)
-  "Copy directory files from SOURCE to DEST ensuring subdirectories."
+  "Link directory files from SOURCE to DEST ensuring subdirectories."
   (let ((stack (list source))
         (base source)
         (absolute-source))
@@ -707,12 +724,12 @@ version control nor listed in EXTERNALS-LIST."
                (source-sans-base (substring source (length base)))
                (package-file-name
                 (if (archive--core-package-empty-dest-p dest)
-                    ;; Copy to root with it's original filename.
+                    ;; Link to root with its original filename.
                     source-sans-base
                   (concat
                    ;; Prepend the destination, allowing for directory rename.
                    (file-name-as-directory dest) source-sans-base))))
-          (archive--core-package-copy-file
+          (archive--core-package-link-file
            source package-file-name
            emacs-repo-root package-root exclude-regexp))))))
 
@@ -740,16 +757,16 @@ version control nor listed in EXTERNALS-LIST."
              ;; Files may be just a string, normalize.
              (list file-patterns)
            file-patterns))))
-    (message "Copying files for package: %s" name)
+    (message "Linking files for package: %s" name)
     (when (file-directory-p package-root)
       (delete-directory package-root t))
     (make-directory package-root t)
     (dolist (file-pattern file-patterns)
       (pcase-let* ((`(,file . ,dest) file-pattern))
         (if (file-directory-p (expand-file-name file emacs-repo-root))
-            (archive--core-package-copy-directory
+            (archive--core-package-link-directory
              file dest emacs-repo-root package-root exclude-regexp)
-          (archive--core-package-copy-file
+          (archive--core-package-link-file
            file dest emacs-repo-root package-root exclude-regexp))))))
 
 (defun archive-add/remove/update-externals ()
@@ -757,14 +774,15 @@ version control nor listed in EXTERNALS-LIST."
   (let ((externals-list
          (with-current-buffer (find-file-noselect "externals-list")
            (read (buffer-string)))))
-    (archive--cleanup-packages externals-list)
-    (archive--sync-emacs-repo)
-    (pcase-dolist ((and definition `(,name ,kind ,_url)) externals-list)
-      (pcase kind
-        (`:subtree nil)               ;Nothing to do.
-        (`:external (archive--external-package-sync name))
-        (`:core (archive--core-package-sync definition))
-        (_ (message "Unknown external package kind `%S' for %s" kind name))))))
+    (let ((with-core (archive--sync-emacs-repo)))
+      (archive--cleanup-packages externals-list with-core)
+      (pcase-dolist ((and definition `(,name ,kind ,_url)) externals-list)
+        (pcase kind
+          (`:subtree nil)               ;Nothing to do.
+          (`:external (archive--external-package-sync name))
+          (`:core (when with-core (archive--core-package-sync definition)))
+          (_ (message "Unknown external package kind `%S' for %s"
+                      kind name)))))))
 
 (provide 'archive-contents)
 ;;; archive-contents.el ends here
diff --git a/admin/ert-support.el b/admin/ert-support.el
new file mode 100644
index 0000000..93d1af8
--- /dev/null
+++ b/admin/ert-support.el
@@ -0,0 +1,54 @@
+;; The contents of this file are subject to the GPL License, Version 3.0.
+
+;; Copyright (C) 2016, Free Software Foundation, Inc.
+
+;; This program is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+(defun ert-support-package-install (top-directory package)
+  ;; blitz default value and set up from elpa.
+  (setq package-archives
+        `(("local-elpa" . ,(concat top-directory "/archive/packages"))))
+  (setq package-user-dir
+        (make-temp-file "elpa-test" t))
+  (package-initialize)
+  (package-refresh-contents)
+  (package-install package))
+
+(defun ert-support-test-find-tests (package-directory package)
+  (or
+   (directory-files package-directory nil ".*-test.el$")
+   (directory-files package-directory nil ".*-tests.el$")
+   (let ((dir-test
+          (concat package-directory "/test")))
+     (when (file-exists-p dir-test)
+       (directory-files dir-test)))
+   (let ((dir-tests
+          (concat package-directory "/tests")))
+     (when (file-exists-p dir-tests)
+       (directory-files dir-tests)))))
+
+(defun ert-support-load-tests (package-directory package)
+  (mapc
+   (lambda(file)
+     (message "Loading test file... %s" (concat package-directory file))
+     (load-file (concat package-directory file)))
+   (ert-support-test-find-tests package-directory package)))
+
+(defun ert-support-test-package (top-directory package)
+  (ert-support-package-install top-directory package)
+  (ert-support-load-tests
+   (concat top-directory "/packages/" (symbol-name package) "/")
+   package)
+
+  (ert-run-tests-batch-and-exit t))