02/103: python-build-system: Use PEP 517-compatible builds.

[guix-commits]

lbraun pushed a commit to branch wip-python-pep517
in repository guix.

commit 42bd781c2853b6d76984422c444c7452c2bb75fc
Author: Lars-Dominik Braun <lars@6xq.net>
AuthorDate: Mon Nov 22 15:11:44 2021 +0100

    python-build-system: Use PEP 517-compatible builds.
    
    This is effectively an entire rewrite of python-build-system. It supports
    all PEP 517-compatible build backends.
    
    * gnu/packages/python-commencement.scm: New file containing new Python
    toolchain package(s).
    * gnu/local.mk (GNU_SYSTEM_MODULES): Add it.
    * gnu/packages/python-xyz.scm (python-setuptools): Move…
    * gnu/packages/python-build.scm (python-setuptools): …here.
    (python-setuptools-bootstrap): New variable.
    (python-wheel): Break bootstrap cycle.
    (python-wheel-bootstrap): New variable.
    * gnu/packages/python.scm (python-2.7): Do not install setuptools and pip.
    * guix/build-system/python.scm (%python-build-system-modules): Use
    (guix build json).
    (default-python): Default to python-toolchain-for-build.
    (lower): Add default wheel output, remove test-target and
    use-setuptools? flags, add build-backend, test-backend and test-flags.
    * gnu/packages/boost.scm, gnu/packages/cups.scm,
    gnu/packages/graphics.scm, gnu/packages/qt.scm: Add (guix build json)
    to #:imported-modules.
    * guix/build/python-build-system.scm: Rewrite build system.
---
 gnu/local.mk                         |   1 +
 gnu/packages/boost.scm               |   1 +
 gnu/packages/cups.scm                |   1 +
 gnu/packages/graphics.scm            |   1 +
 gnu/packages/python-build.scm        |  99 +++++++-
 gnu/packages/python-commencement.scm |  87 +++++++
 gnu/packages/python-xyz.scm          |  44 ----
 gnu/packages/python.scm              |   2 +-
 gnu/packages/qt.scm                  |   1 +
 guix/build-system/python.scm         |  22 +-
 guix/build/python-build-system.scm   | 446 +++++++++++++++++++++++------------
 11 files changed, 495 insertions(+), 210 deletions(-)

diff --git a/gnu/local.mk b/gnu/local.mk
index 0bae6ffa63..fcbe8277fb 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -486,6 +486,7 @@ GNU_SYSTEM_MODULES =                                \
   %D%/packages/python.scm                      \
   %D%/packages/python-build.scm                        \
   %D%/packages/python-check.scm                        \
+  %D%/packages/python-commencement.scm         \
   %D%/packages/python-compression.scm          \
   %D%/packages/python-crypto.scm               \
   %D%/packages/python-science.scm              \
diff --git a/gnu/packages/boost.scm b/gnu/packages/boost.scm
index b53b1f4257..e2a8b62436 100644
--- a/gnu/packages/boost.scm
+++ b/gnu/packages/boost.scm
@@ -87,6 +87,7 @@
        ("tcsh" ,tcsh)))
     (arguments
      `(#:imported-modules ((guix build python-build-system)
+                           (guix build json)
                            ,@%gnu-build-system-modules)
        #:modules (((guix build python-build-system) #:select (python-version))
                   ,@%gnu-build-system-modules)
diff --git a/gnu/packages/cups.scm b/gnu/packages/cups.scm
index 6342f85404..182ce7eba7 100644
--- a/gnu/packages/cups.scm
+++ b/gnu/packages/cups.scm
@@ -572,6 +572,7 @@ should only be used as part of the Guix cups-pk-helper 
service.")
          "--enable-qt5" "--disable-qt4")
 
        #:imported-modules ((guix build python-build-system)
+                           (guix build json)
                            ,@%gnu-build-system-modules)
        #:modules ((guix build gnu-build-system)
                   (guix build utils)
diff --git a/gnu/packages/graphics.scm b/gnu/packages/graphics.scm
index 3f83c72599..3b2d6bf095 100644
--- a/gnu/packages/graphics.scm
+++ b/gnu/packages/graphics.scm
@@ -773,6 +773,7 @@ exception-handling library.")
     (build-system cmake-build-system)
     (arguments
      `(#:imported-modules ((guix build python-build-system)
+                           (guix build json)
                            ,@%cmake-build-system-modules)
        #:configure-flags '("-D2GEOM_BUILD_SHARED=ON"
                            "-D2GEOM_BOOST_PYTHON=ON"
diff --git a/gnu/packages/python-build.scm b/gnu/packages/python-build.scm
index 1db5a6b335..1a35738f95 100644
--- a/gnu/packages/python-build.scm
+++ b/gnu/packages/python-build.scm
@@ -28,7 +28,9 @@
   #:use-module (guix build-system python)
   #:use-module (guix download)
   #:use-module (guix git-download)
-  #:use-module (guix packages))
+  #:use-module (guix packages)
+  #:use-module (gnu packages)
+  #:use-module (gnu packages python))
 
 ;;; Commentary:
 ;;;
@@ -38,6 +40,69 @@
 ;;;
 ;;; Code:
 
+(define-public python-setuptools
+  (package
+    (name "python-setuptools")
+    (version "52.0.0")
+    (source
+     (origin
+      (method url-fetch)
+      (uri (pypi-uri "setuptools" version))
+      (sha256
+       (base32
+        "15ibjdjhkwgj6qbmpsxikkqdfsb1550z46fly7dm15ah4bk1wfpv"))
+      (modules '((guix build utils)))
+      (snippet
+       '(begin
+          ;; Remove included binaries which are used to build self-extracting
+          ;; installers for Windows.
+          ;; TODO: Find some way to build them ourself so we can include them.
+          (for-each delete-file (find-files "setuptools" 
"^(cli|gui).*\\.exe$"))
+          #t))))
+    (outputs '("out" "wheel"))
+    (build-system python-build-system)
+    ;; FIXME: Tests require pytest, which itself relies on setuptools.
+    ;; One could bootstrap with an internal untested setuptools.
+    (arguments
+     `(#:tests? #f
+       #:python ,python-wrapper ; Break cycle with default build system’s 
setuptools dependency.
+       #:phases (modify-phases %standard-phases
+                  ;; Use this setuptools’ sources to bootstrap themselves.
+                  (add-before 'build 'set-PYTHONPATH
+                    (lambda _
+                      (format #t "current working dir ~s~%" (getcwd))
+                      (setenv "GUIX_PYTHONPATH"
+                              (string-append ".:" (getenv "GUIX_PYTHONPATH")))
+                      #t)))))
+    ;; Required to build wheels.
+    (propagated-inputs `(("python-wheel" ,python-wheel)))
+    (home-page "https://pypi.org/project/setuptools/";)
+    (synopsis
+     "Library designed to facilitate packaging Python projects")
+    (description
+     "Setuptools is a fully-featured, stable library designed to facilitate
+packaging Python projects, where packaging includes:
+Python package and module definitions,
+distribution package metadata,
+test hooks,
+project installation,
+platform-specific details,
+Python 3 support.")
+    ;; TODO: setuptools now bundles the following libraries:
+    ;; packaging, pyparsing, six and appdirs. How to unbundle?
+    (license (list license:psfl        ; setuptools itself
+                   license:expat       ; six, appdirs, pyparsing
+                   license:asl2.0      ; packaging is dual ASL2/BSD-2
+                   license:bsd-2))))
+    ;(properties `((python2-variant . ,(delay python2-setuptools))))))
+
+;; Break loop between python-setuptools and python-wheel.
+(define-public python-setuptools-bootstrap
+  (package
+    (inherit python-setuptools)
+    (name "python-setuptools-bootstrap")
+    (propagated-inputs `(("python-wheel" ,python-wheel-bootstrap)))))
+
 (define-public python-wheel
   (package
     (name "python-wheel")
@@ -49,12 +114,11 @@
         (sha256
          (base32
           "0ii6f34rvpjg3nmw4bc2h7fhdsy38y1h93hghncfs5akfrldmj8h"))))
+    (outputs '("out" "wheel"))
     (build-system python-build-system)
     (arguments
-     ;; FIXME: The test suite runs "python setup.py bdist_wheel", which in turn
-     ;; fails to find the newly-built bdist_wheel library, even though it is
-     ;; available on PYTHONPATH.  What search path is consulted by setup.py?
-     '(#:tests? #f))
+     `(#:python ,python-wrapper)) ; Break cycle with 
python-toolchain-for-build.
+    (native-inputs `(("python-setuptools" ,python-setuptools-bootstrap)))
     (home-page "https://bitbucket.org/pypa/wheel/";)
     (synopsis "Format for built Python packages")
     (description
@@ -67,6 +131,31 @@ scripts to their final locations) at any later time.  Wheel 
files can be
 installed with a newer @code{pip} or with wheel's own command line utility.")
     (license license:expat)))
 
+(define-public python-wheel-bootstrap
+  (package
+    (inherit python-wheel)
+    (name "python-wheel-bootstrap")
+    (build-system copy-build-system)
+    (native-inputs '()) ; Break cycle to setuptools.
+    (arguments
+     `(#:install-plan
+       ;; XXX: Do not hard-code Python version.
+       '(("wheel" "lib/python3.9/site-packages/wheel"))
+       #:phases
+       (modify-phases %standard-phases
+         ;; Add metadata for setuptools, so it will find the wheel-building 
code.
+         (add-after 'install 'install-metadata
+           (lambda* (#:key outputs #:allow-other-keys)
+             (let* ((out (assoc-ref outputs "out"))
+                    (site-dir (string-append out 
"/lib/python3.9/site-packages"))
+                    (metadata-dir (string-append site-dir "/wheel.egg-info")))
+               (mkdir-p metadata-dir)
+               (call-with-output-file (string-append metadata-dir 
"/entry_points.txt")
+                 (lambda (port)
+                   (format port "~
+                           [distutils.commands]~@
+                           bdist_wheel = 
wheel.bdist_wheel:bdist_wheel~%")))))))))))
+
 (define-public python2-wheel
   (package-with-python2 python-wheel))
 
diff --git a/gnu/packages/python-commencement.scm 
b/gnu/packages/python-commencement.scm
new file mode 100644
index 0000000000..1860897bfd
--- /dev/null
+++ b/gnu/packages/python-commencement.scm
@@ -0,0 +1,87 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 
Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2014 Andreas Enge <andreas@enge.fr>
+;;; Copyright © 2012 Nikita Karetnikov <nikita@karetnikov.org>
+;;; Copyright © 2014, 2015, 2017 Mark H Weaver <mhw@netris.org>
+;;; Copyright © 2017, 2018, 2019, 2021 Efraim Flashner <efraim@flashner.co.il>
+;;; Copyright © 2018 Tobias Geerinckx-Rice <me@tobias.gr>
+;;; Copyright © 2018, 2019, 2020 Jan (janneke) Nieuwenhuizen <janneke@gnu.org>
+;;; Copyright © 2019, 2020 Marius Bakke <mbakke@fastmail.com>
+;;; Copyright © 2020 Timothy Sample <samplet@ngyro.com>
+;;; Copyright © 2020 Guy Fleury Iteriteka <gfleury@disroot.org>
+;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com>
+;;; Copyright © 2021 Lars-Dominik Braun <lars@6xq.net>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (gnu packages python-commencement)
+  #:use-module ((guix licenses) #:prefix license:)
+  #:use-module (guix download)
+  #:use-module (guix packages)
+  #:use-module (guix build-system trivial)
+  #:use-module (gnu packages)
+  #:use-module (gnu packages python)
+  #:use-module (gnu packages python-build)
+  #:use-module (gnu packages python-xyz)
+  #:use-module (srfi srfi-1)
+  #:use-module (srfi srfi-26))
+
+(define-public python-toolchain
+  (package
+    (name "python-toolchain")
+    (version (package-version python))
+    (source #f)
+    (build-system trivial-build-system)
+    (arguments
+     '(#:modules ((guix build union))
+       #:builder (begin
+                   (use-modules (ice-9 match)
+                                (srfi srfi-1)
+                                (srfi srfi-26)
+                                (guix build union))
+
+                   (let ((out (assoc-ref %outputs "out")))
+                     (union-build out (filter-map (match-lambda
+                                                ((_ . directory) directory))
+                                              %build-inputs))
+                     #t))))
+    (inputs
+     `(("python" ,python-wrapper)
+       ("python-setuptools" ,python-setuptools)
+       ("python-pip" ,python-pip)
+       ("python-virtualenv" ,python-virtualenv)))
+    (native-search-paths
+     (package-native-search-paths python))
+    (search-paths
+     (package-search-paths python))
+    (license (package-license python)) ; XXX
+    (synopsis "Python toolchain")
+    (description
+     "Python toolchain including Python itself, setuptools and pip.  Use this
+package if you need a fully-fledged Python toolchain instead of just the
+interpreter.")
+    (home-page (package-home-page python))))
+
+;; Python 3 toolchain for python-build-system. We cannot use python-toolchain
+;; here, since we’d need to bootstrap python-pip somehow.
+(define-public python-toolchain-for-build
+  (package
+    (inherit python-toolchain)
+    (name "python-toolchain-for-build")
+    (inputs
+      `(("python" ,python-wrapper)
+        ("python-setuptools" ,python-setuptools)))))
+
diff --git a/gnu/packages/python-xyz.scm b/gnu/packages/python-xyz.scm
index 2af4794019..cc29d8225d 100644
--- a/gnu/packages/python-xyz.scm
+++ b/gnu/packages/python-xyz.scm
@@ -1694,50 +1694,6 @@ other machines, such as over the network.")
        `(("python-enum34" ,python2-enum34)
          ,@(package-propagated-inputs base))))))
 
-(define-public python-setuptools
-  (package
-    (name "python-setuptools")
-    (version "52.0.0")
-    (source
-     (origin
-      (method url-fetch)
-      (uri (pypi-uri "setuptools" version))
-      (sha256
-       (base32
-        "15ibjdjhkwgj6qbmpsxikkqdfsb1550z46fly7dm15ah4bk1wfpv"))
-      (modules '((guix build utils)))
-      (snippet
-       '(begin
-          ;; Remove included binaries which are used to build self-extracting
-          ;; installers for Windows.
-          ;; TODO: Find some way to build them ourself so we can include them.
-          (for-each delete-file (find-files "setuptools" 
"^(cli|gui).*\\.exe$"))
-          #t))))
-    (build-system python-build-system)
-    ;; FIXME: Tests require pytest, which itself relies on setuptools.
-    ;; One could bootstrap with an internal untested setuptools.
-    (arguments
-     `(#:tests? #f))
-    (home-page "https://pypi.org/project/setuptools/";)
-    (synopsis
-     "Library designed to facilitate packaging Python projects")
-    (description
-     "Setuptools is a fully-featured, stable library designed to facilitate
-packaging Python projects, where packaging includes:
-Python package and module definitions,
-distribution package metadata,
-test hooks,
-project installation,
-platform-specific details,
-Python 3 support.")
-    ;; TODO: setuptools now bundles the following libraries:
-    ;; packaging, pyparsing, six and appdirs. How to unbundle?
-    (license (list license:psfl        ; setuptools itself
-                   license:expat       ; six, appdirs, pyparsing
-                   license:asl2.0      ; packaging is dual ASL2/BSD-2
-                   license:bsd-2))
-    (properties `((python2-variant . ,(delay python2-setuptools))))))
-
 ;; Newer versions of setuptools no longer support Python 2.
 (define-public python2-setuptools
   (package
diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm
index f0d2fd6eb8..f2c5613f2f 100644
--- a/gnu/packages/python.scm
+++ b/gnu/packages/python.scm
@@ -183,7 +183,7 @@
        (list "--enable-shared"          ;allow embedding
              "--with-system-expat"      ;for XML support
              "--with-system-ffi"        ;build ctypes
-             "--with-ensurepip=install" ;install pip and setuptools
+             "--with-ensurepip=no"      ;do not install pip and setuptools
              "--with-computed-gotos"    ;main interpreter loop optimization
              "--enable-unicode=ucs4"
 
diff --git a/gnu/packages/qt.scm b/gnu/packages/qt.scm
index 01bf961bbf..0a96856371 100644
--- a/gnu/packages/qt.scm
+++ b/gnu/packages/qt.scm
@@ -2082,6 +2082,7 @@ module provides support functions to the automatically 
generated code.")
                   ((guix build python-build-system) #:select (python-version))
                   ,@%gnu-build-system-modules)
        #:imported-modules ((guix build python-build-system)
+                           (guix build json)
                            ,@%gnu-build-system-modules)
        #:phases
        (modify-phases %standard-phases
diff --git a/guix/build-system/python.scm b/guix/build-system/python.scm
index efade6f74b..d846d320f1 100644
--- a/guix/build-system/python.scm
+++ b/guix/build-system/python.scm
@@ -62,13 +62,14 @@ extension, such as '.tar.gz'."
 (define %python-build-system-modules
   ;; Build-side modules imported by default.
   `((guix build python-build-system)
+    (guix build json)
     ,@%gnu-build-system-modules))
 
 (define (default-python)
   "Return the default Python package."
   ;; Lazily resolve the binding to avoid a circular dependency.
-  (let ((python (resolve-interface '(gnu packages python))))
-    (module-ref python 'python-wrapper)))
+  (let ((python (resolve-interface '(gnu packages python-commencement))))
+    (module-ref python 'python-toolchain-for-build)))
 
 (define (default-python2)
   "Return the default Python 2 package."
@@ -165,26 +166,26 @@ pre-defined variants."
          (build-inputs `(("python" ,python)
                          ("sanity-check.py" ,(local-file sanity-check.py))
                          ,@native-inputs))
-         (outputs outputs)
+         (outputs (append outputs '(wheel)))
          (build python-build)
          (arguments (strip-keyword-arguments private-keywords arguments)))))
 
 (define* (python-build name inputs
                        #:key source
                        (tests? #t)
-                       (test-target "test")
-                       (use-setuptools? #t)
                        (configure-flags ''())
+                       (build-backend #f)
+                       (test-backend #f)
+                       (test-flags #f)
                        (phases '%standard-phases)
-                       (outputs '("out"))
+                       (outputs '("out" "wheel"))
                        (search-paths '())
                        (system (%current-system))
                        (guile #f)
                        (imported-modules %python-build-system-modules)
                        (modules '((guix build python-build-system)
                                   (guix build utils))))
-  "Build SOURCE using PYTHON, and with INPUTS.  This assumes that SOURCE
-provides a 'setup.py' file as its build system."
+  "Build SOURCE using PYTHON, and with INPUTS."
   (define build
     (with-imported-modules imported-modules
       #~(begin
@@ -194,9 +195,10 @@ provides a 'setup.py' file as its build system."
               #~(python-build #:name #$name
                               #:source #+source
                               #:configure-flags #$configure-flags
-                              #:use-setuptools? #$use-setuptools?
                               #:system #$system
-                              #:test-target #$test-target
+                              #:build-backend #$build-backend
+                              #:test-backend #$test-backend
+                              #:test-flags #$test-flags
                               #:tests? #$tests?
                               #:phases #$(if (pair? phases)
                                              (sexp->gexp phases)
diff --git a/guix/build/python-build-system.scm 
b/guix/build/python-build-system.scm
index 08871f60cd..53af460ac2 100644
--- a/guix/build/python-build-system.scm
+++ b/guix/build/python-build-system.scm
@@ -30,11 +30,16 @@
 (define-module (guix build python-build-system)
   #:use-module ((guix build gnu-build-system) #:prefix gnu:)
   #:use-module (guix build utils)
+  #:use-module (guix build json)
   #:use-module (ice-9 match)
   #:use-module (ice-9 ftw)
   #:use-module (ice-9 format)
+  #:use-module (ice-9 rdelim)
+  #:use-module (ice-9 regex)
   #:use-module (srfi srfi-1)
   #:use-module (srfi srfi-26)
+  #:use-module (srfi srfi-34)
+  #:use-module (srfi srfi-35)
   #:export (%standard-phases
             add-installed-pythonpath
             site-packages
@@ -43,96 +48,45 @@
 
 ;; Commentary:
 ;;
-;; Builder-side code of the standard Python package build procedure.
+;; PEP 517-compatible build system for Python packages.
 ;;
+;; PEP 517 mandates the use of a TOML file called pyproject.toml at the
+;; project root, describing build and runtime dependencies, as well as the
+;; build system, which can be different from setuptools. This module uses
+;; that file to extract the build system used and call its wheel-building
+;; entry point build_wheel (see 'build). setuptools’ wheel builder is
+;; used as a fallback if either no pyproject.toml exists or it does not
+;; declare a build-system. It supports config_settings through the
+;; standard #:configure-flags argument.
 ;;
-;; Backgound about the Python installation methods
+;; This wheel, which is just a ZIP file with a file structure defined
+;; by PEP 427 (https://www.python.org/dev/peps/pep-0427/), is then unpacked
+;; and its contents are moved to the appropriate locations in 'install.
 ;;
-;; In Python there are different ways to install packages: distutils,
-;; setuptools, easy_install and pip.  All of these are sharing the file
-;; setup.py, introduced with distutils in Python 2.0. The setup.py file can be
-;; considered as a kind of Makefile accepting targets (or commands) like
-;; "build" and "install".  As of autumn 2016 the recommended way to install
-;; Python packages is using pip.
+;; Then entry points, as defined by the PyPa Entry Point Specification
+;; (https://packaging.python.org/specifications/entry-points/) are read
+;; from a file called entry_points.txt in the package’s site-packages
+;; subdirectory and scripts are written to bin/. These are not part of a
+;; wheel and expected to be created by the installing utility.
 ;;
-;; For both distutils and setuptools, running "python setup.py install" is the
-;; way to install Python packages.  With distutils the "install" command
-;; basically copies all packages into <prefix>/lib/pythonX.Y/site-packages.
-;;
-;; Some time later "setuptools" was established to enhance distutils.  To use
-;; setuptools, the developer imports setuptools in setup.py.  When importing
-;; setuptools, the original "install" command gets overwritten by setuptools'
-;; "install" command.
-;;
-;; The command-line tools easy_install and pip are both capable of finding and
-;; downloading the package source from PyPI (the Python Package Index).  Both
-;; of them import setuptools and execute the "setup.py" file under their
-;; control.  Thus the "setup.py" behaves as if the developer had imported
-;; setuptools within setup.py - even is still using only distutils.
-;;
-;; Setuptools' "install" command (to be more precise: the "easy_install"
-;; command which is called by "install") will put the path of the currently
-;; installed version of each package and it's dependencies (as declared in
-;; setup.py) into an "easy-install.pth" file.  In Guix each packages gets its
-;; own "site-packages" directory and thus an "easy-install.pth" of its own.
-;; To avoid conflicts, the python build system renames the file to
-;; <packagename>.pth in the phase rename-pth-file.  To ensure that Python will
-;; process the .pth file, easy_install also creates a basic "site.py" in each
-;; "site-packages" directory.  The file is the same for all packages, thus
-;; there is no need to rename it.  For more information about .pth files and
-;; the site module, please refere to
-;; https://docs.python.org/3/library/site.html.
-;;
-;; The .pth files contain the file-system paths (pointing to the store) of all
-;; dependencies.  So the dependency is hidden in the .pth file but is not
-;; visible in the file-system.  Now if packages A and B both required packages
-;; P, but in different versions, Guix will not detect this when installing
-;; both A and B to a profile. (For details and example see
-;; https://lists.gnu.org/archive/html/guix-devel/2016-10/msg01233.html.)
-;;
-;; Pip behaves a bit different then easy_install: it always executes
-;; "setup.py" with the option "--single-version-externally-managed" set.  This
-;; makes setuptools' "install" command run the original "install" command
-;; instead of the "easy_install" command, so no .pth file (and no site.py)
-;; will be created.  The "site-packages" directory only contains the package
-;; and the related .egg-info directory.
-;;
-;; This is exactly what we need for Guix and this is what we mimic in the
-;; install phase below.
-;;
-;; As a draw back, the magic of the .pth file of linking to the other required
-;; packages is gone and these packages have now to be declared as
-;; "propagated-inputs".
-;;
-;; Note: Importing setuptools also adds two sub-commands: "install_egg_info"
-;; and "install_scripts".  These sub-commands are executed even if
-;; "--single-version-externally-managed" is set, thus the .egg-info directory
-;; and the scripts defined in entry-points will always be created.
-
-
-(define setuptools-shim
-  ;; Run setup.py with "setuptools" being imported, which will patch
-  ;; "distutils". This is needed for packages using "distutils" instead of
-  ;; "setuptools" since the former does not understand the
-  ;; "--single-version-externally-managed" flag.
-  ;; Python code taken from pip 9.0.1 pip/utils/setuptools_build.py
-  (string-append
-   "import setuptools, tokenize;__file__='setup.py';"
-   "f=getattr(tokenize, 'open', open)(__file__);"
-   "code=f.read().replace('\\r\\n', '\\n');"
-   "f.close();"
-   "exec(compile(code, __file__, 'exec'))"))
-
-(define (call-setuppy command params use-setuptools?)
-  (if (file-exists? "setup.py")
-      (begin
-         (format #t "running \"python setup.py\" with command ~s and 
parameters ~s~%"
-                command params)
-         (if use-setuptools?
-             (apply invoke "python" "-c" setuptools-shim
-                    command params)
-             (apply invoke "python" "./setup.py" command params)))
-      (error "no setup.py found")))
+;; Caveats:
+;; - There is no support for in-tree build backends.
+
+;; Base error type.
+(define-condition-type &python-build-error &error
+  python-build-error?)
+
+;; Raised when 'check cannot find a valid test system in the inputs.
+(define-condition-type &test-system-not-found &python-build-error
+  test-system-not-found?)
+
+;; Raised when multiple wheels are created by 'build.
+(define-condition-type &cannot-extract-multiple-wheels &python-build-error
+  cannot-extract-multiple-wheels?)
+
+;; Raised, when no wheel has been built by the build system.
+(define-condition-type &no-wheels-built &python-build-error
+  no-wheels-built?)
 
 (define* (sanity-check #:key tests? inputs outputs #:allow-other-keys)
   "Ensure packages depending on this package via setuptools work properly,
@@ -143,23 +97,83 @@ without errors."
     (with-directory-excursion "/tmp"
       (invoke "python" sanity-check.py (site-packages inputs outputs)))))
 
-(define* (build #:key use-setuptools? #:allow-other-keys)
+(define* (build #:key outputs build-backend configure-flags #:allow-other-keys)
   "Build a given Python package."
-  (call-setuppy "build" '() use-setuptools?)
+
+  (define (pyproject.toml->build-backend file)
+    "Look up the build backend in a pyproject.toml file."
+    (call-with-input-file file
+      (lambda (in)
+        (let loop ((line (read-line in 'concat)))
+          (if (eof-object? line)
+              #f
+              (let ((m (string-match "build-backend = [\"'](.+)[\"']" line)))
+                (if m (match:substring m 1)
+                    (loop (read-line in 'concat)))))))))
+
+  (let* ((wheel-output (assoc-ref outputs "wheel"))
+         (wheel-dir (if wheel-output wheel-output "dist"))
+         ;; There is no easy way to get data from Guile into Python via
+         ;; s-expressions, but we have JSON serialization already, which Python
+         ;; also supports out-of-the-box.
+         (config-settings (call-with-output-string (cut write-json 
configure-flags <>)))
+         ;; python-setuptools’ default backend supports setup.py *and*
+         ;; pyproject.toml. Allow overriding this automatic detection via
+         ;; build-backend.
+         (auto-build-backend (if (file-exists? "pyproject.toml")
+                               (pyproject.toml->build-backend "pyproject.toml")
+                               #f))
+         ;; Use build system detection here and not in importer, because a) we
+         ;; have alot of legacy packages and b) the importer cannot update 
arbitrary
+         ;; fields in case a package switches its build system.
+         (use-build-backend (or
+                              build-backend
+                              auto-build-backend
+                              "setuptools.build_meta")))
+    (format #t "Using '~a' to build wheels, auto-detected '~a', override 
'~a'.~%"
+               use-build-backend auto-build-backend build-backend)
+    (mkdir-p wheel-dir)
+    ;; Call the PEP 517 build function, which drops a .whl into wheel-dir.
+    (invoke "python" "-c" "import sys, importlib, json
+config_settings = json.loads (sys.argv[3])
+builder = importlib.import_module(sys.argv[1])
+builder.build_wheel(sys.argv[2], config_settings=config_settings)"
+            use-build-backend wheel-dir config-settings))
   #t)
 
-(define* (check #:key tests? test-target use-setuptools? #:allow-other-keys)
+(define* (check #:key inputs outputs tests? test-backend test-flags 
#:allow-other-keys)
   "Run the test suite of a given Python package."
   (if tests?
-      ;; Running `setup.py test` creates an additional .egg-info directory in
-      ;; build/lib in some cases, e.g. if the source is in a sub-directory
-      ;; (given with `package_dir`). This will by copied to the output, too,
-      ;; so we need to remove.
-      (let ((before (find-files "build" "\\.egg-info$" #:directories? #t)))
-        (call-setuppy test-target '() use-setuptools?)
-        (let* ((after (find-files "build" "\\.egg-info$" #:directories? #t))
-               (inter (lset-difference string=? after before)))
-          (for-each delete-file-recursively inter)))
+    ;; Unfortunately with PEP 517 there is no common method to specify test
+    ;; systems. Guess test system based on inputs instead.
+    (let* ((pytest (which "pytest"))
+           (nosetests (which "nosetests"))
+           (nose2 (which "nose2"))
+           (have-setup-py (file-exists? "setup.py"))
+           (use-test-backend
+            (or
+              test-backend
+              ;; Prefer pytest
+              (if pytest 'pytest #f)
+              (if nosetests 'nose #f)
+              (if nose2 'nose2 #f)
+              ;; But fall back to setup.py, which should work for most
+              ;; packages. XXX: would be nice not to depend on setup.py here? 
fails
+              ;; more often than not to find any tests at all. Maybe we can run
+              ;; `python -m unittest`?
+              (if have-setup-py 'setup.py #f))))
+        (format #t "Using ~a~%" use-test-backend)
+        (match use-test-backend
+          ('pytest
+           (apply invoke (cons pytest (or test-flags '("-vv")))))
+          ('nose
+           (apply invoke (cons nosetests (or test-flags '("-v")))))
+          ('nose2
+           (apply invoke (cons nose2 (or test-flags '("-v" 
"--pretty-assert")))))
+          ('setup.py
+           (apply invoke (append '("python" "setup.py") (or test-flags 
'("test" "-v")))))
+          ;; The developer should explicitly disable tests in this case.
+          (else (raise (condition (&test-system-not-found))))))
       (format #t "test suite not run~%"))
   #t)
 
@@ -196,33 +210,166 @@ running checks after installing the package."
                                 "/bin:"
                                 (getenv "PATH"))))
 
-(define* (install #:key inputs outputs (configure-flags '()) use-setuptools?
-                  #:allow-other-keys)
-  "Install a given Python package."
-  (let* ((out (python-output outputs))
+(define* (install #:key inputs outputs (configure-flags '()) 
#:allow-other-keys)
+  "Install a wheel file according to PEP 427"
+  ;; See 
https://www.python.org/dev/peps/pep-0427/#installing-a-wheel-distribution-1-0-py32-none-any-whl
+  (let* ((site-dir (site-packages inputs outputs))
          (python (assoc-ref inputs "python"))
-         (major-minor (map string->number
-                           (take (string-split (python-version python) #\.) 
2)))
-         (<3.7? (match major-minor
-                   ((major minor)
-                    (or (< major 3) (and (= major 3) (< minor 7))))))
-         (params (append (list (string-append "--prefix=" out)
-                               "--no-compile")
-                         (if use-setuptools?
-                             ;; distutils does not accept these flags
-                             (list "--single-version-externally-managed"
-                                   "--root=/")
-                             '())
-                         configure-flags)))
-    (call-setuppy "install" params use-setuptools?)
-    ;; Rather than produce potentially non-reproducible .pyc files on Pythons
-    ;; older than 3.7, whose 'compileall' module lacks the
-    ;; '--invalidation-mode' option, do not generate any.
-    (unless <3.7?
-      (invoke "python" "-m" "compileall" "--invalidation-mode=unchecked-hash"
-              out))))
-
-(define* (wrap #:key inputs outputs #:allow-other-keys)
+         (out (assoc-ref outputs "out")))
+    (define (extract file)
+      "Extract wheel (ZIP file) into site-packages directory"
+      ;; Use Python’s zipfile to avoid extra dependency
+      (invoke "python" "-m" "zipfile" "-e" file site-dir))
+
+    (define python-hashbang
+      (string-append "#!" python "/bin/python"))
+
+    (define* (merge-directories source destination #:optional (post-move #f))
+      "Move all files in SOURCE into DESTINATION, merging the two directories."
+      (format #t "Merging directory ~a into ~a~%" source destination)
+      (for-each
+        (lambda (file)
+          (format #t "~a/~a -> ~a/~a~%" source file destination file)
+          (mkdir-p destination)
+          (rename-file
+              (string-append source "/" file)
+              (string-append destination "/" file))
+          (when post-move
+            (post-move file)))
+        (scandir source (negate (cut member <> '("." "..")))))
+      (rmdir source))
+
+    (define (expand-data-directory directory)
+      "Move files from all .data subdirectories to their respective
+destinations."
+      ;; Python’s distutils.command.install defines this mapping from source to
+      ;; destination mapping.
+      (let ((source (string-append directory "/scripts"))
+            (destination (string-append out "/bin")))
+        (when (file-exists? source)
+          (merge-directories
+           source
+           destination
+           (lambda (f)
+             (let ((dest-path (string-append destination "/" f)))
+               (chmod dest-path #o755)
+               (substitute* dest-path (("#!python") python-hashbang)))))))
+      ;; XXX: Why does Python not use share/ here?
+      (let ((source (string-append directory "/data"))
+            (destination (string-append out "/share")))
+        (when (file-exists? source)
+          (merge-directories source destination)))
+      (let* ((distribution (car (string-split (basename directory) #\-)))
+            (source (string-append directory "/headers"))
+            (destination (string-append out "/include/python" (python-version 
python) "/" distribution)))
+        (when (file-exists? source)
+          (merge-directories source destination))))
+    
+  (define (list-directories base predicate)
+    ;; Cannot use find-files here, because it’s recursive.
+    (scandir
+      base
+      (lambda (name)
+        (let ((stat (lstat (string-append base "/" name))))
+        (and
+          (not (member name '("." "..")))
+          (eq? (stat:type stat) 'directory)
+          (predicate name stat))))))
+
+  (let* ((wheel-output (assoc-ref outputs "wheel"))
+         (wheel-dir (if wheel-output wheel-output "dist"))
+         (wheels (find-files wheel-dir "\\.whl$"))) ; XXX: do not recurse
+    (cond
+    ((> (length wheels) 1) ; This code does not support multiple wheels
+                                ; yet, because their outputs would have to be
+                                ; merged properly.
+      (raise (condition (&cannot-extract-multiple-wheels))))
+      ((= (length wheels) 0)
+       (raise (condition (&no-wheels-built)))))
+    (for-each extract wheels))
+  (let ((datadirs (map
+                    (cut string-append site-dir "/" <>)
+                    (list-directories site-dir (file-name-predicate 
"\\.data$")))))
+    (for-each (lambda (directory)
+                (expand-data-directory directory)
+                (rmdir directory))
+              datadirs))
+  #t))
+
+(define* (compile-bytecode #:key inputs outputs (configure-flags '()) 
#:allow-other-keys)
+  "Compile installed byte-code in site-packages."
+  (let ((site-dir (site-packages inputs outputs)))
+    (invoke "python" "-m" "compileall" site-dir)
+    ;; XXX: We could compile with -O and -OO too here, at the cost of more 
space.
+    #t))
+
+(define* (create-entrypoints #:key inputs outputs (configure-flags '()) 
#:allow-other-keys)
+  "Implement Entry Points Specification
+(https://packaging.python.org/specifications/entry-points/) by PyPa,
+which creates runnable scripts in bin/ from entry point specification
+file entry_points.txt. This is necessary, because wheels do not contain
+these binaries and installers are expected to create them."
+
+  (define (entry-points.txt->entry-points file)
+    "Specialized parser for Python configfile-like files, in particular
+entry_points.txt. Returns a list of console_script and gui_scripts
+entry points."
+    (call-with-input-file file
+      (lambda (in)
+        (let loop ((line (read-line in))
+                   (inside #f)
+                   (result '()))
+          (if (eof-object? line)
+            result
+            (let* ((group-match (string-match "^\\[(.+)\\]$" line))
+                  (group-name (if group-match (match:substring group-match 1) 
#f))
+                  (next-inside
+                   (if (not group-name)
+                     inside
+                     (or
+                       (string=? group-name "console_scripts")
+                       (string=? group-name "gui_scripts"))))
+                  (item-match (string-match "^([^ =]+)\\s*=\\s*([^:]+):(.+)$" 
line)))
+              (if (and inside item-match)
+                (loop (read-line in) next-inside (cons (list
+                                                        (match:substring 
item-match 1)
+                                                        (match:substring 
item-match 2)
+                                                        (match:substring 
item-match 3))
+                                                         result))
+                (loop (read-line in) next-inside result))))))))
+
+  (define (create-script path name module function)
+    "Create a Python script from an entry point’s NAME, MODULE and
+  FUNCTION and return write it to PATH/NAME."
+    (let ((interpreter (which "python3"))
+          (file-path (string-append path "/" name)))
+      (format #t "Creating entry point for '~a.~a' at '~a'.~%" module function
+                 file-path)
+      (call-with-output-file file-path
+        (lambda (port)
+          ;; Technically the script could also include search-paths,
+          ;; but having a generic 'wrap phases also handles manually
+          ;; written entry point scripts.
+          (format port "#!~a
+# Auto-generated entry point script.
+import sys
+import ~a as mod
+sys.exit (mod.~a ())~%" interpreter module function)))
+        (chmod file-path #o755)))
+
+  (let* ((site-dir (site-packages inputs outputs))
+         (out (assoc-ref outputs "out"))
+         (bin-dir (string-append out "/bin"))
+         (entry-point-files (find-files site-dir "^entry_points.txt$")))
+    (mkdir-p bin-dir)
+    (for-each
+      (lambda (f)
+        (for-each
+          (lambda (ep) (apply create-script (cons bin-dir ep)))
+          (entry-points.txt->entry-points f)))
+      entry-point-files)))
+
+(define* (wrap #:key inputs outputs search-paths #:allow-other-keys)
   (define (list-of-files dir)
     (find-files dir (lambda (file stat)
                       (and (eq? 'regular (stat:type stat))
@@ -241,6 +388,20 @@ running checks after installing the package."
   (define %sh (delay (search-input-file inputs "bin/bash")))
   (define (sh) (force %sh))
 
+    (define input-directories
+    ;; The "source" input can be a directory, but we don't want it for search
+    ;; paths.  See <https://issues.guix.gnu.org/44924>.
+    (match (alist-delete "source" inputs)
+      (((_ . dir) ...)
+       dir)))
+
+    (for-each (match-lambda
+             ((env-var (files ...) separator type pattern)
+              (display (search-path-as-list files input-directories
+                                     #:type type
+                                     #:pattern pattern))))
+            search-paths)
+
   (let* ((var `("GUIX_PYTHONPATH" prefix
                 ,(search-path-as-string->list
                   (or (getenv "GUIX_PYTHONPATH") "")))))
@@ -250,29 +411,12 @@ running checks after installing the package."
                             files)))
               bindirs)))
 
-(define* (rename-pth-file #:key name inputs outputs #:allow-other-keys)
-  "Rename easy-install.pth to NAME.pth to avoid conflicts between packages
-installed with setuptools."
-  ;; Even if the "easy-install.pth" is not longer created, we kept this phase.
-  ;; There still may be packages creating an "easy-install.pth" manually for
-  ;; some good reason.
-  (let* ((site-packages (site-packages inputs outputs))
-         (easy-install-pth (string-append site-packages "/easy-install.pth"))
-         (new-pth (string-append site-packages "/" name ".pth")))
-    (when (file-exists? easy-install-pth)
-      (rename-file easy-install-pth new-pth))))
-
-(define* (ensure-no-mtimes-pre-1980 #:rest _)
-  "Ensure that there are no mtimes before 1980-01-02 in the source tree."
-  ;; Rationale: patch-and-repack creates tarballs with timestamps at the POSIX
-  ;; epoch, 1970-01-01 UTC.  This causes problems with Python packages,
-  ;; because Python eggs are ZIP files, and the ZIP format does not support
-  ;; timestamps before 1980.
-  (let ((early-1980 315619200))  ; 1980-01-02 UTC
-    (ftw "." (lambda (file stat flag)
-               (unless (<= early-1980 (stat:mtime stat))
-                 (utime file early-1980 early-1980))
-               #t))))
+(define* (set-SOURCE-DATE-EPOCH #:rest _)
+  "Set the 'SOURCE_DATE_EPOCH' environment variable.  This is used by tools
+that incorporate timestamps as a way to tell them to use a fixed timestamp.
+See https://reproducible-builds.org/specs/source-date-epoch/.";
+  (setenv "SOURCE_DATE_EPOCH" "315619200") ;; python-wheel respects this 
variable and sets pre-1980 times on files in zip files, which is unsupported
+  #t)
 
 (define* (enable-bytecode-determinism #:rest _)
   "Improve determinism of pyc files."
@@ -299,11 +443,11 @@ by Cython."
   ;; prefix directory.  The check phase is moved after the installation phase
   ;; to ease testing the built package.
   (modify-phases gnu:%standard-phases
-    (add-after 'unpack 'ensure-no-mtimes-pre-1980 ensure-no-mtimes-pre-1980)
-    (add-after 'ensure-no-mtimes-pre-1980 'enable-bytecode-determinism
+    (add-after 'unpack 'enable-bytecode-determinism
       enable-bytecode-determinism)
     (add-after 'enable-bytecode-determinism 'ensure-no-cythonized-files
       ensure-no-cythonized-files)
+    (replace 'set-SOURCE-DATE-EPOCH set-SOURCE-DATE-EPOCH)
     (delete 'bootstrap)
     (delete 'configure)                 ;not needed
     (replace 'build build)
@@ -313,9 +457,11 @@ by Cython."
     (add-after 'add-install-to-pythonpath 'add-install-to-path
       add-install-to-path)
     (add-after 'add-install-to-path 'wrap wrap)
+    ;; must be before tests, so they can use installed packages’ entry points.
+    (add-before 'wrap 'create-entrypoints create-entrypoints)
     (add-after 'wrap 'check check)
     (add-after 'check 'sanity-check sanity-check)
-    (add-before 'strip 'rename-pth-file rename-pth-file)))
+    (add-before 'check 'compile-bytecode compile-bytecode)))
 
 (define* (python-build #:key inputs (phases %standard-phases)
                        #:allow-other-keys #:rest args)