[bug#51319] [PATCH 1/2] build-system/julia: Enable Julia Pkg to find installed packages

[Efraim Flashner]

Thanks for the patch series! I'm really looking forward to this working
out in Guix, being able to use Julia to download as-yet unpackaged julia
packages will make using Julia much easier.

A couple of comments inline in the patch:

On Thu, Oct 21, 2021 at 01:09:58PM +0000, Jean-Baptiste Volatier via 
Guix-patches via wrote:
> * guix/build/julia-build-system.scm (link-depot): New phase.
> Julia built-in package manager (Pkg) looks for packages in
> JULIA_DEPOT_PATH/packages/PACKAGENAME/XXXX where XXXX is
> a string encoding package UUID and SHA1 of files.
> The link-depot phase creates a link at the correct location
> to allow Pkg to find packages that were already installed by
> Guix.
> (%package-path): modified package path from packages/ to
> loadpath/
> (project.toml->uuid): retrive package uuid from TOML file
> 
> * gnu/packages/julia-xyz.scm: correct paths to reflect
> change of %package-path
> 
> * gnu/packages/julia.scm: correct paths to reflect change of
> %package-path
> ---
>  gnu/packages/julia-xyz.scm        |  2 +-
>  gnu/packages/julia.scm            |  2 +-
>  guix/build/julia-build-system.scm | 46 ++++++++++++++++++++++++++++---
>  3 files changed, 44 insertions(+), 6 deletions(-)
> 
> diff --git a/gnu/packages/julia-xyz.scm b/gnu/packages/julia-xyz.scm
> index c1ebc0fba1..1e407a2277 100644
> --- a/gnu/packages/julia-xyz.scm
> +++ b/gnu/packages/julia-xyz.scm
> @@ -4610,7 +4610,7 @@ (define-public julia-uris
>             (lambda* (#:key source outputs #:allow-other-keys)
>               (let ((out (assoc-ref outputs "out")))
>                 (chdir
> -                (string-append out "/share/julia/packages/URIs/test")))
> +                (string-append out "/share/julia/loadpath/URIs/test")))
>               #t)))))
>      ;; required for tests
>      (inputs `(("julia-json" ,julia-json)))
> diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
> index 411f2e2e10..5b9bf91ed6 100644
> --- a/gnu/packages/julia.scm
> +++ b/gnu/packages/julia.scm
> @@ -673,7 +673,7 @@ (define-public julia
>      (native-search-paths
>        (list (search-path-specification
>                (variable "JULIA_LOAD_PATH")
> -              (files (list "share/julia/packages/")))
> +              (files (list "share/julia/loadpath/")))
>              (search-path-specification
>                (variable "JULIA_DEPOT_PATH")
>                (files (list "share/julia/")))))
> diff --git a/guix/build/julia-build-system.scm 
> b/guix/build/julia-build-system.scm
> index d74acf2a05..c7f2119974 100644
> --- a/guix/build/julia-build-system.scm
> +++ b/guix/build/julia-build-system.scm
> @@ -1,5 +1,6 @@
>  ;;; GNU Guix --- Functional package management for GNU
>  ;;; Copyright © 2019, 2020 Nicolò Balzarotti <nicolo@nixo.xyz>
> +;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me>
>  ;;;
>  ;;; This file is part of GNU Guix.
>  ;;;
> @@ -20,10 +21,13 @@
>  (define-module (guix build julia-build-system)
>    #:use-module ((guix build gnu-build-system) #:prefix gnu:)
>    #:use-module (guix build utils)
> +  #:use-module (rnrs io ports)
>    #:use-module (ice-9 match)
>    #:use-module (ice-9 regex)
>    #:use-module (ice-9 rdelim)
> +  #:use-module (ice-9 popen)
>    #:export (%standard-phases
> +            %package-path
>              julia-create-package-toml
>              julia-build))
> 
> @@ -37,7 +41,7 @@ (define (invoke-julia code)
>    (invoke "julia" "-e" code))
> 
>  ;; subpath where we store the package content
> -(define %package-path "/share/julia/packages/")
> +(define %package-path "/share/julia/loadpath/")
> 
>  (define (project.toml->name file)
>    "Look for Julia package name in the TOML file FILE (usually named
> @@ -51,6 +55,18 @@ (define (project.toml->name file)
>                (if m (match:substring m 1)
>                    (loop (read-line in 'concat)))))))))
> 
> +(define (project.toml->uuid file)
> +  "Look for Julia package uuid in the TOML file FILE (usually named
> +Project.toml)."
> +  (call-with-input-file file
> +    (lambda (in)
> +      (let loop ((line (read-line in 'concat)))
> +        (if (eof-object? line)
> +            #f
> +            (let ((m (string-match "uuid\\s*=\\s*\"(.*)\"" line)))
> +              (if m (match:substring m 1)
> +                  (loop (read-line in 'concat)))))))))
> +
>  (define* (install #:key source inputs outputs julia-package-name
>                    #:allow-other-keys)
>    (let* ((out (assoc-ref outputs "out"))
> @@ -73,7 +89,7 @@ (define* (precompile #:key source inputs outputs 
> julia-package-name
>      (setenv "JULIA_DEPOT_PATH" builddir)
>      ;; Add new package dir to the load path.
>      (setenv "JULIA_LOAD_PATH"
> -            (string-append builddir "packages/" ":"
> +            (string-append builddir "loadpath/" ":"
>                             (or (getenv "JULIA_LOAD_PATH")
>                                 "")))
>      ;; Actual precompilation:
> @@ -97,15 +113,36 @@ (define* (check #:key tests? source inputs outputs 
> julia-package-name
>        (setenv "SOURCE_DATE_EPOCH" "1")
>        (setenv "JULIA_DEPOT_PATH" builddir)
>        (setenv "JULIA_LOAD_PATH"
> -              (string-append builddir "packages/" ":"
> +              (string-append builddir "loadpath/" ":"
>                               (or (getenv "JULIA_LOAD_PATH")
>                                   "")))
>        (setenv "HOME" "/tmp")
>        (invoke "julia" "--depwarn=yes"
> -              (string-append builddir "packages/"
> +              (string-append builddir "loadpath/"
>                               package "/test/runtests.jl"))))
>    #t)
> 
> +(define* (link-depot #:key source inputs outputs julia-package-name
> +                     #:allow-other-keys)

I had to wrap this with '(when (file-exists? "Project.toml")' since we
do have some packages that pre-date the use of Project.toml. Building
julia-bufferedstreams will test both a package with and without a
Project.toml.

> +  (let* ((out (assoc-ref outputs "out"))
> +         (package-name (or
> +                        julia-package-name
> +                        (project.toml->name "Project.toml")))
> +         (package-dir (string-append out %package-path package-name))
> +         (uuid (project.toml->uuid "Project.toml"))
> +         (pipe (open-pipe* OPEN_READ "julia" "-e"
> +                           (format #f "using Pkg;
> +println(Base.version_slug(Base.UUID(\"~a\"),
> +                          Base.SHA1(Pkg.GitTools.tree_hash(\".\"))))" uuid 
> package-dir)))

Here you have a '~a' to use uuid but package-dir isn't used anywhere. Is
package-dir not needed or is there a missing substitution?

> +         (slug (string-trim-right (get-string-all pipe))))
> +    ;; When installing a package, julia looks first at in the 
> JULIA_DEPOT_PATH
> +    ;; for a path like packages/PACKAGE/XXXX
> +    ;; Where XXXX is a slug encoding the package UUID and SHA1 of the files
> +    ;; Here we create a link with the correct path to enable julia to find 
> the package
> +    (mkdir-p (string-append out "/share/julia/packages/" package-name))
> +    (symlink package-dir (string-append out "/share/julia/packages/" 
> package-name "/" slug)))
> +  #t)
> +
>  (define (julia-create-package-toml outputs source
>                                     name uuid version
>                                     deps)
> @@ -138,6 +175,7 @@ (define %standard-phases
>      (delete 'check) ; tests must be run after installation
>      (replace 'install install)
>      (add-after 'install 'precompile precompile)
> +    (add-after 'unpack 'link-depot link-depot)
>      (add-after 'install 'check check)
>      ;; TODO: In the future we could add a "system-image-generation" phase
>      ;; where we use PackageCompiler.jl to speed up package loading times
> --
> 2.33.1
> 

-- 
Efraim Flashner   <efraim@flashner.co.il>   רנשלפ םירפא
GPG key = A28B F40C 3E55 1372 662D  14F7 41AA E7DC CA3D 8351
Confidentiality cannot be guaranteed on emails sent or received unencrypted

signature.asc
Description: PGP signature