[bug#73197] gnu: julia: Update to 1.9.3.

Message ID 87ikpc5z6g.fsf@gmail.com
State New
Headers
Series [bug#73197] gnu: julia: Update to 1.9.3. |

Commit Message

Simon Tournier Feb. 14, 2025, 3:22 p.m. UTC
  Hi,

On Mon, 13 Jan 2025 at 19:31, Simon Tournier <zimon.toutoune@gmail.com> wrote:

> For the record, it’s about base/loading.jl:
>
> --8<---------------cut here---------------start------------->8---
>      try
>          toplevel_load[] = false
>          # perform the search operation to select the module file require intends to load
> -        path = locate_package(pkg)
> +        path = locate_package(pkg, env)
>          if path === nothing
>              throw(ArgumentError("""
>                  Package $pkg is required but does not seem to be installed:
> --8<---------------cut here---------------end--------------->8---

Welcome to a debugging session using the most advanced technique: print! :-)

Attached the base/loading.jl with many ’println’.

Well, that’s very interesting because it seems related to some Julia
internals.  Somehow, the order of LOAD_PATH matters!

In all the dance, the code enters once and it founds it!

--8<---------------cut here---------------start------------->8---
ENTRY _require: Call locate_package( ConstructionBase [187b0558-2788-49d3-abe0-74a17ed4e7c9] , /gnu/store/garmz1dz382ib25p2ckl0kj9rjay4xd4-julia-unitful-1.12.2/share/julia/loadpath/ )

[...]

RESULT _require: path = /gnu/store/yg4bz2kb7dqb6haaj0v7hj4mrd8rn27l-julia-constructionbase-1.3.0/share/julia/loadpath/ConstructionBase/src/ConstructionBase.jl
--8<---------------cut here---------------end--------------->8---

Then, it enters later with the same call… And bang!

--8<---------------cut here---------------start------------->8---
ENTRY _require: Call locate_package( ConstructionBase [187b0558-2788-49d3-abe0-74a17ed4e7c9] , /gnu/store/garmz1dz382ib25p2ckl0kj9rjay4xd4-julia-unitful-1.12.2/share/julia/loadpath/ )

[...]

RESULT _require: path = nothing
ERROR: LoadError: ArgumentError: Package ConstructionBase [187b0558-2788-49d3-abe0-74a17ed4e7c9] is required but does not seem to be installed:
 - Run `Pkg.instantiate()` to install all recorded dependencies.
--8<---------------cut here---------------end--------------->8---

What’s different?  The call loops over load_path(), something like:

--8<---------------cut here---------------start------------->8---
locate_package_env: START LOOP; over load_path() = ["/gnu/store/garmz1dz382ib25p2ckl0kj9rjay4xd4-julia-unitful-1.12.2/share/julia/environments/v1.9/Project.toml", "/gnu/store/cp0fbzd4j1znnp4fjpsknap0ksnp2xk1-julia-1.9.3/share/julia/stdlib/v1.9", "/gnu/store/yg4bz2kb7dqb6haaj0v7hj4mrd8rn27l-julia-constructionbase-1.3.0/share/julia/loadpath/", "/gnu/store/garmz1dz382ib25p2ckl0kj9rjay4xd4-julia-unitful-1.12.2/share/julia/loadpath/"]

locate_package_env: loop = 1
locate_package_env: env = /gnu/store/garmz1dz382ib25p2ckl0kj9rjay4xd4-julia-unitful-1.12.2/share/julia/environments/v1.9/Project.toml
locate_package_env: path = nothing

locate_package_env: loop = 2
locate_package_env: env = /gnu/store/cp0fbzd4j1znnp4fjpsknap0ksnp2xk1-julia-1.9.3/share/julia/stdlib/v1.9
locate_package_env: path = nothing

locate_package_env: loop = 3
locate_package_env: env = /gnu/store/yg4bz2kb7dqb6haaj0v7hj4mrd8rn27l-julia-constructionbase-1.3.0/share/julia/loadpath/
locate_package_env: path = /gnu/store/yg4bz2kb7dqb6haaj0v7hj4mrd8rn27l-julia-constructionbase-1.3.0/share/julia/loadpath/ConstructionBase/src/ConstructionBase.jl
--8<---------------cut here---------------end--------------->8---

And for the second case, tandam!

--8<---------------cut here---------------start------------->8---
locate_package_env: START LOOP; over load_path() = ["/tmp/.julia/environments/v1.9/Project.toml", "/gnu/store/cp0fbzd4j1znnp4fjpsknap0ksnp2xk1-julia-1.9.3/share/julia/stdlib/v1.9", "/gnu/store/garmz1dz382ib25p2ckl0kj9rjay4xd4-julia-unitful-1.12.2/share/julia/loadpath/", "/gnu/store/yg4bz2kb7dqb6haaj0v7hj4mrd8rn27l-julia-constructionbase-1.3.0/share/julia/loadpath/"]

locate_package_env: loop = 1
locate_package_env: env = /tmp/.julia/environments/v1.9/Project.toml
locate_package_env: path = nothing

locate_package_env: loop = 2
locate_package_env: env = /gnu/store/cp0fbzd4j1znnp4fjpsknap0ksnp2xk1-julia-1.9.3/share/julia/stdlib/v1.9
locate_package_env: path = nothing

locate_package_env: loop = 3
locate_package_env: env = /gnu/store/garmz1dz382ib25p2ckl0kj9rjay4xd4-julia-unitful-1.12.2/share/julia/loadpath/
locate_package_env: path = nothing
locate_package_env: BREAK: /gnu/store/garmz1dz382ib25p2ckl0kj9rjay4xd4-julia-unitful-1.12.2/share/julia/loadpath/ == /gnu/store/garmz1dz382ib25p2ckl0kj9rjay4xd4-julia-unitful-1.12.2/share/julia/loadpath/
locate_package_env:      : loading_extension = false
locate_package_env:      : precompiling_extension = false
locate_package_env: END LOOP

locate_package_env: pkg = ConstructionBase [187b0558-2788-49d3-abe0-74a17ed4e7c9]
locate_package_env: Sys.STDLIB = /gnu/store/cp0fbzd4j1znnp4fjpsknap0ksnp2xk1-julia-1.9.3/share/julia/stdlib/v1.9
locate_package_env: Call manifest_uuid_path( /gnu/store/cp0fbzd4j1znnp4fjpsknap0ksnp2xk1-julia-1.9.3/share/julia/stdlib/v1.9 , ConstructionBase [187b0558-2788-49d3-abe0-74a17ed4e7c9] )
locate_package_env: mbypath = nothing String? false
locate_package_env: RETURN: path = nothing ; env = /gnu/store/garmz1dz382ib25p2ckl0kj9rjay4xd4-julia-unitful-1.12.2/share/julia/loadpath/
locate_package: RETURN: nothing
_require: Post locate_package( ConstructionBase [187b0558-2788-49d3-abe0-74a17ed4e7c9] , /gnu/store/garmz1dz382ib25p2ckl0kj9rjay4xd4-julia-unitful-1.12.2/share/julia/loadpath/ )
RESULT _require: path = nothing
--8<---------------cut here---------------end--------------->8---

The attentive reader notes that load_path() is a list of 4 elements –the
last contains what we want– but a test breaks the loop.

Well, I do not know the details about the reasons of all that.  Maybe
something worth to ask upstream. :-)

Well, I’ll (force) push shortly a trivial fix to the branch
wip-julia-upgrade. 

Cheers,
simon

PS:  For the record, some elements:

1. Checkout wip-julia-upgrade
2. cp $(./pre-inst-env guix build julia -S) /tmp/
3. Uncompress the source and copy (e.g., /home/simon/tmp/julia-1.9.3)
4. Tweak the base/loading.jl file (and other as base/initdefs.jl)
5. Use this source with:

    (source
     (local-file "/home/simon/tmp/julia-1.9.3"
                 #:recursive? #t))

6. guix shell -CPNWD guix --expose=/home/simon/tmp/=/home/simon/tmp/
7. Build: ./pre-inst-env guix build --no-grafts --without-tests=julia julia
8. Tweak guix/build-system/julia-build-system.scm
9. Loop ./pre-inst-env guix build --no-grafts --without-tests=julia julia-unitful

Hope that helps.

--
  

Comments

Simon Tournier Feb. 14, 2025, 9:10 p.m. UTC | #1
Hi,

On Fri, 14 Feb 2025 at 16:22, Simon Tournier <zimon.toutoune@gmail.com> wrote:

> Well, I’ll (force) push shortly a trivial fix to the branch
> wip-julia-upgrade.

I did. \o/

And waiting the rebuild by ci.guix, I’m rebuilding some on my own.  And
I obverse weird behaviour…

For example, on the top of 5f31ce18b3.

--8<---------------cut here---------------start------------->8---
$ ./pre-inst-env guix build --no-grafts      \
     --without-tests=julia-argcheck          \
     --without-tests=julia-documenter        \
     --without-tests=julia-genericschur      \
     --without-tests=julia-static            \
     --without-tests=julia-statsbase         \
     --without-tests=julia-infinity          \
     --without-tests=julia-pycall            \
     --without-tests=julia-http              \
     --without-tests=julia-staticarrays      \
     --without-tests=julia-finitedifferences \
   julia-benchmarktools

/gnu/store/pdj7ilvyh4i96jd5av97vrlghpxc25ni-julia-benchmarktools-1.3.2
--8<---------------cut here---------------end--------------->8---

So far, so good!  Here all ’without-tests’ are packages that need to be
updated or need a fix in their test suite–e.g., one test fails because
hardware or other as it happens time to time with Julia packages.

Now, let do the same but using the ’julia’ package without its own
tests.  It should be pass, no?

--8<---------------cut here---------------start------------->8---
$ ./pre-inst-env guix build --no-grafts      \
     --without-tests=julia-argcheck          \
     --without-tests=julia-documenter        \
     --without-tests=julia-genericschur      \
     --without-tests=julia-static            \
     --without-tests=julia-statsbase         \
     --without-tests=julia-infinity          \
     --without-tests=julia-pycall            \
     --without-tests=julia-http              \
     --without-tests=julia-staticarrays      \
     --without-tests=julia-finitedifferences \
   --without-tests=julia                     \
   julia-benchmarktools 

starting phase `precompile'
ERROR: LoadError: ArgumentError: Package Parsers [69de0a69-1ddd-5017-9359-2bf0b02dc9f0] is required but does not seem to be installed:
 - Run `Pkg.instantiate()` to install all recorded dependencies.
--8<---------------cut here---------------end--------------->8---

Euh, I’m probably doing wrong something somewhere…  But it does not jump
to my eyes what.  Any idea?

For the record, here the path:

    $ guix graph --path julia-benchmarktools julia-parsers
    julia-benchmarktools@1.3.2
    julia-json@0.21.3
    julia-parsers@2.2.4

And note that julia-parsers is propagated by julia-json then
julia-benchmarktools propagates julia-json.

Therefore, the full ’julia’ package finds julia-parsers but not all the
same except running the ’check’ phase of julia’.

What do I miss about package transformation?

That’s said, it’s weird behaviour because I also observe the converse:
something that passes using --without-tests=julia but fails with full
’julia’.

Bah… my creativity starts to be lacking…

Cheers,
simon

PS: What about Nix?

1. I don’t clearly understand what they do.
2. So much Julia code… Hum, I’m not convinced that’s the way…

https://github.com/NixOS/nixpkgs/tree/master/pkgs/development/julia-modules
https://github.com/NixOS/nixpkgs/tree/master/pkgs/development/compilers/julia
  
King, Spencer via Guix-patches" via Feb. 18, 2025, 10:30 p.m. UTC | #2
Dear Simon,

Thanks for working on this ! I've tried to follow the nix install
process. Here are some (incomplete) notes.
For brievity's sake, only the Zlib package will be shown in the files
below.

1. A list of dependencies is stored into a YAML:

8<-----------------------------------------------------------------------

cat
/nix/store/3c8lw0flmvvl5hlkc8c7xcwvwj1kbzp2-julia-package-closure.yml

- name: Zlib_jll

  uuid: 83775a58-1f1d-513f-b197-d71354ab007a

  version: 1.2.13+0

  depends_on:

    "Libdl": "8f399da3-3557-5675-b5ff-fb832c97cbdb"

8<-----------------------------------------------------------------------

2. Each dependency is a derivation and transformed into a git repo

8<-----------------------------------------------------------------------

building
'/nix/store/icap989di414ax9zpq1b07ca4szn1v6j-Zlib_jll.jl-866bc71.drv'...

exporting https://github.com/JuliaBinaryWrappers/Zlib_jll.jl.git (rev
866bc7103cfb6233c35d6f1b2c6bd23ef9f6fe5f) into
/nix/store/pi31a0w9xm3qych5yfdw3mg93sxl41rq-Zlib_jll.jl-866bc71

Initialized empty Git repository in
/nix/store/pi31a0w9xm3qych5yfdw3mg93sxl41rq-Zlib_jll.jl-866bc71/.git/

remote: Enumerating objects: 26, done.

remote: Counting objects: 100% (7/7), done.

remote: Compressing objects: 100% (6/6), done.

remote: Total 26 (delta 1), reused 1 (delta 1), pack-reused 19 (from
1)

Unpacking objects: 100% (26/26), 5.55 KiB | 1.85 MiB/s, done.

From https://github.com/JuliaBinaryWrappers/Zlib_jll.jl

* branch           
866bc7103cfb6233c35d6f1b2c6bd23ef9f6fe5f -> FETCH_HEAD

Switched to a new branch 'fetchgit'

removing `.git'...

8<-----------------------------------------------------------------------

3. Package location are overriden, first into a JSON

8<-----------------------------------------------------------------------

$ cat /nix/store/nf666fq31xz8f7hx9qi9ygicf4hdfx4i-Overrides.json

    "uuid-83775a58-1f1d-513f-b197-d71354ab007a": {

        "Zlib": {

            "path":
"/nix/store/6d4hpjr4f04zq9090kd9kkffa9j2q7gj-Zlib",

            "sha1":
"b86177a36c8ba482120ab766b6670177dffd72f3"

        }

    },

8<-----------------------------------------------------------------------

4. ... then in a TOML

8<-----------------------------------------------------------------------

$ cat /nix/store/gnwl6k320mv0pnb8hpcdww40hypp3i7f-Overrides.toml

b86177a36c8ba482120ab766b6670177dffd72f3 =
"/nix/store/6d4hpjr4f04zq9090kd9kkffa9j2q7gj-Zlib"

8<-----------------------------------------------------------------------

5. A local registry is created. Somewhere, the following code is run

8<-----------------------------------------------------------------------

Pkg.Registry.add(Pkg.RegistrySpec(path=\"/nix/store/p5dy9z5869qnmmawsid5dcy1nqy1k5sn-minimal-julia-registry`.

8<-----------------------------------------------------------------------

The registry looks like this:

8<-----------------------------------------------------------------------

$ tree
/nix/store/p5dy9z5869qnmmawsid5dcy1nqy1k5sn-minimal-julia-registry

├── A

│   ├── ArrowTypes

│   │   ├── Compat.toml

│   │   ├── Deps.toml

│   │   ├── Package.toml

│   │   └── Versions.toml

8<-----------------------------------------------------------------------

with Compat.toml

8<-----------------------------------------------------------------------

["1-1.1"]

julia = "1.3.0-1"

["1.2-2"]

julia = "1"

8<-----------------------------------------------------------------------

with Deps.toml

8<-----------------------------------------------------------------------

[1-2]

UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

["2.1-2"]

Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"

8<-----------------------------------------------------------------------

with Package.toml

8<-----------------------------------------------------------------------

name = "ArrowTypes"

uuid = "31f734f8-188a-4ce0-8406-c8a06bd891cd"

repo =
"file:///nix/store/ndqz9cikh67hqwxh4l7vji07764p4xpk-julia-ArrowTypes-2.3.0"

subdir = "src/ArrowTypes"

["2.3.0"]

git-tree-sha1 = "404265cd8128a2515a81d5eae16de90fdef05101"

8<-----------------------------------------------------------------------

6. I'm not sure how the registry is used at this step so here the log
output if you can make sense out of it

8<-----------------------------------------------------------------------

Copying registry from
`/nix/store/p5dy9z5869qnmmawsid5dcy1nqy1k5sn-minimal-julia-registry`

Copied registry `General` to
`/nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/depot/registries/General`

Adding packages: CSV

Resolving package versions...

Cloning [e2d170a0-9d28-54be-80f0-106bbe20a464] DataValueInterfaces
from
file:///nix/store/0gdzfghqzs56ilzb9dx239fij4hny9i3-julia-DataValueInterfaces-1.0.0

Installed DataValueInterfaces ───────── v1.0.0

...

Updating
`/nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/project/Project.toml`

[336ed68f] + CSV v0.10.14

Updating
`/nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/project/Manifest.toml`

Removing registry `General` from
/nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/depot/registries/General

building
'/nix/store/0k35snxa68lrkap77s376gqx8ppxr8yn-julia-1.9.4-env.drv'...

8<-----------------------------------------------------------------------

Hope that helps,

Alexis
  
Simon Tournier Feb. 20, 2025, 11:02 a.m. UTC | #3
Hi Alexis,

On Tue, 18 Feb 2025 at 23:30, Alexis Praga via Guix-patches via <guix-patches@gnu.org> wrote:

>                              I've tried to follow the nix install
> process. Here are some (incomplete) notes.

Thanks!  It helps.


> 5. A local registry is created. Somewhere, the following code is run

On Monday, I thought about something like that.  However, I’m note sure
that one package can write to another outputs than its own.  Something
to check…

My idea was to create a “local” registry per package containing all the
requirements.  Relying heavily on symbolic links, as profile does.

Today, it works that way:

--8<---------------cut here---------------start------------->8---
$ tree -L 2 $(guix build julia-zygote)/share/julia
/gnu/store/n64x6qrxy7b2lci9aar3pv6kikp1rgan-julia-zygote-0.6.41/share/julia
├── compiled
│   └── v1.8
├── loadpath
│   └── Zygote
└── packages
    └── Zygote
--8<---------------cut here---------------end--------------->8---

where julia-zygote contains this definition:

    (propagated-inputs
     (list julia-abstractffts
           julia-chainrules
           julia-chainrulescore
           julia-diffrules
           julia-fillarrays
           julia-forwarddiff
           julia-irtools
           julia-logexpfunctions
           julia-macrotools
           julia-nanmath
           julia-requires
           julia-specialfunctions
           julia-zygoterules))

In other words, when running the ’check’ phase, we setup the LOAD_PATH
and DEPOT_PATH via the environment variable JULIA_{LOAD,DEPOT}_PATH.

However, stuff changed with v1.9 as explained here [1].  Maybe I’m
missing something with all that.  My current understanding: tweaking
these environment variables will not be enough.

Now, let enter inside “guix shell julia-zygote”:

--8<---------------cut here---------------start------------->8---
$ tree -L 2 $GUIX_ENVIRONMENT/share/julia
/gnu/store/asis32i2wf3jr9854nh1yxjf6fsipk62-profile/share/julia
├── compiled
│   └── v1.8
├── loadpath
│   ├── AbstractFFTs -> /gnu/store/hlnlr2xkd06mrcg72v7ilynqv36z9xqi-julia-abstractffts-1.0.1/share/julia/loadpath/AbstractFFTs
│   ├── Aqua -> /gnu/store/jb4vzsn12nbfqyr8inhz2a570fv56x4i-julia-aqua-0.5.5/share/julia/loadpath/Aqua
│   ├── Calculus -> /gnu/store/k2ghzcx087sssllkk1b3c8xxjwbr3n43-julia-calculus-0.5.1/share/julia/loadpath/Calculus
│   ├── ChainRules -> /gnu/store/sj99c9z4siq87qhmix0xr5yyzkcphg80-julia-chainrules-1.35.0/share/julia/loadpath/ChainRules
│   ├── ChainRulesCore -> /gnu/store/h09v7lvgahl7rx95aj0rz92hc34xpmpn-julia-chainrulescore-1.12.2/share/julia/loadpath/ChainRulesCore

[...]

│   ├── StaticArrays -> /gnu/store/xn3jfkssqa0lggnrr3qkaicvawyn4wda-julia-staticarrays-1.2.13/share/julia/loadpath/StaticArrays
│   ├── Zygote -> /gnu/store/n64x6qrxy7b2lci9aar3pv6kikp1rgan-julia-zygote-0.6.41/share/julia/loadpath/Zygote
│   └── ZygoteRules -> /gnu/store/ska4a8nxxl9qmv1hgb4ah5147lngq588-julia-zygoterules-0.2.2/share/julia/loadpath/ZygoteRules
└── packages
    ├── AbstractFFTs -> /gnu/store/hlnlr2xkd06mrcg72v7ilynqv36z9xqi-julia-abstractffts-1.0.1/share/julia/packages/AbstractFFTs
    ├── Aqua -> /gnu/store/jb4vzsn12nbfqyr8inhz2a570fv56x4i-julia-aqua-0.5.5/share/julia/packages/Aqua
    ├── Calculus -> /gnu/store/k2ghzcx087sssllkk1b3c8xxjwbr3n43-julia-calculus-0.5.1/share/julia/packages/Calculus
    ├── ChainRules -> /gnu/store/sj99c9z4siq87qhmix0xr5yyzkcphg80-julia-chainrules-1.35.0/share/julia/packages/ChainRules
    ├── ChainRulesCore -> /gnu/store/h09v7lvgahl7rx95aj0rz92hc34xpmpn-julia-chainrulescore-1.12.2/share/julia/packages/ChainRulesCore

[...]

    ├── StaticArrays -> /gnu/store/xn3jfkssqa0lggnrr3qkaicvawyn4wda-julia-staticarrays-1.2.13/share/julia/packages/StaticArrays
    ├── Zygote -> /gnu/store/n64x6qrxy7b2lci9aar3pv6kikp1rgan-julia-zygote-0.6.41/share/julia/packages/Zygote
    └── ZygoteRules -> /gnu/store/ska4a8nxxl9qmv1hgb4ah5147lngq588-julia-zygoterules-0.2.2/share/julia/packages/ZygoteRules

65 directories, 0 files
--8<---------------cut here---------------end--------------->8---

My idea is to do the same at least for the ’check’ phase,


Thanks again for trying with Nix.  It helps!  I’ll read again all these
details, food for creativity. ;-)

Cheers,
simon

1: [bug#73197] [PATCH] gnu: julia: Update to 1.9.3.
Simon Tournier <zimon.toutoune@gmail.com>
Mon, 13 Jan 2025 19:31:16 +0100
id:87bjwa4lcb.fsf@gmail.com
https://issues.guix.gnu.org/73197
https://issues.guix.gnu.org/msgid/87bjwa4lcb.fsf@gmail.com
https://yhetil.org/guix/87bjwa4lcb.fsf@gmail.com
  
King, Spencer via Guix-patches" via Feb. 20, 2025, 11:10 p.m. UTC | #4
Hi Simon,

As a follow-up, here's how nix uses the local depot.
It wraps julia by setting the following variable to subfolders of the "meta" derivation 

1. JULIA_DEPOT_PATH to ~/.julia and the local depot
home/alex/.julia:/nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/depot

2. JULIA_PROJECT_PATH to a custom project
/nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/project

3. JULIA_LOAD_PATH to the Project.toml
@:/nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/project/Project.toml:@v#.#:@stdlib
(not sure what the weird characters are about, like '@')

The depot itself has the following structure
- source code in clones/ with UUID
- compiled library in compiled
- source code in packages/ with package name (again !)

❯ tree /nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/depot/ -L 2
/nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/depot/
├── artifacts
│   └── Overrides.toml
├── clones
    ├── 944b1d66-785c-5afd-91f1-9de20f533193
    │   ├── LICENSE.md
    │   ├── Project.toml
    │   ├── README.md
    │   ├── src
    │   │   ├── CodecZlib.jl
    │   │   ├── compression.jl
    │   │   ├── decompression.jl
    │   │   └── libz.jl
    │   └── test
    │       ├── abra.gz
    │       ├── foo.txt.gz
    │       └── runtests.jl
│  ── [...]
|── compiled
|   └── v1.9
|       ├── CodecZlib
|       │   ├── 1TI30_fZ84u.ji
|       │   └── 1TI30_fZ84u.so
|       ├── [...]
├── logs
│   └── manifest_usage.toml
├── packages
│   ├── CodecZlib
|        └── Ndt0H
|          ├── LICENSE.md
|          ├── Project.toml
|          ├── README.md
|          ├── src
|          │   ├── CodecZlib.jl
|          │   ├── compression.jl
|          │   ├── decompression.jl
|          │   └── libz.jl
|          └── test
|              ├── abra.gz
|              ├── foo.txt.gz
|              └── runtests.jl
|      └── [...]
|   ├── registries
└── scratchspaces
    └── 44cfe95a-1eb2-52ea-b672-e2afdf69b78f

Finally, the project itself has a TOML file defining the package we want to install (in my case CSV) and the Manifest.toml for this package (I think)

❯ tree /nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/project/
/nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/project/
├── Manifest.toml
└── Project.toml

❯ cat /nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/project/Project.toml
     [deps]
     CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"

❯ head -n 10 /nix/store/ap871sp56qx74k4hz2nlnjvpys8pdh7j-julia-depot/project/Manifest.toml
     # This file is machine-generated - editing it directly is not advised

     julia_version = "1.9.4"
     manifest_format = "2.0"
     project_hash = "de8e0bb32b50d890eb0519dc7725e6a1f422aa74"

     [[deps.Artifacts]]
     uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

     [[deps.Base64]]


Hope that helps,

Alexis
  
Simon Tournier March 5, 2025, 1:59 p.m. UTC | #5
Hi,

On Fri, 14 Feb 2025 at 22:10, zimoun <zimon.toutoune@gmail.com> wrote:

> Bah… my creativity starts to be lacking…

Unexpected failures by Cuirass?!

For the record, if we consider the last evaluation [1] of the branch
wip-julia-upgrade on the top of 5f31c [2], Cuirass reports this failure
[3] about julia-abstractffts.

--8<---------------cut here---------------start------------->8---
starting phase `check'
running tests from '/gnu/store/w5iqv2y9yg12plyks3ppywyvq2ffirl8-julia-abstractffts-1.0.1/share/julia/loadpath/AbstractFFTs/test/runtests.jl'
ERROR: LoadError: ArgumentError: Package ConstructionBase [187b0558-2788-49d3-abe0-74a17ed4e7c9] is required but does not seem to be installed:
 - Run `Pkg.instantiate()` to install all recorded dependencies.

Stacktrace:
--8<---------------cut here---------------end--------------->8---

Hum?!  What do I miss?

Cuirass says the derivation and the output are:

    /gnu/store/pyvn3lmxizbhyvh29h8ladx8ycpsxbi6-julia-abstractffts-1.0.1.drv
    /gnu/store/w5iqv2y9yg12plyks3ppywyvq2ffirl8-julia-abstractffts-1.0.1

And on my local machine, using Guix 056910e, it just builds and the
’check’ phase just passes.

--8<---------------cut here---------------start------------->8---
$ guix build --check \
   /gnu/store/pyvn3lmxizbhyvh29h8ladx8ycpsxbi6-julia-abstractffts-1.0.1.drv

The following derivation will be built:
  /gnu/store/pyvn3lmxizbhyvh29h8ladx8ycpsxbi6-julia-abstractffts-1.0.1.drv
building /gnu/store/pyvn3lmxizbhyvh29h8ladx8ycpsxbi6-julia-abstractffts-1.0.1.drv...
starting phase `separate-from-pid1'
build process now running as PID 18
phase `separate-from-pid1' succeeded after 0.0 seconds

[...]

starting phase `check'
running tests from '/gnu/store/w5iqv2y9yg12plyks3ppywyvq2ffirl8-julia-abstractffts-1.0.1/share/julia/loadpath/AbstractFFTs/test/runtests.jl'
Test Summary: | Pass  Total  Time
rfft sizes    |    5      5  0.2s
Test Summary: | Pass  Total  Time
Custom Plan   |    4      4  0.5s
Test Summary:   | Pass  Total  Time
Shift functions |   14     14  0.3s
Test Summary:   | Pass  Total  Time
FFT Frequencies |   67     67  0.7s
Test Summary: | Pass  Total  Time
normalization |    1      1  0.0s
phase `check' succeeded after 13.2 seconds
starting phase `patch-shebangs'
phase `patch-shebangs' succeeded after 0.0 seconds
starting phase `strip'
phase `strip' succeeded after 0.0 seconds
starting phase `validate-runpath'
phase `validate-runpath' succeeded after 0.0 seconds
starting phase `validate-documentation-location'
phase `validate-documentation-location' succeeded after 0.0 seconds
starting phase `delete-info-dir-file'
phase `delete-info-dir-file' succeeded after 0.0 seconds
starting phase `patch-dot-desktop-files'
phase `patch-dot-desktop-files' succeeded after 0.0 seconds
starting phase `make-dynamic-linker-cache'
phase `make-dynamic-linker-cache' succeeded after 0.0 seconds
starting phase `install-license-files'
installing 1 license files from '.'
phase `install-license-files' succeeded after 0.0 seconds
starting phase `reset-gzip-timestamps'
phase `reset-gzip-timestamps' succeeded after 0.0 seconds
starting phase `compress-documentation'
phase `compress-documentation' succeeded after 0.0 seconds
guix build: error: derivation `/gnu/store/pyvn3lmxizbhyvh29h8ladx8ycpsxbi6-julia-abstractffts-1.0.1.drv' may not be deterministic: output `/gnu/store/w5iqv2y9yg12plyks3ppywyvq2ffirl8-julia-abstractffts-1.0.1' differs
--8<---------------cut here---------------end--------------->8---

The non-deterministic is expected; another story.


What could be twisted?  How is it possible?

Cheers,
simon

1: https://ci.guix.gnu.org/eval/2041608
2: https://git.savannah.gnu.org/cgit/guix.git/log/?id=5f31ce18b3655264e3d101eada5382bf22ac1cfa
3: https://ci.guix.gnu.org/build/9309544/details
  
Ludovic Courtès March 7, 2025, 11:20 p.m. UTC | #6
Hello,

Simon Tournier <zimon.toutoune@gmail.com> skribis:

> And on my local machine, using Guix 056910e, it just builds and the
> ’check’ phase just passes.
>
> $ guix build --check \
>    /gnu/store/pyvn3lmxizbhyvh29h8ladx8ycpsxbi6-julia-abstractffts-1.0.1.drv

It fails on my laptop just like
<https://ci.guix.gnu.org/build/9309544/details> shows:

--8<---------------cut here---------------start------------->8---
starting phase `check'
running tests from '/gnu/store/w5iqv2y9yg12plyks3ppywyvq2ffirl8-julia-abstractffts-1.0.1/share/julia/loadpath/AbstractFFTs/test/runtests.jl'
ERROR: LoadError: ArgumentError: Package ConstructionBase [187b0558-2788-49d3-abe0-74a17ed4e7c9] is required but does not seem to be installed:
 - Run `Pkg.instantiate()` to install all recorded dependencies.
--8<---------------cut here---------------end--------------->8---

If you don’t get that failure on your own machine, perhaps there’s
something non-deterministic here?  I wouldn’t expect this type of error
to be non-deterministic, but who knows.

Tricky!

Ludo’.
  

Patch

--- /tmp/julia-1.9.3/base/loading.jl	1970-01-01 01:00:01.000000000 +0100
+++ /home/simon/tmp/julia-1.9.3/base/loading.jl	2025-02-14 15:38:59.879047968 +0100
@@ -391,6 +391,7 @@ 
 identify_package(name::String)                = _nothing_or_first(identify_package_env(name))
 
 function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)
+    println("locate_package_env: ARGS: ", pkg, " , ", stopenv)
     cache = LOADING_CACHE[]
     if cache !== nothing
         pathenv = get(cache.located, (pkg, stopenv), nothing)
@@ -399,6 +400,7 @@ 
     path = nothing
     env′ = nothing
     if pkg.uuid === nothing
+        println("locate_package_env: Case: nothing... TODO")
         for env in load_path()
             env′ = env
             # look for the toplevel pkg `pkg.name` in this entry
@@ -418,9 +420,18 @@ 
             end
         end
     else
+        println("locate_package_env: pkg = ", pkg)
+        println("locate_package_env: START LOOP; over load_path() = ", load_path())
+        #println("locate_package_env: number of loop", length(load_path))
+        loop = 1
         for env in load_path()
+            println("locate_package_env: loop = ", loop)
+            println("locate_package_env: env = ", env)
             env′ = env
+            println("locate_package_env: Call manifest_uuid_path( ", env, " , ", pkg, " )")
             path = manifest_uuid_path(env, pkg)
+            println("locate_package_env: Post path =  manifest_uuid_path( ", env, " , ", pkg, " )")
+            println("locate_package_env: path = ", path)
             # missing is used as a sentinel to stop looking further down in envs
             if path === missing
                 path = nothing
@@ -431,14 +442,27 @@ 
                 @goto done
             end
             if !(loading_extension || precompiling_extension)
+                stopenv == env && println("locate_package_env: BREAK: ", stopenv, " == ", env)
+                stopenv == env && println("locate_package_env:      : loading_extension = ", loading_extension)
+                stopenv == env && println("locate_package_env:      : precompiling_extension = ", precompiling_extension)
                 stopenv == env && break
             end
+            loop = loop + 1
         end
+        println("locate_package_env: END LOOP; No more env")
         # Allow loading of stdlibs if the name/uuid are given
         # e.g. if they have been explicitly added to the project/manifest
+        println("locate_package_env: pkg = ", pkg)
+        println("locate_package_env: Sys.STDLIB = ", Sys.STDLIB)
+        println("locate_package_env: Call manifest_uuid_path( ", Sys.STDLIB, " , ", pkg, " )")
         mbypath = manifest_uuid_path(Sys.STDLIB, pkg)
+        println("locate_package_env: Post mbypath = manifest_uuid_path( ", Sys.STDLIB, " , ", pkg, " )")
+        println("locate_package_env: mbypath = ", mbypath, " String? ", mbypath isa String)
         if mbypath isa String
+            println("locate_package_env: Call entry_path( ", mbypath, " , ", pkg.name, " )")
             path = entry_path(mbypath, pkg.name)
+            println("locate_package_env: Post path = entry_path( ", mbypath, " , ", pkg.name, " )")
+            println("locate_package_env: path = ", path)
             @goto done
         end
     end
@@ -446,6 +470,7 @@ 
     if cache !== nothing
         cache.located[(pkg, stopenv)] = path, env′
     end
+    println("locate_package_env: RETURN: path = ", path, " ; env = ", env′)
     return path, env′
 end
 
@@ -464,7 +489,11 @@ 
 ```
 """
 function locate_package(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)::Union{Nothing,String}
-    _nothing_or_first(locate_package_env(pkg, stopenv))
+    println("locate_package: ARGS: ", pkg, " , ", stopenv)
+    println("locate_package: Call locate_package_env(", pkg, " , ", stopenv, " )")
+    ret = _nothing_or_first(locate_package_env(pkg, stopenv))
+    println("locate_package: RETURN: ", ret)
+    return ret
 end
 
 """
@@ -581,9 +610,14 @@ 
 #  - `path`: the path of an explicit project file
 function env_project_file(env::String)::Union{Bool,String}
     @lock require_lock begin
+    println("env_project_file: ARGS: ", env)
     cache = LOADING_CACHE[]
     if cache !== nothing
+        println("env_project_file: cache = ", cache)
+        println("env_project_file: cache.env_project_file = ", cache.env_project_file)
         project_file = get(cache.env_project_file, env, nothing)
+        println("env_project_file: project_file = ", project_file)
+        project_file === nothing || println("env_project_file: RETURN: ", project_file)
         project_file === nothing || return project_file
     end
     if isdir(env)
@@ -596,6 +630,7 @@ 
     if cache !== nothing
         cache.env_project_file[env] = project_file
     end
+    println("env_project_file: RETURN: ", project_file)
     return project_file
     end
 end
@@ -651,20 +686,31 @@ 
 end
 
 function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String,Missing}
+    println("manifest_uuid_path: ARGS: ", env, " , ", pkg)
+    println("manifest_uuid_path: Call env_project_file( ", env, " )")
     project_file = env_project_file(env)
+    println("manifest_uuid_path: Post project_file = env_project_file( ", env, " )")
+    println("manifest_uuid_path: project_file = ", project_file, " String? ", project_file isa String)
     if project_file isa String
         proj = project_file_name_uuid(project_file, pkg.name)
         if proj == pkg
             # if `pkg` matches the project, return the project itself
-            return project_file_path(project_file)
+            ret = project_file_path(project_file)
+            println("manifest_uuid_path: RETURN: (1) ", ret)
+            return
         end
         mby_ext = project_file_ext_path(project_file, pkg.name)
+        mby_ext === nothing || println("manifest_uuid_path: RETURN: (2) ", mby_ext)
         mby_ext === nothing || return mby_ext
         # look for manifest file and `where` stanza
-        return explicit_manifest_uuid_path(project_file, pkg)
+        ret = explicit_manifest_uuid_path(project_file, pkg)
+        println("manifest_uuid_path: RETURN: (3) ", ret)
+        return ret
     elseif project_file
         # if env names a directory, search it
-        return implicit_manifest_uuid_path(env, pkg)
+        ret = implicit_manifest_uuid_path(env, pkg)
+        println("manifest_uuid_path: RETURN: (4) ", ret)
+        return ret
     end
     return nothing
 end
@@ -1758,6 +1804,7 @@ 
 
 # Returns `nothing` or the new(ish) module
 function _require(pkg::PkgId, env=nothing)
+    println("_require: ARGS: ", pkg, " , ", env)
     assert_havelock(require_lock)
     # handle recursive calls to require
     loading = get(package_locks, pkg, false)
@@ -1772,7 +1819,10 @@ 
     try
         toplevel_load[] = false
         # perform the search operation to select the module file require intends to load
+        println("ENTRY _require: Call locate_package( ", pkg, " , ", env, " )")
         path = locate_package(pkg, env)
+        println("_require: Post locate_package( ", pkg, " , ", env, " )")
+        println("RESULT _require: path = ", path)
         if path === nothing
             throw(ArgumentError("""
                 Package $pkg is required but does not seem to be installed: