From 481eb308406efd6bd0f5a01f986758922c84747b Mon Sep 17 00:00:00 2001 From: Sam Kaplan Date: Tue, 8 Nov 2022 06:36:21 -0600 Subject: [PATCH 1/8] add libstdcxx version-bounds option --- docs/src/pythoncall.md | 18 ++++++++++++++++++ src/cpython/context.jl | 43 +++++++++++++++++++++++++----------------- test/context.jl | 14 ++++++++++++++ 3 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 test/context.jl diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index 2ffd3bc1..f138ed18 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -285,6 +285,24 @@ into it. If you want to use a pre-existing Conda environment, see the previous s If `conda`, `mamba` or `micromamba` is not in your `PATH` you will also need to set `JULIA_CONDAPKG_EXE` to its path. +#### If you installed a newer version of libstdc++ +On Linux, Julia comes bundled with its own copy of *libstdc++*. Therefore, when interacting with +CondaPkg, PythonCall injects a dependency to bound the allowed versions of the `libstdcxx-ng` +Conda package. To override this value (e.g. if the user replaced the bundled libstdc++ with something +newer), use: + +```julia +[PythonCall] +ENV["JULIA_CONDAPKG_LIBSTDCXX_VERSION_BOUND"] = ">=3.4,<=12" +``` + +To figure out installed version, run +```bash +strings /path/to/julia/lib/julia/libstdc++.so.6 | grep GLIBCXX +``` +Then look at +for the GCC version compatible with the GLIBCXX version. + ## [Installing Python packages](@id python-deps) Assuming you haven't [opted out](@ref pythoncall-config), PythonCall uses diff --git a/src/cpython/context.jl b/src/cpython/context.jl index 176fba12..7c2ae77b 100644 --- a/src/cpython/context.jl +++ b/src/cpython/context.jl @@ -30,6 +30,31 @@ function _atpyexit() return end +# By default, ensure libstdc++ in the Conda environment is compatible with +# the one linked in Julia. This is platform/version dependent, so needs to +# occur at runtime. +# +# Allow the user to override the default. This is useful when the version +# of libstdcxx linked in Julia is customized in the local installation of +# Julia. +# +# To figure out cxx_version for a given Julia version, run +# strings /path/to/julia/lib/julia/libstdc++.so.6 | grep GLIBCXX +# then look at +# https://gcc.gnu.org/onlinedocs/gcc-12.1.0/libstdc++/manual/manual/abi.html +# for the highest GCC version compatible with the highest GLIBCXX version. +function get_libstdcxx_version_bound() + if Base.VERSION <= v"1.6.2" + # GLIBCXX_3.4.26 + cxx_version = ">=3.4,<9.2" + else + # GLIBCXX_3.4.29 + # checked up to v1.8.0 + cxx_version = ">=3.4,<11.4" + end + get(ENV, "JULIA_CONDAPKG_LIBSTDCXX_VERSION_BOUND", cxx_version) +end + function init_context() CTX.is_embedded = haskey(ENV, "JULIA_PYTHONCALL_LIBPTR") @@ -60,23 +85,7 @@ function init_context() exe_path::String else if Sys.islinux() - # Ensure libstdc++ in the Conda environment is compatible with the one - # linked in Julia. This is platform/version dependent, so needs to occur at - # runtime. - # - # To figure out cxx_version for a given Julia version, run - # strings /path/to/julia/lib/julia/libstdc++.so.6 | grep GLIBCXX - # then look at - # https://gcc.gnu.org/onlinedocs/gcc-12.1.0/libstdc++/manual/manual/abi.html - # for the highest GCC version compatible with the highest GLIBCXX version. - if Base.VERSION <= v"1.6.2" - # GLIBCXX_3.4.26 - cxx_version = ">=3.4,<9.2" - else - # GLIBCXX_3.4.29 - # checked up to v1.8.0 - cxx_version = ">=3.4,<11.4" - end + cxx_version = get_libstdcxx_version_bound() CondaPkg.add("libstdcxx-ng", version=cxx_version, channel="conda-forge", temp=true, file=joinpath(@__DIR__, "..", "..", "CondaPkg.toml"), resolve=false) end # By default, we use Python installed by CondaPkg. diff --git a/test/context.jl b/test/context.jl new file mode 100644 index 00000000..3426e990 --- /dev/null +++ b/test/context.jl @@ -0,0 +1,14 @@ +@testitem "libstdc++ version" begin + cxxversion = PythonCall.get_libstdcxx_version_bound() + + if VERSION <= v"1.6.2" + @test cxxversion == ">=3.4,<9.2" + else + @test cxxversion == ">=3.4,<11.4" + end + + ENV["JULIA_CONDAPKG_LIBSTDCXX_VERSION_BOUND"] = ">=3.4,<=12" + + cxxversion = PythonCall.get_libstdcxx_version_bound() + @test cxxversion == ">3.4,<=12" +end \ No newline at end of file From 2ef05717d8f7a8db81245b5f814c2282d753d561 Mon Sep 17 00:00:00 2001 From: Sam Kaplan Date: Fri, 11 Nov 2022 11:16:12 -0600 Subject: [PATCH 2/8] Update src/cpython/context.jl Co-authored-by: Elliot Saba --- src/cpython/context.jl | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/cpython/context.jl b/src/cpython/context.jl index 7c2ae77b..30394640 100644 --- a/src/cpython/context.jl +++ b/src/cpython/context.jl @@ -44,14 +44,29 @@ end # https://gcc.gnu.org/onlinedocs/gcc-12.1.0/libstdc++/manual/manual/abi.html # for the highest GCC version compatible with the highest GLIBCXX version. function get_libstdcxx_version_bound() - if Base.VERSION <= v"1.6.2" - # GLIBCXX_3.4.26 - cxx_version = ">=3.4,<9.2" - else - # GLIBCXX_3.4.29 - # checked up to v1.8.0 - cxx_version = ">=3.4,<11.4" - end + # This list comes from: https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html + # Start with GCC 4.8, as it's extremely difficult to build Julia with anything older + vers_mapping = Dict( + 18 => v"4.8.0", + 19 => v"4.8.3", + 21 => v"5.1.0", + 22 => v"6.1.0", + 23 => v"7.1.0", + 24 => v"7.2.0", + 25 => v"8.1.0", + 26 => v"9.1.0", + 27 => v"9.2.0", + 28 => v"9.3.0", + 29 => v"11.1.0", + 30 => v"12.1.0", + 31 => v"13.1.0", + ) + # Get the libstdcxx version that is currently loaded in this Julia process + loaded_libstdcxx_version = Base.BinaryPlatforms.detect_libstdcxx_version() + # Map it through to get a GCC version; if the version is unknown, we simply return + # the highest GCC version we know about, which should be a fairly safe choice. + max_version = get(vers_mapping, loaded_libstdcxx_version.patch, vers_mapping[maximum(keys(vers_mapping))]) + cxx_version = ">=3.4,<=$(max_version.major).$(max_version.minor)" get(ENV, "JULIA_CONDAPKG_LIBSTDCXX_VERSION_BOUND", cxx_version) end From bf9cc5be58ecc559ce596161a5a2bca603c802c9 Mon Sep 17 00:00:00 2001 From: samkaplan Date: Fri, 11 Nov 2022 17:43:29 +0000 Subject: [PATCH 3/8] adapt to @staticfloat's changes --- docs/src/pythoncall.md | 8 ++++---- test/context.jl | 12 ++---------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index f138ed18..d6e16b0b 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -286,10 +286,10 @@ If `conda`, `mamba` or `micromamba` is not in your `PATH` you will also need to `JULIA_CONDAPKG_EXE` to its path. #### If you installed a newer version of libstdc++ -On Linux, Julia comes bundled with its own copy of *libstdc++*. Therefore, when interacting with -CondaPkg, PythonCall injects a dependency to bound the allowed versions of the `libstdcxx-ng` -Conda package. To override this value (e.g. if the user replaced the bundled libstdc++ with something -newer), use: +PythonCall injects a dependency to bound the allowed versions of the `libstdcxx-ng` +Conda package. It finds the bound by runtime discovery of the libstdcxx version. To +override this value (e.g. if the user replaced the bundled libstdc++ with something +else, use: ```julia [PythonCall] diff --git a/test/context.jl b/test/context.jl index 3426e990..fcd50086 100644 --- a/test/context.jl +++ b/test/context.jl @@ -1,14 +1,6 @@ @testitem "libstdc++ version" begin - cxxversion = PythonCall.get_libstdcxx_version_bound() - - if VERSION <= v"1.6.2" - @test cxxversion == ">=3.4,<9.2" - else - @test cxxversion == ">=3.4,<11.4" - end - ENV["JULIA_CONDAPKG_LIBSTDCXX_VERSION_BOUND"] = ">=3.4,<=12" - cxxversion = PythonCall.get_libstdcxx_version_bound() - @test cxxversion == ">3.4,<=12" + cxxversion = PythonCall.C.get_libstdcxx_version_bound() + @test cxxversion == ">=3.4,<=12" end \ No newline at end of file From daf0ff0194146d30701143aa0e4b68258e2af67a Mon Sep 17 00:00:00 2001 From: Sam Kaplan Date: Sun, 13 Nov 2022 11:01:21 -0600 Subject: [PATCH 4/8] fixup --- docs/src/pythoncall.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index d6e16b0b..cd728f96 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -288,8 +288,7 @@ If `conda`, `mamba` or `micromamba` is not in your `PATH` you will also need to #### If you installed a newer version of libstdc++ PythonCall injects a dependency to bound the allowed versions of the `libstdcxx-ng` Conda package. It finds the bound by runtime discovery of the libstdcxx version. To -override this value (e.g. if the user replaced the bundled libstdc++ with something -else, use: +override this value, use: ```julia [PythonCall] From 97b7cfaaa299763bf8e75c17b8215e00490a9a89 Mon Sep 17 00:00:00 2001 From: Sam Kaplan Date: Thu, 17 Nov 2022 09:33:04 -0600 Subject: [PATCH 5/8] JULIA_CONDAPKG_LIBSTDCXX_VERSION_BOUND -> JULIA_PYTHONCALL_LIBSTDCXX_VERSION_BOUND --- docs/src/pythoncall.md | 2 +- src/cpython/context.jl | 2 +- test/context.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index cd728f96..ec023836 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -292,7 +292,7 @@ override this value, use: ```julia [PythonCall] -ENV["JULIA_CONDAPKG_LIBSTDCXX_VERSION_BOUND"] = ">=3.4,<=12" +ENV["JULIA_PYTHONCALL_LIBSTDCXX_VERSION_BOUND"] = ">=3.4,<=12" ``` To figure out installed version, run diff --git a/src/cpython/context.jl b/src/cpython/context.jl index 30394640..aa392176 100644 --- a/src/cpython/context.jl +++ b/src/cpython/context.jl @@ -67,7 +67,7 @@ function get_libstdcxx_version_bound() # the highest GCC version we know about, which should be a fairly safe choice. max_version = get(vers_mapping, loaded_libstdcxx_version.patch, vers_mapping[maximum(keys(vers_mapping))]) cxx_version = ">=3.4,<=$(max_version.major).$(max_version.minor)" - get(ENV, "JULIA_CONDAPKG_LIBSTDCXX_VERSION_BOUND", cxx_version) + get(ENV, "JULIA_PYTHONCALL_LIBSTDCXX_VERSION_BOUND", cxx_version) end function init_context() diff --git a/test/context.jl b/test/context.jl index fcd50086..7bc9c44f 100644 --- a/test/context.jl +++ b/test/context.jl @@ -1,5 +1,5 @@ @testitem "libstdc++ version" begin - ENV["JULIA_CONDAPKG_LIBSTDCXX_VERSION_BOUND"] = ">=3.4,<=12" + ENV["JULIA_PYTHONCALL_LIBSTDCXX_VERSION_BOUND"] = ">=3.4,<=12" cxxversion = PythonCall.C.get_libstdcxx_version_bound() @test cxxversion == ">=3.4,<=12" From dc55bd5eb3a56431117b59013eaddf6efe595821 Mon Sep 17 00:00:00 2001 From: Sam Kaplan Date: Fri, 18 Nov 2022 11:10:13 -0600 Subject: [PATCH 6/8] tweak look-up table for version mapping --- src/cpython/context.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cpython/context.jl b/src/cpython/context.jl index aa392176..e893b263 100644 --- a/src/cpython/context.jl +++ b/src/cpython/context.jl @@ -49,6 +49,7 @@ function get_libstdcxx_version_bound() vers_mapping = Dict( 18 => v"4.8.0", 19 => v"4.8.3", + 20 => v"4.9.0", 21 => v"5.1.0", 22 => v"6.1.0", 23 => v"7.1.0", @@ -56,9 +57,9 @@ function get_libstdcxx_version_bound() 25 => v"8.1.0", 26 => v"9.1.0", 27 => v"9.2.0", - 28 => v"9.3.0", - 29 => v"11.1.0", - 30 => v"12.1.0", + 28 => v"9.5.0", + 29 => v"11.3.0", + 30 => v"12.2.0", 31 => v"13.1.0", ) # Get the libstdcxx version that is currently loaded in this Julia process From e7c8dac5642b28fa0c312ad0ff4fae9ce6658133 Mon Sep 17 00:00:00 2001 From: Sam Kaplan Date: Fri, 18 Nov 2022 11:44:48 -0600 Subject: [PATCH 7/8] handle the case where Julia is unable to discover the version of libstdc++ --- docs/src/pythoncall.md | 2 +- src/cpython/context.jl | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index ec023836..ec9a0a8a 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -287,7 +287,7 @@ If `conda`, `mamba` or `micromamba` is not in your `PATH` you will also need to #### If you installed a newer version of libstdc++ PythonCall injects a dependency to bound the allowed versions of the `libstdcxx-ng` -Conda package. It finds the bound by runtime discovery of the libstdcxx version. To +Conda package. It finds the bound by runtime discovery of the libstdc++ version. To override this value, use: ```julia diff --git a/src/cpython/context.jl b/src/cpython/context.jl index e893b263..63c1e323 100644 --- a/src/cpython/context.jl +++ b/src/cpython/context.jl @@ -64,11 +64,18 @@ function get_libstdcxx_version_bound() ) # Get the libstdcxx version that is currently loaded in this Julia process loaded_libstdcxx_version = Base.BinaryPlatforms.detect_libstdcxx_version() - # Map it through to get a GCC version; if the version is unknown, we simply return - # the highest GCC version we know about, which should be a fairly safe choice. - max_version = get(vers_mapping, loaded_libstdcxx_version.patch, vers_mapping[maximum(keys(vers_mapping))]) - cxx_version = ">=3.4,<=$(max_version.major).$(max_version.minor)" - get(ENV, "JULIA_PYTHONCALL_LIBSTDCXX_VERSION_BOUND", cxx_version) + + if loaded_libstdcxx_version !== nothing + # Map it through to get a GCC version; if the version is unknown, we simply return + # the highest GCC version we know about, which should be a fairly safe choice. + max_version = get(vers_mapping, loaded_libstdcxx_version.patch, vers_mapping[maximum(keys(vers_mapping))]) + return get(ENV, "JULIA_PYTHONCALL_LIBSTDCXX_VERSION_BOUND", ">=3.4,<=$(max_version.major).$(max_version.minor)") + elseif haskey(ENV, "JULIA_PYTHONCALL_LIBSTDCXX_VERSION_BOUND") + return ENV["JULIA_PYTHONCALL_LIBSTDCXX_VERSION_BOUND"] + else + # Julia does not link against any version of libstdc++ known to Julia (e.g. using clang instead, or something not in the 3.4.x series) + return nothing + end end function init_context() @@ -102,7 +109,11 @@ function init_context() else if Sys.islinux() cxx_version = get_libstdcxx_version_bound() - CondaPkg.add("libstdcxx-ng", version=cxx_version, channel="conda-forge", temp=true, file=joinpath(@__DIR__, "..", "..", "CondaPkg.toml"), resolve=false) + if cxx_version !== nothing + CondaPkg.add("libstdcxx-ng", version=cxx_version, channel="conda-forge", temp=true, file=joinpath(@__DIR__, "..", "..", "CondaPkg.toml"), resolve=false) + end + # if cxx_version is nothing, then we assume that Julia does not link against libstdcxx-ng, and so we do not + # enforce a version bound. end # By default, we use Python installed by CondaPkg. exe_path = Sys.iswindows() ? joinpath(CondaPkg.envdir(), "python.exe") : joinpath(CondaPkg.envdir(), "bin", "python") From 0cd0a325f1a3a548d321c08ae3d5dbe1761c72a4 Mon Sep 17 00:00:00 2001 From: Sam Kaplan Date: Fri, 18 Nov 2022 12:00:24 -0600 Subject: [PATCH 8/8] fixup code comment --- src/cpython/context.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpython/context.jl b/src/cpython/context.jl index 63c1e323..cb3851e3 100644 --- a/src/cpython/context.jl +++ b/src/cpython/context.jl @@ -112,7 +112,7 @@ function init_context() if cxx_version !== nothing CondaPkg.add("libstdcxx-ng", version=cxx_version, channel="conda-forge", temp=true, file=joinpath(@__DIR__, "..", "..", "CondaPkg.toml"), resolve=false) end - # if cxx_version is nothing, then we assume that Julia does not link against libstdcxx-ng, and so we do not + # if cxx_version is nothing, then we assume that Julia does not link against any version ob libstdc++ known by Julia, and so we do not # enforce a version bound. end # By default, we use Python installed by CondaPkg.