From c9716ea16db0c3cf4f0242e1b6aa9900a7be8c3e Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Tue, 17 Dec 2019 14:15:01 -0700 Subject: [PATCH 1/5] ghc.exactDeps/envDeps built with unpatched ghc The `overlays/ghc.nix` overrides were not applied to the `ghc` used to by the `exactDeps` and `envDeps` `passthru` derivations. This resulted in each ghc building twice with only one of them being cached by hydra. This change fixes this by moving the exactDeps and envDeps for ghc into sub directories of the output of the ghc derivation itself. --- builder/make-config-files.nix | 12 ++++---- compiler/ghc/default.nix | 8 ++--- overlays/bootstrap.nix | 58 +++++++++++++++++------------------ 3 files changed, 36 insertions(+), 42 deletions(-) diff --git a/builder/make-config-files.nix b/builder/make-config-files.nix index 7a5385cdd8..d8db9e37c5 100644 --- a/builder/make-config-files.nix +++ b/builder/make-config-files.nix @@ -12,7 +12,7 @@ let # Every library component built with `comp-builder.nix` includes an `exactDep` # and `envDep` directory with precomputed values used here. - # GHC derivations include `exactDep` and `envDep` derivations that have + # GHC derivations include `exactDep` and `envDep` directories that have # the same information for each of the built in packages. # exactDep will pass --exact-configuration to the `SETUP_HS confiugre` command. @@ -38,9 +38,9 @@ let ''; catGhcPkgExactDep = p: '' - if [ -e ${ghc.exactDeps}/${p} ]; then - cat ${ghc.exactDeps}/${p}/configure-flags >> $out/configure-flags - cat ${ghc.exactDeps}/${p}/cabal.config >> $out/cabal.config + if [ -e ${ghc}/exactDeps/${p} ]; then + cat ${ghc}/exactDeps/${p}/configure-flags >> $out/configure-flags + cat ${ghc}/exactDeps/${p}/cabal.config >> $out/cabal.config fi ''; @@ -49,8 +49,8 @@ let ''; catGhcPkgEnvDep = p: '' - if [ -e ${ghc.envDeps}/${p} ]; then - cat ${ghc.envDeps}/${p} >> $out/ghc-environment + if [ -e ${ghc}/envDeps/${p} ]; then + cat ${ghc}/envDeps/${p} >> $out/ghc-environment fi ''; diff --git a/compiler/ghc/default.nix b/compiler/ghc/default.nix index 192523e0fd..deae5ed4b4 100644 --- a/compiler/ghc/default.nix +++ b/compiler/ghc/default.nix @@ -13,7 +13,7 @@ , libiconv ? null, ncurses -, exactDeps, envDeps +, installDeps , # GHC can be built with system libffi or a bundled one. libffi ? null @@ -300,7 +300,7 @@ in let configured-src = stdenv.mkDerivation (rec { egrep --quiet '^#!' <(head -n 1 $i) || continue sed -i -e '2i export PATH="$PATH:${stdenv.lib.makeBinPath [ targetPackages.stdenv.cc.bintools coreutils ]}"' $i done - ''; + '' + installDeps targetPrefix; passthru = { inherit bootPkgs targetPrefix; @@ -315,10 +315,6 @@ in let configured-src = stdenv.mkDerivation (rec { # Used to detect non haskell-nix compilers (accedental use of nixpkgs compilers can lead to unexpected errors) isHaskellNixCompiler = true; - - # These are used in make-config-files.nix - exactDeps = exactDeps drv; - envDeps = envDeps drv; } // extra-passthru; meta = { diff --git a/overlays/bootstrap.nix b/overlays/bootstrap.nix index 8114093c71..d03d9ef32b 100644 --- a/overlays/bootstrap.nix +++ b/overlays/bootstrap.nix @@ -1,32 +1,30 @@ self: super: let - exactDeps = ghc: self.runCommand "ghc-exactdep-${ghc.version}" { nativeBuildInputs = [ ghc ]; } '' - for P in $(${ghc.targetPrefix}ghc-pkg list --simple-output | sed 's/-[0-9][0-9.]*//g'); do - mkdir -p $out/$P - touch $out/$P/configure-flags - touch $out/$P/cabal.config - - if id=$(${ghc.targetPrefix}ghc-pkg field $P id --simple-output); then - echo "--dependency=$P=$id" >> $out/$P/configure-flags - elif id=$(${ghc.targetPrefix}ghc-pkg field "z-$P-z-*" id --simple-output); then - name=$(${ghc.targetPrefix}ghc-pkg field "z-$P-z-*" name --simple-output) + installDeps = targetPrefix: '' + for P in $($out/bin/${targetPrefix}ghc-pkg list --simple-output | sed 's/-[0-9][0-9.]*//g'); do + mkdir -p $out/exactDeps/$P + touch $out/exactDeps/$P/configure-flags + touch $out/exactDeps/$P/cabal.config + + if id=$($out/bin/${targetPrefix}ghc-pkg field $P id --simple-output); then + echo "--dependency=$P=$id" >> $out/exactDeps/$P/configure-flags + elif id=$($out/bin/${targetPrefix}ghc-pkg field "z-$P-z-*" id --simple-output); then + name=$($out/bin/${targetPrefix}ghc-pkg field "z-$P-z-*" name --simple-output) # so we are dealing with a sublib. As we build sublibs separately, the above # query should be safe. - echo "--dependency=''${name#z-$P-z-}=$id" >> $out/$P/configure-flags + echo "--dependency=''${name#z-$P-z-}=$id" >> $out/exactDeps/$P/configure-flags fi - if ver=$(${ghc.targetPrefix}ghc-pkg field $P version --simple-output); then - echo "constraint: $P == $ver" >> $out/$P/cabal.config - echo "constraint: $P installed" >> $out/$P/cabal.config + if ver=$($out/bin/${targetPrefix}ghc-pkg field $P version --simple-output); then + echo "constraint: $P == $ver" >> $out/exactDeps/$P/cabal.config + echo "constraint: $P installed" >> $out/exactDeps/$P/cabal.config fi done - ''; - envDeps = ghc: self.runCommand "ghc-envdep-${ghc.version}" { nativeBuildInputs = [ ghc ]; } '' - mkdir -p $out - for P in $(${ghc.targetPrefix}ghc-pkg list --simple-output | sed 's/-[0-9.]*//g'); do - touch $out/$P - if id=$(${ghc.targetPrefix}ghc-pkg field $P id --simple-output); then - echo "package-id $id" >> $out/$P + mkdir -p $out/evalDeps + for P in $($out/bin/${targetPrefix}ghc-pkg list --simple-output | sed 's/-[0-9.]*//g'); do + touch $out/evalDeps/$P + if id=$($out/bin/${targetPrefix}ghc-pkg field $P id --simple-output); then + echo "package-id $id" >> $out/evalDeps/$P fi done ''; @@ -105,7 +103,7 @@ in { ghc844 = self.callPackage ../compiler/ghc { extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc844; }; - inherit bootPkgs sphinx exactDeps envDeps; + inherit bootPkgs sphinx installDeps; buildLlvmPackages = self.buildPackages.llvmPackages_5; llvmPackages = self.llvmPackages_5; @@ -123,7 +121,7 @@ in { ghc861 = self.callPackage ../compiler/ghc { extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc861; }; - inherit bootPkgs sphinx exactDeps envDeps; + inherit bootPkgs sphinx installDeps; buildLlvmPackages = self.buildPackages.llvmPackages_5; llvmPackages = self.llvmPackages_5; @@ -139,7 +137,7 @@ in { ghc862 = self.callPackage ../compiler/ghc { extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc862; }; - inherit bootPkgs sphinx exactDeps envDeps; + inherit bootPkgs sphinx installDeps; buildLlvmPackages = self.buildPackages.llvmPackages_5; llvmPackages = self.llvmPackages_5; @@ -156,7 +154,7 @@ in { ghc863 = self.callPackage ../compiler/ghc { extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc863; }; - inherit bootPkgs sphinx exactDeps envDeps; + inherit bootPkgs sphinx installDeps; buildLlvmPackages = self.buildPackages.llvmPackages_5; llvmPackages = self.llvmPackages_5; @@ -173,7 +171,7 @@ in { ghc864 = self.callPackage ../compiler/ghc { extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc864; }; - inherit bootPkgs sphinx exactDeps envDeps; + inherit bootPkgs sphinx installDeps; buildLlvmPackages = self.buildPackages.llvmPackages_5; llvmPackages = self.llvmPackages_5; @@ -190,7 +188,7 @@ in { ghc865 = self.callPackage ../compiler/ghc { extra-passthru = { buildGHC = self.buildPackages.haskell-nix.compiler.ghc865; }; - inherit bootPkgs sphinx exactDeps envDeps; + inherit bootPkgs sphinx installDeps; buildLlvmPackages = self.buildPackages.llvmPackages_5; llvmPackages = self.llvmPackages_5; @@ -264,10 +262,10 @@ in { # We mark these compilers as boot compilers to make sure they are only used # where a boot compiler is expected. compiler = builtins.mapAttrs (_: v: - v // { + v.overrideAttrs (drv: { + postInstall = (drv.postInstall or "") + installDeps ""; + }) // { isHaskellNixBootCompiler = true; - exactDeps = exactDeps v; - envDeps = envDeps v; } ) (import ../compiler/old-ghc-nix { pkgs = self; }); From 5d9676d2c18da3bd42cd2498ee7989638820f6f7 Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Tue, 17 Dec 2019 14:24:44 -0700 Subject: [PATCH 2/5] ifdLevel 0 --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 98cedb1827..c103021f15 100644 --- a/release.nix +++ b/release.nix @@ -2,7 +2,7 @@ , scrubJobs ? true , haskell-nix ? { outPath = ./.; rev = "abcdef"; } , nixpkgsArgs ? {} -, ifdLevel ? 3 +, ifdLevel ? 0 }: let defaultNixpkgs = import ./nixpkgs {}; in From 94aadae19d070c95c6a654055aa4d35cb26d6cb6 Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Tue, 17 Dec 2019 18:23:06 -0700 Subject: [PATCH 3/5] ifdLevel 1 --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index c103021f15..2468a0e9df 100644 --- a/release.nix +++ b/release.nix @@ -2,7 +2,7 @@ , scrubJobs ? true , haskell-nix ? { outPath = ./.; rev = "abcdef"; } , nixpkgsArgs ? {} -, ifdLevel ? 0 +, ifdLevel ? 1 }: let defaultNixpkgs = import ./nixpkgs {}; in From b9aa1bb9dc40a2994b0f78baf735c7991a0eef0f Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Tue, 17 Dec 2019 20:49:15 -0700 Subject: [PATCH 4/5] ifdLevel 2 --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 2468a0e9df..4cfc162e13 100644 --- a/release.nix +++ b/release.nix @@ -2,7 +2,7 @@ , scrubJobs ? true , haskell-nix ? { outPath = ./.; rev = "abcdef"; } , nixpkgsArgs ? {} -, ifdLevel ? 1 +, ifdLevel ? 2 }: let defaultNixpkgs = import ./nixpkgs {}; in From fd4704bdf92d51da53b9477642fbcf0924cf99b8 Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Tue, 17 Dec 2019 21:15:49 -0700 Subject: [PATCH 5/5] ifdLevel 3 --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 4cfc162e13..98cedb1827 100644 --- a/release.nix +++ b/release.nix @@ -2,7 +2,7 @@ , scrubJobs ? true , haskell-nix ? { outPath = ./.; rev = "abcdef"; } , nixpkgsArgs ? {} -, ifdLevel ? 2 +, ifdLevel ? 3 }: let defaultNixpkgs = import ./nixpkgs {}; in