From 84c6d36767fb26bbbd92b1802d5432258065d400 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Sat, 23 Jan 2021 11:27:34 +0000 Subject: [PATCH 1/6] Parallelize benchmarks --- ghcide/bench/hist/Main.hs | 2 +- shake-bench/src/Development/Benchmark/Rules.hs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ghcide/bench/hist/Main.hs b/ghcide/bench/hist/Main.hs index e9e87693b9..6df7bafd51 100644 --- a/ghcide/bench/hist/Main.hs +++ b/ghcide/bench/hist/Main.hs @@ -66,7 +66,7 @@ type instance RuleResult GetExample = Maybe Example type instance RuleResult GetExamples = [Example] main :: IO () -main = shakeArgs shakeOptions {shakeChange = ChangeModtimeAndDigest} $ do +main = shakeArgs shakeOptions {shakeChange = ChangeModtimeAndDigest, shakeThreads = 0} $ do createBuildSystem $ \resource -> do configStatic <- liftIO $ readConfigIO config let build = outputFolder configStatic diff --git a/shake-bench/src/Development/Benchmark/Rules.hs b/shake-bench/src/Development/Benchmark/Rules.hs index 2b78fcdac0..fdfb79fbe5 100644 --- a/shake-bench/src/Development/Benchmark/Rules.hs +++ b/shake-bench/src/Development/Benchmark/Rules.hs @@ -188,14 +188,14 @@ buildRules build MkBuildRules{..} = do [build -/- "binaries/*/" <> executableName ,build -/- "binaries/*/ghc.path" ] &%> \[out, ghcPath] -> do - let [_, _binaries, _ver, _] = splitDirectories out + let [_, _binaries, ver, _] = splitDirectories out liftIO $ createDirectoryIfMissing True $ dropFileName out commitid <- readFile' $ takeDirectory out "commitid" - cmd_ $ "git worktree add bench-temp " ++ commitid + cmd_ $ "git worktree add bench-temp-" ++ ver ++ " " ++ commitid buildSystem <- askOracle $ GetBuildSystem () - flip actionFinally (cmd_ ("git worktree remove bench-temp --force" :: String)) $ do - ghcLoc <- liftIO $ findGhc buildSystem "bench-temp" - buildProject buildSystem [Cwd "bench-temp"] (".." takeDirectory out) + flip actionFinally (cmd_ ("git worktree remove bench-temp-" <> ver <> " --force" :: String)) $ do + ghcLoc <- liftIO $ findGhc buildSystem ver + buildProject buildSystem [Cwd $ "bench-temp-" <> ver] (".." takeDirectory out) writeFile' ghcPath ghcLoc -------------------------------------------------------------------------------- From 7f731470d50e90bb4a74b34ac497d0f2db6253f9 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Sat, 23 Jan 2021 11:27:47 +0000 Subject: [PATCH 2/6] [benchmarks] Collect heap profiles --- ghcide/bench/hist/Main.hs | 3 +++ ghcide/ghcide.cabal | 3 ++- .../src/Development/Benchmark/Rules.hs | 20 +++++++++++++------ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/ghcide/bench/hist/Main.hs b/ghcide/bench/hist/Main.hs index 6df7bafd51..c23e2ca7e9 100644 --- a/ghcide/bench/hist/Main.hs +++ b/ghcide/bench/hist/Main.hs @@ -74,6 +74,7 @@ main = shakeArgs shakeOptions {shakeChange = ChangeModtimeAndDigest, shakeThread benchRules build resource (MkBenchRules (askOracle $ GetSamples ()) benchGhcide "ghcide") csvRules build svgRules build + eventlogRules build action $ allTargets build ghcideBuildRules :: MkBuildRules BuildSystem @@ -122,6 +123,7 @@ buildGhcide Cabal args out = do ,"--install-method=copy" ,"--overwrite-policy=always" ,"--ghc-options=-rtsopts" + ,"--ghc-options=-eventlog" ] buildGhcide Stack args out = @@ -131,6 +133,7 @@ buildGhcide Stack args out = ,"ghcide:ghcide" ,"--copy-bins" ,"--ghc-options=-rtsopts" + ,"--ghc-options=-eventlog" ] benchGhcide diff --git a/ghcide/ghcide.cabal b/ghcide/ghcide.cabal index 1a7806d61c..62a75f56ca 100644 --- a/ghcide/ghcide.cabal +++ b/ghcide/ghcide.cabal @@ -219,7 +219,8 @@ benchmark benchHist hs-source-dirs: bench/hist bench/lib other-modules: Experiments.Types build-tool-depends: - ghcide:ghcide-bench + ghcide:ghcide-bench, + eventlog2html:eventlog2html default-extensions: BangPatterns DeriveFunctor diff --git a/shake-bench/src/Development/Benchmark/Rules.hs b/shake-bench/src/Development/Benchmark/Rules.hs index fdfb79fbe5..e1a572cae9 100644 --- a/shake-bench/src/Development/Benchmark/Rules.hs +++ b/shake-bench/src/Development/Benchmark/Rules.hs @@ -49,6 +49,7 @@ module Development.Benchmark.Rules benchRules, MkBenchRules(..), BenchProject(..), csvRules, svgRules, + eventlogRules, allTargets, GetExample(..), GetExamples(..), IsExample(..), RuleResultForExample, @@ -134,11 +135,11 @@ allTargets buildFolder = do ++ [ buildFolder getExampleName ex T.unpack (humanName ver) - escaped (escapeExperiment e) <.> mode <.> "svg" + escaped (escapeExperiment e) <.> mode | e <- experiments, ex <- examples, ver <- versions, - mode <- ["", "diff"] + mode <- ["svg", "diff.svg","eventlog.html"] ] -------------------------------------------------------------------------------- @@ -224,9 +225,11 @@ benchRules build benchResource MkBenchRules{..} = do priority 0 $ [ build -/- "*/*/*.csv", build -/- "*/*/*.benchmark-gcStats", + build -/- "*/*/*.eventlog", + build -/- "*/*/*.hp", build -/- "*/*/*.log" ] - &%> \[outcsv, outGc, outLog] -> do + &%> \[outcsv, outGc, outEventLog, outHp, outLog] -> do let [_, exampleName, ver, exp] = splitDirectories outcsv example <- fromMaybe (error $ "Unknown example " <> exampleName) <$> askOracle (GetExample exampleName) @@ -234,7 +237,7 @@ benchRules build benchResource MkBenchRules{..} = do setupRes <- setupProject liftIO $ createDirectoryIfMissing True $ dropFileName outcsv let exePath = build "binaries" ver executableName - exeExtraArgs = ["+RTS", "-I0.5", "-S" <> takeFileName outGc, "-RTS"] + exeExtraArgs = ["+RTS", "-l-a", "-h", "-ol" <> outEventLog, "-S" <> outGc, "-RTS"] ghcPath = build "binaries" ver "ghc.path" experiment = Escaped $ dropExtension exp need [exePath, ghcPath] @@ -247,8 +250,8 @@ benchRules build benchResource MkBenchRules{..} = do RemEnv "GHC_PACKAGE_PATH", AddPath [takeDirectory ghcPath, "."] [] ] - BenchProject{..} - cmd_ Shell $ "mv *.benchmark-gcStats " <> dropFileName outcsv + BenchProject {..} + cmd_ Shell $ "mv ghcide.hp " <> dropFileName outcsv dropExtension exp <.> "hp" -- extend csv output with allocation data csvContents <- liftIO $ lines <$> readFile outcsv @@ -378,6 +381,11 @@ svgRules build = do title = show (unescapeExperiment exp) <> " - live bytes over time" plotDiagram False diagram out +eventlogRules :: FilePattern -> Rules () +eventlogRules build = do + build -/- "*/*/*.eventlog.html" %> \out -> do + need [dropExtension out] + cmd_ $ "eventlog2html " <> dropExtension out -------------------------------------------------------------------------------- -------------------------------------------------------------------------------- From d8cfb2f8da6019648d6644bdd4655f08da379466 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Sat, 23 Jan 2021 13:00:38 +0000 Subject: [PATCH 3/6] [benchmarks] pass RTS options to the real thing! We were passing RTS options to the wrong thing --- ghcide/bench/hist/Main.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ghcide/bench/hist/Main.hs b/ghcide/bench/hist/Main.hs index c23e2ca7e9..9098663ac5 100644 --- a/ghcide/bench/hist/Main.hs +++ b/ghcide/bench/hist/Main.hs @@ -146,11 +146,11 @@ benchGhcide samples buildSystem args BenchProject{..} = do "--samples=" <> show samples, "--csv=" <> outcsv, "--ghcide=" <> exePath, + "--ghcide-options=" <> unwords exeExtraArgs, "--select", unescaped (unescapeExperiment experiment) ] ++ exampleToOptions example ++ [ "--stack" | Stack == buildSystem - ] ++ - exeExtraArgs + ] From daec91d0d137d3618281d69821985df895335f54 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Sat, 23 Jan 2021 14:55:06 +0000 Subject: [PATCH 4/6] Drop the 8.6.5 benchmark since the vanilla rts doesn't support +RTS -h --- .github/mergify.yml | 1 - .github/workflows/bench.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/mergify.yml b/.github/mergify.yml index f7811f0ca9..c209562aef 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -7,7 +7,6 @@ pull_request_rules: conditions: - status-success=bench (8.10.2, ubuntu-latest) - status-success=bench (8.8.4, ubuntu-latest) - - status-success=bench (8.6.5, ubuntu-latest) - status-success=nix (default, ubuntu-latest) - status-success=nix (default, macOS-latest) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 4976092c45..e3c6b85aea 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - ghc: ['8.10.2', '8.8.4', '8.6.5'] + ghc: ['8.10.2', '8.8.4'] os: [ubuntu-latest] steps: From 0f09e6652b521bee3868a3a42f7ed20be0171477 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Sat, 23 Jan 2021 14:59:39 +0000 Subject: [PATCH 5/6] Tone down timeout --- ghcide/bench/hist/Main.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ghcide/bench/hist/Main.hs b/ghcide/bench/hist/Main.hs index 9098663ac5..943bff5c84 100644 --- a/ghcide/bench/hist/Main.hs +++ b/ghcide/bench/hist/Main.hs @@ -140,7 +140,7 @@ benchGhcide :: Natural -> BuildSystem -> [CmdOption] -> BenchProject Example -> Action () benchGhcide samples buildSystem args BenchProject{..} = do command_ args "ghcide-bench" $ - [ "--timeout=3000", + [ "--timeout=300", "--no-clean", "-v", "--samples=" <> show samples, From 2d7bd0c0fc13749e7a2737270c1341c66e9c68a0 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Sat, 23 Jan 2021 15:31:04 +0000 Subject: [PATCH 6/6] Review feedbacks --- shake-bench/src/Development/Benchmark/Rules.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shake-bench/src/Development/Benchmark/Rules.hs b/shake-bench/src/Development/Benchmark/Rules.hs index e1a572cae9..8efc5c355d 100644 --- a/shake-bench/src/Development/Benchmark/Rules.hs +++ b/shake-bench/src/Development/Benchmark/Rules.hs @@ -84,7 +84,7 @@ import GHC.Stack (HasCallStack) import qualified Graphics.Rendering.Chart.Backend.Diagrams as E import Graphics.Rendering.Chart.Easy ((.=)) import qualified Graphics.Rendering.Chart.Easy as E -import System.Directory (findExecutable, createDirectoryIfMissing) +import System.Directory (createDirectoryIfMissing, findExecutable, renameFile) import System.FilePath import qualified Text.ParserCombinators.ReadP as P import Text.Read (Read (..), get, @@ -251,7 +251,7 @@ benchRules build benchResource MkBenchRules{..} = do AddPath [takeDirectory ghcPath, "."] [] ] BenchProject {..} - cmd_ Shell $ "mv ghcide.hp " <> dropFileName outcsv dropExtension exp <.> "hp" + liftIO $ renameFile "ghcide.hp" $ dropFileName outcsv dropExtension exp <.> "hp" -- extend csv output with allocation data csvContents <- liftIO $ lines <$> readFile outcsv @@ -385,7 +385,7 @@ eventlogRules :: FilePattern -> Rules () eventlogRules build = do build -/- "*/*/*.eventlog.html" %> \out -> do need [dropExtension out] - cmd_ $ "eventlog2html " <> dropExtension out + cmd_ ("eventlog2html" :: String) [dropExtension out] -------------------------------------------------------------------------------- --------------------------------------------------------------------------------