Merge ghc-head into wip/hi-haddock

.travis.yml

@@ -1,3 +1,6 @@
+# NOTE: manually changes were made to an otherwise autogenerated script. This is to
+#       query GHC CI artifacts instead of going via Herbert's PPA 
+#
 # This Travis job script has been generated by a script via
 #
 #   make_travis_yml_2.hs 'haddock.cabal'
@@ -28,46 +31,50 @@ before_cache:
 
 matrix:
   include:
-    - compiler: "ghc-head"
-      env: GHCHEAD=true
-    # env: TEST=--disable-tests BENCH=--disable-benchmarks
-      addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head,ghc-head], sources: [hvr-ghc]}}
+    - os: linux
+      addons: {apt: {packages: [ghc-ppa-tools,cabal-install-head], sources: [hvr-ghc]}}
+      env:
+        - GHC_ZIP='https://gitlab.haskell.org/ghc/ghc/-/jobs/artifacts/master/download?job=validate-x86_64-linux-deb8'
 
 before_install:
- - HC=${CC}
+ # Manually install GHC validate artifact
+ - travis_retry curl -L $GHC_ZIP --output artifact.zip
+ - unzip artifact.zip
+ - tar xpf ghc.tar.xz --strip-components 1
+ - ./configure
+ - sudo make V=1 install
+
+ # Set up some vars
+ - HC=ghc
  - HCPKG=${HC/ghc/ghc-pkg}
- - unset CC
- - PATH=/opt/ghc/bin:/opt/ghc-ppa-tools/bin:$PATH
+ - PATH=/usr/local/bin:/opt/ghc/bin:/opt/ghc-ppa-tools/bin:$PATH
  - PKGNAME='haddock'
 
 install:
  - cabal --version
  - echo "$(${HC} --version) [$(${HC} --print-project-git-commit-id 2> /dev/null || echo '?')]"
- - BENCH=${BENCH---enable-benchmarks}
- - TEST=${TEST---enable-tests}
- - GHCHEAD=${GHCHEAD-false}
+ - BENCH=--enable-benchmarks
+ - TEST=--enable-tests
  - travis_retry cabal update -v
  - sed -i 's/^jobs:/-- jobs:/' ${HOME}/.cabal/config
  - rm -fv cabal.project.local
  - rm -f cabal.project.freeze
  # Overlay Hackage Package Index for GHC HEAD: https://github.com/hvr/head.hackage
  - |
-   if $GHCHEAD; then
-     sed -i 's/-- allow-newer: .*/allow-newer: *:base/' ${HOME}/.cabal/config
-     for pkg in $($HCPKG list --simple-output); do pkg=$(echo $pkg | sed 's/-[^-]*$//'); sed -i "s/allow-newer: /allow-newer: *:$pkg, /" ${HOME}/.cabal/config; done
+   sed -i 's/-- allow-newer: .*/allow-newer: *:base/' ${HOME}/.cabal/config
+   for pkg in $($HCPKG list --simple-output); do pkg=$(echo $pkg | sed 's/-[^-]*$//'); sed -i "s/allow-newer: /allow-newer: *:$pkg, /" ${HOME}/.cabal/config; done
 
-     echo 'repository head.hackage'                                                        >> ${HOME}/.cabal/config
-     echo '   url: http://head.hackage.haskell.org/'                                       >> ${HOME}/.cabal/config
-     echo '   secure: True'                                                                >> ${HOME}/.cabal/config
-     echo '   root-keys: 07c59cb65787dedfaef5bd5f987ceb5f7e5ebf88b904bbd4c5cbdeb2ff71b740' >> ${HOME}/.cabal/config
-     echo '              2e8555dde16ebd8df076f1a8ef13b8f14c66bad8eafefd7d9e37d0ed711821fb' >> ${HOME}/.cabal/config
-     echo '              8f79fd2389ab2967354407ec852cbe73f2e8635793ac446d09461ffb99527f6e' >> ${HOME}/.cabal/config
-     echo '   key-threshold: 3'                                                            >> ${HOME}/.cabal.config
+   echo 'repository head.hackage'                                                        >> ${HOME}/.cabal/config
+   echo '   url: http://head.hackage.haskell.org/'                                       >> ${HOME}/.cabal/config
+   echo '   secure: True'                                                                >> ${HOME}/.cabal/config
+   echo '   root-keys: 07c59cb65787dedfaef5bd5f987ceb5f7e5ebf88b904bbd4c5cbdeb2ff71b740' >> ${HOME}/.cabal/config
+   echo '              2e8555dde16ebd8df076f1a8ef13b8f14c66bad8eafefd7d9e37d0ed711821fb' >> ${HOME}/.cabal/config
+   echo '              8f79fd2389ab2967354407ec852cbe73f2e8635793ac446d09461ffb99527f6e' >> ${HOME}/.cabal/config
+   echo '   key-threshold: 3'                                                            >> ${HOME}/.cabal.config
 
-     grep -Ev -- '^\s*--' ${HOME}/.cabal/config | grep -Ev '^\s*$'
+   grep -Ev -- '^\s*--' ${HOME}/.cabal/config | grep -Ev '^\s*$'
 
-     cabal new-update head.hackage -v
-   fi
+   cabal new-update head.hackage -v
  - travis_retry cabal new-build -w ${HC} ${TEST} ${BENCH} --dep -j2 --allow-newer --constraint 'setup.Cabal installed' all
  - travis_retry cabal new-build -w ${HC} --disable-tests --disable-benchmarks --dep -j2 --allow-newer --constraint 'setup.Cabal installed' all
 
@@ -76,8 +83,8 @@ install:
 script:
  - if [ -f configure.ac ]; then autoreconf -i; fi
  - rm -rf dist/
- - cabal sdist # test that a source-distribution can be generated
- - cd dist/
+ - cabal new-sdist # test that a source-distribution can be generated
+ - cd dist-newstyle/sdist/
  - SRCTAR=(${PKGNAME}-*.tar.gz)
  - SRC_BASENAME="${SRCTAR/%.tar.gz}"
  - tar -xvf "./$SRC_BASENAME.tar.gz"

CHANGES.md

@@ -6,6 +6,12 @@
 
  * Support inline markup in markdown-style links (#875)
 
+ * The hyperlinker backend has been re-engineered to use HIE files
+   and display type annotations on expressions (#977)
+
+ * The hyperlinker backend lexer is now more incremental, faster, and
+   more memory efficient (#977)
+
 ## Changes in version 2.22.0
 
  * Make `--package-version` optional for `--hoogle` (#899)

doc/invoking.rst

@@ -504,7 +504,7 @@ The following options are available:
 
 .. option:: --bypass-interface-version-check
 
-    **DANGEROUS** Causes Haddock to ignore the interface verions of
+    **DANGEROUS** Causes Haddock to ignore the interface versions of
     binary Haddock interface files. This can make Haddock crash during
     deserialization of interface files.
 

haddock-api/haddock-api.cabal

@@ -46,7 +46,7 @@ library
                , Cabal           ^>= 2.4.0
                , ghc             ^>= 8.7
                , ghc-paths       ^>= 0.1.0.9
-               , haddock-library ^>= 1.7.0
+               , haddock-library ^>= 1.8.0
                , xhtml           ^>= 3000.2.2
 
   -- Versions for the dependencies below are transitively pinned by
@@ -59,6 +59,7 @@ library
                , directory
                , filepath
                , ghc-boot
+               , ghc-boot-th
                , transformers
 
   hs-source-dirs: src
@@ -97,7 +98,6 @@ library
     Haddock.Backends.HaddockDB
     Haddock.Backends.Hoogle
     Haddock.Backends.Hyperlinker
-    Haddock.Backends.Hyperlinker.Ast
     Haddock.Backends.Hyperlinker.Parser
     Haddock.Backends.Hyperlinker.Renderer
     Haddock.Backends.Hyperlinker.Types
@@ -130,7 +130,6 @@ test-suite spec
     Haddock
     Haddock.Backends.Hoogle
     Haddock.Backends.Hyperlinker
-    Haddock.Backends.Hyperlinker.Ast
     Haddock.Backends.Hyperlinker.Renderer
     Haddock.Backends.Hyperlinker.Utils
     Haddock.Backends.LaTeX
@@ -171,7 +170,7 @@ test-suite spec
   build-depends: Cabal           ^>= 2.4
                , ghc             ^>= 8.7
                , ghc-paths       ^>= 0.1.0.9
-               , haddock-library ^>= 1.7.0
+               , haddock-library ^>= 1.8.0
                , xhtml           ^>= 3000.2.2
                , hspec           >= 2.4.4 && < 2.6
                , QuickCheck      ^>= 2.11
@@ -187,6 +186,7 @@ test-suite spec
                , directory
                , filepath
                , ghc-boot
+               , ghc-boot-th
                , transformers
 
   build-tool-depends:

haddock-api/resources/html/solarized.css

@@ -53,3 +53,45 @@ a:link, a:visited {
 a:hover, a.hover-highlight {
 	background-color: #eee8d5;
 }
+
+span.annot{
+    position:relative;
+    color:#000;
+    text-decoration:none
+  }
+
+span.annot:hover{z-index:25; background-color:#ff0}
+
+span.annot span.annottext{
+  display: none;
+  border-radius: 5px 5px;
+
+  -moz-border-radius: 5px;
+  -webkit-border-radius: 5px;
+
+  box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.1);
+  -webkit-box-shadow: 5px 5px rgba(0, 0, 0, 0.1);
+  -moz-box-shadow: 5px 5px rgba(0, 0, 0, 0.1);
+
+  position: absolute;
+  left: 1em; top: 2em;
+  z-index: 99;
+  margin-left: 5;
+  background: #FFFFAA;
+  border: 2px solid #FFAD33;
+  padding: 0.8em 1em;
+}
+
+span.annot:hover span.annottext{
+  display:block;
+}
+
+/* This bridges the gap so you can mouse into the tooltip without it disappearing */
+span.annot span.annottext:before{
+  content: "";
+  position: absolute;
+  left: -1em; top: -1em;
+  background: #FFFFFF00;
+  z-index:-1;
+  padding: 2em 2em;
+}

haddock-api/src/Haddock.hs

@@ -39,6 +39,7 @@ import Haddock.Version
 import Haddock.InterfaceFile
 import Haddock.Options
 import Haddock.Utils
+import Haddock.GhcUtils (modifySessionDynFlags, setOutputDir)
 
 import Control.Monad hiding (forM_)
 import Data.Foldable (forM_, foldl')
@@ -66,6 +67,8 @@ import qualified GHC.Paths as GhcPaths
 import Paths_haddock_api (getDataDir)
 import System.Directory (doesDirectoryExist)
 #endif
+import System.Directory (getTemporaryDirectory)
+import System.FilePath ((</>))
 
 import Text.ParserCombinators.ReadP (readP_to_S)
 import GHC hiding (verbosity)
@@ -164,14 +167,23 @@ haddockWithGhc ghc args = handleTopExceptions $ do
   -- bypass the interface version check
   let noChecks = Flag_BypassInterfaceVersonCheck `elem` flags
 
+  -- Create a temporary directory and redirect GHC output there (unless user
+  -- requested otherwise).
+  --
+  -- Output dir needs to be set before calling 'depanal' since 'depanal' uses it
+  -- to compute output file names that are stored in the 'DynFlags' of the
+  -- resulting 'ModSummary's.
+  let withDir | Flag_NoTmpCompDir `elem` flags = id
+              | otherwise = withTempOutputDir
+
   unless (Flag_NoWarnings `elem` flags) $ do
     hypSrcWarnings flags
     forM_ (warnings args) $ \warning -> do
       hPutStrLn stderr warning
     when noChecks $
       hPutStrLn stderr noCheckWarning
 
-  ghc flags' $ do
+  ghc flags' $ withDir $ do
     dflags <- getDynFlags
 
     forM_ (optShowInterfaceFile flags) $ \path -> liftIO $ do
@@ -202,6 +214,15 @@ haddockWithGhc ghc args = handleTopExceptions $ do
       -- Render even though there are no input files (usually contents/index).
       liftIO $ renderStep dflags flags sinceQual qual packages []
 
+-- | Run the GHC action using a temporary output directory
+withTempOutputDir :: Ghc a -> Ghc a
+withTempOutputDir action = do
+  tmp <- liftIO getTemporaryDirectory
+  x   <- liftIO getProcessID
+  let dir = tmp </> ".haddock-" ++ show x
+  modifySessionDynFlags (setOutputDir dir)
+  withTempDir dir action
+
 -- | Create warnings about potential misuse of -optghc
 warnings :: [String] -> [String]
 warnings = map format . filter (isPrefixOf "-optghc")
@@ -221,8 +242,9 @@ withGhc flags action = do
   let handleSrcErrors action' = flip handleSourceError action' $ \err -> do
         printException err
         liftIO exitFailure
+      needHieFiles = Flag_HyperlinkedSource `elem` flags
 
-  withGhc' libDir (ghcFlags flags) (\_ -> handleSrcErrors action)
+  withGhc' libDir needHieFiles (ghcFlags flags) (\_ -> handleSrcErrors action)
 
 
 readPackagesAndProcessModules :: [Flag] -> [String]
@@ -444,24 +466,10 @@ readInterfaceFiles name_cache_accessor pairs bypass_version_check = do
 
 -- | Start a GHC session with the -haddock flag set. Also turn off
 -- compilation and linking. Then run the given 'Ghc' action.
-withGhc' :: String -> [String] -> (DynFlags -> Ghc a) -> IO a
-withGhc' libDir flags ghcActs = runGhc (Just libDir) $ do
-  dynflags  <- getSessionDynFlags
-  dynflags' <- parseGhcFlags
-                 (foldl' gopt_set dynflags
-                         [ -- Include docstrings in .hi-files.
-                           Opt_Haddock
-
-                           -- Ignore any aspects of .hi-files except docs.
-                         , Opt_SkipIfaceVersionCheck
-
-                           -- If we can't use an old .hi-file, save the new one.
-                         , Opt_WriteInterface
-                         ]) {
-    hscTarget = HscNothing,
-    ghcMode   = CompManager,
-    ghcLink   = NoLink
-    }
+withGhc' :: String -> Bool -> [String] -> (DynFlags -> Ghc a) -> IO a
+withGhc' libDir needHieFiles flags ghcActs = runGhc (Just libDir) $ do
+  dynflags' <- parseGhcFlags =<< getSessionDynFlags
+
   -- We disable pattern match warnings because than can be very
   -- expensive to check
   let dynflags'' = unsetPatternMatchWarnings $
@@ -492,11 +500,30 @@ withGhc' libDir flags ghcActs = runGhc (Just libDir) $ do
     parseGhcFlags dynflags = do
       -- TODO: handle warnings?
 
-      let flags' = filterRtsFlags flags
-      (dynflags', rest, _) <- parseDynamicFlags dynflags (map noLoc flags')
+      let extra_opts =
+            [ Opt_Haddock
+              -- Include docstrings in .hi-files.
+
+            , Opt_SkipIfaceVersionCheck
+              -- Ignore any aspects of .hi-files except docs.
+
+            , Opt_WriteInterface
+              -- If we can't use an old .hi-file, save the new one.
+            ] ++
+            [ Opt_WriteHie | needHieFiles
+              -- Generate .hie-files
+            ]
+          dynflags' = (foldl' gopt_set dynflags extra_opts)
+                        { hscTarget = HscNothing
+                        , ghcMode   = CompManager
+                        , ghcLink   = NoLink
+                        }
+          flags' = filterRtsFlags flags
+
+      (dynflags'', rest, _) <- parseDynamicFlags dynflags' (map noLoc flags')
       if not (null rest)
         then throwE ("Couldn't parse GHC options: " ++ unwords flags')
-        else return dynflags'
+        else return dynflags''
 
 unsetPatternMatchWarnings :: DynFlags -> DynFlags
 unsetPatternMatchWarnings dflags =