From 242f7645de08dfce6fa170334c8196691e21c61f Mon Sep 17 00:00:00 2001 From: "B. Wilson" Date: Mon, 10 Jan 2022 03:12:13 +0900 Subject: [PATCH 1/6] Reboot project to overhaul the make2 build system. The goal is to provide a build process that is friendly to package maintainers for arbitrary Linux distributions, while maintaining the current status quo for non-Linux targets. In particular, the project aims to improved the following perceived deficiences of current make2: 1. Ad-hoc, idiosyncratic shell scripts; 2. Fragile custom code for compiler detection; 3. Ignores host's CFLAGS, LDFLAGS etc; 4. Non-obvious availability of other build flags, e.g. `USE_OPENMP` etc; 5. Non-error exit code on failure; 6. Unsavory reputation amongst existing package maintainers. The above issues are adressable by updates to the build scripts alone. However, there are further issues that require supporting changes elsewhere: 7. Build non-determinism (due to `__DATE__` and `__TIME__` macro usage); 8. Hostile to FHS-conforming installs. Importantly, in the course of working on the above, we impose some important constraints on ourself: 1. Build system changes must not introduce unintended changes in build products; 2. The current, officially supported build environments must not be broken; 3. Changes should be as conservative as possible to achieve goals. This commit introduces a temporary meta-build tool, `jbuild.scm`, to help maintain the above mentioned constraints. See that file for details. --- jbuild.scm | 395 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 395 insertions(+) create mode 100644 jbuild.scm diff --git a/jbuild.scm b/jbuild.scm new file mode 100644 index 000000000..e4a308dcf --- /dev/null +++ b/jbuild.scm @@ -0,0 +1,395 @@ +;;; GNU Guix development package. +;;; +;;; To build and install, close this repository, and from the project root run +;;; the following: +;;; +;;; guix package --install-from-file=jbuild.scm +;;; +;;; To bootstrap a development environment container, the following command is +;;; helpful: +;;; +;;; guix shell --container --development --file=jbuild.scm +;;; + +(define-module (jsoftware) + #:use-module ((ice-9 popen) #:select (open-pipe* + close-pipe)) + #:use-module ((ice-9 rdelim) #:select (read-line)) + #:use-module ((ice-9 match) #:select (match match-lambda)) + #:use-module ((srfi srfi-1) #:select (any every)) + #:use-module ((srfi srfi-26) #:select (cut)) + #:use-module ((guix gexp) #:select (gexp + local-file + program-file)) + #:use-module ((guix records) #:select (define-record-type*)) + #:use-module ((guix licenses) #:prefix license: #:select (gpl3+)) + #:use-module ((guix utils) #:select (target-arm? target-64bit?)) + #:use-module ((guix build-system gnu) #:select (gnu-build-system + %gnu-build-system-modules)) + #:use-module ((guix build-system trivial) #:select (trivial-build-system)) + #:use-module ((guix build utils) #:select (substitute* with-directory-excursion)) + #:use-module ((guix packages) #:select (base32 + origin + package + package/inherit + package-properties + package-version)) + #:use-module ((gnu packages) #:select (search-patches)) + #:use-module ((gnu packages libedit) #:select (libedit)) + #:use-module ((gnu packages llvm) #:select (clang-toolchain + libomp)) + #:use-module ((gnu packages maths) #:select (sleef))) + + +(define %source-dir (dirname (current-filename))) + +(define (git-version) + "Return a version string suitable for development builds." + (let* ((pipe (with-directory-excursion %source-dir + (open-pipe* OPEN_READ "git" "describe" "--always" "--tags"))) + (version (read-line pipe))) + (close-pipe pipe) + version)) + +(define (git-user) + "Return a user info string scraped from Git." + (let* ((name-pipe (with-directory-excursion %source-dir + (open-pipe* OPEN_READ "git" "config" "user.name"))) + (email-pipe (with-directory-excursion %source-dir + (open-pipe* OPEN_READ "git" "config" "user.email"))) + (name (read-line name-pipe)) + (email (read-line email-pipe)) + (status (every identity (map close-pipe `(,name-pipe ,email-pipe))))) + (format #f "~a <~a>" name email))) + +;; Predicate intended for SELECT? argument of local-file procedure. Returns +;; true if and only if file is tracked by git. +(define git-file? + (let* ((pipe (with-directory-excursion %source-dir + (open-pipe* OPEN_READ "git" "ls-files"))) + (files (let loop ((lines '())) + (match (read-line pipe) + ((? eof-object?) (reverse lines)) + (line (loop (cons line lines)))))) + (status (close-pipe pipe))) + (lambda (file stat) + (match (stat:type stat) + ('directory #t) + ((or 'regular 'symlink) (any (cut string-suffix? <> file) files)) + (_ #f))))) + + +(define (ijconsole) + "Generate a G-exp script that detects AVX/AVX2 support at runtime and + executes jconsole with the appropriate libj.so and profile.ijs." + (program-file "ijconsole" + #~(begin + (use-modules ((ice-9 rdelim) #:select (read-line)) + ((ice-9 regex) #:select (regexp-match? string-match))) + + ;; Assume that this script will be installed under bin/. + (define %basedir (dirname (dirname (current-filename)))) + + (define (cpu-feature-line? string) + (string-prefix? "flags" string)) + + (define (contains-word? word string) + (regexp-match? + (string-match (string-join `("\\<" ,word "\\>") "") + string))) + + (define (has-cpu-feature? feature) + (with-input-from-file "/proc/cpuinfo" + (lambda () + (catch 'found + (lambda () + (let loop ((line (read-line))) + (cond ((eof-object? line) #f) + ((and (cpu-feature-line? line) + (contains-word? feature line)) + (throw 'found)) + (else (loop (read-line)))))) + (const #t))))) + + (let* ((jconsole (string-append %basedir "/libexec/j/jconsole")) + (libj (format #f "~a/lib/j/libj-~a.so" %basedir + (cond ((has-cpu-feature? "avx2") "avx2") + ((has-cpu-feature? "avx") "avx") + (else "")))) + (jprofile (string-append %basedir "/etc/j/profile.ijs"))) + (apply execl jconsole "ijconsole" "-lib" libj "-jprofile" jprofile + (cdr (command-line))))))) + + +(define* (j-package #:key (builder (git-user)) + (version (git-version)) + (type 'devel) + (extra-inputs '()) + (extra-envars '())) + (package + (name "j") + (version version) + (source (local-file %source-dir #:recursive? #t #:select? git-file?)) + (build-system gnu-build-system) + (native-inputs `(("clang-toolchain" ,clang-toolchain))) + (inputs (cons* `("libedit" ,libedit) + `("libomp" ,libomp) + `("ijconsole" ,(ijconsole)) + extra-inputs)) + (arguments + `(#:tests? #f + #:modules (((ice-9 ftw) #:select (scandir)) + ((ice-9 popen) #:select (open-pipe* close-pipe)) + ((ice-9 regex) #:select (match:substring string-match)) + ((ice-9 threads) #:select (parallel par-for-each)) + ((srfi srfi-26) #:select (cut)) + ((srfi srfi-1) #:select (fold)) + ,@%gnu-build-system-modules) + #:phases + ;; Upstream's build system consists of ad-hoc scripts that build + ;; build up (very complicated) environment variables to pass to make. + ;; The basic build process looks like this: + ;; + ;; 1) Copy jsrc/jversion-x.h to jsrc/jversion.h and edit values; + ;; 2) Set jplatform and j64x environment variables; + ;; 3) Run make2/build_jconsole.sh and make2/build_libj.sh; + ;; + ;; However, upstream expects users to run J directly from the source + ;; directory; they do not supply a make `install' target. Thus it + ;; takes some massaging to install files in FHS-style directories. + (modify-phases %standard-phases + ;; In particular, we have to set up + ;; + ;; 1) jsrc/jversion.h as in a typical build; + ;; 2) jlibrary/bin/profilex.ijs to point to writable directories; + ;; 3) make2/build_*.sh to respect standard build conventions; + ;; 4) jsrc/jconsole.c to fix libedit dlopen; and + ;; 5) Hard coded references to addons directory. + (replace 'configure + (lambda* (#:key target inputs outputs #:allow-other-keys) + (let* ((clang-toolchain (assoc-ref inputs "clang-toolchain")) + (clang (string-append clang-toolchain "/bin/clang")) + (libedit (assoc-ref inputs "libedit")) + (out (assoc-ref outputs "out"))) + ;; Set up build constants + (copy-file "jsrc/jversion-x.h" "jsrc/jversion.h") + (substitute* "jsrc/jversion.h" + (("^#define jversion.*$") + (format #f "#define jversion ~s\n" ,version)) + (("^#define jtype.*$") + (format #f "#define jtype ~s\n" (symbol->string ',type))) + (("^#define jbuilder.*$") + (format #f "#define jbuilder ~s\n" ,builder))) + ;; Create profilex.ijs overrides to point to the correct + ;; store items. Note that we set ~install and ~addons + ;; directories to reside under ~user to allow installing + ;; and loading addons. TODO: Guix-ify J addons as well. + (call-with-output-file "jlibrary/bin/profilex.ijs" + (lambda (port) + (display + (string-join + (list + "share=. '/share/j',~ ({.~ _2 { I.@:=&'/') BINPATH" + "system=. share,'/system'" + "tools=. share,'/tools'" + ;; Upstream defaults to spamming $HOME with + ;; unhidden userdata directories. Set this to be + ;; $HOME/.j// instead + "'jtype jversion'=. (3&{,{.) <;._2 ,&'/' 9!:14''" + "jversion=. ({.~ i.&'-') jversion" + "jsuffix=. >@{&('';'-beta') jtype -: 'beta'" + "user=. home,'/.j/',jversion,jsuffix" + "addons=. user,'/addons'" + "break=. user,'/break'" + "config=. user,'/config'" + "install=. user,'/install'" + "snap=. user,'/snap'" + "temp=. user,'/temp'" + "\n") + "\n") + port))) + ;; Munge the build scripts into reason: + ;; 1. Short-circuit the fragile compiler detection; + ;; 2. Make sure to include our CFLAGS and LFLAGS; and + ;; 3. Propagate script errors to top level. + (for-each + (lambda (file) + (with-directory-excursion "make2" + (substitute* file + ;; The `compiler' variable doesn't point to the actual + ;; compiler. It is just a switch to tell the build + ;; scripts whether to use gcc- or clang-specific + ;; flags. + (("^compiler=.*$") "compiler=clang\n") + (("^LDFLAGS=\"" def) (string-append def "$LDFLAGS ")) + (("^(common=\")(\\$USETHREAD.*)$" _ def rest) + (string-append def "$CFLAGS " rest)) + (("^#!.*" shebang) + (string-append shebang "set -o errexit\n"))))) + '("build_jconsole.sh" "build_libj.sh")) + ;; The jconsole manually loads libedit with dlopen. The path + ;; must be absolute to correctly point to our input. + (substitute* "jsrc/jconsole.c" + (("libedit\\.so\\.[0-9]" so-file) + (format #f "~a/lib/~a" libedit so-file))) + ;; The ~addons/dev directory supplies tentative J-script + ;; definitions of new J engine functionality. Since we point + ;; ~addons under the ~user directory, we move it under + ;; ~system instead, which sits as-is in the output. + (with-directory-excursion "jsrc" + (for-each + (lambda (file) + (substitute* file (("~addons/dev") "~system/dev"))) + (scandir "." + (lambda (f) (eq? (stat:type (stat f)) 'regular))))) + ;; Implementation of 9!:14 records build time which breaks + ;; build reproducibility. Note that upstream code depends on + ;; the exact format of these strings, so we need to mimic the + ;; standard. + (substitute* "jsrc/j.c" + (("__DATE__") "\"Jan 01 1970\"") + (("__TIME__") "\"00:00:00\"")) + ;; Upstream recommends using clang, with GCC support being + ;; second-class, often resulting in build failures. + (setenv "CC" clang)))) + ;; The build output depends primarily on the values of the + ;; `jplatform' and `j64x' environment variables. If the target is + ;; ARM, then `jplatform' is "raspberry", otherwise it is `linux'. + ;; In addition to 32- and 64- bit versions, `j64x' controlls + ;; whether AVX or AVX2 variants of libj are built. + ;; + ;; However, build targets are not fine-grained enough to + ;; distinguish between CPU features. Thus we build and install all + ;; variants of libj, expecting jconsole to be called with a wrapper + ;; script that detects AVX features and loads the appropriate libj + ;; at runtime. + (replace 'build + (lambda _ + (setenv "USE_OPENMP" "1") + (setenv "USE_THREAD" "1") + (for-each (lambda (var-val) (apply setenv var-val)) + (quote ,extra-envars)) + ;; The build scripts assume that PWD is make2. + (with-directory-excursion "make2" + (let* ((platform ,(if (target-arm?) "raspberry" "linux")) + (jplat (string-append "jplatform=" platform)) + (target-bit ,(if (target-64bit?) "64" "32")) + (jbit (string-append "j64x=" "j" target-bit)) + (jbit-avx (string-append jbit "avx")) + (jbit-avx2 (string-append jbit "avx2"))) + (parallel + ;; Since jconsole doesn't depend on AVX features, we just + ;; build it once. + (invoke "env" jplat jbit "./build_jconsole.sh") + (invoke "env" jplat jbit "./build_libj.sh") + (if ,(target-64bit?) + (parallel + (invoke "env" jplat jbit-avx "./build_libj.sh") + (invoke "env" jplat jbit-avx2 + "./build_libj.sh")))))))) + ;; The test suite is expected to be run as follows for each variant + ;; of libj that we build: + ;; + ;; $ echo 'RUN ddall' | jconsole test/tsu.ijs + ;; + ;; This requires a working jconsole with accessible jlibrary files. + ;; We simply place these all under test/bin. + (replace 'check + (lambda* (#:key tests? #:allow-other-keys) + (when tests? + (let ((jplatform ,(if (target-arm?) "raspberry" "linux"))) + (mkdir-p "test/bin") + (for-each + (lambda (dir) + (let ((source (string-append "jlibrary/" dir)) + (dest (string-append "test/bin/" dir))) + (begin + (mkdir-p dest) + (copy-recursively source dest)))) + '("system" "tools" "addons")) + ;; The jlibrary/dev directory only sometimes exists, but + ;; needs to be copied into the ~system directory when it + ;; does. + (for-each + (lambda (dev-dir) + (if (access? dev-dir R_OK) + (copy-recursively dev-dir "test/bin/system/dev"))) + '("jlibrary/dev" "jlibrary/addons/dev")) + (par-for-each + (lambda (dir) + (let* ((jbin (string-append "bin/" jplatform)) + (jbit ,(if (target-64bit?) "j64" "j32")) + (jconsole (string-append jbin "/" jbit + "/jconsole")) + (source (string-append jbin "/" dir)) + (dest (string-append "test/bin/" dir))) + (begin + (mkdir-p dest) + (copy-recursively source dest) + (install-file "jlibrary/bin/profile.ijs" dest) + (install-file jconsole dest) + (let* ((jc (string-append dest "/jconsole")) + (tests "test/tsu.ijs") + (port (open-pipe* OPEN_WRITE jc tests))) + (display "RUN ddall\n" port) + (when (not (zero? (status:exit-val + (close-pipe port)))) + (error "Some J build tests failed.")))))) + (scandir (string-append "bin/" jplatform) + (negate (cut member <> '("." ".."))))) + #t)))) + ;; Now that everything is built, installation is fairly + ;; straightforward, following FHS conventions. The only quirk is + ;; that we install jconsole under /libexec to make room for the + ;; wrapper replacement under /bin. + (replace 'install + (lambda* (#:key outputs inputs #:allow-other-keys) + (let* ((jplat ,(if (target-arm?) "raspberry" "linux")) + (jbit ,(if (target-64bit?) "j64" "j32")) + (interp (string-join `("bin" ,jplat ,jbit "jconsole") "/")) + (ijconsole (assoc-ref inputs "ijconsole")) + (vname (match:substring (string-match "[0-9]+" ,version))) + (out (assoc-ref outputs "out")) + (bin (string-append out "/bin")) + (etc (string-append out "/etc/j")) + (lib (string-append out "/lib/j")) + (libexec (string-append out "/libexec/j")) + (share (string-append out "/share/j")) + (system (string-append share "/system")) + (dev (string-append system "/dev"))) + (mkdir-p bin) + (copy-file ijconsole (string-append bin "/ijconsole-" vname)) + (mkdir-p lib) + (for-each + (lambda (jarch) + (let* ((jbin (string-join `("bin" ,jplat ,jarch) "/")) + (javx-match (string-match "avx.*" jarch)) + (javx (if (not javx-match) "" + (match:substring javx-match))) + (sep (if javx-match "-" "")) + (source (string-append jbin "/libj.so")) + (dest (format #f "~a/libj~a~a.so" lib sep javx))) + (copy-file source dest))) + (scandir (string-append "bin/" jplat) + (negate (cut member <> '("." ".."))))) + (install-file interp libexec) + (copy-recursively "jlibrary/system" system) + (for-each + (lambda (source-dev) + (if (access? source-dev R_OK) + (copy-recursively source-dev dev))) + '("jlibrary/dev" "jlibrary/addons/dev")) + (install-file "jlibrary/bin/profile.ijs" etc) + (install-file "jlibrary/bin/profilex.ijs" etc))))))) + (home-page "https://www.jsoftware.com/") + (synopsis "Ascii-only, array programming language in the APL family") + (description + "J is a high-level, general-purpose programming language that is +particularly suited to the mathematical, statistical, and logical analysis of +data. It is a powerful tool for developing algorithms and exploring problems +that are not already well understood.") + (license license:gpl3+))) + +(j-package #:extra-inputs `(("sleef" ,sleef)) + #:extra-envars `(("USE_SLEEF_SRC" "0") + ("LDFLAGS" "-lsleef"))) From 8cc6265cf21b99c0df1a63f3dfb4c10763700b30 Mon Sep 17 00:00:00 2001 From: "B. Wilson" Date: Sun, 24 Apr 2022 13:32:11 +0900 Subject: [PATCH 2/6] jbuild.scm: Instrument in automatic build verification Executing jbuild.scm as a script will build J from source, hash the output, and display the result, with a comparison against a known good target hash. Note that, unfortunately, the diff in this commit got mixed with a large re-indentation fix, but the bulk of semantic changes take place below the `;;; Execution' section. --- jbuild.scm | 729 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 424 insertions(+), 305 deletions(-) mode change 100644 => 100755 jbuild.scm diff --git a/jbuild.scm b/jbuild.scm old mode 100644 new mode 100755 index e4a308dcf..81e7053ba --- a/jbuild.scm +++ b/jbuild.scm @@ -1,3 +1,5 @@ +#!/usr/bin/env -S guix repl +!# ;;; GNU Guix development package. ;;; ;;; To build and install, close this repository, and from the project root run @@ -15,14 +17,29 @@ #:use-module ((ice-9 popen) #:select (open-pipe* close-pipe)) #:use-module ((ice-9 rdelim) #:select (read-line)) + #:use-module ((ice-9 regex) #:select (string-match)) #:use-module ((ice-9 match) #:select (match match-lambda)) - #:use-module ((srfi srfi-1) #:select (any every)) + #:use-module ((srfi srfi-1) #:select (any concatenate every zip)) #:use-module ((srfi srfi-26) #:select (cut)) + #:use-module ((guix base32) #:select (bytevector->base32-string)) + #:use-module ((guix colors) #:select (color + colorize-string)) + #:use-module ((guix derivations) #:select (built-derivations + derivation-outputs + derivation-output-path)) #:use-module ((guix gexp) #:select (gexp + file-union local-file - program-file)) + plain-file + program-file + with-imported-modules)) + #:use-module ((guix hash) #:select (file-hash*)) #:use-module ((guix records) #:select (define-record-type*)) #:use-module ((guix licenses) #:prefix license: #:select (gpl3+)) + #:use-module ((guix monads) #:select (mbegin + mapm + mlet* + return)) #:use-module ((guix utils) #:select (target-arm? target-64bit?)) #:use-module ((guix build-system gnu) #:select (gnu-build-system %gnu-build-system-modules)) @@ -31,12 +48,19 @@ #:use-module ((guix packages) #:select (base32 origin package + package->derivation package/inherit + package-grafts package-properties package-version)) + #:use-module ((guix store) #:select (%store-monad + open-connection + run-with-store + store-lift)) #:use-module ((gnu packages) #:select (search-patches)) #:use-module ((gnu packages libedit) #:select (libedit)) #:use-module ((gnu packages llvm) #:select (clang-toolchain + clang-toolchain-12 libomp)) #:use-module ((gnu packages maths) #:select (sleef))) @@ -46,8 +70,10 @@ (define (git-version) "Return a version string suitable for development builds." (let* ((pipe (with-directory-excursion %source-dir - (open-pipe* OPEN_READ "git" "describe" "--always" "--tags"))) - (version (read-line pipe))) + (open-pipe* OPEN_READ "git" "describe" "--always" + "--tags" + "--abbrev=0"))) + (version (string-append (read-line pipe) "+git"))) (close-pipe pipe) version)) @@ -70,6 +96,10 @@ (files (let loop ((lines '())) (match (read-line pipe) ((? eof-object?) (reverse lines)) + ((? (lambda (file) ; skip this file + (string-match (current-filename) + (canonicalize-path file)))) + (loop lines)) (line (loop (cons line lines)))))) (status (close-pipe pipe))) (lambda (file stat) @@ -79,317 +109,406 @@ (_ #f))))) -(define (ijconsole) - "Generate a G-exp script that detects AVX/AVX2 support at runtime and - executes jconsole with the appropriate libj.so and profile.ijs." - (program-file "ijconsole" - #~(begin - (use-modules ((ice-9 rdelim) #:select (read-line)) - ((ice-9 regex) #:select (regexp-match? string-match))) +;; G-exp script that detects AVX/AVX2 support at runtime and executes jconsole +;; with the appropriate libj.so and profile.ijs." +(define ijconsole + (with-imported-modules '((guix cpu) + (guix memoization) + (guix profiling) + (guix sets) + (srfi srfi-26)) + (program-file "ijconsole" + #~(begin + (use-modules ((guix cpu) #:select (cpu-flags current-cpu)) + ((guix sets) #:select (set-contains?)) + ((srfi srfi-26) #:select (cute))) - ;; Assume that this script will be installed under bin/. - (define %basedir (dirname (dirname (current-filename)))) + ;; Assume that this script will be installed under bin/. + (define %basedir (dirname (dirname (current-filename)))) - (define (cpu-feature-line? string) - (string-prefix? "flags" string)) + (let* ((jconsole (string-append %basedir "/libexec/j/jconsole")) + (cpu-has-flag? + (cute set-contains? (cpu-flags (current-cpu)) <>)) + (libj (format #f "~a/lib/j/libj~a.so" %basedir + (cond ((cpu-has-flag? "avx2") "-avx2") + ((cpu-has-flag? "avx") "-avx") + (else "")))) + (jprofile (string-append %basedir "/etc/j/profile.ijs"))) + (apply execl jconsole "ijconsole" "-lib" libj "-jprofile" jprofile + (cdr (command-line)))))))) - (define (contains-word? word string) - (regexp-match? - (string-match (string-join `("\\<" ,word "\\>") "") - string))) - (define (has-cpu-feature? feature) - (with-input-from-file "/proc/cpuinfo" - (lambda () - (catch 'found - (lambda () - (let loop ((line (read-line))) - (cond ((eof-object? line) #f) - ((and (cpu-feature-line? line) - (contains-word? feature line)) - (throw 'found)) - (else (loop (read-line)))))) - (const #t))))) +(define jsoftware-auxiliary-files + (file-union "jsoftware-auxiliary-files" + `(("bin/ijconsole" ,ijconsole) + ;; profilex.ijs overrides ~install and ~addons directories to reside + ;; under the user-writable ~user. This allows local-install of addons + ;; via pacman. TODO: Guix-ify J addons as well. + ("etc/j/profilex.ijs" + ,(plain-file "profilex.ijs" + (string-join + '("'jtype jversion'=. (3&{,{.) <;._2 ,&'/' 9!:14''" + "basedir=. ({.~ _2 { I.@:=&'/') BINPATH" - (let* ((jconsole (string-append %basedir "/libexec/j/jconsole")) - (libj (format #f "~a/lib/j/libj-~a.so" %basedir - (cond ((has-cpu-feature? "avx2") "avx2") - ((has-cpu-feature? "avx") "avx") - (else "")))) - (jprofile (string-append %basedir "/etc/j/profile.ijs"))) - (apply execl jconsole "ijconsole" "-lib" libj "-jprofile" jprofile - (cdr (command-line))))))) + "share=. basedir,'/share/j'" + "system=. share,'/system'" + "tools=. share,'/tools'" + "user=. home,'/.config/j/',jversion" + "addons=. user,'/addons'" + "break=. user,'/break'" + "config=. user,'/config'" + "install=. user,'/install'" + "snap=. user,'/snap'" + "temp=. user,'/temp'") + "\n")))))) -(define* (j-package #:key (builder (git-user)) - (version (git-version)) - (type 'devel) - (extra-inputs '()) - (extra-envars '())) - (package - (name "j") - (version version) - (source (local-file %source-dir #:recursive? #t #:select? git-file?)) - (build-system gnu-build-system) - (native-inputs `(("clang-toolchain" ,clang-toolchain))) - (inputs (cons* `("libedit" ,libedit) - `("libomp" ,libomp) - `("ijconsole" ,(ijconsole)) - extra-inputs)) - (arguments - `(#:tests? #f - #:modules (((ice-9 ftw) #:select (scandir)) - ((ice-9 popen) #:select (open-pipe* close-pipe)) - ((ice-9 regex) #:select (match:substring string-match)) - ((ice-9 threads) #:select (parallel par-for-each)) - ((srfi srfi-26) #:select (cut)) - ((srfi srfi-1) #:select (fold)) - ,@%gnu-build-system-modules) - #:phases - ;; Upstream's build system consists of ad-hoc scripts that build - ;; build up (very complicated) environment variables to pass to make. - ;; The basic build process looks like this: - ;; - ;; 1) Copy jsrc/jversion-x.h to jsrc/jversion.h and edit values; - ;; 2) Set jplatform and j64x environment variables; - ;; 3) Run make2/build_jconsole.sh and make2/build_libj.sh; - ;; - ;; However, upstream expects users to run J directly from the source - ;; directory; they do not supply a make `install' target. Thus it - ;; takes some massaging to install files in FHS-style directories. - (modify-phases %standard-phases - ;; In particular, we have to set up - ;; - ;; 1) jsrc/jversion.h as in a typical build; - ;; 2) jlibrary/bin/profilex.ijs to point to writable directories; - ;; 3) make2/build_*.sh to respect standard build conventions; - ;; 4) jsrc/jconsole.c to fix libedit dlopen; and - ;; 5) Hard coded references to addons directory. - (replace 'configure - (lambda* (#:key target inputs outputs #:allow-other-keys) - (let* ((clang-toolchain (assoc-ref inputs "clang-toolchain")) - (clang (string-append clang-toolchain "/bin/clang")) - (libedit (assoc-ref inputs "libedit")) - (out (assoc-ref outputs "out"))) - ;; Set up build constants - (copy-file "jsrc/jversion-x.h" "jsrc/jversion.h") - (substitute* "jsrc/jversion.h" - (("^#define jversion.*$") - (format #f "#define jversion ~s\n" ,version)) - (("^#define jtype.*$") - (format #f "#define jtype ~s\n" (symbol->string ',type))) - (("^#define jbuilder.*$") - (format #f "#define jbuilder ~s\n" ,builder))) - ;; Create profilex.ijs overrides to point to the correct - ;; store items. Note that we set ~install and ~addons - ;; directories to reside under ~user to allow installing - ;; and loading addons. TODO: Guix-ify J addons as well. - (call-with-output-file "jlibrary/bin/profilex.ijs" - (lambda (port) - (display - (string-join - (list - "share=. '/share/j',~ ({.~ _2 { I.@:=&'/') BINPATH" - "system=. share,'/system'" - "tools=. share,'/tools'" - ;; Upstream defaults to spamming $HOME with - ;; unhidden userdata directories. Set this to be - ;; $HOME/.j// instead - "'jtype jversion'=. (3&{,{.) <;._2 ,&'/' 9!:14''" - "jversion=. ({.~ i.&'-') jversion" - "jsuffix=. >@{&('';'-beta') jtype -: 'beta'" - "user=. home,'/.j/',jversion,jsuffix" - "addons=. user,'/addons'" - "break=. user,'/break'" - "config=. user,'/config'" - "install=. user,'/install'" - "snap=. user,'/snap'" - "temp=. user,'/temp'" - "\n") - "\n") - port))) - ;; Munge the build scripts into reason: - ;; 1. Short-circuit the fragile compiler detection; - ;; 2. Make sure to include our CFLAGS and LFLAGS; and - ;; 3. Propagate script errors to top level. - (for-each - (lambda (file) - (with-directory-excursion "make2" - (substitute* file - ;; The `compiler' variable doesn't point to the actual - ;; compiler. It is just a switch to tell the build - ;; scripts whether to use gcc- or clang-specific - ;; flags. - (("^compiler=.*$") "compiler=clang\n") - (("^LDFLAGS=\"" def) (string-append def "$LDFLAGS ")) - (("^(common=\")(\\$USETHREAD.*)$" _ def rest) - (string-append def "$CFLAGS " rest)) - (("^#!.*" shebang) - (string-append shebang "set -o errexit\n"))))) - '("build_jconsole.sh" "build_libj.sh")) - ;; The jconsole manually loads libedit with dlopen. The path - ;; must be absolute to correctly point to our input. - (substitute* "jsrc/jconsole.c" - (("libedit\\.so\\.[0-9]" so-file) - (format #f "~a/lib/~a" libedit so-file))) - ;; The ~addons/dev directory supplies tentative J-script - ;; definitions of new J engine functionality. Since we point - ;; ~addons under the ~user directory, we move it under - ;; ~system instead, which sits as-is in the output. - (with-directory-excursion "jsrc" - (for-each +(define* (j-package #:key + (version (git-version)) + (release-type 'release) + (extra-inputs '()) + (extra-envars '()) + (toolchain clang-toolchain) + (compiler-path "/bin/clang") + (builder (git-user))) + (package + (name "j") + (version version) + (source (local-file %source-dir #:recursive? #t #:select? git-file?)) + (build-system gnu-build-system) + (native-inputs `(("toolchain" ,toolchain))) + (inputs (cons* libedit libomp jsoftware-auxiliary-files extra-inputs)) + (arguments + `(#:parallel-tests? #f + #:tests? #f + #:modules (((ice-9 ftw) #:select (scandir)) + ((ice-9 popen) #:select (open-pipe* close-pipe)) + ((ice-9 regex) #:select (match:substring string-match)) + ((ice-9 threads) #:select (par-for-each parallel)) + ((srfi srfi-26) #:select (cut)) + ((srfi srfi-1) #:select (fold)) + ,@%gnu-build-system-modules) + #:phases + ;; Upstream's build system consists of ad-hoc scripts that build build + ;; up (very complicated) environment variables to pass to make. The + ;; basic build process looks like this: + ;; + ;; 1) Copy jsrc/jversion-x.h to jsrc/jversion.h and edit values; + ;; 2) Set jplatform and j64x environment variables; + ;; 3) Run make2/build_jconsole.sh and make2/build_libj.sh; + ;; + ;; However, upstream expects users to run J directly from the source + ;; directory; they do not supply a make `install' target. Thus it takes + ;; some massaging to install files in FHS-style directories. + + (modify-phases %standard-phases + ;; In particular, we have to set up + ;; + ;; 1) jsrc/jversion.h as in a typical build; + ;; 2) jlibrary/bin/profilex.ijs to point to writable directories; + ;; 3) make2/build_*.sh to respect standard build conventions; + ;; 4) jsrc/jconsole.c to fix libedit dlopen; and + ;; 5) Hard coded references to addons directory. + (replace 'configure + (lambda* (#:key target inputs outputs #:allow-other-keys) + (let* ((toolchain (assoc-ref inputs "toolchain")) + (compiler (string-append toolchain ,compiler-path)) + (libedit (assoc-ref inputs "libedit")) + (out (assoc-ref outputs "out"))) + ;; Set up build constants + (copy-file "jsrc/jversion-x.h" "jsrc/jversion.h") + (substitute* "jsrc/jversion.h" + (("^#define jversion.*$") + (format #f "#define jversion ~s\n" ,version)) + (("^#define jtype.*$") + (format #f "#define jtype ~s\n" "debug")) + (("^#define jbuilder.*$") + (format #f "#define jbuilder ~s\n" ,builder))) + ;; Munge the build scripts into reason: + ;; 1. Short-circuit the fragile compiler detection; + ;; 2. Make sure to include our CFLAGS and LFLAGS; and + ;; 3. Propagate script errors to top level. + (with-directory-excursion "make2" + (for-each (lambda (file) - (substitute* file (("~addons/dev") "~system/dev"))) - (scandir "." - (lambda (f) (eq? (stat:type (stat f)) 'regular))))) - ;; Implementation of 9!:14 records build time which breaks - ;; build reproducibility. Note that upstream code depends on - ;; the exact format of these strings, so we need to mimic the - ;; standard. - (substitute* "jsrc/j.c" - (("__DATE__") "\"Jan 01 1970\"") - (("__TIME__") "\"00:00:00\"")) - ;; Upstream recommends using clang, with GCC support being - ;; second-class, often resulting in build failures. - (setenv "CC" clang)))) - ;; The build output depends primarily on the values of the - ;; `jplatform' and `j64x' environment variables. If the target is - ;; ARM, then `jplatform' is "raspberry", otherwise it is `linux'. - ;; In addition to 32- and 64- bit versions, `j64x' controlls - ;; whether AVX or AVX2 variants of libj are built. - ;; - ;; However, build targets are not fine-grained enough to - ;; distinguish between CPU features. Thus we build and install all - ;; variants of libj, expecting jconsole to be called with a wrapper - ;; script that detects AVX features and loads the appropriate libj - ;; at runtime. - (replace 'build - (lambda _ - (setenv "USE_OPENMP" "1") - (setenv "USE_THREAD" "1") - (for-each (lambda (var-val) (apply setenv var-val)) - (quote ,extra-envars)) - ;; The build scripts assume that PWD is make2. - (with-directory-excursion "make2" - (let* ((platform ,(if (target-arm?) "raspberry" "linux")) - (jplat (string-append "jplatform=" platform)) - (target-bit ,(if (target-64bit?) "64" "32")) - (jbit (string-append "j64x=" "j" target-bit)) - (jbit-avx (string-append jbit "avx")) - (jbit-avx2 (string-append jbit "avx2"))) - (parallel - ;; Since jconsole doesn't depend on AVX features, we just - ;; build it once. - (invoke "env" jplat jbit "./build_jconsole.sh") - (invoke "env" jplat jbit "./build_libj.sh") - (if ,(target-64bit?) - (parallel - (invoke "env" jplat jbit-avx "./build_libj.sh") - (invoke "env" jplat jbit-avx2 - "./build_libj.sh")))))))) - ;; The test suite is expected to be run as follows for each variant - ;; of libj that we build: - ;; - ;; $ echo 'RUN ddall' | jconsole test/tsu.ijs - ;; - ;; This requires a working jconsole with accessible jlibrary files. - ;; We simply place these all under test/bin. - (replace 'check - (lambda* (#:key tests? #:allow-other-keys) - (when tests? - (let ((jplatform ,(if (target-arm?) "raspberry" "linux"))) - (mkdir-p "test/bin") - (for-each - (lambda (dir) - (let ((source (string-append "jlibrary/" dir)) - (dest (string-append "test/bin/" dir))) - (begin - (mkdir-p dest) - (copy-recursively source dest)))) - '("system" "tools" "addons")) - ;; The jlibrary/dev directory only sometimes exists, but - ;; needs to be copied into the ~system directory when it - ;; does. - (for-each - (lambda (dev-dir) - (if (access? dev-dir R_OK) - (copy-recursively dev-dir "test/bin/system/dev"))) - '("jlibrary/dev" "jlibrary/addons/dev")) - (par-for-each - (lambda (dir) - (let* ((jbin (string-append "bin/" jplatform)) - (jbit ,(if (target-64bit?) "j64" "j32")) - (jconsole (string-append jbin "/" jbit - "/jconsole")) - (source (string-append jbin "/" dir)) + (substitute* file + ;; The `compiler' variable doesn't point to the actual + ;; compiler. It is just a switch to tell the build + ;; scripts whether to use gcc- or clang-specific flags. + (("^compiler=.*$") "compiler=clang\n") + (("^LDFLAGS=\"" def) (string-append def "$LDFLAGS ")) + + ;; Prepend our $CFLAGS to 'common', which holds CFLAGS + ;; common between all compilers. Note: We are careful to + ;; only add these flags once, ignoring lines like + ;; common="$common ...". The __GUARD__ dance works around + ;; the lack of regex negative lookahead. + (("^common=\"\\$common") "__GUARD__") + (("^(common=\")(.*)$" _ def rest) + (string-append def "$CFLAGS " rest)) + (("^__GUARD__") "common=\"$common") + + ;; Make sure errors exit with non-zero exit code. + (("^exit\n$") "exit 1\n") + + ;; Allow build scripts to pass arguments to make. The + ;; scripts pass a large number of variables to make, and + ;; the build step needs to invoke the `clean' target. + (("^(make -f .*)\n$" _ make-cmd) + (string-append make-cmd " \"$@\"\n")) + + (("^#!.*" shebang) + (string-append shebang "set -o errexit\n")))) + '("build_jconsole.sh" "build_libj.sh"))) + ;; The jconsole manually loads libedit with dlopen. The path + ;; must be absolute to correctly point to our input. + (substitute* "jsrc/jconsole.c" + (("libedit\\.so\\.[0-9]" so-file) + (format #f "~a/lib/~a" libedit so-file))) + ;; The ~addons/dev directory supplies tentative J-script + ;; definitions of new J engine functionality. Since we point + ;; ~addons under the ~user directory, we move it under ~system + ;; instead, which sits as-is in the output. + (with-directory-excursion "jsrc" + (for-each + (lambda (file) + (substitute* file (("~addons/dev") "~system/dev"))) + (scandir "." + (lambda (f) (eq? (stat:type (stat f)) 'regular))))) + ;; Implementation of 9!:14 records build time which breaks build + ;; reproducibility. Note that upstream code depends on the + ;; exact format of these strings, so we need to mimic the + ;; standard. + (substitute* "jsrc/j.c" + (("__DATE__") "\"Jan 01 1970\"") + (("__TIME__") "\"00:00:00\"")) + ;; Upstream recommends using clang, with GCC support being + ;; second-class, often resulting in build failures. + (setenv "CC" compiler)))) + + ;; The build output depends primarily on the values of the `jplatform' + ;; and `j64x' environment variables. If the target is ARM, then + ;; `jplatform' is "raspberry", otherwise it is `linux'. In addition + ;; to 32- and 64- bit versions, `j64x' controlls whether AVX or AVX2 + ;; variants of libj are built. + ;; + ;; However, build targets are not fine-grained enough to distinguish + ;; between CPU features. Thus we build and install all variants of + ;; libj, expecting jconsole to be called with a wrapper script that + ;; detects AVX features and loads the appropriate libj at runtime. + (replace 'build + (lambda* (#:key parallel-build? #:allow-other-keys) + (setenv "CFLAGS" "-g") + (setenv "USE_OPENMP" "1") + (setenv "USE_THREAD" "1") + (for-each (lambda (var-val) (apply setenv var-val)) + (quote ,extra-envars)) + ;; The build scripts assume that PWD is make2. + (with-directory-excursion "make2" + (let* ((platform ,(if (target-arm?) "raspberry" "linux")) + (target-bit ,(if (target-64bit?) "64" "32")) + (run-script + (lambda (script platform jbit . args) + (apply invoke + (cons* "env" + (string-append "jplatform=" platform) + (string-append "j64x=" jbit) + script args)))) + (run (lambda* (script #:key (variant "")) + (let* ((jbit (string-append "j" target-bit variant)) + (m (string-match "build_(.*)\\.sh" script)) + (c (match:substring m 1)) + (makefile (string-append "makefile-" c))) + (begin + ;; Scripts use `jplatform' and `j64x' + ;; environment variables to determine what + ;; binary to produce. + (run-script script platform jbit) + ;; Object files lingering around from previous + ;; builds will incorrectly propagate to + ;; others. + (run-script script platform jbit "clean")))))) + (let-syntax ((run-steps + (syntax-rules () + ((run-steps exp ...) + (if parallel-build? + (parallel exp ...) + (begin exp ...)))))) + (run-steps + ;; Since jconsole doesn't depend on AVX features, we just + ;; build it once. + (run "./build_jconsole.sh") + (run "./build_libj.sh") + (run "./build_tsdll.sh") + (if ,(target-64bit?) + (run-steps + (run "./build_libj.sh" #:variant "avx") + (run "./build_tsdll.sh" #:variant "avx") + (run "./build_libj.sh" #:variant "avx2") + (run "./build_tsdll.sh" #:variant "avx2"))))))))) + ;; The test suite is expected to be run as follows for each variant of + ;; libj that we build: + ;; + ;; $ echo 'RUN ddall' | jconsole test/tsu.ijs + ;; + ;; This requires a working jconsole with accessible jlibrary files. We + ;; simply place these all under test/bin. + (replace 'check + (lambda* (#:key tests? parallel-tests? #:allow-other-keys) + (when tests? + (let ((platform ,(if (target-arm?) "raspberry" "linux")) + (for-each* (if parallel-tests? par-for-each for-each))) + (mkdir-p "test/bin") + (for-each + (lambda (dir) + (let ((source (string-append "jlibrary/" dir)) (dest (string-append "test/bin/" dir))) - (begin - (mkdir-p dest) - (copy-recursively source dest) - (install-file "jlibrary/bin/profile.ijs" dest) - (install-file jconsole dest) - (let* ((jc (string-append dest "/jconsole")) - (tests "test/tsu.ijs") - (port (open-pipe* OPEN_WRITE jc tests))) - (display "RUN ddall\n" port) - (when (not (zero? (status:exit-val - (close-pipe port)))) - (error "Some J build tests failed.")))))) - (scandir (string-append "bin/" jplatform) - (negate (cut member <> '("." ".."))))) - #t)))) - ;; Now that everything is built, installation is fairly - ;; straightforward, following FHS conventions. The only quirk is - ;; that we install jconsole under /libexec to make room for the - ;; wrapper replacement under /bin. - (replace 'install - (lambda* (#:key outputs inputs #:allow-other-keys) - (let* ((jplat ,(if (target-arm?) "raspberry" "linux")) - (jbit ,(if (target-64bit?) "j64" "j32")) - (interp (string-join `("bin" ,jplat ,jbit "jconsole") "/")) - (ijconsole (assoc-ref inputs "ijconsole")) - (vname (match:substring (string-match "[0-9]+" ,version))) - (out (assoc-ref outputs "out")) - (bin (string-append out "/bin")) - (etc (string-append out "/etc/j")) - (lib (string-append out "/lib/j")) - (libexec (string-append out "/libexec/j")) - (share (string-append out "/share/j")) - (system (string-append share "/system")) - (dev (string-append system "/dev"))) - (mkdir-p bin) - (copy-file ijconsole (string-append bin "/ijconsole-" vname)) - (mkdir-p lib) - (for-each - (lambda (jarch) - (let* ((jbin (string-join `("bin" ,jplat ,jarch) "/")) - (javx-match (string-match "avx.*" jarch)) - (javx (if (not javx-match) "" - (match:substring javx-match))) - (sep (if javx-match "-" "")) - (source (string-append jbin "/libj.so")) - (dest (format #f "~a/libj~a~a.so" lib sep javx))) - (copy-file source dest))) - (scandir (string-append "bin/" jplat) - (negate (cut member <> '("." ".."))))) - (install-file interp libexec) - (copy-recursively "jlibrary/system" system) - (for-each - (lambda (source-dev) - (if (access? source-dev R_OK) - (copy-recursively source-dev dev))) - '("jlibrary/dev" "jlibrary/addons/dev")) - (install-file "jlibrary/bin/profile.ijs" etc) - (install-file "jlibrary/bin/profilex.ijs" etc))))))) - (home-page "https://www.jsoftware.com/") - (synopsis "Ascii-only, array programming language in the APL family") - (description - "J is a high-level, general-purpose programming language that is + (begin + (mkdir-p dest) + (copy-recursively source dest)))) + '("system" "tools" "addons")) + ;; The jlibrary/dev directory only sometimes exists, but needs + ;; to be copied into the ~system directory when it does. + (for-each + (lambda (dev-dir) + (if (file-exists? dev-dir) + (copy-recursively dev-dir "test/bin/system/dev"))) + '("jlibrary/dev" "jlibrary/addons/dev")) + (for-each* + (lambda (dir) + (let* ((bin (string-append "bin/" platform)) + (jbit ,(if (target-64bit?) "j64" "j32")) + (jconsole (string-append bin "/" jbit "/jconsole")) + (source (string-append bin "/" dir)) + (dest (string-append "test/bin/" dir))) + (begin + (mkdir-p dest) + (copy-recursively source dest) + (install-file "jlibrary/bin/profile.ijs" dest) + (install-file jconsole dest) + ;; Some tests verify firesystem interaction and + ;; require the ability to create ~user, which normally + ;; resides under HOME. Note, this path should be + ;; unique to each test run, to prevent collisions + ;; between tests. + (setenv "HOME" (string-append (getcwd) "/" dest)) + (let* ((jconsole (string-append dest "/jconsole")) + (tests "test/tsu.ijs") + (port (open-pipe* OPEN_WRITE jconsole tests))) + ;; This input to tsu.ijs runs the tests, ensuring + ;; that failures induce a non-zero exit while also + ;; printing out the name of the failed tests. + (display "(exit@# [ echo) RUN ddall\n" port) + (when (not (zero? (status:exit-val + (close-pipe port)))) + (error "Some J build tests failed.")))))) + (scandir (string-append "bin/" platform) + (negate (cut member <> '("." ".."))))) + #t)))) + ;; Now that everything is built, installation is fairly + ;; straightforward, following FHS conventions. The only quirk is that + ;; we install jconsole under /libexec to make room for the wrapper + ;; replacement under /bin. + (replace 'install + (lambda* (#:key outputs inputs #:allow-other-keys) + (let* ((platform ,(if (target-arm?) "raspberry" "linux")) + (jbit ,(if (target-64bit?) "j64" "j32")) + (out (assoc-ref outputs "out")) + (bin (string-append out "/bin")) + (etc (string-append out "/etc/j")) + (lib (string-append out "/lib/j")) + (libexec (string-append out "/libexec/j")) + (share (string-append out "/share/j")) + (system (string-append share "/system")) + (dev (string-append system "/dev"))) + (mkdir-p bin) + (copy-file (search-input-file inputs "bin/ijconsole") + (string-append bin "/ijconsole-" ,version)) + (mkdir-p lib) + (for-each + (lambda (jarch) + (let* ((jbin (string-join `("bin" ,platform ,jarch) "/")) + (javx-match (string-match "avx.*" jarch)) + (javx (if (not javx-match) "" + (match:substring javx-match))) + (sep (if javx-match "-" "")) + (source (string-append jbin "/libj.so")) + (dest (format #f "~a/libj~a~a.so" lib sep javx))) + (copy-file source dest))) + (scandir (string-append "bin/" platform) + (negate (cut member <> '("." ".."))))) + (install-file (string-append "bin/" platform + "/" jbit "/jconsole") + libexec) + (copy-recursively "jlibrary/system" system) + (for-each + (lambda (source-dev) + (if (access? source-dev R_OK) + (copy-recursively source-dev dev))) + '("jlibrary/dev" "jlibrary/addons/dev")) + (install-file "jlibrary/bin/profile.ijs" etc) + (install-file (search-input-file inputs "etc/j/profilex.ijs") + etc))))))) + (home-page "https://www.jsoftware.com/") + (synopsis "Ascii-only, array programming language in the APL family") + (description + "J is a high-level, general-purpose programming language that is particularly suited to the mathematical, statistical, and logical analysis of -data. It is a powerful tool for developing algorithms and exploring problems +data. It is a powerful tool for developing algorithms and exploring problems that are not already well understood.") (license license:gpl3+))) -(j-package #:extra-inputs `(("sleef" ,sleef)) - #:extra-envars `(("USE_SLEEF_SRC" "0") - ("LDFLAGS" "-lsleef"))) + +;;; Execution +(define (path-hash-result path target-hash) + (let* ((hash (bytevector->base32-string (file-hash* path)))) + (list path target-hash hash (equal? hash target-hash)))) + +(define (build-package package) + (run-with-store (open-connection) + (mlet* %store-monad + ((drv (package->derivation package)) + (grafts ((store-lift package-grafts) package))) + (mbegin %store-monad + (built-derivations (list drv)) + (return (map (compose derivation-output-path cdr) + (derivation-outputs drv))))))) + +(define (build-packages package . packages) + (concatenate (map build-package (cons package packages)))) + +(define (print-build-results paths target-hashes) + (for-each + (match-lambda + ((path target-hash hash match?) + ;(format #t "~a\n\t~a\t~a\n\t\t~a\n" path match? target-hash hash) + (display + (string-append + (colorize-string path (color DARK)) "\n" + "\t" (colorize-string "Target Hash: " (color DARK)) target-hash "\n" + "\t" (colorize-string "Actual Hash: " (color DARK)) hash "\n" + "\t" (colorize-string "Hashes Match? " (color DARK)) + (if match? + (colorize-string "YES" (color GREEN)) + (colorize-string "NO" (color RED))) + "\n")))) + (map path-hash-result paths target-hashes))) + +(define j-base-args + `(#:toolchain ,clang-toolchain-12 + #:extra-inputs ,(list sleef) + #:extra-envars (("USE_SLEEF_SRC" "0") + ("LDFLAGS" "-lsleef")))) + +(define j (apply j-package j-base-args)) +(define %j-target-hash "lr47z4x2bw7xjllupd5ekr3ulzzcgy3uhhgyxypd5gkfch46qdha") + +(match (program-arguments) + ;; Don't execute anything if loading in repl + ((cmd "repl" . args) #t) + ;; Build and show results if executing as a script + ((? (lambda (args) + (equal? (canonicalize-path (car args)) (current-filename)))) + (print-build-results (build-packages j) (list %j-target-hash))) + ;; Otherwise, assume we're feeding to a guix command + (_ j)) From 8484555c14615ddb84e5ea8bdce3adb9b32a5dc4 Mon Sep 17 00:00:00 2001 From: "B. Wilson" Date: Sun, 24 Apr 2022 17:11:56 +0900 Subject: [PATCH 3/6] make3: Initialize make3 as a copy of make2 Our starting point for make3 is make2. We anticipate that trialing make3 will necessitate it sitting parallel to make3, at least temporarily, and so have opted to copy make2 instead of munging it directly. --- jbuild.scm | 12 +- make3/build_all.sh | 44 ++++ make3/build_jconsole.sh | 223 +++++++++++++++++ make3/build_jnative.sh | 200 +++++++++++++++ make3/build_libj.sh | 524 ++++++++++++++++++++++++++++++++++++++++ make3/build_tsdll.sh | 198 +++++++++++++++ make3/clean.sh | 29 +++ make3/cpbin.sh | 96 ++++++++ make3/macos-libomp.txt | 15 ++ make3/make.txt | 69 ++++++ make3/makefile-jconsole | 29 +++ make3/makefile-jnative | 29 +++ make3/makefile-libj | 190 +++++++++++++++ make3/makefile-tsdll | 28 +++ 14 files changed, 1680 insertions(+), 6 deletions(-) create mode 100755 make3/build_all.sh create mode 100755 make3/build_jconsole.sh create mode 100755 make3/build_jnative.sh create mode 100755 make3/build_libj.sh create mode 100755 make3/build_tsdll.sh create mode 100755 make3/clean.sh create mode 100755 make3/cpbin.sh create mode 100644 make3/macos-libomp.txt create mode 100644 make3/make.txt create mode 100644 make3/makefile-jconsole create mode 100644 make3/makefile-jnative create mode 100644 make3/makefile-libj create mode 100644 make3/makefile-tsdll diff --git a/jbuild.scm b/jbuild.scm index 81e7053ba..1d3888754 100755 --- a/jbuild.scm +++ b/jbuild.scm @@ -195,7 +195,7 @@ ;; ;; 1) Copy jsrc/jversion-x.h to jsrc/jversion.h and edit values; ;; 2) Set jplatform and j64x environment variables; - ;; 3) Run make2/build_jconsole.sh and make2/build_libj.sh; + ;; 3) Run make3/build_jconsole.sh and make3/build_libj.sh; ;; ;; However, upstream expects users to run J directly from the source ;; directory; they do not supply a make `install' target. Thus it takes @@ -206,7 +206,7 @@ ;; ;; 1) jsrc/jversion.h as in a typical build; ;; 2) jlibrary/bin/profilex.ijs to point to writable directories; - ;; 3) make2/build_*.sh to respect standard build conventions; + ;; 3) make3/build_*.sh to respect standard build conventions; ;; 4) jsrc/jconsole.c to fix libedit dlopen; and ;; 5) Hard coded references to addons directory. (replace 'configure @@ -228,7 +228,7 @@ ;; 1. Short-circuit the fragile compiler detection; ;; 2. Make sure to include our CFLAGS and LFLAGS; and ;; 3. Propagate script errors to top level. - (with-directory-excursion "make2" + (with-directory-excursion "make3" (for-each (lambda (file) (substitute* file @@ -303,8 +303,8 @@ (setenv "USE_THREAD" "1") (for-each (lambda (var-val) (apply setenv var-val)) (quote ,extra-envars)) - ;; The build scripts assume that PWD is make2. - (with-directory-excursion "make2" + ;; The build scripts assume that PWD is make3. + (with-directory-excursion "make3" (let* ((platform ,(if (target-arm?) "raspberry" "linux")) (target-bit ,(if (target-64bit?) "64" "32")) (run-script @@ -501,7 +501,7 @@ that are not already well understood.") ("LDFLAGS" "-lsleef")))) (define j (apply j-package j-base-args)) -(define %j-target-hash "lr47z4x2bw7xjllupd5ekr3ulzzcgy3uhhgyxypd5gkfch46qdha") +(define %j-target-hash "tihgvwf7oqyi3cg5p4gregjby42odlrkobumroa6nb7cuphrsrya") (match (program-arguments) ;; Don't execute anything if loading in repl diff --git a/make3/build_all.sh b/make3/build_all.sh new file mode 100755 index 000000000..ecf80a829 --- /dev/null +++ b/make3/build_all.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# build all binaries + +realpath() +{ + oldpath=`pwd` + if ! cd $1 > /dev/null 2>&1; then + cd ${1##*/} > /dev/null 2>&1 + echo $( pwd -P )/${1%/*} + else + pwd -P + fi + cd $oldpath > /dev/null 2>&1 +} + +cd "$(realpath "$0")" +echo "entering `pwd`" + +./clean.sh + +if [ "`uname`" != "Darwin" ] && ( [ "`uname -m`" = "armv6l" ] || [ "`uname -m`" = "aarch64" ] ); then +jplatform="${jplatform:=raspberry}" +elif [ "`uname`" = "Darwin" ]; then +jplatform="${jplatform:=darwin}" +else +jplatform="${jplatform:=linux}" +fi +if [ "`uname -m`" = "x86_64" ]; then +if [ -z "${jplatform##*linux*}" ]; then +j64x="${j64x:=j64avx}" +else +j64x="${j64x:=j64}" +fi +elif [ "`uname -m`" = "aarch64" ]; then +j64x="${j64x:=j64}" +elif [ "`uname -m`" = "arm64" ] && [ -z "${jplatform##*darwin*}" ]; then +j64x="${j64x:=j64arm}" +else +j64x="${j64x:=j32}" +fi + +jplatform=$jplatform j64x=$j64x ./build_jconsole.sh +jplatform=$jplatform j64x=$j64x ./build_libj.sh +jplatform=$jplatform j64x=$j64x ./build_tsdll.sh diff --git a/make3/build_jconsole.sh b/make3/build_jconsole.sh new file mode 100755 index 000000000..e648eb2c7 --- /dev/null +++ b/make3/build_jconsole.sh @@ -0,0 +1,223 @@ +#!/bin/sh + +realpath() +{ + oldpath=`pwd` + if ! cd $1 > /dev/null 2>&1; then + cd ${1##*/} > /dev/null 2>&1 + echo $( pwd -P )/${1%/*} + else + pwd -P + fi + cd $oldpath > /dev/null 2>&1 +} + +cd "$(realpath "$0")" +echo "entering `pwd`" + +if [ "`uname`" != "Darwin" ] && ( [ "`uname -m`" = "armv6l" ] || [ "`uname -m`" = "aarch64" ] ); then +jplatform="${jplatform:=raspberry}" +elif [ "`uname`" = "Darwin" ]; then +jplatform="${jplatform:=darwin}" +else +jplatform="${jplatform:=linux}" +fi +if [ "`uname -m`" = "x86_64" ]; then +j64x="${j64x:=j64avx}" +elif [ "`uname -m`" = "aarch64" ]; then +j64x="${j64x:=j64}" +elif [ "`uname -m`" = "arm64" ] && [ -z "${jplatform##*darwin*}" ]; then +j64x="${j64x:=j64arm}" +else +j64x="${j64x:=j32}" +fi +USE_LINENOISE="${USE_LINENOISE:=1}" + +# gcc 5 vs 4 - killing off linux asm routines (overflow detection) +# new fast code uses builtins not available in gcc 4 +# use -DC_NOMULTINTRINSIC to continue to use more standard c in version 4 +# too early to move main linux release package to gcc 5 + +if [ -z "${jplatform##*darwin*}" ]; then +if [ -z "${j64x##*j64arm*}" ]; then +macmin="-arch arm64 -mmacosx-version-min=11" +else +macmin="-arch x86_64 -mmacosx-version-min=10.6" +fi +fi + +if [ "x$CC" = x'' ] ; then +if [ -f "/usr/bin/cc" ]; then +CC=cc +else +if [ -f "/usr/bin/clang" ]; then +CC=clang +else +CC=gcc +fi +fi +export CC +fi +# compiler=`$CC --version | head -n 1` +compiler=$(readlink -f $(command -v $CC) 2> /dev/null || echo $CC) +echo "CC=$CC" +echo "compiler=$compiler" + +if [ -z "${compiler##*gcc*}" ] || [ -z "${CC##*gcc*}" ]; then +# gcc +common="$OPENMP -fPIC -O2 -fvisibility=hidden -fno-strict-aliasing \ + -Werror -Wextra -Wno-unknown-warning-option \ + -Wno-cast-function-type \ + -Wno-clobbered \ + -Wno-empty-body \ + -Wno-format-overflow \ + -Wno-implicit-fallthrough \ + -Wno-maybe-uninitialized \ + -Wno-missing-field-initializers \ + -Wno-parentheses \ + -Wno-pointer-sign \ + -Wno-shift-negative-value \ + -Wno-sign-compare \ + -Wno-type-limits \ + -Wno-uninitialized \ + -Wno-unused-parameter \ + -Wno-unused-value " + +else +# clang +common="$OPENMP -fPIC -O2 -fvisibility=hidden -fno-strict-aliasing \ + -Werror -Wextra -Wno-unknown-warning-option \ + -Wsign-compare \ + -Wtautological-constant-out-of-range-compare \ + -Wuninitialized \ + -Wno-char-subscripts \ + -Wno-consumed \ + -Wno-delete-non-abstract-non-virtual-dtor \ + -Wno-empty-body \ + -Wno-implicit-float-conversion \ + -Wno-implicit-int-float-conversion \ + -Wno-int-in-bool-context \ + -Wno-missing-braces \ + -Wno-parentheses \ + -Wno-pass-failed \ + -Wno-pointer-sign \ + -Wno-string-plus-int \ + -Wno-unknown-pragmas \ + -Wno-unsequenced \ + -Wno-unused-function \ + -Wno-unused-parameter \ + -Wno-unused-value \ + -Wno-unused-variable " + +fi + +TARGET=jconsole + +if [ -z "${j64x##*32*}" ]; then +USE_EMU_AVX=0 +else +USE_EMU_AVX="${USE_EMU_AVX:=1}" +fi +if [ $USE_EMU_AVX -eq 1 ] ; then +common="$common -DEMU_AVX=1" +fi + +USE_PYXES="${USE_PYXES:=1}" +if [ $USE_PYXES -eq 1 ] ; then +common="$common -DPYXES=1" +else +common="$common -DPYXES=0" +fi + +if [ "$USE_LINENOISE" -ne "1" ] ; then +common="$common -DREADLINE" +else +common="$common -DREADLINE -DUSE_LINENOISE" +OBJSLN="linenoise.o" +fi + +case $jplatform\_$j64x in + +linux_j32) +CFLAGS="$common -m32 -msse2 -mfpmath=sse " +LDFLAGS=" -m32 -ldl $LDTHREAD" +;; +linux_j64) +CFLAGS="$common" +LDFLAGS=" -ldl $LDTHREAD" +;; +linux_j64avx) +CFLAGS="$common" +LDFLAGS=" -ldl $LDTHREAD" +;; +linux_j64avx2) +CFLAGS="$common" +LDFLAGS=" -ldl $LDTHREAD" +;; +raspberry_j32) +CFLAGS="$common -marm -march=armv6 -mfloat-abi=hard -mfpu=vfp -DRASPI" +LDFLAGS=" -ldl $LDTHREAD" +;; +raspberry_j64) +CFLAGS="$common -march=armv8-a+crc -DRASPI" +LDFLAGS=" -ldl $LDTHREAD" +;; +darwin_j32) +CFLAGS="$common -m32 -msse2 -mfpmath=sse $macmin" +LDFLAGS=" -ldl $LDTHREAD -m32 $macmin " +;; +#-mmacosx-version-min=10.5 +darwin_j64) +CFLAGS="$common $macmin" +LDFLAGS=" -ldl $LDTHREAD $macmin " +;; +darwin_j64avx) +CFLAGS="$common $macmin" +LDFLAGS=" -ldl $LDTHREAD $macmin " +;; +darwin_j64avx2) +CFLAGS="$common $macmin" +LDFLAGS=" -ldl $LDTHREAD $macmin " +;; +darwin_j64arm) # darwin arm +CFLAGS="$common $macmin -march=armv8-a+crc " +LDFLAGS=" -ldl $LDTHREAD $macmin " +;; +windows_j32) +TARGET=jconsole.exe +CFLAGS="$common -m32 " +LDFLAGS=" -m32 -Wl,--stack=0x1000000,--subsystem,console -static-libgcc $LDTHREAD" +;; +windows_j64) +TARGET=jconsole.exe +CFLAGS="$common" +LDFLAGS=" -Wl,--stack=0x1000000,--subsystem,console -static-libgcc $LDTHREAD" +;; +windows_j64avx) +TARGET=jconsole.exe +CFLAGS="$common" +LDFLAGS=" -Wl,--stack=0x1000000,--subsystem,console -static-libgcc $LDTHREAD" +;; +windows_j64avx2) +TARGET=jconsole.exe +CFLAGS="$common" +LDFLAGS=" -Wl,--stack=0x1000000,--subsystem,console -static-libgcc $LDTHREAD" +;; +*) +echo no case for those parameters +exit +esac + +echo "CFLAGS=$CFLAGS" + +if [ ! -f ../jsrc/jversion.h ] ; then + cp ../jsrc/jversion-x.h ../jsrc/jversion.h +fi + +mkdir -p ../bin/$jplatform/$j64x +mkdir -p obj/$jplatform/$j64x/ +cp makefile-jconsole obj/$jplatform/$j64x/. +export CFLAGS LDFLAGS TARGET OBJSLN jplatform j64x +cd obj/$jplatform/$j64x/ +make -f makefile-jconsole +cd - diff --git a/make3/build_jnative.sh b/make3/build_jnative.sh new file mode 100755 index 000000000..db0e66cb5 --- /dev/null +++ b/make3/build_jnative.sh @@ -0,0 +1,200 @@ +#!/bin/sh + +realpath() +{ + oldpath=`pwd` + if ! cd $1 > /dev/null 2>&1; then + cd ${1##*/} > /dev/null 2>&1 + echo $( pwd -P )/${1%/*} + else + pwd -P + fi + cd $oldpath > /dev/null 2>&1 +} + +cd "$(realpath "$0")" +echo "entering `pwd`" + +if [ "`uname`" != "Darwin" ] && ( [ "`uname -m`" = "armv6l" ] || [ "`uname -m`" = "aarch64" ] ); then +jplatform="${jplatform:=raspberry}" +elif [ "`uname`" = "Darwin" ]; then +jplatform="${jplatform:=darwin}" +else +jplatform="${jplatform:=linux}" +fi +if [ "`uname -m`" = "x86_64" ]; then +j64x="${j64x:=j64avx}" +elif [ "`uname -m`" = "aarch64" ]; then +j64x="${j64x:=j64}" +elif [ "`uname -m`" = "arm64" ] && [ -z "${jplatform##*darwin*}" ]; then +j64x="${j64x:=j64arm}" +else +j64x="${j64x:=j32}" +fi + +# gcc 5 vs 4 - killing off linux asm routines (overflow detection) +# new fast code uses builtins not available in gcc 4 +# use -DC_NOMULTINTRINSIC to continue to use more standard c in version 4 +# too early to move main linux release package to gcc 5 + +if [ -z "${jplatform##*darwin*}" ]; then +if [ -z "${j64x##*j64arm*}" ]; then +macmin="-arch arm64 -mmacosx-version-min=11" +else +macmin="-arch x86_64 -mmacosx-version-min=10.6" +fi +fi + +if [ "x$CC" = x'' ] ; then +if [ -f "/usr/bin/cc" ]; then +CC=cc +else +if [ -f "/usr/bin/clang" ]; then +CC=clang +else +CC=gcc +fi +fi +export CC +fi +# compiler=`$CC --version | head -n 1` +compiler=$(readlink -f $(command -v $CC) 2> /dev/null || echo $CC) +echo "CC=$CC" +echo "compiler=$compiler" + +if [ -z "${compiler##*gcc*}" ] || [ -z "${CC##*gcc*}" ]; then +# gcc +common="$OPENMP -fPIC -O2 -fvisibility=hidden -fno-strict-aliasing \ + -Werror -Wextra -Wno-unknown-warning-option \ + -Wno-cast-function-type \ + -Wno-clobbered \ + -Wno-empty-body \ + -Wno-format-overflow \ + -Wno-implicit-fallthrough \ + -Wno-maybe-uninitialized \ + -Wno-missing-field-initializers \ + -Wno-parentheses \ + -Wno-pointer-sign \ + -Wno-shift-negative-value \ + -Wno-sign-compare \ + -Wno-type-limits \ + -Wno-uninitialized \ + -Wno-unused-parameter \ + -Wno-unused-value " + +else +# clang +common="$OPENMP -fPIC -O2 -fvisibility=hidden -fno-strict-aliasing \ + -Werror -Wextra -Wno-unknown-warning-option \ + -Wsign-compare \ + -Wtautological-constant-out-of-range-compare \ + -Wuninitialized \ + -Wno-char-subscripts \ + -Wno-consumed \ + -Wno-delete-non-abstract-non-virtual-dtor \ + -Wno-empty-body \ + -Wno-implicit-float-conversion \ + -Wno-implicit-int-float-conversion \ + -Wno-int-in-bool-context \ + -Wno-missing-braces \ + -Wno-parentheses \ + -Wno-pass-failed \ + -Wno-pointer-sign \ + -Wno-string-plus-int \ + -Wno-unknown-pragmas \ + -Wno-unsequenced \ + -Wno-unused-function \ + -Wno-unused-parameter \ + -Wno-unused-value \ + -Wno-unused-variable " + +fi + +USE_PYXES="${USE_PYXES:=1}" +if [ $USE_PYXES -eq 1 ] ; then +common="$common -DPYXES=1" +LDTHREAD=" -pthread " +else +common="$common -DPYXES=0" +fi + +if [ -z "${j64x##*32*}" ]; then +USE_EMU_AVX=0 +else +USE_EMU_AVX="${USE_EMU_AVX:=1}" +fi +if [ $USE_EMU_AVX -eq 1 ] ; then +common="$common -DEMU_AVX=1" +fi + +case $jplatform\_$j64x in + +linux_j32) +TARGET=libjnative.so +CFLAGS="$common -m32 -msse2 -mfpmath=sse -I$JAVA_HOME/include -I$JAVA_HOME/include/linux " +LDFLAGS=" -shared -Wl,-soname,libjnative.so -m32 " +;; +linux_j64) +TARGET=libjnative.so +CFLAGS="$common -I$JAVA_HOME/include -I$JAVA_HOME/include/linux " +LDFLAGS=" -shared -Wl,-soname,libjnative.so " +;; +linux_j64avx) +TARGET=libjnative.so +CFLAGS="$common -I$JAVA_HOME/include -I$JAVA_HOME/include/linux " +LDFLAGS=" -shared -Wl,-soname,libjnative.so " +;; +linux_j64avx2) +TARGET=libjnative.so +CFLAGS="$common -I$JAVA_HOME/include -I$JAVA_HOME/include/linux " +LDFLAGS=" -shared -Wl,-soname,libjnative.so " +;; +raspberry_j32) +TARGET=libjnative.so +CFLAGS="$common -marm -march=armv6 -mfloat-abi=hard -mfpu=vfp -I$JAVA_HOME/include -I$JAVA_HOME/include/linux " +LDFLAGS=" -shared -Wl,-soname,libjnative.so " +;; +raspberry_j64) +TARGET=libjnative.so +CFLAGS="$common -march=armv8-a+crc -I$JAVA_HOME/include -I$JAVA_HOME/include/linux " +LDFLAGS=" -shared -Wl,-soname,libjnative.so " +;; +darwin_j32) +TARGET=libjnative.dylib +CFLAGS="$common -m32 -msse2 -mfpmath=sse $macmin -I$JAVA_HOME/include -I$JAVA_HOME/include/darwin " +LDFLAGS=" -m32 $macmin -dynamiclib " +;; +darwin_j64) +TARGET=libjnative.dylib +CFLAGS="$common $macmin -I$JAVA_HOME/include -I$JAVA_HOME/include/darwin " +LDFLAGS=" $macmin -dynamiclib " +;; +darwin_j64avx) +TARGET=libjnative.dylib +CFLAGS="$common $macmin -I$JAVA_HOME/include -I$JAVA_HOME/include/darwin " +LDFLAGS=" $macmin -dynamiclib " +;; +darwin_j64avx2) +TARGET=libjnative.dylib +CFLAGS="$common $macmin -I$JAVA_HOME/include -I$JAVA_HOME/include/darwin " +LDFLAGS=" $macmin -dynamiclib " +;; +darwin_j64arm) # darwin arm +TARGET=libjnative.dylib +CFLAGS="$common $macmin -march=armv8-a+crc -I$JAVA_HOME/include -I$JAVA_HOME/include/darwin " +LDFLAGS=" $macmin -dynamiclib " +;; +*) +echo no case for those parameters +exit +esac + +echo "CFLAGS=$CFLAGS" + +mkdir -p ../bin/$jplatform/$j64x +mkdir -p obj/$jplatform/$j64x/ +cp makefile-jnative obj/$jplatform/$j64x/. +export CFLAGS LDFLAGS TARGET jplatform j64x +cd obj/$jplatform/$j64x/ +make -f makefile-jnative +cd - diff --git a/make3/build_libj.sh b/make3/build_libj.sh new file mode 100755 index 000000000..c69b72ca9 --- /dev/null +++ b/make3/build_libj.sh @@ -0,0 +1,524 @@ +#!/bin/sh + +realpath() +{ + oldpath=`pwd` + if ! cd $1 > /dev/null 2>&1; then + cd ${1##*/} > /dev/null 2>&1 + echo $( pwd -P )/${1%/*} + else + pwd -P + fi + cd $oldpath > /dev/null 2>&1 +} + +cd "$(realpath "$0")" +echo "entering `pwd`" + +if [ "`uname`" != "Darwin" ] && ( [ "`uname -m`" = "armv6l" ] || [ "`uname -m`" = "aarch64" ] ); then +jplatform="${jplatform:=raspberry}" +elif [ "`uname`" = "Darwin" ]; then +jplatform="${jplatform:=darwin}" +else +jplatform="${jplatform:=linux}" +fi +if [ "`uname -m`" = "x86_64" ]; then +j64x="${j64x:=j64avx}" +elif [ "`uname -m`" = "aarch64" ]; then +j64x="${j64x:=j64}" +elif [ "`uname -m`" = "arm64" ] && [ -z "${jplatform##*darwin*}" ]; then +j64x="${j64x:=j64arm}" +else +j64x="${j64x:=j32}" +fi + +# gcc 5 vs 4 - killing off linux asm routines (overflow detection) +# new fast code uses builtins not available in gcc 4 +# use -DC_NOMULTINTRINSIC to continue to use more standard c in version 4 +# too early to move main linux release package to gcc 5 + +if [ -z "${jplatform##*darwin*}" ]; then +if [ -z "${j64x##*j64arm*}" ]; then +macmin="-arch arm64 -mmacosx-version-min=11" +NO_SHA_ASM="${NO_SHA_ASM:=1}" +else +macmin="-arch x86_64 -mmacosx-version-min=10.6" +fi +fi + +if [ "x$CC" = x'' ] ; then +if [ -f "/usr/bin/cc" ]; then +CC=cc +else +if [ -f "/usr/bin/clang" ]; then +CC=clang +else +CC=gcc +fi +fi +export CC +fi +# compiler=`$CC --version | head -n 1` +compiler=$(readlink -f $(command -v $CC) 2> /dev/null || echo $CC) +echo "CC=$CC" +echo "compiler=$compiler" + +USE_OPENMP="${USE_OPENMP:=0}" +if [ $USE_OPENMP -eq 1 ] ; then +if [ -z "${jplatform##*darwin*}" ]; then +if [ -z "${j64x##*j64arm*}" ]; then +# assume libomp installed at /opt/homebrew/ +OPENMP=" -Xpreprocessor -fopenmp -I/opt/homebrew/include " +LDOPENMP=" -L/opt/homebrew/lib -Wl,-rpath,/opt/homebrew/lib -lomp " +else +# assume libomp installed at /usr/local/ +OPENMP=" -Xpreprocessor -fopenmp -I/usr/local/include " +LDOPENMP=" -L/usr/local/lib -Wl,-rpath,/usr/local/lib -lomp " +fi +else +OPENMP=" -fopenmp " +LDOPENMP=" -fopenmp " +if [ -z "${compiler##*gcc*}" ] || [ -z "${CC##*gcc*}" ]; then +LDOPENMP32=" -l:libgomp.so.1 " # gcc +else +if [ -f /etc/redhat-release ] ; then +LDOPENMP32=" -l:libomp.so " # clang +else +LDOPENMP32=" -l:libomp.so.5 " # clang +fi +fi +fi +fi + +if [ -z "${compiler##*gcc*}" ] || [ -z "${CC##*gcc*}" ]; then +# gcc +common="$OPENMP -fPIC -O2 -falign-functions=4 -fvisibility=hidden -fno-strict-aliasing -fwrapv -fno-stack-protector -flax-vector-conversions \ + -Werror -Wextra -Wno-unknown-warning-option \ + -Wno-cast-function-type \ + -Wno-clobbered \ + -Wno-empty-body \ + -Wno-format-overflow \ + -Wno-implicit-fallthrough \ + -Wno-int-to-pointer-cast \ + -Wno-maybe-uninitialized \ + -Wno-missing-field-initializers \ + -Wno-overflow \ + -Wno-parentheses \ + -Wno-pointer-sign \ + -Wno-pointer-to-int-cast \ + -Wno-return-local-addr \ + -Wno-shift-count-overflow \ + -Wno-shift-negative-value \ + -Wno-sign-compare \ + -Wno-string-plus-int \ + -Wno-type-limits \ + -Wno-uninitialized \ + -Wno-unused-parameter \ + -Wno-unused-value " + +else +# clang +common="$OPENMP -fPIC -O2 -fvisibility=hidden -fno-strict-aliasing -fwrapv \ + -Werror -Wextra -Wno-unknown-warning-option \ + -Wsign-compare \ + -Wtautological-constant-out-of-range-compare \ + -Wuninitialized \ + -Wno-char-subscripts \ + -Wno-consumed \ + -Wno-delete-non-abstract-non-virtual-dtor \ + -Wno-empty-body \ + -Wno-implicit-float-conversion \ + -Wno-implicit-int-float-conversion \ + -Wno-int-in-bool-context \ + -Wno-missing-braces \ + -Wno-null-pointer-subtraction \ + -Wno-parentheses \ + -Wno-pass-failed \ + -Wno-pointer-sign \ + -Wno-pointer-to-int-cast \ + -Wno-sometimes-uninitialized \ + -Wno-string-plus-int \ + -Wno-unknown-pragmas \ + -Wno-unsequenced \ + -Wno-unused-but-set-variable \ + -Wno-unused-function \ + -Wno-unused-parameter \ + -Wno-unused-value \ + -Wno-unused-variable " + +fi + +USE_BOXEDSPARSE="${USE_BOXEDSPARSE:=0}" +if [ $USE_BOXEDSPARSE -eq 1 ] ; then +common="$common -DBOXEDSPARSE" +fi + +USE_PYXES="${USE_PYXES:=1}" +if [ $USE_PYXES -eq 1 ] ; then +common="$common -DPYXES=1" +LDTHREAD=" -pthread " +else +common="$common -DPYXES=0" +fi + +USE_SLEEF_SRC="${USE_SLEEF_SRC:=1}" +if [ -z "${j64x##*32*}" ] && [ -z "${jplatform##*raspberry*}" ]; then +# USE_SLEEF="${USE_SLEEF:=1}" +USE_SLEEF=0 +else +USE_SLEEF="${USE_SLEEF:=1}" +fi +if [ $USE_SLEEF -eq 1 ] ; then +common="$common -DSLEEF=1" +else +USE_SLEEF_SRC=0 +fi + +if [ -z "${j64x##*32*}" ]; then +USE_EMU_AVX=0 +else +USE_EMU_AVX="${USE_EMU_AVX:=1}" +fi +if [ $USE_EMU_AVX -eq 1 ] ; then +common="$common -DEMU_AVX=1" +fi + +NO_SHA_ASM="${NO_SHA_ASM:=0}" + +if [ $NO_SHA_ASM -ne 0 ] ; then + +common="$common -DNO_SHA_ASM" + +else + +SRC_ASM_LINUX=" \ + keccak1600-x86_64-elf.o \ + sha1-x86_64-elf.o \ + sha256-x86_64-elf.o \ + sha512-x86_64-elf.o " + +SRC_ASM_LINUX32=" \ + keccak1600-mmx-elf.o \ + sha1-586-elf.o \ + sha256-586-elf.o \ + sha512-586-elf.o " + +SRC_ASM_RASPI=" \ + keccak1600-armv8-elf.o \ + sha1-armv8-elf.o \ + sha256-armv8-elf.o \ + sha512-armv8-elf.o " + +SRC_ASM_RASPI32=" \ + keccak1600-armv4-elf.o \ + sha1-armv4-elf.o \ + sha256-armv4-elf.o \ + sha512-armv4-elf.o " + +SRC_ASM_MAC=" \ + keccak1600-x86_64-macho.o \ + sha1-x86_64-macho.o \ + sha256-x86_64-macho.o \ + sha512-x86_64-macho.o " + +SRC_ASM_MAC32=" \ + keccak1600-mmx-macho.o \ + sha1-586-macho.o \ + sha256-586-macho.o \ + sha512-586-macho.o " + +OBJS_ASM_WIN=" \ + ../../../../openssl-asm/keccak1600-x86_64-nasm.o \ + ../../../../openssl-asm/sha1-x86_64-nasm.o \ + ../../../../openssl-asm/sha256-x86_64-nasm.o \ + ../../../../openssl-asm/sha512-x86_64-nasm.o " + +OBJS_ASM_WIN32=" \ + ../../../../openssl-asm/keccak1600-mmx-nasm.o \ + ../../../../openssl-asm/sha1-586-nasm.o \ + ../../../../openssl-asm/sha256-586-nasm.o \ + ../../../../openssl-asm/sha512-586-nasm.o " + +fi + +OBJS_BASE64=" \ + ../../../../base64/lib/arch/avx2/codec-avx2.o \ + ../../../../base64/lib/arch/generic/codec-generic.o \ + ../../../../base64/lib/arch/neon32/codec-neon32.o \ + ../../../../base64/lib/arch/neon64/codec-neon64.o \ + ../../../../base64/lib/arch/ssse3/codec-ssse3.o \ + ../../../../base64/lib/arch/sse41/codec-sse41.o \ + ../../../../base64/lib/arch/sse42/codec-sse42.o \ + ../../../../base64/lib/arch/avx/codec-avx.o \ + ../../../../base64/lib/lib.o \ + ../../../../base64/lib/codec_choose.o \ + ../../../../base64/lib/tables/tables.o \ +" + +case $jplatform\_$j64x in + +linux_j32) # linux x86 +TARGET=libj.so +# faster, but sse2 not available for 32-bit amd cpu +# sse does not support mfpmath=sse in 32-bit gcc +CFLAGS="$common -m32 -msse2 -mfpmath=sse " +# slower, use 387 fpu and truncate extra precision +# CFLAGS="$common -m32 -ffloat-store " +LDFLAGS=" -shared -Wl,-soname,libj.so -m32 -lm -ldl $LDOPENMP32 $LDTHREAD" +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_LINUX32}" +GASM_FLAGS="-m32" +FLAGS_SLEEF=" -DENABLE_SSE2 " +FLAGS_BASE64="" +;; + +linux_j64) # linux intel 64bit nonavx +TARGET=libj.so +CFLAGS="$common -msse3 " +LDFLAGS=" -shared -Wl,-soname,libj.so -lm -ldl $LDOPENMP $LDTHREAD" +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_LINUX}" +GASM_FLAGS="" +FLAGS_SLEEF=" -DENABLE_SSE2 " +FLAGS_BASE64="" +;; + +linux_j64avx) # linux intel 64bit avx +TARGET=libj.so +CFLAGS="$common -DC_AVX=1 " +LDFLAGS=" -shared -Wl,-soname,libj.so -lm -ldl $LDOPENMP $LDTHREAD" +CFLAGS_SIMD=" -mavx " +OBJS_FMA=" gemm_int-fma.o " +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_LINUX}" +GASM_FLAGS="" +FLAGS_SLEEF=" -DENABLE_AVX " +FLAGS_BASE64=" -DHAVE_SSSE3=1 -DHAVE_AVX=1 " +;; + +linux_j64avx2) # linux intel 64bit avx2 +TARGET=libj.so +CFLAGS="$common -DC_AVX=1 -DC_AVX2=1 " +LDFLAGS=" -shared -Wl,-soname,libj.so -lm -ldl $LDOPENMP $LDTHREAD" +CFLAGS_SIMD=" -march=haswell -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mmovbe -mpopcnt " +OBJS_FMA=" gemm_int-fma.o " +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_LINUX}" +GASM_FLAGS="" +FLAGS_SLEEF=" -DENABLE_AVX2 " +FLAGS_BASE64=" -DHAVE_AVX2=1 " +;; + +raspberry_j32) # linux raspbian arm +TARGET=libj.so +CFLAGS="$common -Wno-overflow -marm -march=armv6 -mfloat-abi=hard -mfpu=vfp -DRASPI " +LDFLAGS=" -shared -Wl,-soname,libj.so -lm -ldl $LDOPENMP $LDTHREAD" +SRC_ASM="${SRC_ASM_RASPI32}" +GASM_FLAGS="" +FLAGS_SLEEF=" -DENABLE_VECEXT " # ENABLE_NEON32 single precision, useless +FLAGS_BASE64="" +;; + +raspberry_j64) # linux arm64 +TARGET=libj.so +CFLAGS="$common -march=armv8-a+crc -DRASPI -DC_CRC32C=1 " +LDFLAGS=" -shared -Wl,-soname,libj.so -lm -ldl $LDOPENMP $LDTHREAD" +OBJS_AESARM=" aes-arm.o " +SRC_ASM="${SRC_ASM_RASPI}" +GASM_FLAGS="" +FLAGS_SLEEF=" -DENABLE_ADVSIMD " +FLAGS_BASE64=" -DHAVE_NEON64=1 " +;; + +darwin_j32) # darwin x86 +TARGET=libj.dylib +CFLAGS="$common -m32 -msse2 -mfpmath=sse $macmin" +LDFLAGS=" -dynamiclib -lm -ldl $LDOPENMP $LDTHREAD -m32 $macmin" +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_MAC32}" +GASM_FLAGS="-m32 $macmin" +FLAGS_SLEEF=" -DENABLE_SSE2 " +FLAGS_BASE64="" +;; + +darwin_j64) # darwin intel 64bit nonavx +TARGET=libj.dylib +CFLAGS="$common $macmin -msse3 " +LDFLAGS=" -dynamiclib -lm -ldl $LDOPENMP $LDTHREAD $macmin" +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_MAC}" +GASM_FLAGS="$macmin" +FLAGS_SLEEF=" -DENABLE_SSE2 " +FLAGS_BASE64="" +;; + +darwin_j64avx) # darwin intel 64bit +TARGET=libj.dylib +CFLAGS="$common $macmin -DC_AVX=1 " +LDFLAGS=" -dynamiclib -lm -ldl $LDOPENMP $LDTHREAD $macmin" +CFLAGS_SIMD=" -mavx " +OBJS_FMA=" gemm_int-fma.o " +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_MAC}" +GASM_FLAGS="$macmin" +FLAGS_SLEEF=" -DENABLE_AVX " +FLAGS_BASE64=" -DHAVE_SSSE3=1 -DHAVE_AVX=1 " +;; + +darwin_j64avx2) # darwin intel 64bit +TARGET=libj.dylib +CFLAGS="$common $macmin -DC_AVX=1 -DC_AVX2=1 " +LDFLAGS=" -dynamiclib -lm -ldl $LDOPENMP $LDTHREAD $macmin" +CFLAGS_SIMD=" -march=haswell -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mmovbe -mpopcnt " +OBJS_FMA=" gemm_int-fma.o " +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_MAC}" +GASM_FLAGS="$macmin" +FLAGS_SLEEF=" -DENABLE_AVX2 " +FLAGS_BASE64=" -DHAVE_AVX2=1 " +;; + +darwin_j64arm) # darwin arm +TARGET=libj.dylib +CFLAGS="$common $macmin $common -march=armv8-a+crc -DC_CRC32C=1 " +LDFLAGS=" -dynamiclib -lm -ldl $LDOPENMP $LDTHREAD $macmin" +OBJS_AESARM=" aes-arm.o " +SRC_ASM="" +GASM_FLAGS="$macmin" +FLAGS_SLEEF=" -DENABLE_ADVSIMD " +FLAGS_BASE64=" -DHAVE_NEON64=1 " +;; + +windows_j32) # windows x86 +jolecom="${jolecom:=0}" +if [ $jolecom -eq 1 ] ; then +DOLECOM="-DOLECOM" +fi +TARGET=j.dll +# faster, but sse2 not available for 32-bit amd cpu +# sse does not support mfpmath=sse in 32-bit gcc +CFLAGS="$common $DOLECOM -m32 -msse2 -mfpmath=sse -D_FILE_OFFSET_BITS=64 -D_JDLL " +# slower, use 387 fpu and truncate extra precision +# CFLAGS="$common -m32 -ffloat-store " +LDFLAGS=" -shared -Wl,--enable-stdcall-fixup -lm -static-libgcc -static-libstdc++ $LDOPENMP32 $LDTHREAD" +if [ $jolecom -eq 1 ] ; then +DLLOBJS=" jdll.o jdllcomx.o " +LIBJDEF=" ../../../../dllsrc/jdll.def " +else +DLLOBJS=" jdll.o " +LIBJDEF=" ../../../../dllsrc/jdll2.def " +fi +LIBJRES=" jdllres.o " +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_WIN32}" +OBJS_ASM="${OBJS_ASM_WIN32}" +GASM_FLAGS="" +FLAGS_SLEEF=" -DENABLE_SSE2 " +FLAGS_BASE64="" +;; + +windows_j64) # windows intel 64bit nonavx +jolecom="${jolecom:=0}" +if [ $jolecom -eq 1 ] ; then +DOLECOM="-DOLECOM" +fi +TARGET=j.dll +CFLAGS="$common -msse3 $DOLECOM -D_FILE_OFFSET_BITS=64 -D_JDLL " +LDFLAGS=" -shared -Wl,--enable-stdcall-fixup -lm -static-libgcc -static-libstdc++ $LDOPENMP $LDTHREAD" +if [ $jolecom -eq 1 ] ; then +DLLOBJS=" jdll.o jdllcomx.o " +LIBJDEF=" ../../../../dllsrc/jdll.def " +else +DLLOBJS=" jdll.o " +LIBJDEF=" ../../../../dllsrc/jdll2.def " +fi +LIBJRES=" jdllres.o " +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_WIN}" +OBJS_ASM="${OBJS_ASM_WIN}" +GASM_FLAGS="" +FLAGS_SLEEF=" -DENABLE_SSE2 " +FLAGS_BASE64="" +;; + +windows_j64avx) # windows intel 64bit avx +jolecom="${jolecom:=0}" +if [ $jolecom -eq 1 ] ; then +DOLECOM="-DOLECOM" +fi +TARGET=j.dll +CFLAGS="$common $DOLECOM -DC_AVX=1 -D_FILE_OFFSET_BITS=64 -D_JDLL " +LDFLAGS=" -shared -Wl,--enable-stdcall-fixup -lm -static-libgcc -static-libstdc++ $LDOPENMP $LDTHREAD" +CFLAGS_SIMD=" -mavx " +if [ $jolecom -eq 1 ] ; then +DLLOBJS=" jdll.o jdllcomx.o " +LIBJDEF=" ../../../../dllsrc/jdll.def " +else +DLLOBJS=" jdll.o " +LIBJDEF=" ../../../../dllsrc/jdll2.def " +fi +LIBJRES=" jdllres.o " +OBJS_FMA=" gemm_int-fma.o " +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_WIN}" +OBJS_ASM="${OBJS_ASM_WIN}" +GASM_FLAGS="" +FLAGS_SLEEF=" -DENABLE_AVX " +FLAGS_BASE64=" -DHAVE_SSSE3=1 -DHAVE_AVX=1 " +;; + +windows_j64avx2) # windows intel 64bit avx +jolecom="${jolecom:=0}" +if [ $jolecom -eq 1 ] ; then +DOLECOM="-DOLECOM" +fi +TARGET=j.dll +CFLAGS="$common $DOLECOM -DC_AVX=1 -DC_AVX2=1 -D_FILE_OFFSET_BITS=64 -D_JDLL " +LDFLAGS=" -shared -Wl,--enable-stdcall-fixup -lm -static-libgcc -static-libstdc++ $LDOPENMP $LDTHREAD" +CFLAGS_SIMD=" -march=haswell -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mmovbe -mpopcnt " +if [ $jolecom -eq 1 ] ; then +DLLOBJS=" jdll.o jdllcomx.o " +LIBJDEF=" ../../../../dllsrc/jdll.def " +else +DLLOBJS=" jdll.o " +LIBJDEF=" ../../../../dllsrc/jdll2.def " +fi +LIBJRES=" jdllres.o " +OBJS_FMA=" gemm_int-fma.o " +OBJS_AESNI=" aes-ni.o " +SRC_ASM="${SRC_ASM_WIN}" +OBJS_ASM="${OBJS_ASM_WIN}" +GASM_FLAGS="" +FLAGS_SLEEF=" -DENABLE_AVX2 " +FLAGS_BASE64=" -DHAVE_AVX2=1 " +;; + +*) +echo no case for those parameters +exit +esac + +if [ $USE_SLEEF -eq 1 ] ; then +if [ $USE_SLEEF_SRC -eq 1 ] ; then +OBJS_SLEEF=" \ + ../../../../sleef/src/common/arraymap.o \ + ../../../../sleef/src/common/common.o \ + ../../../../sleef/src/libm/rempitab.o \ + ../../../../sleef/src/libm/sleefsimddp.o \ + " +fi +fi + +echo "CFLAGS=$CFLAGS" + +if [ ! -f ../jsrc/jversion.h ] ; then + cp ../jsrc/jversion-x.h ../jsrc/jversion.h +fi + +mkdir -p ../bin/$jplatform/$j64x +mkdir -p obj/$jplatform/$j64x/ +cp makefile-libj obj/$jplatform/$j64x/. +export CFLAGS LDFLAGS TARGET CFLAGS_SIMD GASM_FLAGS FLAGS_SLEEF FLAGS_BASE64 DLLOBJS LIBJDEF LIBJRES OBJS_BASE64 OBJS_FMA OBJS_AESNI OBJS_AESARM OBJS_SLEEF OBJS_ASM SRC_ASM jplatform j64x +cd obj/$jplatform/$j64x/ +make -f makefile-libj +cd - diff --git a/make3/build_tsdll.sh b/make3/build_tsdll.sh new file mode 100755 index 000000000..6ba16556c --- /dev/null +++ b/make3/build_tsdll.sh @@ -0,0 +1,198 @@ +#!/bin/sh + +realpath() +{ + oldpath=`pwd` + if ! cd $1 > /dev/null 2>&1; then + cd ${1##*/} > /dev/null 2>&1 + echo $( pwd -P )/${1%/*} + else + pwd -P + fi + cd $oldpath > /dev/null 2>&1 +} + +cd "$(realpath "$0")" +echo "entering `pwd`" + +if [ "`uname`" != "Darwin" ] && ( [ "`uname -m`" = "armv6l" ] || [ "`uname -m`" = "aarch64" ] ); then +jplatform="${jplatform:=raspberry}" +elif [ "`uname`" = "Darwin" ]; then +jplatform="${jplatform:=darwin}" +else +jplatform="${jplatform:=linux}" +fi +if [ "`uname -m`" = "x86_64" ]; then +j64x="${j64x:=j64avx}" +elif [ "`uname -m`" = "aarch64" ]; then +j64x="${j64x:=j64}" +elif [ "`uname -m`" = "arm64" ] && [ -z "${jplatform##*darwin*}" ]; then +j64x="${j64x:=j64arm}" +else +j64x="${j64x:=j32}" +fi + +# gcc 5 vs 4 - killing off linux asm routines (overflow detection) +# new fast code uses builtins not available in gcc 4 +# use -DC_NOMULTINTRINSIC to continue to use more standard c in version 4 +# too early to move main linux release package to gcc 5 + +if [ -z "${jplatform##*darwin*}" ]; then +if [ -z "${j64x##*j64arm*}" ]; then +macmin="-arch arm64 -mmacosx-version-min=11" +else +macmin="-arch x86_64 -mmacosx-version-min=10.6" +fi +fi + +if [ "x$CC" = x'' ] ; then +if [ -f "/usr/bin/cc" ]; then +CC=cc +else +if [ -f "/usr/bin/clang" ]; then +CC=clang +else +CC=gcc +fi +fi +export CC +fi +# compiler=`$CC --version | head -n 1` +compiler=$(readlink -f $(command -v $CC) 2> /dev/null || echo $CC) +echo "CC=$CC" +echo "compiler=$compiler" + +if [ -z "${compiler##*gcc*}" ] || [ -z "${CC##*gcc*}" ]; then +# gcc +common="$OPENMP -fPIC -O2 -fvisibility=hidden -fno-strict-aliasing \ + -Werror -Wextra -Wno-unknown-warning-option \ + -Wno-cast-function-type \ + -Wno-clobbered \ + -Wno-empty-body \ + -Wno-format-overflow \ + -Wno-implicit-fallthrough \ + -Wno-maybe-uninitialized \ + -Wno-missing-field-initializers \ + -Wno-parentheses \ + -Wno-pointer-sign \ + -Wno-shift-negative-value \ + -Wno-sign-compare \ + -Wno-type-limits \ + -Wno-uninitialized \ + -Wno-unused-parameter \ + -Wno-unused-value " + +else +# clang +common="$OPENMP -fPIC -O2 -fvisibility=hidden -fno-strict-aliasing \ + -Werror -Wextra -Wno-unknown-warning-option \ + -Wsign-compare \ + -Wtautological-constant-out-of-range-compare \ + -Wuninitialized \ + -Wno-char-subscripts \ + -Wno-consumed \ + -Wno-delete-non-abstract-non-virtual-dtor \ + -Wno-empty-body \ + -Wno-implicit-float-conversion \ + -Wno-implicit-int-float-conversion \ + -Wno-int-in-bool-context \ + -Wno-missing-braces \ + -Wno-parentheses \ + -Wno-pass-failed \ + -Wno-pointer-sign \ + -Wno-string-plus-int \ + -Wno-unknown-pragmas \ + -Wno-unsequenced \ + -Wno-unused-function \ + -Wno-unused-parameter \ + -Wno-unused-value \ + -Wno-unused-variable " + +fi + +case $jplatform\_$j64x in + +linux_j32) # linux x86 +TARGET=libtsdll.so +# faster, but sse2 not available for 32-bit amd cpu +# sse does not support mfpmath=sse in 32-bit gcc +CFLAGS="$common -m32 -msse2 -mfpmath=sse -DC_NOMULTINTRINSIC " +# slower, use 387 fpu and truncate extra precision +# CFLAGS="$common -m32 -ffloat-store " +LDFLAGS=" -shared -Wl,-soname,libtsdll.so -m32 -lm -ldl" +;; + +linux_j64) # linux intel 64bit nonavx +TARGET=libtsdll.so +CFLAGS="$common " +LDFLAGS=" -shared -Wl,-soname,libtsdll.so -lm -ldl" +;; + +linux_j64avx) # linux intel 64bit avx +TARGET=libtsdll.so +CFLAGS="$common " +LDFLAGS=" -shared -Wl,-soname,libtsdll.so -lm -ldl" +;; + +linux_j64avx2) # linux intel 64bit avx +TARGET=libtsdll.so +CFLAGS="$common " +LDFLAGS=" -shared -Wl,-soname,libtsdll.so -lm -ldl" +;; + +raspberry_j32) # linux raspbian arm +TARGET=libtsdll.so +CFLAGS="$common -marm -march=armv6 -mfloat-abi=hard -mfpu=vfp -DRASPI -DC_NOMULTINTRINSIC " +LDFLAGS=" -shared -Wl,-soname,libtsdll.so -lm -ldl" +;; + +raspberry_j64) # linux arm64 +TARGET=libtsdll.so +CFLAGS="$common -march=armv8-a+crc -DRASPI " +LDFLAGS=" -shared -Wl,-soname,libtsdll.so -lm -ldl" +;; + +darwin_j32) # darwin x86 +TARGET=libtsdll.dylib +CFLAGS="$common -m32 -msse2 -mfpmath=sse $macmin" +LDFLAGS=" -dynamiclib -lm -ldl -m32 $macmin" +;; + +darwin_j64) # darwin intel 64bit nonavx +TARGET=libtsdll.dylib +CFLAGS="$common $macmin" +LDFLAGS=" -dynamiclib -lm -ldl $macmin" +;; + +darwin_j64avx) # darwin intel 64bit +TARGET=libtsdll.dylib +CFLAGS="$common $macmin " +LDFLAGS=" -dynamiclib -lm -ldl $macmin" +;; + +darwin_j64avx2) # darwin intel 64bit +TARGET=libtsdll.dylib +CFLAGS="$common $macmin " +LDFLAGS=" -dynamiclib -lm -ldl $macmin" +;; + +darwin_j64arm) # darwin arm +TARGET=libtsdll.dylib +CFLAGS="$common $macmin -march=armv8-a+crc " +LDFLAGS=" -dynamiclib -lm -ldl $macmin" +;; + +*) +echo no case for those parameters +exit +esac + +echo "CFLAGS=$CFLAGS" + +mkdir -p ../bin/$jplatform/$j64x +mkdir -p obj/$jplatform/$j64x/ +cp makefile-tsdll obj/$jplatform/$j64x/. +export CFLAGS LDFLAGS TARGET jplatform j64x +cd obj/$jplatform/$j64x/ +make -f makefile-tsdll +cd - diff --git a/make3/clean.sh b/make3/clean.sh new file mode 100755 index 000000000..0a6935a33 --- /dev/null +++ b/make3/clean.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# rm all *.o for clean builds - makefile dependencies are not set + +realpath() +{ + oldpath=`pwd` + if ! cd $1 > /dev/null 2>&1; then + cd ${1##*/} > /dev/null 2>&1 + echo $( pwd -P )/${1%/*} + else + pwd -P + fi + cd $oldpath > /dev/null 2>&1 +} + +cd "$(realpath "$0")" +echo "entering `pwd`" + +find ../jsrc -name "*.o" -type f -delete || true +find ../dllsrc -name "*.o" -type f -delete || true +find ../sleef/src -name "*.o" -type f -delete || true +find ../base64 -name "*.o" -type f -delete || true +find obj -name "*.o" -type f -delete || true + +find ../jsrc -name "*.tmp" -type f -delete || true +find ../dllsrc -name "*.tmp" -type f -delete || true +find ../sleef/src -name "*.tmp" -type f -delete || true +find ../base64 -name "*.tmp" -type f -delete || true +find obj -name "*.tmp" -type f -delete || true || true diff --git a/make3/cpbin.sh b/make3/cpbin.sh new file mode 100755 index 000000000..acf1bda2e --- /dev/null +++ b/make3/cpbin.sh @@ -0,0 +1,96 @@ +#!/bin/sh + +# copy binaries in bin/ to jlibrary/bin + +cop(){ +# $1 src +# $2 srclib +# $3 dest +# $4 destlib +# MUST rename/remove object first; overwrite cause cache error +if [ -f "../bin/${jplatform}/$1/$2" ]; then +if [ -f "../jlibrary/$3/$4" ]; then +mv -f "../jlibrary/$3/$4" "/tmp/$4.old.$$" +fi +echo \# cp "../bin/${jplatform}/$1/$2" "../jlibrary/$3/$4" +cp "../bin/${jplatform}/$1/$2" "../jlibrary/$3/$4" +fi +} + +cd "`dirname "$0"`" + +if [ "`uname -m`" = "armv6l" ] || [ "`uname -m`" = "aarch64" ] || [ "$RASPI" = 1 ]; then +jplatform="${jplatform:=raspberry}" +elif [ "`uname`" = "Darwin" ]; then +jplatform="${jplatform:=darwin}" +else +jplatform="${jplatform:=linux}" +fi + +echo \# jplatform $jplatform + +if [ $jplatform = "darwin" ]; then + +# macos 64-bit +if [ -f "../bin/${jplatform}/j64/jconsole" ] && [ -f "../bin/${jplatform}/j64arm/jconsole" ]; then +# fat binary +if [ -f "../jlibrary/bin/jconsole" ]; then +mv -f "../jlibrary/bin/jconsole" "/tmp/jconsole.old.$$" +fi +echo \# lipo "../bin/${jplatform}/j64/jconsole" "../bin/${jplatform}/j64arm/jconsole" -create -output "../jlibrary/bin/jconsole-mac" +lipo "../bin/${jplatform}/j64/jconsole" "../bin/${jplatform}/j64arm/jconsole" -create -output "../jlibrary/bin/jconsole-mac" +cp ../jlibrary/bin/jconsole-mac ../jlibrary/bin/jconsole +elif [ -f "../bin/${jplatform}/j64/jconsole" ]; then +cop j64 jconsole bin jconsole-mac +cp ../jlibrary/bin/jconsole-mac ../jlibrary/bin/jconsole +elif [ -f "../bin/${jplatform}/j64arm/jconsole" ]; then +cop j64arm jconsole bin jconsole-mac +cp ../jlibrary/bin/jconsole-mac ../jlibrary/bin/jconsole +fi + +if [ -f "../bin/${jplatform}/j64/libtsdll.dylib" ] && [ -f "../bin/${jplatform}/j64arm/libtsdll.dylib" ]; then +# fat binary +if [ -f "../jlibrary/bin/libtsdll.dylib" ]; then +mv -f "../jlibrary/bin/libtsdll.dylib" "/tmp/libtsdll.dylib.old.$$" +fi +echo \# lipo "../bin/${jplatform}/j64/libtsdll.dylib" "../bin/${jplatform}/j64arm/libtsdll.dylib" -create -output "../jlibrary/bin/libtsdll.dylib" +lipo "../bin/${jplatform}/j64/libtsdll.dylib" "../bin/${jplatform}/j64arm/libtsdll.dylib" -create -output "../jlibrary/bin/libtsdll.dylib" +elif [ -f "../bin/${jplatform}/j64/libtsdll.dylib" ]; then +cop j64 libtsdll.dylib bin libtsdll.dylib +elif [ -f "../bin/${jplatform}/j64arm/libtsdll.dylib" ]; then +cop j64arm libtsdll.dylib bin libtsdll.dylib +fi + +if [ -f "../bin/${jplatform}/j64/libj.dylib" ] && [ -f "../bin/${jplatform}/j64arm/libj.dylib" ]; then +# fat binary +if [ -f "../jlibrary/bin/libj.dylib" ]; then +mv -f "../jlibrary/bin/libj.dylib" "/tmp/libj.dylib.old.$$" +fi +echo \# lipo "../bin/${jplatform}/j64/libj.dylib" "../bin/${jplatform}/j64arm/libj.dylib" -create -output "../jlibrary/bin/libj.dylib" +lipo "../bin/${jplatform}/j64/libj.dylib" "../bin/${jplatform}/j64arm/libj.dylib" -create -output "../jlibrary/bin/libj.dylib" +elif [ -f "../bin/${jplatform}/j64/libj.dylib" ]; then +cop j64 libj.dylib bin libj.dylib +elif [ -f "../bin/${jplatform}/j64arm/libj.dylib" ]; then +cop j64arm libj.dylib bin libj.dylib +fi + +cop j64avx libj.dylib bin libjavx.dylib +cop j64avx2 libj.dylib bin libjavx2.dylib + +else + +# linux/raspberry 64-bit +cop j64 jconsole bin jconsole +cop j64 jconsole bin jconsole-lx +cop j64 libtsdll.so bin libtsdll.so +cop j64 libj.so bin libj.so +cop j64avx libj.so bin libjavx.so +cop j64avx2 libj.so bin libjavx2.so + +# linux/raspberry 32-bit +cop j32 jconsole bin32 jconsole +cop j32 libtsdll.so bin32 libtsdll.so +cop j32 libj.so bin32 libj.so + +fi + diff --git a/make3/macos-libomp.txt b/make3/macos-libomp.txt new file mode 100644 index 000000000..729607a7f --- /dev/null +++ b/make3/macos-libomp.txt @@ -0,0 +1,15 @@ +get/build cmake if not yet + +$ brew install cmake + +How to get/build libomp. + +$ cd +$ svn co http://llvm.org/svn/llvm-project/openmp/trunk libomp +$ cd libomp +$ mkdir -p build +$ cmake -DCMAKE_INSTALL_PREFIX=/usr/local/opt/libomp +$ make && make install + +need to deploy this run-time +/usr/local/opt/libomp/lib/libomp.dylib diff --git a/make3/make.txt b/make3/make.txt new file mode 100644 index 000000000..dcd83f3b3 --- /dev/null +++ b/make3/make.txt @@ -0,0 +1,69 @@ +how to build/test j32/j64/j64avx/j64avx2 for linux/darwin/raspberry/... + +************************************************************************** +# commands are assumed to run in make2 folder + +# make sure shell scripts are executable +chmod +x *.sh + +# ensure proper jsrc/jversion.h (default jversion-x.h template) + +# !!! make2 build commands will be confused if there are *.o files in jsrc +# !!! makefiles do not have h file dependencies !!! +# after h file changes, run ./clean.sh to rm all *.o files for clean build +./clean.sh + +# general format for build_... commands +# don't miss the ./ part when typing commands +jplatform=(..) j64x=(..) ./build_libj.sh + +where +jplatform : linux darwin raspberry (default host) +j64x : j64 j64avx j64avx2 j64arm j32 (default host or j64avx) + +# binaries generated inside the bin folder under jsource + +# eg. build j64 binaries +jplatform=linux j64x=j64 ./build_libj.sh + +# target apple m1 +jplatform=darwin j64x=j64arm ./build_libj.sh + +# target apple intel/m1 +jplatform=darwin j64x=j64 ./build_libj.sh + +# target apple intel +jplatform=darwin j64x=j64avx ./build_libj.sh +jplatform=darwin j64x=j64avx2 ./build_libj.sh + +# linux and j64 is the default on 64-bit linux, so this is fine +./build_libj.sh + +# the easiest way to build all binaries +# this will run ./clean.sh and can detect the host platform +./build_all.sh + +# target apple m1 +j64x=j64arm ./build_all.sh + +# target apple intel/m1 +j64x=j64 ./build_all.sh + +# target apple intel +j64x=j64avx ./build_all.sh +j64x=j64avx2 ./build_all.sh + +# test +# copy binaries to jlibrary/bin folder +./cpbin.sh + +# cd the jlibrary/bin or jlibrary/bin32 folder and run depending on the cpu architecture +# avx2 / x86 / arm +./jconsole ../../test/tsu.ijs +# avx +./jconsole -lib libjavx.so ../../test/tsu.ijs +# nonavx +./jconsole -lib libj-nonavx.so ../../test/tsu.ijs + +# and follow instructions there + diff --git a/make3/makefile-jconsole b/make3/makefile-jconsole new file mode 100644 index 000000000..7cd24b66b --- /dev/null +++ b/make3/makefile-jconsole @@ -0,0 +1,29 @@ +vpath % ../../../../jsrc + +ifndef jplatform +jplatform=linux +endif +ifndef j64x +j64x=j64 +endif + +ODIR=../../../../bin/$(jplatform)/$(j64x) + +.SUFFIXES: .o + +OBJS= \ + jconsole.o \ + jeload.o + +%.o: %.c + $(CC) -c -o $@ $< $(CFLAGS) + +$(ODIR)/$(TARGET) : $(OBJS) $(OBJSLN) + $(CC) -o $@ $(OBJS) $(OBJSLN) $(LDFLAGS) + +all: $(ODIR)/$(TARGET) + +clean: + rm -f $(OBJS) $(OBJSLN) linenoise.o + +.PHONY: all clean diff --git a/make3/makefile-jnative b/make3/makefile-jnative new file mode 100644 index 000000000..1adce4f39 --- /dev/null +++ b/make3/makefile-jnative @@ -0,0 +1,29 @@ +vpath % ../../../../jsrc + +ifndef jplatform +jplatform=linux +endif +ifndef j64x +j64x=j64 +endif + +ODIR=../../../../bin/$(jplatform)/$(j64x) + +.SUFFIXES: .o + +OBJS= \ + andjnative.o \ + jeload.o + +%.o: %.c + $(CC) -c -o $@ $< $(CFLAGS) + +$(ODIR)/$(TARGET) : $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAGS) + +all: $(ODIR)/$(TARGET) + +clean: + rm -f $(OBJS) + +.PHONY: all clean diff --git a/make3/makefile-libj b/make3/makefile-libj new file mode 100644 index 000000000..012b85b1d --- /dev/null +++ b/make3/makefile-libj @@ -0,0 +1,190 @@ +vpath % ../../../../jsrc:../../../../jsrc/blis:../../../../jsrc/openssl/sha:../../../../jsrc/openssl/sha/asm:../../../../dllsrc + +ifndef jplatform +jplatform=linux +endif +ifndef j64x +j64x=j64 +endif + +ODIR=../../../../bin/$(jplatform)/$(j64x) + +.SUFFIXES: .o + +OBJS= \ + a.o \ + ab.o \ + aes-c.o \ + aes-sse2.o \ + af.o \ + ai.o \ + am.o \ + am1.o \ + amn.o \ + ao.o \ + ap.o \ + ar.o \ + as.o \ + au.o \ + gemm_c-ref.o \ + gemm_int-aarch64.o \ + gemm_int-avx.o \ + gemm_int-sse2.o \ + gemm_vec-ref.o \ + c.o \ + ca.o \ + cc.o \ + cd.o \ + cf.o \ + cg.o \ + ch.o \ + cip.o \ + cl.o \ + cp.o \ + cpdtsp.o \ + cpuinfo.o \ + cr.o \ + crs.o \ + ct.o \ + cu.o \ + cv.o \ + cx.o \ + d.o \ + dc.o \ + dss.o \ + dstop.o \ + dsusp.o \ + dtoa.o \ + f.o \ + f2.o \ + fbu.o \ + gemm.o \ + i.o \ + io.o \ + j.o \ + jdlllic.o \ + k.o \ + m.o \ + mbx.o \ + mt.o \ + p.o \ + pv.o \ + px.o \ + r.o \ + rl.o \ + rt.o \ + s.o \ + sc.o \ + sl.o \ + sn.o \ + t.o \ + u.o \ + v.o \ + v0.o \ + v1.o \ + v2.o \ + va1.o \ + va1ss.o \ + va2.o \ + va2s.o \ + va2ss.o \ + vamultsp.o \ + vb.o \ + vbang.o \ + vbit.o \ + vcant.o \ + vcat.o \ + vcatsp.o \ + vchar.o \ + vcomp.o \ + vcompsc.o \ + vd.o \ + vdx.o \ + ve.o \ + vf.o \ + vfft.o \ + vfrom.o \ + vfromsp.o \ + vg.o \ + vgauss.o \ + vgcomp.o \ + vgranking.o \ + vgsort.o \ + vgsp.o \ + vi.o \ + viavx.o \ + viix.o \ + visp.o \ + vm.o \ + vo.o \ + vp.o \ + vq.o \ + vrand.o \ + vrep.o \ + vs.o \ + vsb.o \ + vt.o \ + vu.o \ + vx.o \ + vz.o \ + w.o \ + wc.o \ + wn.o \ + ws.o \ + x.o \ + x15.o \ + xa.o \ + xaes.o \ + xb.o \ + xc.o \ + xcrc.o \ + xd.o \ + xf.o \ + xfmt.o \ + xh.o \ + xi.o \ + xl.o \ + xo.o \ + xs.o \ + xsha.o \ + xt.o \ + xu.o \ + crc32c.o \ + str.o \ + keccak1600.o \ + md4_dgst.o \ + md4_one.o \ + md5_dgst.o \ + md5_one.o \ + openssl-util.o \ + sha1_one.o \ + sha256.o \ + sha3.o \ + sha512.o + +%.o: %.cpp + $(CXX) -c $(CPPFLAGS) -o$@ $^ + +%.o: %.c + $(CC) -c -o $@ $< $(CFLAGS) $(CFLAGS_SIMD) $(CFLAGS_FMA) $(CFLAGS_AESNI) $(CFLAGS_AESARM) $(CFLAGS_SLEEF) $(CFLAGS_BASE64) + +%.o: %.S + $(CC) $(GASM_FLAGS) -c -o $@ $< + +$(OBJS_FMA) : CFLAGS_FMA := -mfma +$(OBJS_AESNI) : CFLAGS_AESNI := -maes +$(OBJS_AESARM) : CFLAGS_AESARM := -march=armv8-a+crc+crypto +$(OBJS_SLEEF) : CFLAGS_SLEEF := $(FLAGS_SLEEF) -DSLEEF_STATIC_LIBS -DNDEBUG -DDORENAME=1 -Wno-sign-compare -ffp-contract=off -fno-math-errno -fno-trapping-math -I../../../../sleef/src/arch -I../../../../sleef/src/common +$(OBJS_BASE64) : CFLAGS_BASE64 := $(FLAGS_BASE64) +$(ODIR)/$(TARGET) : $(OBJS) $(OBJS_BASE64) $(OBJS_FMA) $(OBJS_AESNI) $(OBJS_AESARM) $(OBJS_SLEEF) $(SRC_ASM) $(DLLOBJS) $(LIBJRES) + $(CC) -o $@ $(OBJS) $(OBJS_FMA) $(OBJS_AESNI) $(OBJS_AESARM) $(OBJS_SLEEF) $(OBJS_BASE64) $(OBJS_ASM) $(SRC_ASM) $(DLLOBJS) $(LIBJDEF) $(LIBJRES) $(LDFLAGS) + +$(LIBJRES): jdll.rc + $(WINDRES) -o$@ $^ + +all: $(ODIR)/$(TARGET) + +clean: + rm -f $(OBJS) $(OBJS_FMA) $(OBJS_AESNI) $(OBJS_AESARM) $(OBJS_SLEEF) $(OBJS_BASE64) $(SRC_ASM) $(DLLOBJS) $(LIBJRES) + +.PHONY: all clean diff --git a/make3/makefile-tsdll b/make3/makefile-tsdll new file mode 100644 index 000000000..1cd9ac063 --- /dev/null +++ b/make3/makefile-tsdll @@ -0,0 +1,28 @@ +vpath % ../../../../jsrc + +ifndef jplatform +jplatform=linux +endif +ifndef j64x +j64x=j64 +endif + +ODIR=../../../../bin/$(jplatform)/$(j64x) + +.SUFFIXES: .o + +OBJS= \ + tsdll.o + +%.o: %.c + $(CC) -c -o $@ $< $(CFLAGS) + +$(ODIR)/$(TARGET) : $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAGS) + +all: $(ODIR)/$(TARGET) + +clean: + rm -f $(OBJS) + +.PHONY: all clean From b9d3f68a4f27fde847e23c267730deb41aaa8410 Mon Sep 17 00:00:00 2001 From: "B. Wilson" Date: Mon, 2 May 2022 16:22:20 +0900 Subject: [PATCH 4/6] make3:fix: Disable parallel builds Unfortunately, we ran into build non-determinism, causing random target-hash mismatches. In particular, we noticed this when a hash mismatch fired after removing some dead code from build_jconsole.sh. By diffing the xtrace, i.e. `set +x`, results with and without the change, we confirmed that the executed commands were exactly the same; however, a quick inspection using diffoscope on the two output directories revealed differences, not in jconsole, but in libj.so. The issue seems to be that build outputs are colocated with source files, meaning that parallel builds end up clobbering each other's object files. --- jbuild.scm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jbuild.scm b/jbuild.scm index 1d3888754..a4244f097 100755 --- a/jbuild.scm +++ b/jbuild.scm @@ -179,7 +179,8 @@ (native-inputs `(("toolchain" ,toolchain))) (inputs (cons* libedit libomp jsoftware-auxiliary-files extra-inputs)) (arguments - `(#:parallel-tests? #f + `(#:parallel-build? #f + #:parallel-tests? #f #:tests? #f #:modules (((ice-9 ftw) #:select (scandir)) ((ice-9 popen) #:select (open-pipe* close-pipe)) @@ -501,7 +502,7 @@ that are not already well understood.") ("LDFLAGS" "-lsleef")))) (define j (apply j-package j-base-args)) -(define %j-target-hash "tihgvwf7oqyi3cg5p4gregjby42odlrkobumroa6nb7cuphrsrya") +(define %j-target-hash "har6llk2rv3bwh3gao3qr3ue3dkmm6n4agvmkjhkes3iku2zjngq") (match (program-arguments) ;; Don't execute anything if loading in repl From 5cdde637681af39848d5daa65390e6cf35542ef7 Mon Sep 17 00:00:00 2001 From: "B. Wilson" Date: Mon, 2 May 2022 18:49:17 +0900 Subject: [PATCH 5/6] make3: Explicitly define guix channel revision For 100% build reproducibility we also need to ensure that we build against a stable set of dependencies. This commit defines an explicit guix channel revision against which the build script runs. In particular, without an explicit channel revision and when run on different machines, the build script is likely to produce different outputs, causing a mismatch with the target hash. In fact, the same may happen on a single machine, if invoked across `guix pull` updates. --- jbuild.scm => make3/build.scm | 67 ++++++++++++++++++----------------- make3/channels.scm | 11 ++++++ 2 files changed, 46 insertions(+), 32 deletions(-) rename jbuild.scm => make3/build.scm (93%) create mode 100644 make3/channels.scm diff --git a/jbuild.scm b/make3/build.scm similarity index 93% rename from jbuild.scm rename to make3/build.scm index a4244f097..fce7f7661 100755 --- a/jbuild.scm +++ b/make3/build.scm @@ -1,4 +1,6 @@ -#!/usr/bin/env -S guix repl +#!/usr/bin/env sh +dir=${0%/*}; dir=${dir:-.} +exec guix time-machine --channels=$dir/channels.scm -- repl "$0" !# ;;; GNU Guix development package. ;;; @@ -65,48 +67,49 @@ #:use-module ((gnu packages maths) #:select (sleef))) -(define %source-dir (dirname (current-filename))) +(define %source-dir + (let ((pipe (open-pipe* OPEN_READ "git" "rev-parse" "--show-toplevel"))) + (read-line pipe))) (define (git-version) "Return a version string suitable for development builds." - (let* ((pipe (with-directory-excursion %source-dir - (open-pipe* OPEN_READ "git" "describe" "--always" + (with-directory-excursion %source-dir + (let* ((pipe (open-pipe* OPEN_READ "git" "describe" "--always" "--tags" - "--abbrev=0"))) - (version (string-append (read-line pipe) "+git"))) - (close-pipe pipe) - version)) + "--abbrev=0")) + (version (string-append (read-line pipe) "+git"))) + (close-pipe pipe) + version))) (define (git-user) "Return a user info string scraped from Git." - (let* ((name-pipe (with-directory-excursion %source-dir - (open-pipe* OPEN_READ "git" "config" "user.name"))) - (email-pipe (with-directory-excursion %source-dir - (open-pipe* OPEN_READ "git" "config" "user.email"))) - (name (read-line name-pipe)) - (email (read-line email-pipe)) - (status (every identity (map close-pipe `(,name-pipe ,email-pipe))))) - (format #f "~a <~a>" name email))) + (with-directory-excursion %source-dir + (let* ((name-pipe (open-pipe* OPEN_READ "git" "config" "user.name")) + (email-pipe (open-pipe* OPEN_READ "git" "config" "user.email")) + (name (read-line name-pipe)) + (email (read-line email-pipe)) + (status (every identity (map close-pipe `(,name-pipe ,email-pipe))))) + (format #f "~a <~a>" name email)))) ;; Predicate intended for SELECT? argument of local-file procedure. Returns ;; true if and only if file is tracked by git. (define git-file? - (let* ((pipe (with-directory-excursion %source-dir - (open-pipe* OPEN_READ "git" "ls-files"))) - (files (let loop ((lines '())) - (match (read-line pipe) - ((? eof-object?) (reverse lines)) - ((? (lambda (file) ; skip this file - (string-match (current-filename) - (canonicalize-path file)))) - (loop lines)) - (line (loop (cons line lines)))))) - (status (close-pipe pipe))) - (lambda (file stat) - (match (stat:type stat) - ('directory #t) - ((or 'regular 'symlink) (any (cut string-suffix? <> file) files)) - (_ #f))))) + (with-directory-excursion %source-dir + (let* ((pipe (open-pipe* OPEN_READ "git" "ls-files")) + (files (let loop ((lines '())) + (match (read-line pipe) + ((? eof-object?) (reverse lines)) + ((? (lambda (file) ; skip this file + (string-match (current-filename) + (canonicalize-path file)))) + (loop lines)) + (line (loop (cons line lines)))))) + (status (close-pipe pipe))) + (lambda (file stat) + (match (stat:type stat) + ('directory #t) + ((or 'regular 'symlink) (any (cut string-suffix? <> file) files)) + (_ #f)))))) ;; G-exp script that detects AVX/AVX2 support at runtime and executes jconsole diff --git a/make3/channels.scm b/make3/channels.scm new file mode 100644 index 000000000..b2adc52ee --- /dev/null +++ b/make3/channels.scm @@ -0,0 +1,11 @@ +(list (channel + (name 'guix) + (url "https://git.savannah.gnu.org/git/guix.git") + (branch "master") + (commit + "c026db3f469d79e92155e7cb883830ccafc4206d") + (introduction + (make-channel-introduction + "9edb3f66fd807b096b48283debdcddccfea34bad" + (openpgp-fingerprint + "BBB0 2DDF 2CEA F6A8 0D1D E643 A2A0 6DF2 A33A 54FA"))))) From 0f3038a73e62d55122cc9e0bbad6b09444a722a9 Mon Sep 17 00:00:00 2001 From: "B. Wilson" Date: Thu, 28 Apr 2022 19:14:29 +0900 Subject: [PATCH 6/6] make3: Switch to env-style shebang Much ink has been spilled over the "correct" shebang. Here, opting for env-style is making a declaration of *intent*: These scripts prioritize environment flexiblity over execution precision. In particular, using a `#!/bin/sh` shebang declares that this script shall be run by the interpreter sitting at the absolute path `/bin/sh`. This style of shebang is useful when needing precise control over the executing interpreter. However, the use case for these scripts means that we have little a priori control over the executing interpreter, and there are legitimate cases where users may wish to use some other shell in preference to `/bin/sh`, e.g. containerized environments. The `#!/usr/bin/env sh` shebang caters to these needs. Of course, there are some cases (quite rare, now) where the env-shebang in this commit will fail [0]; however, we make the decision to de-prioritize these cases over the aforementioned ones. [0]:https://www.in-ulm.de/~mascheck/various/shebang/#env --- make3/build_all.sh | 2 +- make3/build_jconsole.sh | 2 +- make3/build_jnative.sh | 2 +- make3/build_libj.sh | 2 +- make3/build_tsdll.sh | 2 +- make3/clean.sh | 2 +- make3/cpbin.sh | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/make3/build_all.sh b/make3/build_all.sh index ecf80a829..480acd321 100755 --- a/make3/build_all.sh +++ b/make3/build_all.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh # build all binaries realpath() diff --git a/make3/build_jconsole.sh b/make3/build_jconsole.sh index e648eb2c7..fad64aaa3 100755 --- a/make3/build_jconsole.sh +++ b/make3/build_jconsole.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh realpath() { diff --git a/make3/build_jnative.sh b/make3/build_jnative.sh index db0e66cb5..d97126295 100755 --- a/make3/build_jnative.sh +++ b/make3/build_jnative.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh realpath() { diff --git a/make3/build_libj.sh b/make3/build_libj.sh index c69b72ca9..34e66b8c8 100755 --- a/make3/build_libj.sh +++ b/make3/build_libj.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh realpath() { diff --git a/make3/build_tsdll.sh b/make3/build_tsdll.sh index 6ba16556c..c448e7ae1 100755 --- a/make3/build_tsdll.sh +++ b/make3/build_tsdll.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh realpath() { diff --git a/make3/clean.sh b/make3/clean.sh index 0a6935a33..6f7ba23c2 100755 --- a/make3/clean.sh +++ b/make3/clean.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh # rm all *.o for clean builds - makefile dependencies are not set realpath() diff --git a/make3/cpbin.sh b/make3/cpbin.sh index acf1bda2e..67e407c78 100755 --- a/make3/cpbin.sh +++ b/make3/cpbin.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh # copy binaries in bin/ to jlibrary/bin