From ff68ddb032774c65279372c6d4bfab5d8825977e Mon Sep 17 00:00:00 2001 From: Marius Bakke Date: Sun, 30 Jun 2019 12:37:25 +0200 Subject: [PATCH 01/44] gnu: ungoogled-chromium: Build against ICU 64. * gnu/packages/icu4c.scm (icu4c-64): New public variable. * gnu/packages/chromium.scm (ungoogled-chromium)[inputs]: Change ICU4C to ICU4C-64. --- gnu/packages/chromium.scm | 6 +++++- gnu/packages/icu4c.scm | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/gnu/packages/chromium.scm b/gnu/packages/chromium.scm index e3575569565..79828ef589a 100644 --- a/gnu/packages/chromium.scm +++ b/gnu/packages/chromium.scm @@ -714,7 +714,11 @@ from forcing GEXP-PROMISE." ("glib" ,glib) ("gtk+" ,gtk+) ("harfbuzz" ,harfbuzz) - ("icu4c" ,icu4c) + + ;; Build against ICU 64 to prevent a localization problem in version 75 + ;; and later: . + ("icu4c" ,icu4c-64) + ("jsoncpp" ,jsoncpp) ("lcms" ,lcms) ("libevent" ,libevent) diff --git a/gnu/packages/icu4c.scm b/gnu/packages/icu4c.scm index 6e93d6aed94..512967a305c 100644 --- a/gnu/packages/icu4c.scm +++ b/gnu/packages/icu4c.scm @@ -4,6 +4,7 @@ ;;; Copyright © 2016, 2017 Efraim Flashner ;;; Copyright © 2017 Clément Lassieur ;;; Copyright © 2017 Ricardo Wurmus +;;; Copyright © 2019 Marius Bakke ;;; ;;; This file is part of GNU Guix. ;;; @@ -23,6 +24,7 @@ (define-module (gnu packages icu4c) #:use-module (gnu packages) #:use-module (gnu packages perl) + #:use-module (gnu packages python) #:use-module (guix licenses) #:use-module (guix packages) #:use-module (guix download) @@ -81,6 +83,22 @@ C/C++ part.") (license x11) (home-page "http://site.icu-project.org/"))) +(define-public icu4c-64 + (package + (inherit icu4c) + (version "64.2") + (source (origin + (inherit (package-source icu4c)) + (uri (string-append + "http://download.icu-project.org/files/icu4c/" version "/icu4c-" + (string-map (lambda (x) (if (char=? x #\.) #\_ x)) version) + "-src.tgz")) + (sha256 + (base32 "0v0xsf14xwlj125y9fd8lrhsaych4d8liv8gr746zng6g225szb2")))) + (native-inputs + `(;; For tests. + ("python" ,python))))) + (define-public java-icu4j (package (name "java-icu4j") From 1307aa4427160c11d6ab84c5b2e96ea172fa0f1e Mon Sep 17 00:00:00 2001 From: Marius Bakke Date: Sun, 30 Jun 2019 13:36:58 +0200 Subject: [PATCH 02/44] gnu: mbedtls-apache: Update to 2.16.2. * gnu/packages/tls.scm (mbedtls-apache): Update to 2.16.2. --- gnu/packages/tls.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnu/packages/tls.scm b/gnu/packages/tls.scm index 8e20101e510..ea6dfda9315 100644 --- a/gnu/packages/tls.scm +++ b/gnu/packages/tls.scm @@ -834,7 +834,7 @@ then ported to the GNU / Linux environment.") (define-public mbedtls-apache (package (name "mbedtls-apache") - (version "2.16.1") + (version "2.16.2") (source (origin (method url-fetch) @@ -844,7 +844,7 @@ then ported to the GNU / Linux environment.") version "-apache.tgz")) (sha256 (base32 - "08zz88gcb2jmpfsgy5b6qc3li6l39yw1dbimd18aziyd889nvl7b")))) + "1906hbwlkq32075hca4vjad03dcc36aycvmaz8yvhr3ygg6lz0x6")))) (build-system cmake-build-system) (arguments `(#:configure-flags From 5dd0269244da42066133d19055c9eb82b9e9ae20 Mon Sep 17 00:00:00 2001 From: Marius Bakke Date: Sun, 30 Jun 2019 13:37:22 +0200 Subject: [PATCH 03/44] gnu: miniupnpc: Update to 2.1.20190625. * gnu/packages/upnp.scm (miniupnpc): Update to 2.1.20190625. --- gnu/packages/upnp.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnu/packages/upnp.scm b/gnu/packages/upnp.scm index 3f3d80f5769..a085fe97472 100644 --- a/gnu/packages/upnp.scm +++ b/gnu/packages/upnp.scm @@ -32,14 +32,14 @@ (define-public miniupnpc (package (name "miniupnpc") - (version "2.1.20190408") + (version "2.1.20190625") (source (origin (method url-fetch) (uri (string-append "https://miniupnp.tuxfamily.org/files/" name "-" version ".tar.gz")) (sha256 - (base32 "1v0l2m8j7r6jiy871d6v11ls243xqn6s3856iwd3bmk5c37npi50")))) + (base32 "1yqp0d8x5ldjfma5x2vhpg1aaafdg0470ismccixww3rzpbza8w7")))) (build-system gnu-build-system) (native-inputs `(("python" ,python-2))) From 022288ba53bd44c9311c6cffef5e8224a2e74598 Mon Sep 17 00:00:00 2001 From: Ivan Petkov Date: Sun, 30 Jun 2019 07:18:20 -0700 Subject: [PATCH 04/44] guix: import: crate: fix redundant inputs list nesting * guix/import/crate.scm (maybe-cargo-inputs): Remove one level of lists. * guix/import/crate.scm (maybe-cargo-development-inputs): Same. * tests/crate.scm: (crate->guix-package)[package]<#:arguments>: Remove one level of list nesting. --- guix/import/crate.scm | 4 ++-- tests/crate.scm | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/guix/import/crate.scm b/guix/import/crate.scm index 9a73d9fe16d..29318aac0ed 100644 --- a/guix/import/crate.scm +++ b/guix/import/crate.scm @@ -85,14 +85,14 @@ (() '()) ((package-inputs ...) - `((#:cargo-inputs ,package-inputs))))) + `(#:cargo-inputs ,package-inputs)))) (define (maybe-cargo-development-inputs package-names) (match (package-names->package-inputs package-names) (() '()) ((package-inputs ...) - `((#:cargo-development-inputs ,package-inputs))))) + `(#:cargo-development-inputs ,package-inputs)))) (define (maybe-arguments arguments) (match arguments diff --git a/tests/crate.scm b/tests/crate.scm index a4a328d5073..72c3a133502 100644 --- a/tests/crate.scm +++ b/tests/crate.scm @@ -91,7 +91,7 @@ ('build-system 'cargo-build-system) ('arguments ('quasiquote - (('#:cargo-inputs (("rust-bar" ('unquote rust-bar))))))) + ('#:cargo-inputs (("rust-bar" ('unquote rust-bar)))))) ('home-page "http://example.com") ('synopsis "summary") ('description "summary") From a3d1a3487a34820d2a0b3d5e6206e4df1c77c15c Mon Sep 17 00:00:00 2001 From: Marius Bakke Date: Sun, 30 Jun 2019 20:14:58 +0200 Subject: [PATCH 05/44] gnu: gtksourceviewmm: Build against gtksourceview@3. * gnu/packages/gtk.scm (gtksourceviewmm)[propagated-inputs]: Change GTKSOURCEVIEW to GTKSOURCEVIEW-3. --- gnu/packages/gtk.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnu/packages/gtk.scm b/gnu/packages/gtk.scm index 2776bd31a3c..3f835f51e5f 100644 --- a/gnu/packages/gtk.scm +++ b/gnu/packages/gtk.scm @@ -1182,7 +1182,7 @@ extensive documentation, including API reference and a tutorial.") ;; In 'Requires' of gtksourceviewmm-3.0.pc. `(("glibmm" ,glibmm) ("gtkmm" ,gtkmm) - ("gtksourceview" ,gtksourceview))) + ("gtksourceview" ,gtksourceview-3))) (synopsis "C++ interface to the GTK+ 'GtkTextView' widget") (description "gtksourceviewmm is a portable C++ library that extends the standard GTK+ From 037bef7db4f67b1f7df879f8e41a971308ce8128 Mon Sep 17 00:00:00 2001 From: Danny Milosavljevic Date: Mon, 1 Jul 2019 12:08:08 +0200 Subject: [PATCH 06/44] gnu: behave: Fix build. * gnu/packages/check.scm (behave)[propagated-inputs]: Add python-importlib-metadata. --- gnu/packages/check.scm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gnu/packages/check.scm b/gnu/packages/check.scm index 9bc5036cf66..dae2b0e1d91 100644 --- a/gnu/packages/check.scm +++ b/gnu/packages/check.scm @@ -1924,7 +1924,8 @@ backported from Python 2.7 for Python 2.4+.") ("python-pyhamcrest" ,python-pyhamcrest) ("python-pytest" ,python-pytest))) (propagated-inputs - `(("python-six" ,python-six) + `(("python-importlib-metadata" ,python-importlib-metadata) + ("python-six" ,python-six) ("python-parse" ,python-parse) ("python-parse-type" ,python-parse-type))) (arguments From 78c88baad56ae2843d809a8160631a7bae071315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= Date: Mon, 1 Jul 2019 12:29:02 +0200 Subject: [PATCH 07/44] gnu: guile: Add 2.2.6. * gnu/packages/guile.scm (guile-2.2.5): Rename to... (guile-2.2.6): ... this, and update to 2.2.6. --- gnu/packages/guile.scm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gnu/packages/guile.scm b/gnu/packages/guile.scm index f0a53e7c397..6e11a13c562 100644 --- a/gnu/packages/guile.scm +++ b/gnu/packages/guile.scm @@ -286,17 +286,17 @@ without requiring the source code to be rewritten.") (max-silent-time . 36000))))) ;10 hours (needed on ARM ; when heavily loaded) -(define-public guile-2.2.5 +(define-public guile-2.2.6 (package (inherit guile-2.2) - (version "2.2.5") + (version "2.2.6") (source (origin (inherit (package-source guile-2.2)) (uri (string-append "mirror://gnu/guile/guile-" version ".tar.xz")) (sha256 (base32 - "19w5ws1jvs01dpv756qv2cgs37rsnwq1f4f07mj0wra35pqp6c7w")))))) + "1269ymxm56j1z1lvq1y42rm961f2n7rinm3k6l00p9k52hrpcddk")))))) (define-public guile-next ;; This is the upcoming Guile 3.0, with JIT support. From cea5db1b8454c02452bed8cbe6eb89ce37164fe8 Mon Sep 17 00:00:00 2001 From: Ricardo Wurmus Date: Mon, 1 Jul 2019 23:31:24 +0200 Subject: [PATCH 08/44] etc: Do not recommend use of SKS key servers. * etc/guix-install.sh: Recommend downloading the GPG key from Savannah. --- etc/guix-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/guix-install.sh b/etc/guix-install.sh index 604c6832029..aa95cb4e209 100755 --- a/etc/guix-install.sh +++ b/etc/guix-install.sh @@ -104,7 +104,7 @@ chk_require() gpg --list-keys ${OPENPGP_SIGNING_KEY_ID} >/dev/null 2>&1 || ( _err "${ERR}Missing OpenPGP public key. Fetch it with this command:" - echo " gpg --keyserver pool.sks-keyservers.net --recv-keys ${OPENPGP_SIGNING_KEY_ID}" + echo " wget https://sv.gnu.org/people/viewgpg.php?user_id=15145 -qO - | gpg --import -" exit 1 ) } From 2868804de69ed23f396fd8e68d6dd18c40518948 Mon Sep 17 00:00:00 2001 From: Nicolas Goaziou Date: Tue, 2 Jul 2019 00:13:08 +0200 Subject: [PATCH 09/44] gnu: python-duniterpy: Update to 0.54.3. * gnu/packages/finance.scm (python-duniterpy): Update to 0.54.3. --- gnu/packages/finance.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnu/packages/finance.scm b/gnu/packages/finance.scm index cd5d3b7d9a9..573ec0be151 100644 --- a/gnu/packages/finance.scm +++ b/gnu/packages/finance.scm @@ -935,7 +935,7 @@ Luhn and family of ISO/IEC 7064 check digit algorithms. ") (define-public python-duniterpy (package (name "python-duniterpy") - (version "0.54.1") + (version "0.54.3") (source (origin (method git-fetch) @@ -946,7 +946,7 @@ Luhn and family of ISO/IEC 7064 check digit algorithms. ") (file-name (git-file-name name version)) (sha256 (base32 - "15z5wc3ahvv8axyiqmf7hd4y91ahh1x4bfmgsqxwygyhswl1yjq8")))) + "1k3rpfc9zxj9z50cr4zjfyzdla9ap5mj1v1rlcriqmflgb5cmiba")))) (build-system python-build-system) (arguments ;; Tests fail with "AttributeError: module 'attr' has no attribute 's'". From 7242d5792b9aee67da30cbe21b1b20265c12248d Mon Sep 17 00:00:00 2001 From: Nicolas Goaziou Date: Tue, 2 Jul 2019 00:38:36 +0200 Subject: [PATCH 10/44] gnu: giac: Update to 1.5.0-61. * gnu/packages/algebra.scm (giac): Update to 1.5.0-61. --- gnu/packages/algebra.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnu/packages/algebra.scm b/gnu/packages/algebra.scm index 95189401f7f..7e66e42f0ac 100644 --- a/gnu/packages/algebra.scm +++ b/gnu/packages/algebra.scm @@ -331,7 +331,7 @@ precision.") (define-public giac (package (name "giac") - (version "1.5.0-57") + (version "1.5.0-61") (source (origin (method url-fetch) ;; "~parisse/giac" is not used because the maintainer regularly @@ -343,7 +343,7 @@ precision.") "source/giac_" version ".tar.gz")) (sha256 (base32 - "08c93knsisbk9dkyyrignw0wvqbr1sa5czlvk5l307ahxbbmqncf")))) + "050vzpqq77fhky32sbisc0ysimgp60xjv39q7y45jkaabdkmclwh")))) (build-system gnu-build-system) (arguments `(#:modules ((ice-9 ftw) From 0f0651295e742b399af4436f350662afb4a6f7af Mon Sep 17 00:00:00 2001 From: Nicolas Goaziou Date: Tue, 2 Jul 2019 00:45:05 +0200 Subject: [PATCH 11/44] gnu: silkaj: Update to 0.7.2. * gnu/packages/finance.scm (silkaj): Update to 0.7.2. --- gnu/packages/finance.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnu/packages/finance.scm b/gnu/packages/finance.scm index 573ec0be151..03a8285fc12 100644 --- a/gnu/packages/finance.scm +++ b/gnu/packages/finance.scm @@ -993,7 +993,7 @@ main features are: (define-public silkaj (package (name "silkaj") - (version "0.7.0") + (version "0.7.2") (source (origin (method git-fetch) @@ -1003,7 +1003,7 @@ main features are: (file-name (git-file-name name version)) (sha256 (base32 - "0pnd5v15sgxxm114sbs0z24a4dars5hy1nabc9v9ask7kxzrxs9y")))) + "059k2kil2l8jcm4wp86w1z7y8p26rww7d3l5fzds0qq2dzvkvzgs")))) (build-system python-build-system) (arguments `(#:tests? #f)) ;no test From 01589acc5e114e46e314777bc0d60ebfa620e4f8 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:00 -0400 Subject: [PATCH 12/44] import: pypi: Do not consider requirements.txt files. PyPI packages are mandated to have a setup.py file, which contains a listing of the required dependencies. The setuptools/distutils machinery embed metadata in the archives they produce, which contains this information. There is no need nor gain to collect the requirements from a "requirements.txt" file, as it is not the true record of dependencies for PyPI packages and may contain extraneous requirements or not exist at all. * guix/import/pypi.scm (guess-requirements): Update comment. [guess-requirements-from-source]: Do not attempt to parse the file requirements.txt. Streamline logic. * tests/pypi.scm (test-requires.txt): Rename from test-requirements, to hint at the file being tested. ("pypi->guix-package"): Adapt so that the fake package contains a requires.txt file rather than a requirements.txt file. ("pypi->guix-package, wheels"): Likewise. --- guix/import/pypi.scm | 35 +++++++++++++---------------------- tests/pypi.scm | 23 ++++++++++++----------- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 3a20fc4b9b3..8269aa61d71 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -206,35 +206,26 @@ cannot determine package dependencies")) (call-with-temporary-directory (lambda (dir) (let* ((pypi-name (string-take dirname (string-rindex dirname #\-))) - (req-files (list (string-append dirname "/requirements.txt") - (string-append dirname "/" pypi-name ".egg-info" - "/requires.txt"))) - (exit-codes (map (lambda (file-name) - (parameterize ((current-error-port (%make-void-port "rw+")) - (current-output-port (%make-void-port "rw+"))) - (system* "tar" "xf" tarball "-C" dir file-name))) - req-files))) - ;; Only one of these files needs to exist. - (if (any zero? exit-codes) - (match (find-files dir) - ((file . _) - (read-requirements file)) - (() - (warning (G_ "No requirements file found.\n")))) + (requires.txt (string-append dirname "/" pypi-name + ".egg-info" "/requires.txt")) + (exit-code (parameterize ((current-error-port (%make-void-port "rw+")) + (current-output-port (%make-void-port "rw+"))) + (system* "tar" "xf" tarball "-C" dir requires.txt)))) + (if (zero? exit-code) + (read-requirements (string-append dir "/" requires.txt)) (begin - (warning (G_ "Failed to extract requirements files\n")) + (warning + (G_ "Failed to extract file: ~a from source.~%") + requires.txt) '()))))) '()))) - ;; First, try to compute the requirements using the wheel, since that is the - ;; most reliable option. If a wheel is not provided for this package, try - ;; getting them by reading either the "requirements.txt" file or the - ;; "requires.txt" from the egg-info directory from the source tarball. Note - ;; that "requirements.txt" is not mandatory, so this is likely to fail. + ;; First, try to compute the requirements using the wheel, else, fallback to + ;; reading the "requires.txt" from the egg-info directory from the source + ;; tarball. (or (guess-requirements-from-wheel) (guess-requirements-from-source))) - (define (compute-inputs source-url wheel-url tarball) "Given the SOURCE-URL of an already downloaded TARBALL, return a list of name/variable pairs describing the required inputs of this package. Also diff --git a/tests/pypi.scm b/tests/pypi.scm index 6daa44a6e73..a0271fffad7 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -23,7 +23,7 @@ #:use-module (gcrypt hash) #:use-module (guix tests) #:use-module (guix build-system python) - #:use-module ((guix build utils) #:select (delete-file-recursively which)) + #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p)) #:use-module (srfi srfi-64) #:use-module (ice-9 match)) @@ -55,11 +55,12 @@ (define test-source-hash "") -(define test-requirements -"# A comment +(define test-requires.txt "\ +# A comment # A comment after a space bar -baz > 13.37") +baz > 13.37 +") (define test-metadata "{ @@ -107,10 +108,10 @@ baz > 13.37") (match url ("https://example.com/foo-1.0.0.tar.gz" (begin - (mkdir "foo-1.0.0") - (with-output-to-file "foo-1.0.0/requirements.txt" + (mkdir-p "foo-1.0.0/foo.egg-info/") + (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () - (display test-requirements))) + (display test-requires.txt))) (system* "tar" "czvf" file-name "foo-1.0.0/") (delete-file-recursively "foo-1.0.0") (set! test-source-hash @@ -157,11 +158,11 @@ baz > 13.37") (lambda (url file-name) (match url ("https://example.com/foo-1.0.0.tar.gz" - (begin - (mkdir "foo-1.0.0") - (with-output-to-file "foo-1.0.0/requirements.txt" + (begin + (mkdir-p "foo-1.0.0/foo.egg-info/") + (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () - (display test-requirements))) + (display test-requires.txt))) (system* "tar" "czvf" file-name "foo-1.0.0/") (delete-file-recursively "foo-1.0.0") (set! test-source-hash From a853acebe1ccdba51ae509cd8a838a954c7c8bd2 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Mon, 10 Jun 2019 12:20:08 +0900 Subject: [PATCH 13/44] tests: pypi: Mute the output of tar. The output of tar when creating archives for the purpose of tests is not useful, so we mute it. * tests/pypi.scm ("pypi->guix-package"): Mute the output of tar. ("pypi->guix-package, wheels"): Likewise. --- tests/pypi.scm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/pypi.scm b/tests/pypi.scm index a0271fffad7..6df69073dcf 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -112,7 +112,8 @@ baz > 13.37 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () (display test-requires.txt))) - (system* "tar" "czvf" file-name "foo-1.0.0/") + (parameterize ((current-output-port (%make-void-port "rw+"))) + (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") (set! test-source-hash (call-with-input-file file-name port-sha256)))) @@ -163,7 +164,8 @@ baz > 13.37 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () (display test-requires.txt))) - (system* "tar" "czvf" file-name "foo-1.0.0/") + (parameterize ((current-output-port (%make-void-port "rw+"))) + (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") (set! test-source-hash (call-with-input-file file-name port-sha256)))) From c4797121beea74ae93e3ce17677b9e72b8df920d Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:00 -0400 Subject: [PATCH 14/44] import: pypi: Do not parse optional requirements from source. * guix/import/pypi.scm: Export PARSE-REQUIRES.TXT. (clean-requirement): Move procedure to the top level. (guess-requirements): Move the READ-REQUIREMENTS procedure to the top level, and rename it to PARSE-REQUIRES.TXT. Move the CLEAN-REQUIREMENT procedure to the top level. Move the COMMENT? functions inside the PARSE-REQUIRES.TXT procedure. (parse-requires.txt): Add a SECTION-HEADER? predicate, and use it to prevent parsing optional requirements. * tests/pypi.scm (test-requires-with-sections): New variable. ("parse-requires.txt, with sections"): New test. --- guix/import/pypi.scm | 74 ++++++++++++++++++++++++++------------------ tests/pypi.scm | 14 +++++++++ 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 8269aa61d71..d9db876222e 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -47,7 +47,8 @@ #:use-module (guix upstream) #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) - #:export (guix-package->pypi-name + #:export (parse-requires.txt + guix-package->pypi-name pypi-recursive-import pypi->guix-package %pypi-updater)) @@ -117,6 +118,47 @@ package definition." ((package-inputs ...) `((propagated-inputs (,'quasiquote ,package-inputs)))))) +(define (clean-requirement s) + ;; Given a requirement LINE, as can be found in a setuptools requires.txt + ;; file, remove everything other than the actual name of the required + ;; package, and return it. + (cond + ((string-index s (char-set #\space #\> #\= #\<)) => (cut string-take s <>)) + (else s))) + +(define (parse-requires.txt requires.txt) + "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of +requirement names." + ;; This is a very incomplete parser, whose job is to select the non-optional + ;; dependencies and strip them out of any version information. + ;; Alternatively, we could implement a PEG parser with the (ice-9 peg) + ;; library and the requirements grammar defined by PEP-0508 + ;; (https://www.python.org/dev/peps/pep-0508/). + + (define (comment? line) + ;; Return #t if the given LINE is a comment, #f otherwise. + (string-prefix? "#" (string-trim line))) + + (define (section-header? line) + ;; Return #t if the given LINE is a section header, #f otherwise. + (string-prefix? "[" (string-trim line))) + + (call-with-input-file requires.txt + (lambda (port) + (let loop ((result '())) + (let ((line (read-line port))) + ;; Stop when a section is encountered, as sections contain optional + ;; (extra) requirements. Non-optional requirements must appear + ;; before any section is defined. + (if (or (eof-object? line) (section-header? line)) + (reverse result) + (cond + ((or (string-null? line) (comment? line)) + (loop result)) + (else + (loop (cons (clean-requirement line) + result)))))))))) + (define (guess-requirements source-url wheel-url tarball) "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list of the required packages specified in the requirements.txt file. TARBALL will @@ -139,34 +181,6 @@ be extracted in a temporary directory." cannot determine package dependencies")) #f))))) - (define (clean-requirement s) - ;; Given a requirement LINE, as can be found in a Python requirements.txt - ;; file, remove everything other than the actual name of the required - ;; package, and return it. - (string-take s - (or (string-index s (lambda (chr) (member chr '(#\space #\> #\= #\<)))) - (string-length s)))) - - (define (comment? line) - ;; Return #t if the given LINE is a comment, #f otherwise. - (eq? (string-ref (string-trim line) 0) #\#)) - - (define (read-requirements requirements-file) - ;; Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list - ;; of name/variable pairs describing the requirements. - (call-with-input-file requirements-file - (lambda (port) - (let loop ((result '())) - (let ((line (read-line port))) - (if (eof-object? line) - result - (cond - ((or (string-null? line) (comment? line)) - (loop result)) - (else - (loop (cons (clean-requirement line) - result)))))))))) - (define (read-wheel-metadata wheel-archive) ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's ;; requirements. @@ -212,7 +226,7 @@ cannot determine package dependencies")) (current-output-port (%make-void-port "rw+"))) (system* "tar" "xf" tarball "-C" dir requires.txt)))) (if (zero? exit-code) - (read-requirements (string-append dir "/" requires.txt)) + (parse-requires.txt (string-append dir "/" requires.txt)) (begin (warning (G_ "Failed to extract file: ~a from source.~%") diff --git a/tests/pypi.scm b/tests/pypi.scm index 6df69073dcf..03455ba6bed 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -62,6 +62,14 @@ bar baz > 13.37 ") +(define test-requires-with-sections "\ +foo ~= 3 +bar != 2 + +[test] +pytest (>=2.5.0) +") + (define test-metadata "{ \"run_requires\": [ @@ -101,6 +109,12 @@ baz > 13.37 (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz" (pypi-uri "cram" "0.7")))))))) +(test-equal "parse-requires.txt, with sections" + '("foo" "bar") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-requires.txt test-requires-with-sections))) + (test-assert "pypi->guix-package" ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch From 803fb336d62ea65990e263ce58d8552e04c9c038 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:01 -0400 Subject: [PATCH 15/44] import: pypi: Improve parsing of requirement specifications. The previous solution was fragile and could leave unwanted characters in a requirement name, such as '[' or ']'. Partially fixes . * guix/import/pypi.scm (use-modules): Export SPECIFICATION->REQUIREMENT-NAME (%requirement-name-regexp): New variable. (clean-requirement): Rename to... (specification->requirement-name): this, which now uses %requirement-name-regexp to select the requirement name from the requirement specification. (parse-requires.txt): Adapt. --- guix/import/pypi.scm | 54 ++++++++++++++++++++++++++++++++------------ tests/pypi.scm | 12 ++++++++++ 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index d9db876222e..6a881bda12c 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -48,6 +48,7 @@ #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) #:export (parse-requires.txt + specification->requirement-name guix-package->pypi-name pypi-recursive-import pypi->guix-package @@ -118,22 +119,47 @@ package definition." ((package-inputs ...) `((propagated-inputs (,'quasiquote ,package-inputs)))))) -(define (clean-requirement s) - ;; Given a requirement LINE, as can be found in a setuptools requires.txt - ;; file, remove everything other than the actual name of the required - ;; package, and return it. - (cond - ((string-index s (char-set #\space #\> #\= #\<)) => (cut string-take s <>)) - (else s))) +(define %requirement-name-regexp + ;; Regexp to match the requirement name in a requirement specification. + + ;; Some grammar, taken from PEP-0508 (see: + ;; https://www.python.org/dev/peps/pep-0508/). + + ;; Using this grammar makes the PEP-0508 regexp easier to understand for + ;; humans. The use of a regexp is preferred to more primitive string + ;; manipulations because we can more directly match what upstream uses + ;; (again, per PEP-0508). The regexp approach is also easier to extend, + ;; should we want to implement more completely the grammar of PEP-0508. + + ;; The unified rule can be expressed as: + ;; specification = wsp* ( url_req | name_req ) wsp* + + ;; where url_req is: + ;; url_req = name wsp* extras? wsp* urlspec wsp+ quoted_marker? + + ;; and where name_req is: + ;; name_req = name wsp* extras? wsp* versionspec? wsp* quoted_marker? + + ;; Thus, we need only matching NAME, which is expressed as: + ;; identifer_end = letterOrDigit | (('-' | '_' | '.' )* letterOrDigit) + ;; identifier = letterOrDigit identifier_end* + ;; name = identifier + (let* ((letter-or-digit "[A-Za-z0-9]") + (identifier-end (string-append "(" letter-or-digit "|" + "[-_.]*" letter-or-digit ")")) + (identifier (string-append "^" letter-or-digit identifier-end "*")) + (name identifier)) + (make-regexp name))) + +(define (specification->requirement-name spec) + "Given a specification SPEC, return the requirement name." + (match:substring + (or (regexp-exec %requirement-name-regexp spec) + (error (G_ "Could not extract requirement name in spec:") spec)))) (define (parse-requires.txt requires.txt) "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of requirement names." - ;; This is a very incomplete parser, whose job is to select the non-optional - ;; dependencies and strip them out of any version information. - ;; Alternatively, we could implement a PEG parser with the (ice-9 peg) - ;; library and the requirements grammar defined by PEP-0508 - ;; (https://www.python.org/dev/peps/pep-0508/). (define (comment? line) ;; Return #t if the given LINE is a comment, #f otherwise. @@ -156,7 +182,7 @@ requirement names." ((or (string-null? line) (comment? line)) (loop result)) (else - (loop (cons (clean-requirement line) + (loop (cons (specification->requirement-name line) result)))))))))) (define (guess-requirements source-url wheel-url tarball) @@ -198,7 +224,7 @@ cannot determine package dependencies")) (hash-ref (list-ref run_requires 0) "requires") '()))) - (map clean-requirement requirements))))) + (map specification->requirement-name requirements))))) (lambda () (delete-file json-file) (rmdir dirname)))))) diff --git a/tests/pypi.scm b/tests/pypi.scm index 03455ba6bed..c40be6c21d7 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -55,6 +55,14 @@ (define test-source-hash "") +(define test-specifications + '("Fizzy [foo, bar]" + "PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1" + "SomethingWithMarker[foo]>1.0;python_version<\"2.7\"" + "requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < \"2.7\"" + "pip @ https://github.com/pypa/pip/archive/1.3.1.zip#\ +sha1=da9234ee9982d4bbb3c72346a6de940a148ea686")) + (define test-requires.txt "\ # A comment # A comment after a space @@ -109,6 +117,10 @@ pytest (>=2.5.0) (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz" (pypi-uri "cram" "0.7")))))))) +(test-equal "specification->requirement-name" + '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip") + (map specification->requirement-name test-specifications)) + (test-equal "parse-requires.txt, with sections" '("foo" "bar") (mock ((ice-9 ports) call-with-input-file From cc9a77cd396f8932bec1b8c7ae9d829ea76f37ec Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:01 -0400 Subject: [PATCH 16/44] import: pypi: Deduplicate requirements. * guix/import/pypi.scm (parse-requires.txt): Remove potential duplicates. --- guix/import/pypi.scm | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 6a881bda12c..ad59a8b731c 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -177,7 +177,11 @@ requirement names." ;; (extra) requirements. Non-optional requirements must appear ;; before any section is defined. (if (or (eof-object? line) (section-header? line)) - (reverse result) + ;; Duplicates can occur, since the same requirement can be + ;; listed multiple times with different conditional markers, e.g. + ;; pytest >= 3 ; python_version >= "3.3" + ;; pytest < 3 ; python_version < "3.3" + (reverse (delete-duplicates result)) (cond ((or (string-null? line) (comment? line)) (loop result)) From c799ad7276dcc771912f4fc7edca400ea48b8ec2 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:02 -0400 Subject: [PATCH 17/44] import: pypi: Support more types of archives. This change enables the PyPI importer to look for requirements in a source archive of a different type than "tar.gz" or "tar.bz2". Also, scan the source archive to find a requires.txt file. * guix/import/pypi.scm: (guess-requirements)[tarball-directory]: Remove procedure. [guess-requirements-from-source]: Use COMRESSED-FILE? to determine if an archive type is supported, and some file extension logic that chooses either "tar" or "unzip" as the extractor. Search for the requires.txt file in the archive instead of using a static, expected location. (guess-requirements): Rename the TARBALL argument to ARCHIVE, to denote the archive format is no longer bound specifically to the Tar format. (compute-inputs): Likewise. * tests/pypi.scm ("pypi->guix-package, no wheel"): Mock the requires.txt at a non-standard location. ("pypi->guix-package, no usable requirement file."): New test. --- guix/import/pypi.scm | 77 +++++++++++++++++++------------------------- tests/pypi.scm | 52 ++++++++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 46 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index ad59a8b731c..a6106ab4ecf 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -39,7 +39,8 @@ #:use-module ((guix build utils) #:select ((package-name->name+version . hyphen-package-name->name+version) - find-files)) + find-files + invoke)) #:use-module (guix import utils) #:use-module ((guix download) #:prefix download:) #:use-module (guix import json) @@ -189,28 +190,11 @@ requirement names." (loop (cons (specification->requirement-name line) result)))))))))) -(define (guess-requirements source-url wheel-url tarball) - "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list -of the required packages specified in the requirements.txt file. TARBALL will +(define (guess-requirements source-url wheel-url archive) + "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list +of the required packages specified in the requirements.txt file. ARCHIVE will be extracted in a temporary directory." - (define (tarball-directory url) - ;; Given the URL of the package's tarball, return the name of the directory - ;; that will be created upon decompressing it. If the filetype is not - ;; supported, return #f. - ;; TODO: Support more archive formats. - (let ((basename (substring url (+ 1 (string-rindex url #\/))))) - (cond - ((string-suffix? ".tar.gz" basename) - (string-drop-right basename 7)) - ((string-suffix? ".tar.bz2" basename) - (string-drop-right basename 8)) - (else - (begin - (warning (G_ "Unsupported archive format: \ -cannot determine package dependencies")) - #f))))) - (define (read-wheel-metadata wheel-archive) ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's ;; requirements. @@ -239,29 +223,34 @@ cannot determine package dependencies")) (call-with-temporary-output-file (lambda (temp port) (if wheel-url - (and (url-fetch wheel-url temp) - (read-wheel-metadata temp)) - #f)))) + (and (url-fetch wheel-url temp) + (read-wheel-metadata temp)) + #f)))) (define (guess-requirements-from-source) ;; Return the package's requirements by guessing them from the source. - (let ((dirname (tarball-directory source-url))) - (if (string? dirname) - (call-with-temporary-directory - (lambda (dir) - (let* ((pypi-name (string-take dirname (string-rindex dirname #\-))) - (requires.txt (string-append dirname "/" pypi-name - ".egg-info" "/requires.txt")) - (exit-code (parameterize ((current-error-port (%make-void-port "rw+")) - (current-output-port (%make-void-port "rw+"))) - (system* "tar" "xf" tarball "-C" dir requires.txt)))) - (if (zero? exit-code) - (parse-requires.txt (string-append dir "/" requires.txt)) - (begin - (warning - (G_ "Failed to extract file: ~a from source.~%") - requires.txt) - '()))))) + (if (compressed-file? source-url) + (call-with-temporary-directory + (lambda (dir) + (parameterize ((current-error-port (%make-void-port "rw+")) + (current-output-port (%make-void-port "rw+"))) + (if (string=? "zip" (file-extension source-url)) + (invoke "unzip" archive "-d" dir) + (invoke "tar" "xf" archive "-C" dir))) + (let ((requires.txt-files + (find-files dir (lambda (abs-file-name _) + (string-match "\\.egg-info/requires.txt$" + abs-file-name))))) + (match requires.txt-files + (() + (warning (G_ "Cannot guess requirements from source archive:\ + no requires.txt file found.~%")) + '()) + (else (parse-requires.txt (first requires.txt-files))))))) + (begin + (warning (G_ "Unsupported archive format; \ +cannot determine package dependencies from source archive: ~a~%") + (basename source-url)) '()))) ;; First, try to compute the requirements using the wheel, else, fallback to @@ -270,13 +259,13 @@ cannot determine package dependencies")) (or (guess-requirements-from-wheel) (guess-requirements-from-source))) -(define (compute-inputs source-url wheel-url tarball) - "Given the SOURCE-URL of an already downloaded TARBALL, return a list of +(define (compute-inputs source-url wheel-url archive) + "Given the SOURCE-URL of an already downloaded ARCHIVE, return a list of name/variable pairs describing the required inputs of this package. Also return the unaltered list of upstream dependency names." (let ((dependencies (remove (cut string=? "argparse" <>) - (guess-requirements source-url wheel-url tarball)))) + (guess-requirements source-url wheel-url archive)))) (values (sort (map (lambda (input) (let ((guix-name (python->package-name input))) diff --git a/tests/pypi.scm b/tests/pypi.scm index c40be6c21d7..b45d2c9d2f9 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -20,6 +20,7 @@ (define-module (test-pypi) #:use-module (guix import pypi) #:use-module (guix base32) + #:use-module (guix memoization) #:use-module (gcrypt hash) #:use-module (guix tests) #:use-module (guix build-system python) @@ -134,8 +135,9 @@ pytest (>=2.5.0) (match url ("https://example.com/foo-1.0.0.tar.gz" (begin - (mkdir-p "foo-1.0.0/foo.egg-info/") - (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" + ;; Unusual requires.txt location should still be found. + (mkdir-p "foo-1.0.0/src/bizarre.egg-info") + (with-output-to-file "foo-1.0.0/src/bizarre.egg-info/requires.txt" (lambda () (display test-requires.txt))) (parameterize ((current-output-port (%make-void-port "rw+"))) @@ -241,4 +243,50 @@ pytest (>=2.5.0) (x (pk 'fail x #f)))))) +(test-assert "pypi->guix-package, no usable requirement file." + ;; Replace network resources with sample data. + (mock ((guix import utils) url-fetch + (lambda (url file-name) + (match url + ("https://example.com/foo-1.0.0.tar.gz" + (mkdir-p "foo-1.0.0/foo.egg-info/") + (parameterize ((current-output-port (%make-void-port "rw+"))) + (system* "tar" "czvf" file-name "foo-1.0.0/")) + (delete-file-recursively "foo-1.0.0") + (set! test-source-hash + (call-with-input-file file-name port-sha256))) + ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) + (_ (error "Unexpected URL: " url))))) + (mock ((guix http-client) http-fetch + (lambda (url . rest) + (match url + ("https://pypi.org/pypi/foo/json" + (values (open-input-string test-json) + (string-length test-json))) + ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) + (_ (error "Unexpected URL: " url))))) + ;; Not clearing the memoization cache here would mean returning the value + ;; computed in the previous test. + (invalidate-memoization! pypi->guix-package) + (match (pypi->guix-package "foo") + (('package + ('name "python-foo") + ('version "1.0.0") + ('source ('origin + ('method 'url-fetch) + ('uri ('pypi-uri "foo" 'version)) + ('sha256 + ('base32 + (? string? hash))))) + ('build-system 'python-build-system) + ('home-page "http://example.com") + ('synopsis "summary") + ('description "summary") + ('license 'license:lgpl2.0)) + (string=? (bytevector->nix-base32-string + test-source-hash) + hash)) + (x + (pk 'fail x #f)))))) + (test-end "pypi") From f0190a5dcdc1882e5a058dc7f12bc7ccb166c63d Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:03 -0400 Subject: [PATCH 18/44] import: pypi: Parse wheel METADATA instead of metadata.json. With newer Wheel releases, there is no more metadata.json file; the METADATA file should be used instead (see: https://github.com/pypa/wheel/issues/195). This change updates our PyPI importer so that it uses the latter. * guix/import/pypi.scm (define-module): Remove unnecessary modules and export the PARSE-WHEEL-METADATA procedure. (parse-wheel-metadata): Add procedure. (guess-requirements): Use it. * tests/pypi.scm (test-metadata): Test it. --- guix/import/pypi.scm | 90 +++++++++++++++++++++++++++----------------- tests/pypi.scm | 60 ++++++++++++++++++++++------- 2 files changed, 101 insertions(+), 49 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index a6106ab4ecf..7cf1e921016 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -21,9 +21,7 @@ ;;; along with GNU Guix. If not, see . (define-module (guix import pypi) - #:use-module (ice-9 binary-ports) #:use-module (ice-9 match) - #:use-module (ice-9 pretty-print) #:use-module (ice-9 regex) #:use-module (ice-9 receive) #:use-module ((ice-9 rdelim) #:select (read-line)) @@ -31,9 +29,6 @@ #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) - #:use-module (rnrs bytevectors) - #:use-module (json) - #:use-module (web uri) #:use-module (guix ui) #:use-module (guix utils) #:use-module ((guix build utils) @@ -49,6 +44,7 @@ #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) #:export (parse-requires.txt + parse-wheel-metadata specification->requirement-name guix-package->pypi-name pypi-recursive-import @@ -177,18 +173,49 @@ requirement names." ;; Stop when a section is encountered, as sections contain optional ;; (extra) requirements. Non-optional requirements must appear ;; before any section is defined. - (if (or (eof-object? line) (section-header? line)) - ;; Duplicates can occur, since the same requirement can be - ;; listed multiple times with different conditional markers, e.g. - ;; pytest >= 3 ; python_version >= "3.3" - ;; pytest < 3 ; python_version < "3.3" - (reverse (delete-duplicates result)) - (cond - ((or (string-null? line) (comment? line)) - (loop result)) - (else - (loop (cons (specification->requirement-name line) - result)))))))))) + (cond + ((or (eof-object? line) (section-header? line)) + ;; Duplicates can occur, since the same requirement can be + ;; listed multiple times with different conditional markers, e.g. + ;; pytest >= 3 ; python_version >= "3.3" + ;; pytest < 3 ; python_version < "3.3" + (reverse (delete-duplicates result))) + ((or (string-null? line) (comment? line)) + (loop result)) + (else + (loop (cons (specification->requirement-name line) + result))))))))) + +(define (parse-wheel-metadata metadata) + "Given METADATA, a Wheel metadata file, return a list of requirement names." + ;; METADATA is a RFC-2822-like, header based file. + + (define (requires-dist-header? line) + ;; Return #t if the given LINE is a Requires-Dist header. + (string-match "^Requires-Dist: " line)) + + (define (requires-dist-value line) + (string-drop line (string-length "Requires-Dist: "))) + + (define (extra? line) + ;; Return #t if the given LINE is an "extra" requirement. + (string-match "extra == '(.*)'" line)) + + (call-with-input-file metadata + (lambda (port) + (let loop ((requirements '())) + (let ((line (read-line port))) + ;; Stop at the first 'Provides-Extra' section: the non-optional + ;; requirements appear before the optional ones. + (cond + ((eof-object? line) + (reverse (delete-duplicates requirements))) + ((and (requires-dist-header? line) (not (extra? line))) + (loop (cons (specification->requirement-name + (requires-dist-value line)) + requirements))) + (else + (loop requirements)))))))) (define (guess-requirements source-url wheel-url archive) "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list @@ -197,25 +224,18 @@ be extracted in a temporary directory." (define (read-wheel-metadata wheel-archive) ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's - ;; requirements. + ;; requirements, or #f if the metadata file contained therein couldn't be + ;; extracted. (let* ((dirname (wheel-url->extracted-directory wheel-url)) - (json-file (string-append dirname "/metadata.json"))) - (and (zero? (system* "unzip" "-q" wheel-archive json-file)) - (dynamic-wind - (const #t) - (lambda () - (call-with-input-file json-file - (lambda (port) - (let* ((metadata (json->scm port)) - (run_requires (hash-ref metadata "run_requires")) - (requirements (if run_requires - (hash-ref (list-ref run_requires 0) - "requires") - '()))) - (map specification->requirement-name requirements))))) - (lambda () - (delete-file json-file) - (rmdir dirname)))))) + (metadata (string-append dirname "/METADATA"))) + (call-with-temporary-directory + (lambda (dir) + (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata)) + (parse-wheel-metadata (string-append dir "/" metadata)) + (begin + (warning + (G_ "Failed to extract file: ~a from wheel.~%") metadata) + #f)))))) (define (guess-requirements-from-wheel) ;; Return the package's requirements using the wheel, or #f if an error diff --git a/tests/pypi.scm b/tests/pypi.scm index b45d2c9d2f9..8b42c2f0710 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -22,6 +22,7 @@ #:use-module (guix base32) #:use-module (guix memoization) #:use-module (gcrypt hash) + #:use-module (guix memoization) #:use-module (guix tests) #:use-module (guix build-system python) #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p)) @@ -79,17 +80,33 @@ bar != 2 pytest (>=2.5.0) ") -(define test-metadata - "{ - \"run_requires\": [ - { - \"requires\": [ - \"bar\", - \"baz (>13.37)\" - ] - } - ] -}") +(define test-metadata "\ +Classifier: Programming Language :: Python :: 3.7 +Requires-Dist: baz ~= 3 +Requires-Dist: bar != 2 +Provides-Extra: test +pytest (>=2.5.0) +") + +(define test-metadata-with-extras " +Classifier: Programming Language :: Python :: 3.7 +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Requires-Dist: wrapt (<2,>=1) +Requires-Dist: bar + +Provides-Extra: dev +Requires-Dist: tox ; extra == 'dev' +Requires-Dist: bumpversion (<1) ; extra == 'dev' +") + +;;; Provides-Extra can appear before Requires-Dist. +(define test-metadata-with-extras-jedi "\ +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Provides-Extra: testing +Requires-Dist: parso (>=0.3.0) +Provides-Extra: testing +Requires-Dist: pytest (>=3.1.0); extra == 'testing' +") (test-begin "pypi") @@ -128,6 +145,18 @@ pytest (>=2.5.0) call-with-input-string) (parse-requires.txt test-requires-with-sections))) +(test-equal "parse-wheel-metadata, with extras" + '("wrapt" "bar") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-wheel-metadata test-metadata-with-extras))) + +(test-equal "parse-wheel-metadata, with extras - Jedi" + '("parso") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-wheel-metadata test-metadata-with-extras-jedi))) + (test-assert "pypi->guix-package" ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch @@ -191,7 +220,7 @@ pytest (>=2.5.0) (mkdir-p "foo-1.0.0/foo.egg-info/") (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () - (display test-requires.txt))) + (display "wrong data to make sure we're testing wheels "))) (parameterize ((current-output-port (%make-void-port "rw+"))) (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") @@ -200,13 +229,13 @@ pytest (>=2.5.0) ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" (begin (mkdir "foo-1.0.0.dist-info") - (with-output-to-file "foo-1.0.0.dist-info/metadata.json" + (with-output-to-file "foo-1.0.0.dist-info/METADATA" (lambda () (display test-metadata))) (let ((zip-file (string-append file-name ".zip"))) ;; zip always adds a "zip" extension to the file it creates, ;; so we need to rename it. - (system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json") + (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA") (rename-file zip-file file-name)) (delete-file-recursively "foo-1.0.0.dist-info"))) (_ (error "Unexpected URL: " url))))) @@ -218,6 +247,9 @@ pytest (>=2.5.0) (string-length test-json))) ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) (_ (error "Unexpected URL: " url))))) + ;; Not clearing the memoization cache here would mean returning the value + ;; computed in the previous test. + (invalidate-memoization! pypi->guix-package) (match (pypi->guix-package "foo") (('package ('name "python-foo") From 73e83730607e34d5f89a7166fd1609342d954633 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Wed, 12 Jun 2019 11:34:23 +0900 Subject: [PATCH 19/44] import: pypi: Fix typo in docstring. * guix/import/pypi.scm (guess-requirements): Fix typo. --- guix/import/pypi.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 7cf1e921016..d861dd960d6 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -218,7 +218,7 @@ requirement names." (loop requirements)))))))) (define (guess-requirements source-url wheel-url archive) - "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list + "Given SOURCE-URL, WHEEL-URL and an ARCHIVE of the package, return a list of the required packages specified in the requirements.txt file. ARCHIVE will be extracted in a temporary directory." From f801c6215d8d5b2b21926d8cc41c0beb1f734108 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Wed, 12 Jun 2019 11:36:39 +0900 Subject: [PATCH 20/44] import: pypi: Completely mute the output of the "unzip" command. * guix/import/pypi.scm (guess-requirements): Completely mute the output of the "unzip" command. --- guix/import/pypi.scm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index d861dd960d6..23a1e69061d 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -230,7 +230,10 @@ be extracted in a temporary directory." (metadata (string-append dirname "/METADATA"))) (call-with-temporary-directory (lambda (dir) - (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata)) + (if (zero? + (parameterize ((current-error-port (%make-void-port "rw+")) + (current-output-port (%make-void-port "rw+"))) + (system* "unzip" wheel-archive "-d" dir metadata))) (parse-wheel-metadata (string-append dir "/" metadata)) (begin (warning From d514276b93ccf168438bbd2892fbc64d5661106c Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 23:12:26 -0400 Subject: [PATCH 21/44] import: pypi: Include optional test inputs as native-inputs. * guix/import/pypi.scm (maybe-inputs): Add INPUT-TYPE argument, and use it. (test-section?): New predicate. (parse-requires.txt): Collect the optional test inputs, and return them as the second element of the returned list. (parse-wheel-metadata): Likewise. (guess-requirements): Adapt. (make-pypi-sexp): Likewise, and include the test inputs requirements as native inputs in the returned package expression. * tests/pypi.scm (test-requires.txt): Include a test section in the test-requires.txt data. (test-requires.txt-beaker): New variable. ("parse-requires.txt"): Adapt. ("parse-requires.txt - Beaker"): New test. ("parse-wheel-metadata, with extras"): Adapt. ("parse-wheel-metadata, with extras - Jedi"): Adapt. ("pypi->guix-package, no wheel"): Re-indent, and add the expected native-inputs. ("pypi->guix-package, wheels"): Likewise. --- guix/import/pypi.scm | 177 ++++++++++++++++++++++++++++--------------- tests/pypi.scm | 79 ++++++++++++------- 2 files changed, 166 insertions(+), 90 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 23a1e69061d..537431dd691 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -4,6 +4,7 @@ ;;; Copyright © 2015, 2016, 2017 Ludovic Courtès ;;; Copyright © 2017 Mathieu Othacehe ;;; Copyright © 2018 Ricardo Wurmus +;;; Copyright © 2019 Maxim Cournoyer ;;; ;;; This file is part of GNU Guix. ;;; @@ -26,6 +27,7 @@ #:use-module (ice-9 receive) #:use-module ((ice-9 rdelim) #:select (read-line)) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) @@ -107,14 +109,15 @@ package on PyPI." ((name version _ ...) (string-append name "-" version ".dist-info")))) -(define (maybe-inputs package-inputs) +(define (maybe-inputs package-inputs input-type) "Given a list of PACKAGE-INPUTS, tries to generate the 'inputs' field of a -package definition." +package definition. INPUT-TYPE, a symbol, is used to populate the name of +the input field." (match package-inputs (() '()) ((package-inputs ...) - `((propagated-inputs (,'quasiquote ,package-inputs)))))) + `((,input-type (,'quasiquote ,package-inputs)))))) (define %requirement-name-regexp ;; Regexp to match the requirement name in a requirement specification. @@ -154,9 +157,19 @@ package definition." (or (regexp-exec %requirement-name-regexp spec) (error (G_ "Could not extract requirement name in spec:") spec)))) +(define (test-section? name) + "Return #t if the section name contains 'test' or 'dev'." + (any (cut string-contains-ci name <>) + '("test" "dev"))) + (define (parse-requires.txt requires.txt) - "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of -requirement names." + "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of lists +of requirements. + +The first list contains the required dependencies while the second the +optional test dependencies. Note that currently, optional, non-test +dependencies are omitted since these can be difficult or expensive to +satisfy." (define (comment? line) ;; Return #t if the given LINE is a comment, #f otherwise. @@ -168,26 +181,49 @@ requirement names." (call-with-input-file requires.txt (lambda (port) - (let loop ((result '())) + (let loop ((required-deps '()) + (test-deps '()) + (inside-test-section? #f) + (optional? #f)) (let ((line (read-line port))) - ;; Stop when a section is encountered, as sections contain optional - ;; (extra) requirements. Non-optional requirements must appear - ;; before any section is defined. (cond - ((or (eof-object? line) (section-header? line)) + ((eof-object? line) ;; Duplicates can occur, since the same requirement can be ;; listed multiple times with different conditional markers, e.g. ;; pytest >= 3 ; python_version >= "3.3" ;; pytest < 3 ; python_version < "3.3" - (reverse (delete-duplicates result))) + (map (compose reverse delete-duplicates) + (list required-deps test-deps))) ((or (string-null? line) (comment? line)) - (loop result)) - (else + (loop required-deps test-deps inside-test-section? optional?)) + ((section-header? line) + ;; Encountering a section means that all the requirements + ;; listed below are optional. Since we want to pick only the + ;; test dependencies from the optional dependencies, we must + ;; track those separately. + (loop required-deps test-deps (test-section? line) #t)) + (inside-test-section? + (loop required-deps + (cons (specification->requirement-name line) + test-deps) + inside-test-section? optional?)) + ((not optional?) (loop (cons (specification->requirement-name line) - result))))))))) + required-deps) + test-deps inside-test-section? optional?)) + (optional? + ;; Skip optional items. + (loop required-deps test-deps inside-test-section? optional?)) + (else + (warning (G_ "parse-requires.txt reached an unexpected \ +condition on line ~a~%") line)))))))) (define (parse-wheel-metadata metadata) - "Given METADATA, a Wheel metadata file, return a list of requirement names." + "Given METADATA, a Wheel metadata file, return a list of lists of +requirements. + +Refer to the documentation of PARSE-REQUIRES.TXT for a description of the +returned value." ;; METADATA is a RFC-2822-like, header based file. (define (requires-dist-header? line) @@ -201,21 +237,29 @@ requirement names." ;; Return #t if the given LINE is an "extra" requirement. (string-match "extra == '(.*)'" line)) + (define (test-requirement? line) + (and=> (match:substring (extra? line) 1) test-section?)) + (call-with-input-file metadata (lambda (port) - (let loop ((requirements '())) + (let loop ((required-deps '()) + (test-deps '())) (let ((line (read-line port))) - ;; Stop at the first 'Provides-Extra' section: the non-optional - ;; requirements appear before the optional ones. (cond ((eof-object? line) - (reverse (delete-duplicates requirements))) + (map (compose reverse delete-duplicates) + (list required-deps test-deps))) ((and (requires-dist-header? line) (not (extra? line))) (loop (cons (specification->requirement-name (requires-dist-value line)) - requirements))) + required-deps) + test-deps)) + ((and (requires-dist-header? line) (test-requirement? line)) + (loop required-deps + (cons (specification->requirement-name (requires-dist-value line)) + test-deps))) (else - (loop requirements)))))))) + (loop required-deps test-deps)))))))) ;skip line (define (guess-requirements source-url wheel-url archive) "Given SOURCE-URL, WHEEL-URL and an ARCHIVE of the package, return a list @@ -268,37 +312,46 @@ be extracted in a temporary directory." (() (warning (G_ "Cannot guess requirements from source archive:\ no requires.txt file found.~%")) - '()) + (list '() '())) (else (parse-requires.txt (first requires.txt-files))))))) (begin (warning (G_ "Unsupported archive format; \ cannot determine package dependencies from source archive: ~a~%") (basename source-url)) - '()))) + (list '() '())))) ;; First, try to compute the requirements using the wheel, else, fallback to ;; reading the "requires.txt" from the egg-info directory from the source - ;; tarball. + ;; archive. (or (guess-requirements-from-wheel) (guess-requirements-from-source))) (define (compute-inputs source-url wheel-url archive) - "Given the SOURCE-URL of an already downloaded ARCHIVE, return a list of -name/variable pairs describing the required inputs of this package. Also + "Given the SOURCE-URL and WHEEL-URL of an already downloaded ARCHIVE, return +a pair of lists, each consisting of a list of name/variable pairs, for the +propagated inputs and the native inputs, respectively. Also return the unaltered list of upstream dependency names." - (let ((dependencies - (remove (cut string=? "argparse" <>) - (guess-requirements source-url wheel-url archive)))) - (values (sort - (map (lambda (input) - (let ((guix-name (python->package-name input))) - (list guix-name (list 'unquote (string->symbol guix-name))))) - dependencies) - (lambda args - (match args - (((a _ ...) (b _ ...)) - (string-ci) deps)) + + (define (requirement->package-name/sort deps) + (sort + (map (lambda (input) + (let ((guix-name (python->package-name input))) + (list guix-name (list 'unquote (string->symbol guix-name))))) + deps) + (lambda args + (match args + (((a _ ...) (b _ ...)) + (string-cipackage-name/sort strip-argparse)) + + (let ((dependencies (guess-requirements source-url wheel-url archive))) + (values (map process-requirements dependencies) + (concatenate dependencies)))) (define (make-pypi-sexp name version source-url wheel-url home-page synopsis description license) @@ -307,29 +360,31 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." (call-with-temporary-output-file (lambda (temp port) (and (url-fetch source-url temp) - (receive (input-package-names upstream-dependency-names) + (receive (guix-dependencies upstream-dependencies) (compute-inputs source-url wheel-url temp) - (values - `(package - (name ,(python->package-name name)) - (version ,version) - (source (origin - (method url-fetch) - - ;; Sometimes 'pypi-uri' doesn't quite work due to mixed - ;; cases in NAME, for instance, as is the case with - ;; "uwsgi". In that case, fall back to a full URL. - (uri (pypi-uri ,(string-downcase name) version)) - (sha256 - (base32 - ,(guix-hash-url temp))))) - (build-system python-build-system) - ,@(maybe-inputs input-package-names) - (home-page ,home-page) - (synopsis ,synopsis) - (description ,description) - (license ,(license->symbol license))) - upstream-dependency-names)))))) + (match guix-dependencies + ((required-inputs test-inputs) + (values + `(package + (name ,(python->package-name name)) + (version ,version) + (source (origin + (method url-fetch) + ;; Sometimes 'pypi-uri' doesn't quite work due to mixed + ;; cases in NAME, for instance, as is the case with + ;; "uwsgi". In that case, fall back to a full URL. + (uri (pypi-uri ,(string-downcase name) version)) + (sha256 + (base32 + ,(guix-hash-url temp))))) + (build-system python-build-system) + ,@(maybe-inputs required-inputs 'propagated-inputs) + ,@(maybe-inputs test-inputs 'native-inputs) + (home-page ,home-page) + (synopsis ,synopsis) + (description ,description) + (license ,(license->symbol license))) + upstream-dependencies)))))))) (define pypi->guix-package (memoize diff --git a/tests/pypi.scm b/tests/pypi.scm index 8b42c2f0710..43d45f1dd8c 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 David Thompson ;;; Copyright © 2016 Ricardo Wurmus +;;; Copyright © 2019 Maxim Cournoyer ;;; ;;; This file is part of GNU Guix. ;;; @@ -68,11 +69,6 @@ sha1=da9234ee9982d4bbb3c72346a6de940a148ea686")) (define test-requires.txt "\ # A comment # A comment after a space -bar -baz > 13.37 -") - -(define test-requires-with-sections "\ foo ~= 3 bar != 2 @@ -80,12 +76,25 @@ bar != 2 pytest (>=2.5.0) ") +;; Beaker contains only optional dependencies. +(define test-requires.txt-beaker "\ +[crypto] +pycryptopp>=0.5.12 + +[cryptography] +cryptography + +[testsuite] +Mock +coverage +") + (define test-metadata "\ Classifier: Programming Language :: Python :: 3.7 Requires-Dist: baz ~= 3 Requires-Dist: bar != 2 Provides-Extra: test -pytest (>=2.5.0) +Requires-Dist: pytest (>=2.5.0) ; extra == 'test' ") (define test-metadata-with-extras " @@ -139,25 +148,31 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip") (map specification->requirement-name test-specifications)) -(test-equal "parse-requires.txt, with sections" - '("foo" "bar") +(test-equal "parse-requires.txt" + (list '("foo" "bar") '("pytest")) (mock ((ice-9 ports) call-with-input-file call-with-input-string) - (parse-requires.txt test-requires-with-sections))) + (parse-requires.txt test-requires.txt))) + +(test-equal "parse-requires.txt - Beaker" + (list '() '("Mock" "coverage")) + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-requires.txt test-requires.txt-beaker))) (test-equal "parse-wheel-metadata, with extras" - '("wrapt" "bar") + (list '("wrapt" "bar") '("tox" "bumpversion")) (mock ((ice-9 ports) call-with-input-file call-with-input-string) (parse-wheel-metadata test-metadata-with-extras))) (test-equal "parse-wheel-metadata, with extras - Jedi" - '("parso") + (list '("parso") '("pytest")) (mock ((ice-9 ports) call-with-input-file call-with-input-string) (parse-wheel-metadata test-metadata-with-extras-jedi))) -(test-assert "pypi->guix-package" +(test-assert "pypi->guix-package, no wheel" ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch (lambda (url file-name) @@ -198,7 +213,10 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' ('propagated-inputs ('quasiquote (("python-bar" ('unquote 'python-bar)) - ("python-baz" ('unquote 'python-baz))))) + ("python-foo" ('unquote 'python-foo))))) + ('native-inputs + ('quasiquote + (("python-pytest" ('unquote 'python-pytest))))) ('home-page "http://example.com") ('synopsis "summary") ('description "summary") @@ -219,25 +237,25 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' (begin (mkdir-p "foo-1.0.0/foo.egg-info/") (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" - (lambda () - (display "wrong data to make sure we're testing wheels "))) + (lambda () + (display "wrong data to make sure we're testing wheels "))) (parameterize ((current-output-port (%make-void-port "rw+"))) (system* "tar" "czvf" file-name "foo-1.0.0/")) - (delete-file-recursively "foo-1.0.0") - (set! test-source-hash - (call-with-input-file file-name port-sha256)))) + (delete-file-recursively "foo-1.0.0") + (set! test-source-hash + (call-with-input-file file-name port-sha256)))) ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" - (begin - (mkdir "foo-1.0.0.dist-info") - (with-output-to-file "foo-1.0.0.dist-info/METADATA" - (lambda () - (display test-metadata))) - (let ((zip-file (string-append file-name ".zip"))) - ;; zip always adds a "zip" extension to the file it creates, - ;; so we need to rename it. - (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA") - (rename-file zip-file file-name)) - (delete-file-recursively "foo-1.0.0.dist-info"))) + (begin + (mkdir "foo-1.0.0.dist-info") + (with-output-to-file "foo-1.0.0.dist-info/METADATA" + (lambda () + (display test-metadata))) + (let ((zip-file (string-append file-name ".zip"))) + ;; zip always adds a "zip" extension to the file it creates, + ;; so we need to rename it. + (system* "zip" "-q" zip-file "foo-1.0.0.dist-info/METADATA") + (rename-file zip-file file-name)) + (delete-file-recursively "foo-1.0.0.dist-info"))) (_ (error "Unexpected URL: " url))))) (mock ((guix http-client) http-fetch (lambda (url . rest) @@ -265,6 +283,9 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' ('quasiquote (("python-bar" ('unquote 'python-bar)) ("python-baz" ('unquote 'python-baz))))) + ('native-inputs + ('quasiquote + (("python-pytest" ('unquote 'python-pytest))))) ('home-page "http://example.com") ('synopsis "summary") ('description "summary") From a5376200541abf8245973e601be246bf65b8b6c7 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Sun, 16 Jun 2019 14:52:25 +0900 Subject: [PATCH 22/44] import: pypi: Update the host URI. * guix/build-system/python.scm (pypi-uri): Update the host URI to "files.pythonhosted.org". --- guix/build-system/python.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guix/build-system/python.scm b/guix/build-system/python.scm index b753940bad7..e39c06528ef 100644 --- a/guix/build-system/python.scm +++ b/guix/build-system/python.scm @@ -50,7 +50,7 @@ "Return a URI string for the Python package hosted on the Python Package Index (PyPI) corresponding to NAME and VERSION. EXTENSION is the file name extension, such as '.tar.gz'." - (string-append "https://pypi.org/packages/source/" + (string-append "https://files.pythonhosted.org/packages/source/" (string-take name 1) "/" name "/" name "-" version extension)) From 4b60ab8c006964d026dee8cf5f1260eba0b2bb81 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Sat, 30 Mar 2019 20:27:35 -0400 Subject: [PATCH 23/44] import: pypi: Preserve package name case when forming pypi-uri. Fixes . * guix/build-system/python.scm (pypi-uri): Update the host URI to "files.pythonhosted.org". * guix/import/pypi.scm (make-pypi-sexp): Preserve the package name case when the source URL calls for it. --- guix/import/pypi.scm | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 537431dd691..ab7a024ee0f 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -368,15 +368,20 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." `(package (name ,(python->package-name name)) (version ,version) - (source (origin - (method url-fetch) - ;; Sometimes 'pypi-uri' doesn't quite work due to mixed - ;; cases in NAME, for instance, as is the case with - ;; "uwsgi". In that case, fall back to a full URL. - (uri (pypi-uri ,(string-downcase name) version)) - (sha256 - (base32 - ,(guix-hash-url temp))))) + (source + (origin + (method url-fetch) + ;; PyPI URL are case sensitive, but sometimes a project + ;; named using mixed case has a URL using lower case, so + ;; we must work around this inconsistency. For actual + ;; examples, compare the URLs of the "Deprecated" and + ;; "uWSGI" PyPI packages. + (uri ,(if (string-contains source-url name) + `(pypi-uri ,name version) + `(pypi-uri ,(string-downcase name) version))) + (sha256 + (base32 + ,(guix-hash-url temp))))) (build-system python-build-system) ,@(maybe-inputs required-inputs 'propagated-inputs) ,@(maybe-inputs test-inputs 'native-inputs) From 48eb71aea807262210c38b5fb675d747adfccff3 Mon Sep 17 00:00:00 2001 From: Jan Nieuwenhuizen Date: Mon, 1 Jul 2019 15:37:10 +0200 Subject: [PATCH 24/44] gnu: emacsy: Update to 0.4.1. * gnu/packages/guile-xyz.scm (emacsy): Update to 0.4.1. --- gnu/packages/guile-xyz.scm | 119 ++++++++++++++----------------------- 1 file changed, 46 insertions(+), 73 deletions(-) diff --git a/gnu/packages/guile-xyz.scm b/gnu/packages/guile-xyz.scm index a8352380f7e..f652a94d2e2 100644 --- a/gnu/packages/guile-xyz.scm +++ b/gnu/packages/guile-xyz.scm @@ -2326,82 +2326,55 @@ more expressive and flexible than the traditional @code{format} procedure.") (license license:bsd-3)))) (define-public emacsy - (let ((commit "7d49cc1425d5d209bdb82cac0d8ea0694b8b3784") - (revision "4")) - (package - (name "emacsy") - (version (string-append "0.1.2-" revision "." (string-take commit 7))) - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://gitlab.com/janneke/emacsy.git") - (commit commit))) - (file-name (string-append name "-" version)) - (sha256 - (base32 - "0k9yns1v8zn135w60sx96nqs2bm2p2dvcvlm987hkw4lbff9ii6i")))) - (build-system gnu-build-system) - (native-inputs - `(("emacsy-webkit-gtk" - ,(origin - (method git-fetch) - (uri (git-reference - (url "https://gitlab.com/janneke/emacsy-webkit-gtk.git") - (commit "35ded1b3e997fd779a17e0c4a2c73741718562d9"))) - (file-name (string-append "emacsy-webkit-gtk" "-" version)) - (sha256 - (base32 - "1gp0li2rbp6in926r3hrww6cnh864pp46v1din2pgmd7vzzl7kg0")))) - ("hello-emacsy" - ,(origin - (method git-fetch) - (uri (git-reference - (url "https://gitlab.com/janneke/hello-emacsy.git") - (commit "2c117e5286a261be4ff24938f3ae1d348396c538"))) - (file-name (string-append "hello-emacsy" "-" version)) - (sha256 - (base32 - "15ykd7s8axcy8ym4v71fgal4x28fxnim0pv0jmpi3dnhizr63zqn")))) - ("autoconf" ,autoconf) - ("automake" ,automake) - ("bzip2" ,bzip2) - ("guile" ,guile-2.2) - ("gettext" ,gnu-gettext) - ("libtool" ,libtool) - ("noweb" ,noweb) - ("perl" ,perl) - ("pkg-config" ,pkg-config) - ("texinfo" ,texinfo) - ("texlive" ,texlive))) - (propagated-inputs - `(("guile-lib" ,guile-lib) - ("guile-readline" ,guile-readline) - ("freeglut" ,freeglut) - ("gssettings-desktop-schemas" ,gsettings-desktop-schemas) - ("webkitgtk" ,webkitgtk))) - (inputs `(("guile" ,guile-2.2))) - (arguments - `(#:phases - (modify-phases %standard-phases - (add-after 'unpack 'unpack-examples - (lambda _ - (copy-recursively (assoc-ref %build-inputs "emacsy-webkit-gtk") - "example/emacsy-webkit-gtk") - (copy-recursively (assoc-ref %build-inputs "hello-emacsy") - "example/hello-emacsy"))) - (add-before 'configure 'setenv - (lambda _ - (setenv "GUILE_AUTO_COMPILE" "0")))))) - (home-page "https://github.com/shanecelis/emacsy/") - (synopsis "Embeddable GNU Emacs-like library using Guile") - (description - "Emacsy is an embeddable GNU Emacs-like library that uses GNU Guile + (package + (name "emacsy") + (version "0.4.1") + (source (origin + (method url-fetch) + (uri (string-append + "https://download.savannah.nongnu.org/releases/" + name "/" name "-" version ".tar.gz")) + (sha256 + (base32 + "1cpb85dl1nibd34c2x2h7vfmjpkgh353p5b1w20v6cs6gmvgg4np")))) + (build-system gnu-build-system) + (native-inputs + `(("autoconf" ,autoconf) + ("automake" ,automake) + ("bzip2" ,bzip2) + ("guile" ,guile-2.2) + ("gettext" ,gnu-gettext) + ("libtool" ,libtool) + ("perl" ,perl) + ("pkg-config" ,pkg-config) + ("texinfo" ,texinfo) + ("texlive" ,texlive))) + (propagated-inputs + `(("dbus-glib" ,dbus-glib) + ("guile" ,guile-2.2) + ("guile-lib" ,guile-lib) + ("guile-readline" ,guile-readline) + ("glib-networking" ,glib-networking) + ("freeglut" ,freeglut) + ("gssettings-desktop-schemas" ,gsettings-desktop-schemas) + ("webkitgtk" ,webkitgtk))) + (arguments + `(#:phases + (modify-phases %standard-phases + (add-before 'configure 'setenv + (lambda _ + (setenv "GUILE_AUTO_COMPILE" "0") + #t))))) + (home-page "https://savannah.nongnu.org/projects/emacsy") + (synopsis "Embeddable GNU Emacs-like library using Guile") + (description + "Emacsy is an embeddable Emacs-like library that uses GNU Guile as extension language. Emacsy can give a C program an Emacsy feel with keymaps, minibuffer, recordable macros, history, tab completion, major and minor modes, etc., and can also be used as a pure Guile library. It -comes with a simple counter example using GLUT and browser examples in C -using gtk+-3 and webkitgtk.") - (license license:gpl3+)))) +comes with a simple counter example using FreeGLUT and browser examples +in C using Gtk+-3 and WebKitGtk.") + (license license:gpl3+))) (define-public guile-jpeg (let ((commit "6a1673578b297c2c1b28e44a76bd5c49e76a5046") From 941ae98eae755a92a0e28d04490ee9a237cc585f Mon Sep 17 00:00:00 2001 From: Rutger Helling Date: Tue, 2 Jul 2019 09:35:01 +0200 Subject: [PATCH 25/44] gnu: faudio: Update to 19.07. * gnu/packages/audio.scm (faudio): Update to 19.07. --- gnu/packages/audio.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnu/packages/audio.scm b/gnu/packages/audio.scm index b8dbc2f00a7..eff4edeb597 100644 --- a/gnu/packages/audio.scm +++ b/gnu/packages/audio.scm @@ -3701,7 +3701,7 @@ library.") (define-public faudio (package (name "faudio") - (version "19.06.07") + (version "19.07") (source (origin (method git-fetch) @@ -3710,7 +3710,7 @@ library.") (commit version))) (file-name (string-append name "-" version "-checkout")) (sha256 - (base32 "1w37qp279lgpyvslwz3wlb4fp0i68ncd411rqdlk5s71b1zz466n")))) + (base32 "1wf6skc5agaikc9qgwk8bx56sad31fafs53lqqn4jmx8i76pl0lw")))) (arguments '(#:tests? #f ; No tests. #:configure-flags '("-DFFMPEG=ON"))) From 71a29277055708b509a477d97c8d5f9815f1b939 Mon Sep 17 00:00:00 2001 From: Amar Singh Date: Sat, 29 Jun 2019 03:09:31 +0530 Subject: [PATCH 26/44] gnu: emacs-shroud: Update to 1.83.4. * gnu/packages/emacs-xyz.scm (emacs-shroud): Update to 1.83.4. Signed-off-by: Efraim Flashner --- gnu/packages/emacs-xyz.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnu/packages/emacs-xyz.scm b/gnu/packages/emacs-xyz.scm index 2a59ae522e1..a27de2f7780 100644 --- a/gnu/packages/emacs-xyz.scm +++ b/gnu/packages/emacs-xyz.scm @@ -550,7 +550,7 @@ for editing Racket's Scribble documentation syntax in Emacs.") (define-public emacs-shroud (package (name "emacs-shroud") - (version "1.15.1") + (version "1.83.4") (source (origin (method git-fetch) @@ -559,7 +559,7 @@ for editing Racket's Scribble documentation syntax in Emacs.") (commit version))) (file-name (git-file-name name version)) (sha256 - (base32 "0wvm4lxqcc1p8v7rpqal3bnqgnpk1gs7v18i83f6cvi5d88jkgdg")))) + (base32 "1yvdjx0kp4y8w5yz2cbqq9n6xl5splvmsyyx8ld1xv0q1c9872nf")))) (build-system emacs-build-system) (propagated-inputs `(("emacs-bui" ,emacs-bui) From 4bac5bba51383f5c01cd5a9a5efe7b9198be86c0 Mon Sep 17 00:00:00 2001 From: Efraim Flashner Date: Tue, 2 Jul 2019 11:42:56 +0300 Subject: [PATCH 27/44] gnu: Wrap program with libraries. * gnu/packages/package-management.scm (stow)[arguments]: Add custom 'wrap-stow phase to wrap stow with necessary inputs. [propagated-inputs]: Move perl-clone, perl-clone-choose, perl-hash-merge to... [inputs]: ... here. --- gnu/packages/package-management.scm | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/gnu/packages/package-management.scm b/gnu/packages/package-management.scm index ca360b38842..d0388e5d436 100644 --- a/gnu/packages/package-management.scm +++ b/gnu/packages/package-management.scm @@ -475,17 +475,28 @@ sub-directory.") (base32 "0h8qr2rxsrkg6d8jxjk68r23jgn1dxdxyp4bnzzinpa8sjhfl905")))) (build-system gnu-build-system) + (arguments + '(#:phases + (modify-phases %standard-phases + (add-after 'install 'wrap-stow + (lambda* (#:key inputs outputs #:allow-other-keys) + (let ((out (assoc-ref outputs "out"))) + (wrap-program (string-append out "/bin/stow") + `("PERL5LIB" ":" prefix + ,(map (lambda (i) (string-append (assoc-ref inputs i) + "/lib/perl5/site_perl")) + '("perl-clone-choose" "perl-clone" "perl-hash-merge")))) + #t)))))) (inputs - `(("perl" ,perl))) + `(("perl" ,perl) + ("perl-clone" ,perl-clone) + ("perl-clone-choose" ,perl-clone-choose) + ("perl-hash-merge" ,perl-hash-merge))) (native-inputs `(("perl-test-simple" ,perl-test-simple) ("perl-test-output" ,perl-test-output) ("perl-capture-tiny" ,perl-capture-tiny) ("perl-io-stringy" ,perl-io-stringy))) - (propagated-inputs - `(("perl-clone-choose" ,perl-clone-choose) - ("perl-clone" ,perl-clone) - ("perl-hash-merge" ,perl-hash-merge))) (home-page "https://www.gnu.org/software/stow/") (synopsis "Managing installed software packages") (description From 697df13f613468f4c572f85d3411dbf60b7bc45a Mon Sep 17 00:00:00 2001 From: Efraim Flashner Date: Tue, 2 Jul 2019 15:04:56 +0300 Subject: [PATCH 28/44] gnu: octave-cli: Sort inputs alphabetically. * gnu/packages/maths.scm (octave-cli)[inputs]: Sort inputs alphabetically. --- gnu/packages/maths.scm | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/gnu/packages/maths.scm b/gnu/packages/maths.scm index 5b31596041b..7d2c4797f12 100644 --- a/gnu/packages/maths.scm +++ b/gnu/packages/maths.scm @@ -1463,30 +1463,30 @@ can solve two kinds of problems: "11wwxpy2q1bhxs2v41bqn05i2sb0905cj1xil6mg8l4k2kka4cq6")))) (build-system gnu-build-system) (inputs - `(("lapack" ,lapack) - ("qhull" ,qhull) - ("readline" ,readline) - ("gl2ps" ,gl2ps) - ("glpk" ,glpk) + `(("alsa-lib" ,alsa-lib) + ("arpack" ,arpack-ng) + ("curl" ,curl) ("fftw" ,fftw) ("fftwf" ,fftwf) - ("arpack" ,arpack-ng) - ("pcre" ,pcre) ("fltk" ,fltk) ("fontconfig" ,fontconfig) ("freetype" ,freetype) + ("gl2ps" ,gl2ps) + ("glpk" ,glpk) + ("glu" ,glu) + ("graphicsmagick" ,graphicsmagick) ("hdf5" ,hdf5) + ("lapack" ,lapack) + ("libsndfile" ,libsndfile) ("libxft" ,libxft) ("mesa" ,mesa) - ("glu" ,glu) - ("zlib" ,zlib) - ("curl" ,curl) - ("texinfo" ,texinfo) - ("graphicsmagick" ,graphicsmagick) - ("suitesparse" ,suitesparse) - ("libsndfile" ,libsndfile) + ("pcre" ,pcre) ("portaudio" ,portaudio) - ("alsa-lib" ,alsa-lib))) + ("qhull" ,qhull) + ("readline" ,readline) + ("suitesparse" ,suitesparse) + ("texinfo" ,texinfo) + ("zlib" ,zlib))) (native-inputs `(("lzip" ,lzip) ("gfortran" ,gfortran) From 35a16f5f6d78f15309b8e8282f46887835db83e0 Mon Sep 17 00:00:00 2001 From: Rutger Helling Date: Tue, 2 Jul 2019 13:45:16 +0200 Subject: [PATCH 29/44] gnu: btrfs-progs: Update to 5.1.1. * gnu/packages/linux.scm (btrfs-progs): Update to 5.1.1. --- gnu/packages/linux.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnu/packages/linux.scm b/gnu/packages/linux.scm index 72b62faf429..b4407a51339 100644 --- a/gnu/packages/linux.scm +++ b/gnu/packages/linux.scm @@ -3544,7 +3544,7 @@ and copy/paste text in the console and in xterm.") (define-public btrfs-progs (package (name "btrfs-progs") - (version "5.1") + (version "5.1.1") (source (origin (method url-fetch) (uri (string-append "mirror://kernel.org/linux/kernel/" @@ -3552,7 +3552,7 @@ and copy/paste text in the console and in xterm.") "btrfs-progs-v" version ".tar.xz")) (sha256 (base32 - "0dgh56pamav8wb9nmabjwdlpcazvqc9pgzwablxn77mqh0qrhkaq")))) + "06xybs7rglxjqkbzl2409acb3rgmnc5zc0xhyaxsc2p1x5yipfcw")))) (build-system gnu-build-system) (outputs '("out" "static")) ; static versions of the binaries in "out" From f62b57ba74ebaee36c554b42933095612d1aee01 Mon Sep 17 00:00:00 2001 From: Marius Bakke Date: Mon, 1 Jul 2019 12:15:48 +0200 Subject: [PATCH 30/44] gnu: re2: Update to 2019-07-01. * gnu/packages/regex.scm (re2): Update to 2019-07-01. --- gnu/packages/regex.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnu/packages/regex.scm b/gnu/packages/regex.scm index a64944080ba..71dac09737a 100644 --- a/gnu/packages/regex.scm +++ b/gnu/packages/regex.scm @@ -30,7 +30,7 @@ (define-public re2 (package (name "re2") - (version "2019-06-01") + (version "2019-07-01") (home-page "https://github.com/google/re2") (source (origin (method git-fetch) @@ -38,7 +38,7 @@ (file-name (git-file-name name version)) (sha256 (base32 - "01613z66wgiffdngbq3031rwd92jf87j93h7y5mn8hlx19gg5k4j")))) + "1ric6gdnf5mqj5iy5f81al49mr3mmjqj3nqi3mw2hjdbbgwkdn71")))) (build-system gnu-build-system) (arguments `(#:modules ((guix build gnu-build-system) From 8b5df106fdd52b8851bbdd978077e8ab257eb840 Mon Sep 17 00:00:00 2001 From: Marius Bakke Date: Tue, 2 Jul 2019 14:52:00 +0200 Subject: [PATCH 31/44] gnu: WebKitGTK: Update to 2.24.3. * gnu/packages/webkit.scm (webkitgtk-2.24): Update to 2.24.3. [source](patches): Remove. * gnu/packages/patches/webkitgtk-sans-gstreamer-gl.patch: Delete file. * gnu/local.mk (dist_patch_DATA): Adjust accordingly. --- gnu/local.mk | 1 - .../patches/webkitgtk-sans-gstreamer-gl.patch | 24 ------------------- gnu/packages/webkit.scm | 5 ++-- 3 files changed, 2 insertions(+), 28 deletions(-) delete mode 100644 gnu/packages/patches/webkitgtk-sans-gstreamer-gl.patch diff --git a/gnu/local.mk b/gnu/local.mk index 0f4cb2a6e43..6e90d886893 100644 --- a/gnu/local.mk +++ b/gnu/local.mk @@ -1370,7 +1370,6 @@ dist_patch_DATA = \ %D%/packages/patches/wavpack-CVE-2018-6767.patch \ %D%/packages/patches/wavpack-CVE-2018-7253.patch \ %D%/packages/patches/wavpack-CVE-2018-7254.patch \ - %D%/packages/patches/webkitgtk-sans-gstreamer-gl.patch \ %D%/packages/patches/weechat-python.patch \ %D%/packages/patches/wicd-bitrate-none-fix.patch \ %D%/packages/patches/wicd-get-selected-profile-fix.patch \ diff --git a/gnu/packages/patches/webkitgtk-sans-gstreamer-gl.patch b/gnu/packages/patches/webkitgtk-sans-gstreamer-gl.patch deleted file mode 100644 index 4577c81edb9..00000000000 --- a/gnu/packages/patches/webkitgtk-sans-gstreamer-gl.patch +++ /dev/null @@ -1,24 +0,0 @@ -Fix build failure when USE_GSTREAMER_GL=off. See -. - -This patch is taken from the upstream source repository: -. - -diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp -index 00a2af6489e..5cb5f7536ac 100644 ---- a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp -+++ b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp -@@ -1000,11 +1000,13 @@ void MediaPlayerPrivateGStreamerBase::updateTextureMapperFlags() - break; - } - -+#if USE(GSTREAMER_GL) - // When the imxvpudecoder is used, the texture sampling of the - // directviv-uploaded texture returns an RGB value, so there's no need to - // convert it. - if (m_videoDecoderPlatform != WebKitGstVideoDecoderPlatform::ImxVPU) - m_textureMapperFlags |= TEXTURE_MAPPER_COLOR_CONVERT_FLAG; -+#endif - } - #endif - diff --git a/gnu/packages/webkit.scm b/gnu/packages/webkit.scm index e9b7ab74a75..6b38998772f 100644 --- a/gnu/packages/webkit.scm +++ b/gnu/packages/webkit.scm @@ -168,15 +168,14 @@ HTML/CSS applications to full-fledged web browsers.") (define-public webkitgtk-2.24 (package/inherit webkitgtk (name "webkitgtk") - (version "2.24.2") + (version "2.24.3") (source (origin (method url-fetch) (uri (string-append "https://www.webkitgtk.org/releases/" name "-" version ".tar.xz")) (sha256 (base32 - "071jnjvjq6wsxx1jh4ql3j53h1nhphs5ga67fa5i9xjvs3qb3701")) - (patches (search-patches "webkitgtk-sans-gstreamer-gl.patch")))) + "0lbcrw5axwrbrajxq7fqywfyh0djqi23ynzb5wi5ghw2grnp83cl")))) (native-inputs `(("gcc" ,gcc-7) ; webkitgtk-2.22 requires gcc-6 or newer ,@(package-native-inputs webkitgtk))) From 7c9fbf3e3dde341b33cfa8dca10001150e54640e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= Date: Tue, 2 Jul 2019 16:41:33 +0200 Subject: [PATCH 32/44] derivations: Update 'build-derivations' docstring. This is a followup to 7c690a47381f645ec5ec0a1fd6ffc34dba1b69c2. * guix/derivations.scm (build-derivations): Update docstring. --- guix/derivations.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/guix/derivations.scm b/guix/derivations.scm index 433b4551a53..ebeac31877e 100644 --- a/guix/derivations.scm +++ b/guix/derivations.scm @@ -1011,8 +1011,8 @@ recursively." (define* (build-derivations store derivations #:optional (mode (build-mode normal))) - "Build DERIVATIONS, a list of objects, .drv file names, or -derivation/output pairs, using the specified MODE." + "Build DERIVATIONS, a list of or objects, +.drv file names, or derivation/output pairs, using the specified MODE." (build-things store (map (match-lambda ((? derivation? drv) (derivation-file-name drv)) From 9616b81e9861c831159a0e1a5993854b9ad3c3e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= Date: Tue, 2 Jul 2019 16:45:12 +0200 Subject: [PATCH 33/44] grafts: 'references-oracle' now takes a derivation input. That way, if we end up calling 'build-derivations', we'll only build the outputs that we really need. * guix/grafts.scm (references-oracle): Rename 'drv' to 'input'. [output-paths]: Remove. Adjust accordingly. (graft-derivation): Adjust call to 'references-oracle'. --- guix/grafts.scm | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/guix/grafts.scm b/guix/grafts.scm index 3b43e11425f..adc7bfafaec 100644 --- a/guix/grafts.scm +++ b/guix/grafts.scm @@ -163,16 +163,11 @@ references. Call REFERENCES to get the list of references." items)))) (remove (cut member <> self) refs))) -(define (references-oracle store drv) - "Return a one-argument procedure that, when passed the file name of DRV's -outputs or their dependencies, returns the list of references of that item. -Use either local info or substitute info; build DRV if no information is -available." - (define (output-paths drv) - (match (derivation->output-paths drv) - (((names . items) ...) - items))) - +(define (references-oracle store input) + "Return a one-argument procedure that, when passed the output file names of +INPUT, a derivation input, or their dependencies, returns the list of +references of that item. Use either local info or substitute info; build +INPUT if no information is available." (define (references* items) (guard (c ((store-protocol-error? c) ;; As a last resort, build DRV and query the references of the @@ -181,13 +176,14 @@ available." ;; Warm up the narinfo cache, otherwise each derivation build ;; will result in one HTTP request to get one narinfo, which is ;; much less efficient than fetching them all upfront. - (substitution-oracle store (list drv)) + (substitution-oracle store + (list (derivation-input-derivation input))) - (and (build-derivations store (list drv)) + (and (build-derivations store (list input)) (map (cut references store <>) items)))) (references/substitutes store items))) - (let loop ((items (output-paths drv)) + (let loop ((items (derivation-input-output-paths input)) (result vlist-null)) (match items (() @@ -324,7 +320,7 @@ DRV, and graft DRV itself to refer to those grafted dependencies." ;; upfront to have as much parallelism as possible when querying substitute ;; info or when building DRV. (define references - (references-oracle store drv)) + (references-oracle store (derivation-input drv outputs))) (match (run-with-state (cumulative-grafts store drv grafts references From 334a2f4def1d4f9dc37718d847923cd941849607 Mon Sep 17 00:00:00 2001 From: Robert Vollmert Date: Thu, 13 Jun 2019 15:50:37 +0200 Subject: [PATCH 34/44] gnu: postgres service: More secure default permissions. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This changes to 'peer' authentication for local socket connections, and password-based authentication for local network connections. * gnu/services/databases.scm (%default-postgres-hba): Change authentication method. Signed-off-by: Ludovic Courtès --- gnu/services/databases.scm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gnu/services/databases.scm b/gnu/services/databases.scm index 7113f1f2a13..ec31489d486 100644 --- a/gnu/services/databases.scm +++ b/gnu/services/databases.scm @@ -5,6 +5,7 @@ ;;; Copyright © 2017 Christopher Baines ;;; Copyright © 2018 Clément Lassieur ;;; Copyright © 2018 Julien Lepiller +;;; Copyright © 2019 Robert Vollmert ;;; ;;; This file is part of GNU Guix. ;;; @@ -91,9 +92,9 @@ (define %default-postgres-hba (plain-file "pg_hba.conf" " -local all all trust -host all all 127.0.0.1/32 trust -host all all ::1/128 trust")) +local all all peer +host all all 127.0.0.1/32 md5 +host all all ::1/128 md5")) (define %default-postgres-ident (plain-file "pg_ident.conf" From 52bb05c70274e5f235901cf8f6d190bfce728e87 Mon Sep 17 00:00:00 2001 From: guy fleury iteriteka Date: Fri, 24 May 2019 11:33:07 +0200 Subject: [PATCH 35/44] gnu: Add chafa. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/image-viewers.scm (chaffa): New variable. Signed-off-by: Ludovic Courtès --- gnu/packages/image-viewers.scm | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/gnu/packages/image-viewers.scm b/gnu/packages/image-viewers.scm index a5bfb28a88a..7d2ff9a6778 100644 --- a/gnu/packages/image-viewers.scm +++ b/gnu/packages/image-viewers.scm @@ -10,6 +10,7 @@ ;;; Copyright © 2018, 2019 Tobias Geerinckx-Rice ;;; Copyright © 2018, 2019 Ricardo Wurmus ;;; Copyright © 2019 Nicolas Goaziou +;;; Copyright © 2019 Guy Fleury Iteriteka ;;; ;;; This file is part of GNU Guix. ;;; @@ -477,3 +478,28 @@ minimalism and usability in mind. Its features include animated GIF controls, file history, rotation/mirroring, and multithreaded preloading.") (license license:gpl3+))) + +(define-public chafa + (package + (name "chafa") + (version "1.0.1") + (source (origin + (method url-fetch) + (uri (string-append "https://hpjansson.org/chafa/releases/chafa-" + version ".tar.xz")) + (sha256 + (base32 + "00cf2z52az0z6bzc3hfm4l8infipy5ck410wqmbaybd2csjr3m29")))) + (build-system gnu-build-system) + (native-inputs + `(("pkg-config" ,pkg-config))) + (inputs + `(("glib" ,glib) + ("imagemagick" ,imagemagick))) + (synopsis "Convert images to ANSI/Unicode characters") + (description + "Chafa is a command-line utility that converts all kinds of images, +including animated GIFs, into ANSI/Unicode character output that can be +displayed in a terminal.") + (home-page "https://hpjansson.org/chafa/") + (license license:lgpl3+))) From c473bd1881d5093789b41cb6927ee3a5ed6d0439 Mon Sep 17 00:00:00 2001 From: Jacob MacDonald Date: Fri, 7 Jun 2019 05:24:12 -0500 Subject: [PATCH 36/44] gnu: Add ghc-wl-pprint-annotated. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/haskell.scm (ghc-wl-pprint-annotated): New variable. Signed-off-by: Ludovic Courtès --- gnu/packages/haskell.scm | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/gnu/packages/haskell.scm b/gnu/packages/haskell.scm index cfbd3bfb76c..f40d92b9f78 100644 --- a/gnu/packages/haskell.scm +++ b/gnu/packages/haskell.scm @@ -11591,4 +11591,34 @@ Replace some ASCII sequences by their Unicode equivalent (turned off by default) @end itemize") (license license:bsd-3))) + +(define-public ghc-wl-pprint-annotated + (package + (name "ghc-wl-pprint-annotated") + (version "0.1.0.1") + (source + (origin + (method url-fetch) + (uri (string-append + "mirror://hackage/package/wl-pprint-annotated/wl-pprint-annotated-" + version + ".tar.gz")) + (sha256 + (base32 + "1br7qyf27iza213inwhf9bm2k6in0zbmfw6w4clqlc9f9cj2nrkb")))) + (build-system haskell-build-system) + (native-inputs + `(("ghc-tasty" ,ghc-tasty) + ("ghc-tasty-hunit" ,ghc-tasty-hunit))) + (home-page + "https://github.com/minad/wl-pprint-annotated#readme") + (synopsis + "Wadler/Leijen pretty printer with annotation support") + (description + "Annotations are useful for coloring. This is a limited version of +@code{wl-pprint-extras} without support for point effects and without the free +monad. Like in @code{annotated-wl-pprint}, only annotations are supported. +Compared to @code{annotated-wl-pprint} this library provides a slightly +modernized interface.") + (license license:bsd-3))) ;;; haskell.scm ends here From cbff89d126bf5985cfa4884f543c0908c437ff41 Mon Sep 17 00:00:00 2001 From: Jacob MacDonald Date: Fri, 7 Jun 2019 06:00:26 -0500 Subject: [PATCH 37/44] gnu: Add ghc-ansi-terminal-0.8. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/haskell.scm (ghc-ansi-terminal-0.8): New variable. Signed-off-by: Ludovic Courtès --- gnu/packages/haskell.scm | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gnu/packages/haskell.scm b/gnu/packages/haskell.scm index f40d92b9f78..a7a8b762c4f 100644 --- a/gnu/packages/haskell.scm +++ b/gnu/packages/haskell.scm @@ -4383,6 +4383,21 @@ allows cursor movement, screen clearing, color output showing or hiding the cursor, and changing the title.") (license license:bsd-3))) +(define-public ghc-ansi-terminal-0.8 + (package (inherit ghc-ansi-terminal) + (name "ghc-ansi-terminal") + (version "0.8.0.4") + (source + (origin + (method url-fetch) + (uri (string-append + "https://hackage.haskell.org/package/ansi-terminal/ansi-terminal-" + version + ".tar.gz")) + (sha256 + (base32 + "0428gq8m3fdnb7ldcsyk97qcch76hcxbgh2666p6f76fs2qbhg7b")))))) + (define-public ghc-vault (package (name "ghc-vault") From 5a499d0f7d5b98443ed0b2c41f2651f66a84ab5e Mon Sep 17 00:00:00 2001 From: Jacob MacDonald Date: Fri, 7 Jun 2019 06:09:24 -0500 Subject: [PATCH 38/44] gnu: ghc-ansi-wl-pprint: Use ghc-ansi-terminal-0.8. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/haskell.scm (ghc-ansi-wl-pprint)[inputs]: Use ghc-ansi-terminal-0.8. Signed-off-by: Ludovic Courtès --- gnu/packages/haskell.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnu/packages/haskell.scm b/gnu/packages/haskell.scm index a7a8b762c4f..2c4baebcd85 100644 --- a/gnu/packages/haskell.scm +++ b/gnu/packages/haskell.scm @@ -4085,7 +4085,7 @@ instances of the @code{Pretty} class.") "0gnb4mkqryv08vncxnj0bzwcnd749613yw3cxfzw6y3nsldp4c56")))) (build-system haskell-build-system) (inputs - `(("ghc-ansi-terminal" ,ghc-ansi-terminal))) + `(("ghc-ansi-terminal" ,ghc-ansi-terminal-0.8))) (home-page "https://github.com/ekmett/ansi-wl-pprint") (synopsis "Wadler/Leijen Pretty Printer for colored ANSI terminal output") (description "This is a pretty printing library based on Wadler's paper From 4e3ebbfb1649063bcc0f350523868c667e6699dd Mon Sep 17 00:00:00 2001 From: Jacob MacDonald Date: Fri, 7 Jun 2019 05:28:36 -0500 Subject: [PATCH 39/44] gnu: ghc-ansi-terminal: Update to 0.9.1. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/haskell.scm (ghc-ansi-terminal): Update to 0.9.1. Signed-off-by: Ludovic Courtès --- gnu/packages/haskell.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnu/packages/haskell.scm b/gnu/packages/haskell.scm index 2c4baebcd85..c16a2915639 100644 --- a/gnu/packages/haskell.scm +++ b/gnu/packages/haskell.scm @@ -4362,7 +4362,7 @@ interface.") (define-public ghc-ansi-terminal (package (name "ghc-ansi-terminal") - (version "0.8.0.4") + (version "0.9.1") (source (origin (method url-fetch) @@ -4372,7 +4372,7 @@ interface.") ".tar.gz")) (sha256 (base32 - "0428gq8m3fdnb7ldcsyk97qcch76hcxbgh2666p6f76fs2qbhg7b")))) + "1yr0ld0kqns3w3j9gl62bdwshvyazidx4dv1qkvq19ivnf08w23l")))) (build-system haskell-build-system) (inputs `(("ghc-colour" ,ghc-colour))) From 3c69d462ce478b2d4e3939afd6b895a3e0534809 Mon Sep 17 00:00:00 2001 From: Jacob MacDonald Date: Fri, 7 Jun 2019 05:36:25 -0500 Subject: [PATCH 40/44] gnu: Add ghc-concurrent-output. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/haskell.scm (ghc-concurrent-output): New variable. Signed-off-by: Ludovic Courtès --- gnu/packages/haskell.scm | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/gnu/packages/haskell.scm b/gnu/packages/haskell.scm index c16a2915639..63d43601824 100644 --- a/gnu/packages/haskell.scm +++ b/gnu/packages/haskell.scm @@ -18,6 +18,7 @@ ;;; Copyright © 2018 Arun Isaac ;;; Copyright © 2018, 2019 Gabriel Hondet ;;; Copyright © 2019 Robert Vollmert +;;; Copyright © 2019 Jacob MacDonald ;;; ;;; This file is part of GNU Guix. ;;; @@ -11607,6 +11608,40 @@ default) @end itemize") (license license:bsd-3))) +(define-public ghc-concurrent-output + (package + (name "ghc-concurrent-output") + (version "1.10.10") + (source + (origin + (method url-fetch) + (uri (string-append + "mirror://hackage/package/concurrent-output/concurrent-output-" + version + ".tar.gz")) + (sha256 + (base32 + "1wnjxnwbc3l853kiiijagzjyb6fmhz3lmkwls24plbximl1qrr22")))) + (build-system haskell-build-system) + (inputs + `(("ghc-async" ,ghc-async) + ("ghc-stm" ,ghc-stm) + ("ghc-exceptions" ,ghc-exceptions) + ("ghc-ansi-terminal" ,ghc-ansi-terminal) + ("ghc-terminal-size" ,ghc-terminal-size))) + (home-page + "https://hackage.haskell.org/package/concurrent-output") + (synopsis + "Ungarble output from several threads or commands") + (description + "Lets multiple threads and external processes concurrently output to the +console, without it getting all garbled up. + +Built on top of that is a way of defining multiple output regions, which are +automatically laid out on the screen and can be individually updated by +concurrent threads. Can be used for progress displays etc.") + (license license:bsd-2))) + (define-public ghc-wl-pprint-annotated (package (name "ghc-wl-pprint-annotated") From 2c4e4bc6583579e0366849d588563d81e9802a0c Mon Sep 17 00:00:00 2001 From: Jacob MacDonald Date: Fri, 7 Jun 2019 05:42:59 -0500 Subject: [PATCH 41/44] gnu: Add ghc-hedgehog. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/haskell-check.scm (ghc-hedgehog): New variable. Signed-off-by: Ludovic Courtès --- gnu/packages/haskell-check.scm | 44 ++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/gnu/packages/haskell-check.scm b/gnu/packages/haskell-check.scm index 550bb13a656..9b010fd5eb5 100644 --- a/gnu/packages/haskell-check.scm +++ b/gnu/packages/haskell-check.scm @@ -822,3 +822,47 @@ minimal dependencies.") cryptographic ciphers, and is used by the test runners of various Haskell implementations of cryptographic ciphers.") (license license:bsd-3))) + +(define-public ghc-hedgehog + (package + (name "ghc-hedgehog") + (version "0.6.1") + (source + (origin + (method url-fetch) + (uri (string-append + "mirror://hackage/package/hedgehog/hedgehog-" + version + ".tar.gz")) + (sha256 + (base32 + "0xz10ycdm5vk9nrcym1fi83k19frfwqz18bz8bnpzwvaj0j41yfj")))) + (build-system haskell-build-system) + (inputs + `(("ghc-ansi-terminal" ,ghc-ansi-terminal-0.8) + ("ghc-async" ,ghc-async) + ("ghc-concurrent-output" ,ghc-concurrent-output) + ("ghc-exceptions" ,ghc-exceptions) + ("ghc-lifted-async" ,ghc-lifted-async) + ("ghc-mmorph" ,ghc-mmorph) + ("ghc-monad-control" ,ghc-monad-control) + ("ghc-pretty-show" ,ghc-pretty-show) + ("ghc-primitive" ,ghc-primitive) + ("ghc-random" ,ghc-random) + ("ghc-resourcet" ,ghc-resourcet) + ("ghc-semigroups" ,ghc-semigroups) + ("ghc-stm" ,ghc-stm) + ("ghc-th-lift" ,ghc-th-lift) + ("ghc-transformers-base" ,ghc-transformers-base) + ("ghc-wl-pprint-annotated" + ,ghc-wl-pprint-annotated))) + (home-page "https://hedgehog.qa") + (synopsis "Property-based testing in the spirt of QuickCheck") + (description + "Hedgehog is a property-based testing system, in the spirit of +QuickCheck. Hedgehog uses integrated shrinking, so shrinks obey the invariants +of generated values by construction. + +To get started quickly, see the examples: +@uref{https://github.com/hedgehogqa/haskell-hedgehog/tree/master/hedgehog-example}") + (license license:bsd-3))) From 252a4d59368d3fea9c63f12592044b856af20afe Mon Sep 17 00:00:00 2001 From: Jacob MacDonald Date: Fri, 7 Jun 2019 05:48:14 -0500 Subject: [PATCH 42/44] gnu: Add ghc-validation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/haskell.scm (ghc-validation): New variable. Signed-off-by: Ludovic Courtès --- gnu/packages/haskell.scm | 54 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/gnu/packages/haskell.scm b/gnu/packages/haskell.scm index 63d43601824..d3ff27412fa 100644 --- a/gnu/packages/haskell.scm +++ b/gnu/packages/haskell.scm @@ -11608,6 +11608,60 @@ default) @end itemize") (license license:bsd-3))) +(define-public ghc-validation + (package + (name "ghc-validation") + (version "1.1") + (source + (origin + (method url-fetch) + (uri (string-append + "mirror://hackage/package/validation/validation-" + version + ".tar.gz")) + (sha256 + (base32 + "1acj7mh3581ks405xswxw6667z7y1y0slisg6jvp6chc191ji9l5")))) + (build-system haskell-build-system) + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'add-setup-script + (lambda _ + ;; The usual "Setup.hs" script is missing from the source. + (with-output-to-file "Setup.hs" + (lambda () + (format #t "import Distribution.Simple~%") + (format #t "main = defaultMain~%")))))))) + (inputs + `(("ghc-semigroups" ,ghc-semigroups) + ("ghc-semigroupoids" ,ghc-semigroupoids) + ("ghc-bifunctors" ,ghc-bifunctors) + ("ghc-lens" ,ghc-lens))) + (native-inputs + `(("ghc-hedgehog" ,ghc-hedgehog) + ("ghc-hunit" ,ghc-hunit))) + (home-page "https://github.com/qfpl/validation") + (synopsis + "Data-type like Either but with an accumulating Applicative") + (description + "A data-type like Either but with differing properties and type-class +instances. + +Library support is provided for this different representation, including +@code{lens}-related functions for converting between each and abstracting over +their similarities. + +The @code{Validation} data type is isomorphic to @code{Either}, but has an +instance of @code{Applicative} that accumulates on the error side. That is to +say, if two (or more) errors are encountered, they are appended using a +@{Semigroup} operation. + +As a consequence of this @code{Applicative} instance, there is no +corresponding @code{Bind} or @code{Monad} instance. @code{Validation} is an +example of, \"An applicative functor that is not a monad.\"") + (license license:bsd-3))) + (define-public ghc-concurrent-output (package (name "ghc-concurrent-output") From c5ab53204eccf2983bfcdc8e6164d02a81e54d23 Mon Sep 17 00:00:00 2001 From: Katherine Cox-Buday Date: Fri, 7 Jun 2019 15:24:58 -0500 Subject: [PATCH 43/44] gnu: Add trivial-utf-8. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/lisp.scm (sbcl-trivial-utf-8, ecl-trivial-utf-8): New variables. Signed-off-by: Ludovic Courtès --- gnu/packages/lisp.scm | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/gnu/packages/lisp.scm b/gnu/packages/lisp.scm index 58813c2d842..37e375d3560 100644 --- a/gnu/packages/lisp.scm +++ b/gnu/packages/lisp.scm @@ -5665,6 +5665,45 @@ basic everyday functions and macros.") (define-public ecl-fare-utils (sbcl-package->ecl-package sbcl-fare-utils)) +(define-public sbcl-trivial-utf-8 + (let ((commit "4d427cfbb1c452436a0efb71c3205c9da67f718f") + (revision "1")) + (package + (name "sbcl-trivial-utf-8") + (version (git-version "0.0.0" revision commit)) + (source + (origin + (method git-fetch) + (uri + (git-reference + (url (string-append "https://gitlab.common-lisp.net/" + "trivial-utf-8/trivial-utf-8.git")) + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1jz27gz8gvqdmvp3k9bxschs6d5b3qgk94qp2bj6nv1d0jc3m1l1")))) + (arguments + ;; Guix incorrectly assumes the "8" is part of the version + ;; number and lobs it off. + `(#:asd-file "trivial-utf-8.asd" + #:asd-system-name "trivial-utf-8")) + (build-system asdf-build-system/sbcl) + (synopsis "UTF-8 input/output library") + (description + "The Babel library solves a similar problem while understanding more +encodings. Trivial UTF-8 was written before Babel existed, but for new +projects you might be better off going with Babel. The one plus that Trivial +UTF-8 has is that it doesn't depend on any other libraries.") + (home-page "https://common-lisp.net/project/trivial-utf-8/") + (license license:bsd-3)))) + +(define-public cl-trivial-utf-8 + (sbcl-package->cl-source-package sbcl-trivial-utf-8)) + +(define-public ecl-trivial-utf-8 + (sbcl-package->ecl-package sbcl-trivial-utf-8)) + (define-public sbcl-idna (package (name "sbcl-idna") From 2e03bbeb7be35faa1cb5b41c119d378d221266f3 Mon Sep 17 00:00:00 2001 From: Zzull Date: Sat, 15 Jun 2019 22:32:19 +0200 Subject: [PATCH 44/44] gnu: Add emacs-multi-term MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/packages/emacs-xyz.scm (emacs-multi-term): New variable Signed-off-by: Ludovic Courtès --- gnu/packages/emacs-xyz.scm | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/gnu/packages/emacs-xyz.scm b/gnu/packages/emacs-xyz.scm index a27de2f7780..d3377009142 100644 --- a/gnu/packages/emacs-xyz.scm +++ b/gnu/packages/emacs-xyz.scm @@ -16340,3 +16340,35 @@ themselves live in an Org-mode file. As such, this leverages the power of Org-mode (the notes may have outlines, latex fragments, babel, etc...) while acting like notes that are made @emph{in} the document.") (license license:gpl3+))) + +(define-public emacs-multi-term + (let ((commit "0804b11e52b960c80f5cd0712ee1e53ae70d83a4")) + (package + (name "emacs-multi-term") + (version "1.2") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/manateelazycat/multi-term.git") + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "0apvidmvb7rv05qjnjhax42ma8wrimik5vxx620dlbv17svz7iyf")))) + (build-system emacs-build-system) + (inputs `(("zsh" ,zsh))) + (home-page "https://github.com/manateelazycat/multi-term") + (synopsis "Manage multiple terminal buffers in Emacs") + (description + "This package enhances @code{term.el} with the following features: + +@enumerate +@item Functions to switch between multiple terminal buffers +@item List of keys to be intercepted by @code{emacs-multi-term} instead of by +the underlying terminal +@item Kills the unused buffer left after exiting the terminal +@item Kills the running sub-processes along with the terminal when killing the +it forcibly +@item Dedicated window for debugging program. +@end enumerate") + (license license:gpl3+))))