diff mbox series

[bug#35813] import: crate: add recursive option

Message ID 20190520182306.11899-1-Karlwfmeakin@gmail.com
State Accepted
Headers show
Series [bug#35813] import: crate: add recursive option | expand

Checks

Context Check Description
cbaines/applying patch fail Apply failed

Commit Message

Karl Meakin May 20, 2019, 6:23 p.m. UTC
* guix/script/import/crate.scm: Add recursive option.
* guix/import/crate.scm (crate-recursive-import): New variable.
---
 doc/guix.texi                 |  8 ++++++++
 guix/import/crate.scm         | 24 ++++++++++++++++--------
 guix/scripts/import/crate.scm | 27 ++++++++++++++++++++++-----
 3 files changed, 46 insertions(+), 13 deletions(-)

Comments

Ludovic Courtès May 24, 2019, 3:42 p.m. UTC | #1
Hi Karl,

Karl Meakin <karl.w.f.meakin@gmail.com> skribis:

> * guix/script/import/crate.scm: Add recursive option.
> * guix/import/crate.scm (crate-recursive-import): New variable.

Thanks for working on this!  Please also mention the guix.texi change
here.

I prefer to let Ivan and Danny comment on this.  My only question is:
would it be possible to augment ‘tests/crate.scm’ with tests for
recursive imports?  That would be great.

Thanks,
Ludo’.
Ivan Petkov May 25, 2019, 7:38 p.m. UTC | #2
Hi Karl,

Thanks for the patch, overall the implementation looks good to me!

> On May 24, 2019, at 8:42 AM, Ludovic Courtès <ludo@gnu.org> wrote:
> 
> would it be possible to augment ‘tests/crate.scm’ with tests for
> recursive imports?  That would be great.

I second Ludo’s comment here, having some tests around this would be great!

Also, have you had a chance to test this by importing a Rust application?
I tried to import hexyl (which has a small dependency closure compared to other
applications), but the final result had some missing crate input definitions
(e.g. Maybe a crate depends on rust-foo, but rust-foo wasn't defined in the
output either). Maybe I did something wrong, but it would be good to confirm!

Also, don't worry about actually trying to build any imported packages as of
yet, you might hit a cycle issue between proc-macro2 and quote. I've got a
patch open[0] for fixing the cargo-build-system to handle this!

--Ivan

[0]: https://debbugs.gnu.org/cgi/bugreport.cgi?bug=35318 <https://debbugs.gnu.org/cgi/bugreport.cgi?bug=35318>
Brian Leung July 20, 2019, 6:30 p.m. UTC | #3
What's the status on this? I was about to start working on a recursive
importer for Rust crates until I noticed this patch.
Ivan Petkov July 20, 2019, 9:43 p.m. UTC | #4
Hi Brian,

> On Jul 20, 2019, at 11:30 AM, Brian Leung <bkleung89@gmail.com> wrote:
> 
> What's the status on this? I was about to start working on a recursive importer for Rust crates until I noticed this patch.

On the patch side a couple of updates are still needed:
* a rebase on the latest master
* an update to also include the native-inputs/dev-dependency inputs in the loop for importing additional crates
* some tests to verify the behavior

Not sure if Karl has a had a chance to update this yet

—Ivan
diff mbox series

Patch

diff --git a/doc/guix.texi b/doc/guix.texi
index ae9ad0739e..636bb7521d 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -8787,6 +8787,14 @@  in Guix.
 Import metadata from the crates.io Rust package repository
 @uref{https://crates.io, crates.io}.
 
+@table @code
+@item --recursive
+@itemx -r
+Traverse the dependency graph of the given upstream package recursively
+and generate package expressions for all those packages that are not yet
+in Guix.
+@end table
+
 @item opam
 @cindex OPAM
 @cindex OCaml
diff --git a/guix/import/crate.scm b/guix/import/crate.scm
index e0b400d054..d8554b0e7a 100644
--- a/guix/import/crate.scm
+++ b/guix/import/crate.scm
@@ -37,6 +37,7 @@ 
   #:use-module (srfi srfi-26)
   #:export (crate->guix-package
             guix-package->crate-name
+            crate-recursive-import
             %crate-updater))
 
 (define (crate-fetch crate-name callback)
@@ -86,8 +87,8 @@ 
 VERSION, INPUTS, NATIVE-INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
   (let* ((port (http-fetch (crate-uri name version)))
          (guix-name (crate-name->package-name name))
-         (inputs (map crate-name->package-name inputs))
-         (native-inputs (map crate-name->package-name native-inputs))
+         (input-packages (map crate-name->package-name inputs))
+         (native-input-packages (map crate-name->package-name native-inputs))
          (pkg `(package
                    (name ,guix-name)
                    (version ,version)
@@ -99,8 +100,8 @@  VERSION, INPUTS, NATIVE-INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
                               (base32
                                ,(bytevector->nix-base32-string (port-sha256 port))))))
                    (build-system cargo-build-system)
-                   ,@(maybe-native-inputs native-inputs "src")
-                   ,@(maybe-inputs inputs "src")
+                   ,@(maybe-native-inputs native-input-packages "src")
+                   ,@(maybe-inputs input-packages "src")
                    (home-page ,(match home-page
                                  (() "")
                                  (_ home-page)))
@@ -111,12 +112,14 @@  VERSION, INPUTS, NATIVE-INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
                                ((license) license)
                                (_ `(list ,@license)))))))
          (close-port port)
-         pkg))
+         (values pkg inputs)))
 
-(define (crate->guix-package crate-name)
-  "Fetch the metadata for CRATE-NAME from crates.io, and return the
+(define crate->guix-package
+  (memoize
+   (lambda* (crate-name _)
+     "Fetch the metadata for CRATE-NAME from crates.io, and return the
 `package' s-expression corresponding to that package, or #f on failure."
-  (crate-fetch crate-name make-crate-sexp))
+     (crate-fetch crate-name make-crate-sexp))))
 
 (define (guix-package->crate-name package)
   "Return the crate name of PACKAGE."
@@ -158,6 +161,11 @@  VERSION, INPUTS, NATIVE-INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
      (version version)
      (urls (list url)))))
 
+(define* (crate-recursive-import package-name)
+  (recursive-import package-name #f
+                    #:repo->guix-package crate->guix-package
+                    #:guix-name crate-name->package-name))
+
 (define %crate-updater
   (upstream-updater
    (name 'crates)
diff --git a/guix/scripts/import/crate.scm b/guix/scripts/import/crate.scm
index cab9a4397b..8fadcdd57c 100644
--- a/guix/scripts/import/crate.scm
+++ b/guix/scripts/import/crate.scm
@@ -27,6 +27,7 @@ 
   #:use-module (srfi srfi-1)
   #:use-module (srfi srfi-11)
   #:use-module (srfi srfi-37)
+  #:use-module (srfi srfi-41)
   #:use-module (ice-9 match)
   #:use-module (ice-9 format)
   #:export (guix-import-crate))
@@ -45,6 +46,8 @@  Import and convert the crate.io package for PACKAGE-NAME.\n"))
   (display (G_ "
   -h, --help             display this help and exit"))
   (display (G_ "
+  -r, --recursive        import packages recursively"))
+  (display (G_ "
   -V, --version          display version information and exit"))
   (newline)
   (show-bug-report-information))
@@ -58,6 +61,9 @@  Import and convert the crate.io package for PACKAGE-NAME.\n"))
          (option '(#\V "version") #f #f
                  (lambda args
                    (show-version-and-exit "guix import crate")))
+         (option '(#\r "recursive") #f #f
+                 (lambda (opt name arg result)
+                   (alist-cons 'recursive #t result)))
          %standard-import-options))
 
 
@@ -83,11 +89,22 @@  Import and convert the crate.io package for PACKAGE-NAME.\n"))
                            (reverse opts))))
     (match args
       ((package-name)
-       (let ((sexp (crate->guix-package package-name)))
-         (unless sexp
-           (leave (G_ "failed to download meta-data for package '~a'~%")
-                  package-name))
-         sexp))
+       (if (assoc-ref opts 'recursive)
+           ;; Recursive import
+           (map (match-lambda
+                  ((and ('package ('name name) . rest) pkg)
+                   `(define-public ,(string->symbol name)
+                      ,pkg))
+                  (_ #f))
+                (reverse
+                 (stream->list
+                  (crate-recursive-import package-name))))
+           ;; Single import
+           (let ((sexp (crate->guix-package package-name #f)))
+             (unless sexp
+               (leave (G_ "failed to download meta-data for package '~a'~%")
+                      package-name))
+             sexp)))
       (()
        (leave (G_ "too few arguments~%")))
       ((many ...)