[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[gwl-devel] [PATCH] workflow: Consider unspecified free inputs when chec
From: |
Kyle Meyer |
Subject: |
[gwl-devel] [PATCH] workflow: Consider unspecified free inputs when checking cache. |
Date: |
Wed, 12 Jun 2019 23:48:21 -0400 |
When deciding whether a process is cached, we take into account the
modification time and size of free inputs, but only those that are
explicitly specified on the command line. As a result, the same
updated input file that would invalidate the cache when given
explicitly to --input does _not_ invalidate the cache when not
specified but instead picked from the current working directory.
Correct this discrepancy by including unspecified free inputs in the
cache prefix calculation.
* gwl/workflows.scm (workflow-run): Pass unspecified free inputs to
make-process->cache-prefix.
---
Note: This is mostly code movement, so something like
git show --color-moved=zebra --color-moved-ws=allow-indentation-change
might be helpful for reviewing.
gwl/workflows.scm | 87 +++++++++++++++++++++++++----------------------
1 file changed, 47 insertions(+), 40 deletions(-)
diff --git a/gwl/workflows.scm b/gwl/workflows.scm
index 50dc2ee..efdf089 100644
--- a/gwl/workflows.scm
+++ b/gwl/workflows.scm
@@ -262,46 +262,51 @@ (define* (workflow-run workflow engine
inputs)
(() '())
(mapping mapping)))
+
+ (define-values (input-names input-files)
+ (match inputs-map
+ (() (values '() '()))
+ (_ (apply values
+ (apply zip inputs-map)))))
+
+ (define unspecified-inputs
+ (lset-difference equal?
+ (workflow-free-inputs workflow)
+ input-names))
+
(define (inputs-valid?)
- (let-values (((input-names input-files)
- (match inputs-map
- (() (values '() '()))
- (_ (apply values
- (apply zip inputs-map))))))
- (match (lset-difference equal?
- (workflow-free-inputs workflow)
- input-names)
- (()
- ;; verify input files
- (match (filter (negate file-exists?) input-files)
- (()
- ;; Link all mapped input files to their target locations
- ;; TODO: ensure that target directories exist.
- (unless (null? inputs-map)
- (for-each (match-lambda
- ((target source)
- (unless (file-exists? target)
- (link source target))))
- inputs-map))
- #t)
- (missing
- (format (current-error-port)
- "Missing files: ~{~% * ~a~}.~%"
- missing)
- #f)))
- (missing
- ;; Try to find the files in the environment.
- ;; XXX Tell user that we pick the files from the current
- ;; working directory.
- ;; XXX These files would need to be mapped into the
- ;; container.
- (let* ((found (filter file-exists? missing))
- (really-missing (lset-difference equal? missing found)))
- (or (null? really-missing)
- (begin (format (current-error-port)
- "Missing inputs: ~{~% * ~a~}.~%Provide them
with --input=NAME=FILE.~%"
- really-missing)
- #f)))))))
+ (match unspecified-inputs
+ (()
+ ;; verify input files
+ (match (filter (negate file-exists?) input-files)
+ (()
+ ;; Link all mapped input files to their target locations
+ ;; TODO: ensure that target directories exist.
+ (unless (null? inputs-map)
+ (for-each (match-lambda
+ ((target source)
+ (unless (file-exists? target)
+ (link source target))))
+ inputs-map))
+ #t)
+ (missing
+ (format (current-error-port)
+ "Missing files: ~{~% * ~a~}.~%"
+ missing)
+ #f)))
+ (missing
+ ;; Try to find the files in the environment.
+ ;; XXX Tell user that we pick the files from the current
+ ;; working directory.
+ ;; XXX These files would need to be mapped into the
+ ;; container.
+ (let* ((found (filter file-exists? missing))
+ (really-missing (lset-difference equal? missing found)))
+ (or (null? really-missing)
+ (begin (format (current-error-port)
+ "Missing inputs: ~{~% * ~a~}.~%Provide them with
--input=NAME=FILE.~%"
+ really-missing)
+ #f))))))
(define ordered-processes
(workflow-run-order workflow #:parallel? parallel?))
(define (run)
@@ -309,7 +314,9 @@ (define* (workflow-run workflow engine
(runner (process-engine-runner engine)))
(define process->cache-prefix
(make-process->cache-prefix workflow
- inputs-map
+ (append inputs-map
+ (map (lambda (x) (list x x))
+ unspecified-inputs))
ordered-processes
engine))
(define cached?
--
2.22.0
- [gwl-devel] [PATCH] workflow: Consider unspecified free inputs when checking cache.,
Kyle Meyer <=