From patchwork Tue Jul 16 09:15:36 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Ludovic_Court=C3=A8s?= X-Patchwork-Id: 66284 Return-Path: X-Original-To: patchwork@mira.cbaines.net Delivered-To: patchwork@mira.cbaines.net Received: by mira.cbaines.net (Postfix, from userid 113) id 2E95F27BBE9; Tue, 16 Jul 2024 10:19:23 +0100 (BST) X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on mira.cbaines.net X-Spam-Level: X-Spam-Status: No, score=-2.7 required=5.0 tests=BAYES_00,DKIM_INVALID, DKIM_SIGNED,MAILING_LIST_MULTI,SPF_HELO_PASS,URIBL_BLOCKED autolearn=ham autolearn_force=no version=3.4.6 Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mira.cbaines.net (Postfix) with ESMTPS id CB36A27BBE2 for ; Tue, 16 Jul 2024 10:19:22 +0100 (BST) Received: from localhost ([::1] helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1sTeKy-0003Cj-2G; Tue, 16 Jul 2024 05:19:12 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1sTeKp-0003AR-Qx for guix-patches@gnu.org; Tue, 16 Jul 2024 05:18:59 -0400 Received: from debbugs.gnu.org ([2001:470:142:5::43]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_128_GCM_SHA256:128) (Exim 4.90_1) (envelope-from ) id 1sTeKp-0003oe-Gs for guix-patches@gnu.org; Tue, 16 Jul 2024 05:18:59 -0400 Received: from Debian-debbugs by debbugs.gnu.org with local (Exim 4.84_2) (envelope-from ) id 1sTeKr-000548-OZ for guix-patches@gnu.org; Tue, 16 Jul 2024 05:19:01 -0400 X-Loop: help-debbugs@gnu.org Subject: [bug#72137] [PATCH 1/2] syscalls: Add =?utf-8?b?4oCYbW9kZeKAmQ==?= parameter to =?utf-8?b?4oCYbG9jay1maWxl4oCZLg==?= Resent-From: Ludovic =?utf-8?q?Court=C3=A8s?= Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Tue, 16 Jul 2024 09:19:01 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 72137 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: patch To: 72137@debbugs.gnu.org Cc: Ludovic =?utf-8?q?Court=C3=A8s?= Received: via spool by 72137-submit@debbugs.gnu.org id=B72137.172112149519386 (code B ref 72137); Tue, 16 Jul 2024 09:19:01 +0000 Received: (at 72137) by debbugs.gnu.org; 16 Jul 2024 09:18:15 +0000 Received: from localhost ([127.0.0.1]:60968 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1sTeK0-00052O-1w for submit@debbugs.gnu.org; Tue, 16 Jul 2024 05:18:15 -0400 Received: from eggs.gnu.org ([209.51.188.92]:53248) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1sTeJx-00051p-Pa for 72137@debbugs.gnu.org; Tue, 16 Jul 2024 05:18:06 -0400 Received: from fencepost.gnu.org ([2001:470:142:3::e]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1sTeHk-0003TP-I9; Tue, 16 Jul 2024 05:15:48 -0400 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=gnu.org; s=fencepost-gnu-org; h=MIME-Version:References:In-Reply-To:Date:Subject:To: From; bh=zymSuu05wUsaPkj+VZXqKU0K5ZcilZM+hbGBPPkqzNs=; b=TMxYjNXVaiIV/YSijzSU sbh6GesukxRXJILCCmwg4LXkOHnamS2s20gifJAJAWwIpYnONOrsSsczXBc/Y82tswD+4aUJgxJbA Kqx5aWfWErGuenxL5M6TCnQxXieq/dwJJv8JUZUTsgufWUnRG6lE1AdXcoNZiPUdE9DNQE6D7RNl3 C5QofR9Fn0XhdtLBgZHODp5oLv07gpUl3kbVIya58LZ9+OA2NSxlbWga4vXLyuwWvnWfBqMeLtBXc oky+u2fTskgLxis2SVw8k9Jfs42wqHWYrvVWmITca/qg9RsGDPmgSOoO5SruJN6MPgFW6gZoPxf9Y RaT0eIEJL+y5hA==; From: Ludovic =?utf-8?q?Court=C3=A8s?= Date: Tue, 16 Jul 2024 11:15:36 +0200 Message-ID: X-Mailer: git-send-email 2.45.2 In-Reply-To: References: MIME-Version: 1.0 X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list X-BeenThere: guix-patches@gnu.org List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: guix-patches-bounces+patchwork=mira.cbaines.net@gnu.org Sender: guix-patches-bounces+patchwork=mira.cbaines.net@gnu.org X-getmail-retrieved-from-mailbox: Patches * guix/build/syscalls.scm (lock-file): Add ‘mode’ parameter and honor it. * tests/syscalls.scm ("lock-file + unlock-file"): New test. Change-Id: I113fb4a8b35dd8782b9c0991574e39a4b4393333 --- guix/build/syscalls.scm | 14 +++++++++----- tests/syscalls.scm | 13 +++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/guix/build/syscalls.scm b/guix/build/syscalls.scm index 39bcffd516..2c20edf058 100644 --- a/guix/build/syscalls.scm +++ b/guix/build/syscalls.scm @@ -1398,14 +1398,18 @@ (define fcntl-flock ;; Presumably we got EAGAIN or so. (throw 'flock-error err)))))) -(define* (lock-file file #:key (wait? #t)) - "Wait and acquire an exclusive lock on FILE. Return an open port." - (let ((port (open-file file "w0"))) - (fcntl-flock port 'write-lock #:wait? wait?) +(define* (lock-file file #:optional (mode "w0") + #:key (wait? #t)) + "Wait and acquire an exclusive lock on FILE. Return an open port according +to MODE." + (let ((port (open-file file mode))) + (fcntl-flock port + (if (output-port? port) 'write-lock 'read-lock) + #:wait? wait?) port)) (define (unlock-file port) - "Unlock PORT, a port returned by 'lock-file'." + "Unlock PORT, a port returned by 'lock-file', and close it." (fcntl-flock port 'unlock) (close-port port) #t) diff --git a/tests/syscalls.scm b/tests/syscalls.scm index 7cf67c060d..13f4f11721 100644 --- a/tests/syscalls.scm +++ b/tests/syscalls.scm @@ -383,6 +383,19 @@ (define perform-container-tests? (close-port file) result))))))))) +(test-equal "lock-file + unlock-file" + 'hello + (call-with-temporary-directory + (lambda (directory) + (let* ((file (in-vicinity directory "lock")) + (out (lock-file file #:wait? #f))) + (display "hello" out) + (unlock-file out) + (let* ((in (lock-file file "r0")) + (content (read in))) + (unlock-file in) + content))))) + (test-equal "set-thread-name" "Syscall Test" (let ((name (thread-name))) From patchwork Tue Jul 16 09:15:37 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Ludovic_Court=C3=A8s?= X-Patchwork-Id: 66285 Return-Path: X-Original-To: patchwork@mira.cbaines.net Delivered-To: patchwork@mira.cbaines.net Received: by mira.cbaines.net (Postfix, from userid 113) id 8B38C27BBEA; Tue, 16 Jul 2024 10:19:27 +0100 (BST) X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on mira.cbaines.net X-Spam-Level: X-Spam-Status: No, score=-2.7 required=5.0 tests=BAYES_00,DKIM_INVALID, DKIM_SIGNED,MAILING_LIST_MULTI,SPF_HELO_PASS,URIBL_BLOCKED autolearn=unavailable autolearn_force=no version=3.4.6 Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mira.cbaines.net (Postfix) with ESMTPS id 91FA227BBE2 for ; Tue, 16 Jul 2024 10:19:26 +0100 (BST) Received: from localhost ([::1] helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1sTeL3-0003FK-LT; Tue, 16 Jul 2024 05:19:13 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1sTeKs-0003CI-1P for guix-patches@gnu.org; Tue, 16 Jul 2024 05:19:06 -0400 Received: from debbugs.gnu.org ([2001:470:142:5::43]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_128_GCM_SHA256:128) (Exim 4.90_1) (envelope-from ) id 1sTeKr-0003ok-O2; Tue, 16 Jul 2024 05:19:01 -0400 Received: from Debian-debbugs by debbugs.gnu.org with local (Exim 4.84_2) (envelope-from ) id 1sTeKs-00054F-88; Tue, 16 Jul 2024 05:19:02 -0400 X-Loop: help-debbugs@gnu.org Subject: [bug#72137] [PATCH 2/2] cache: Avoid cache cleanup storms from concurrent processes. Resent-From: Ludovic =?utf-8?q?Court=C3=A8s?= Original-Sender: "Debbugs-submit" Resent-CC: guix@cbaines.net, dev@jpoiret.xyz, ludo@gnu.org, othacehe@gnu.org, zimon.toutoune@gmail.com, me@tobias.gr, guix-patches@gnu.org Resent-Date: Tue, 16 Jul 2024 09:19:02 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 72137 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: patch To: 72137@debbugs.gnu.org Cc: Ludovic =?utf-8?q?Court=C3=A8s?= , Christopher Baines , Josselin Poiret , Ludovic =?utf-8?q?Court=C3=A8s?= , Mathieu Othacehe , Simon Tournier , Tobias Geerinckx-Rice X-Debbugs-Original-Xcc: Christopher Baines , Josselin Poiret , Ludovic =?utf-8?q?Court=C3=A8s?= , Mathieu Othacehe , Simon Tournier , Tobias Geerinckx-Rice Received: via spool by 72137-submit@debbugs.gnu.org id=B72137.172112151119414 (code B ref 72137); Tue, 16 Jul 2024 09:19:02 +0000 Received: (at 72137) by debbugs.gnu.org; 16 Jul 2024 09:18:31 +0000 Received: from localhost ([127.0.0.1]:60972 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1sTeK8-00052d-5s for submit@debbugs.gnu.org; Tue, 16 Jul 2024 05:18:30 -0400 Received: from eggs.gnu.org ([209.51.188.92]:43328) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1sTeK1-00052E-T8 for 72137@debbugs.gnu.org; Tue, 16 Jul 2024 05:18:10 -0400 Received: from fencepost.gnu.org ([2001:470:142:3::e]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1sTeHm-0003Th-5t; Tue, 16 Jul 2024 05:15:50 -0400 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=gnu.org; s=fencepost-gnu-org; h=MIME-Version:References:In-Reply-To:Date:Subject:To: From; bh=fftxX9awFlRpeAR97JvG8Rfi9msKGGI6CMneubSk01o=; b=Vum5DuPdUl5LqzeQWVLi F13ZgiArr+iRHRNPKxfGUapgP+oGFwRmaUupsUd4BfXAeX4+1oS3LPE3iEAdpXAYVVf2b4R91EDni AvRMsP5zMB+/O3fqx9tUI69QzO49FjThnkBNMOFijNS4rM9G7UjeYWBZp89FlChpyKzoDt8gywHv7 fwnv9ILjTwMWsBO2/uQwEAbeX5xkIItgmPe1nFdnjk88aWKtHpwZfRYRMfUASWerY0kU6CSe7d+dG 9lYGREncTtAlqvltZkeTvWV84inNQnFzumPVjGDNS8BXnhPsbb8G0x7Zj8UB+MTNmAeTAJhwdoGPU H1NpG1X6WWG3tQ==; From: Ludovic =?utf-8?q?Court=C3=A8s?= Date: Tue, 16 Jul 2024 11:15:37 +0200 Message-ID: <26f82a475d4ccd776c64c0a14628597dd65d8908.1721120884.git.ludo@gnu.org> X-Mailer: git-send-email 2.45.2 In-Reply-To: References: MIME-Version: 1.0 X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list X-BeenThere: guix-patches@gnu.org List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: guix-patches-bounces+patchwork=mira.cbaines.net@gnu.org Sender: guix-patches-bounces+patchwork=mira.cbaines.net@gnu.org X-getmail-retrieved-from-mailbox: Patches Reported by Christopher Baines . * guix/cache.scm (maybe-remove-expired-cache-entries): Define ‘expiry-port’; create it with ‘lock-file’. Change ‘last-expiry-date’ accordingly. Write timestamp straight to ‘expiry-port’. * tests/cache.scm ("maybe-remove-expired-cache-entries, cleanup needed but lock taken"): New test. Change-Id: I22441d9d2c4a339d3d3878de131864db5a0ae826 --- guix/cache.scm | 27 ++++++++++++++++++--------- tests/cache.scm | 30 +++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/guix/cache.scm b/guix/cache.scm index 6a91c7d3ef..8b12312c77 100644 --- a/guix/cache.scm +++ b/guix/cache.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013-2017, 2020-2021, 2023 Ludovic Courtès +;;; Copyright © 2013-2017, 2020-2021, 2023-2024 Ludovic Courtès ;;; Copyright © 2022 Simon Tournier ;;; ;;; This file is part of GNU Guix. @@ -19,6 +19,7 @@ (define-module (guix cache) #:use-module ((guix utils) #:select (with-atomic-file-output)) + #:autoload (guix build syscalls) (lock-file unlock-file) #:use-module (srfi srfi-19) #:use-module (srfi srfi-26) #:use-module (ice-9 match) @@ -93,13 +94,19 @@ (define* (maybe-remove-expired-cache-entries cache (define expiry-file (string-append cache "/last-expiry-cleanup")) + (define expiry-port + ;; Get exclusive access to EXPIRY-FILE to avoid "cleanup storms" where + ;; several processes would concurrently decide that time has come to clean + ;; up the same cache. 'lock-file' might throw to 'system-error' or to + ;; 'flock-error'; in either case, assume that we lost the race. + (false-if-exception + (lock-file expiry-file "a+0" #:wait? #f))) + (define last-expiry-date - (catch 'system-error - (lambda () - (or (string->number - (call-with-input-file expiry-file get-string-all)) - 0)) - (const 0))) + (if expiry-port + (or (string->number (get-string-all expiry-port)) + 0) + +inf.0)) (when (obsolete? last-expiry-date now cleanup-period) (remove-expired-cache-entries (cache-entries cache) @@ -108,8 +115,10 @@ (define* (maybe-remove-expired-cache-entries cache #:delete-entry delete-entry) (catch 'system-error (lambda () - (with-atomic-file-output expiry-file - (cute write (time-second now) <>))) + (seek expiry-port 0 SEEK_SET) + (truncate-file expiry-port 0) + (write (time-second now) expiry-port) + (unlock-file expiry-port)) (lambda args ;; ENOENT means CACHE does not exist. (unless (= ENOENT (system-error-errno args)) diff --git a/tests/cache.scm b/tests/cache.scm index d495ace2bd..e8ad083d40 100644 --- a/tests/cache.scm +++ b/tests/cache.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2017, 2020 Ludovic Courtès +;;; Copyright © 2017, 2020, 2024 Ludovic Courtès ;;; Copyright © 2022 Simon Tournier ;;; ;;; This file is part of GNU Guix. @@ -22,7 +22,9 @@ (define-module (test-cache) #:use-module (srfi srfi-1) #:use-module (srfi srfi-19) #:use-module (srfi srfi-64) + #:use-module ((guix build syscalls) #:select (lock-file)) #:use-module ((guix utils) #:select (call-with-temporary-directory)) + #:use-module ((rnrs io ports) #:select (get-string-all)) #:use-module (ice-9 match)) (test-begin "cache") @@ -75,6 +77,32 @@ (define-syntax-rule (test-cache-cleanup cache exp ...) (lambda (port) (display 0 port))))) +(let ((pid #f)) + (test-equal "maybe-remove-expired-cache-entries, cleanup needed but lock taken" + '() + (test-cache-cleanup cache + (let ((in+out (pipe))) + (match (primitive-fork) + (0 (dynamic-wind + (const #t) + (lambda () + (close-port (car in+out)) + (let ((port (lock-file + (string-append cache "/last-expiry-cleanup")))) + (display 0 port) + (display "done!\n" (cdr in+out)) + (close-port (cdr in+out)) + (sleep 100))) + (lambda () + (primitive-exit 0)))) + (n + (set! pid n) + (close-port (cdr in+out)) + (pk 'chr (get-string-all (car in+out))) + (close-port (car in+out))))))) + + (when pid (kill pid SIGKILL))) + (test-equal "maybe-remove-expired-cache-entries, empty cache" '("a" "b" "c") (test-cache-cleanup cache