diff mbox series

[bug#70169,v3,13/13] maint: Ensure generated file reproducibility for dist.

Message ID 0e305324621bdf2c04a6770d37f15efa85dcdb07.1712600307.git.janneke@gnu.org
State New
Headers show
Series Reproducible `make dist' tarball in defiance of Autotools and Gettext | expand

Commit Message

Janneke Nieuwenhuizen April 8, 2024, 6:46 p.m. UTC 
* doc/local.mk (doc-clean): New target.
(DIST_CONFIGURE_FLAGS): New variable.
(auto-clean): Use them in new target.
* Makefile.am (dist-doc-pot-update): Use it in new target.
(dist): Change to depend on it to clean possibly stale files, instead of
doc-pot-update directly.
Add a toplevel check to ensure that Autotools cache is up to date.

Change-Id: I2ff2d88db9fe1e708ab65e33e1f3d7ecee882cb4
---
 Makefile.am  | 28 +++++++++++++++++++++++++---
 doc/local.mk | 29 +++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+), 3 deletions(-)

Comments

pelzflorian (Florian Pelz) April 10, 2024, 4:35 p.m. UTC | #1 
Hello Janneke.

Janneke Nieuwenhuizen <janneke@gnu.org> writes:
> +auto-clean: maintainer-clean-vti doc-clean
> +	rm -f ABOUT-NLS INSTALL
> +	rm -f aclocal.m4 configure libtool Makefile.in
> +	git clean -fdx -- '.am*' build-aux m4 po

Could you enclose the call to git in
“if test -e .git; then … fi” as well?

I’ve switched to wip-tarball, but the hash is different.
It is still quite similar.

florian@floriandesktop ~/src/guix [env]$ diffoscope guix-1.3.0.57741-0e3053.tar.gz /tmp/guix-1.3.0.57741-0e3053.tar.gz 
--- guix-1.3.0.57741-0e3053.tar.gz
+++ /tmp/guix-1.3.0.57741-0e3053.tar.gz
├── guix-1.3.0.57741-0e3053.tar
│ ├── file list
│ │ @@ -215,25 +215,25 @@
│ │  -rw-r--r--   0        0        0    11898 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/local.mk
│ │  -rw-r--r--   0        0        0     2454 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/os-config-bare-bones.texi
│ │  -rw-r--r--   0        0        0     3960 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/os-config-desktop.texi
│ │  -rw-r--r--   0        0        0     2182 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/os-config-lightweight-desktop.texi
│ │  -rw-r--r--   0        0        0      838 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/package-hello.json
│ │  -rw-r--r--   0        0        0      615 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/package-hello.scm
│ │  -rw-r--r--   0        0        0      118 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-1
│ │ --rw-r--r--   0        0        0      123 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-2
│ │ --rw-r--r--   0        0        0      123 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-3
│ │ +-rw-r--r--   0        0        0      125 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-2
│ │ +-rw-r--r--   0        0        0      125 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-3
│ │  -rw-r--r--   0        0        0      118 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-4
│ │ --rw-r--r--   0        0        0      122 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-5
│ │ +-rw-r--r--   0        0        0      124 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-5
│ │  -rw-r--r--   0        0        0      118 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-6
│ │  -rw-r--r--   0        0        0      118 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-vti
│ │  -rw-r--r--   0        0        0      118 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/version-de.texi
│ │ --rw-r--r--   0        0        0      123 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/version-es.texi
│ │ --rw-r--r--   0        0        0      123 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/version-fr.texi
│ │ +-rw-r--r--   0        0        0      125 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/version-es.texi
│ │ +-rw-r--r--   0        0        0      125 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/version-fr.texi
│ │  -rw-r--r--   0        0        0      118 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/version-pt_BR.texi
│ │ --rw-r--r--   0        0        0      122 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/version-ru.texi
│ │ +-rw-r--r--   0        0        0      124 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/version-ru.texi
│ │  -rw-r--r--   0        0        0      118 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/version-zh_CN.texi
│ │  -rw-r--r--   0        0        0      118 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/version.texi
│ │  drwxr-xr-x   0        0        0        0 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/etc/
│ │  -rwxr-xr-x   0        0        0    19022 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/etc/committer.scm.in
│ │  drwxr-xr-x   0        0        0        0 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/etc/completion/
│ │  drwxr-xr-x   0        0        0        0 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/etc/completion/bash/
│ │  -rw-r--r--   0        0        0     9266 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/etc/completion/bash/guix
│ ├── guix-1.3.0.57741-0e3053/doc/stamp-2
│ │ @@ -1,4 +1,4 @@
│ │ -@set UPDATED 29 Februar 2024
│ │ -@set UPDATED-MONTH Februar 2024
│ │ +@set UPDATED 29 February 2024
│ │ +@set UPDATED-MONTH February 2024
│ │  @set EDITION .3.0.57741-0e3053
│ │  @set VERSION .3.0.57741-0e3053
│ ├── guix-1.3.0.57741-0e3053/doc/stamp-3
│ │ @@ -1,4 +1,4 @@
│ │ -@set UPDATED 29 Februar 2024
│ │ -@set UPDATED-MONTH Februar 2024
│ │ +@set UPDATED 29 February 2024
│ │ +@set UPDATED-MONTH February 2024
│ │  @set EDITION .3.0.57741-0e3053
│ │  @set VERSION .3.0.57741-0e3053
│ ├── guix-1.3.0.57741-0e3053/doc/stamp-5
│ │ @@ -1,4 +1,4 @@
│ │ -@set UPDATED 1 Februar 2024
│ │ -@set UPDATED-MONTH Februar 2024
│ │ +@set UPDATED 1 February 2024
│ │ +@set UPDATED-MONTH February 2024
│ │  @set EDITION .3.0.57741-0e3053
│ │  @set VERSION .3.0.57741-0e3053
│ ├── guix-1.3.0.57741-0e3053/doc/version-es.texi
│ │ @@ -1,4 +1,4 @@
│ │ -@set UPDATED 29 Februar 2024
│ │ -@set UPDATED-MONTH Februar 2024
│ │ +@set UPDATED 29 February 2024
│ │ +@set UPDATED-MONTH February 2024
│ │  @set EDITION .3.0.57741-0e3053
│ │  @set VERSION .3.0.57741-0e3053
│ ├── guix-1.3.0.57741-0e3053/doc/version-fr.texi
│ │ @@ -1,4 +1,4 @@
│ │ -@set UPDATED 29 Februar 2024
│ │ -@set UPDATED-MONTH Februar 2024
│ │ +@set UPDATED 29 February 2024
│ │ +@set UPDATED-MONTH February 2024
│ │  @set EDITION .3.0.57741-0e3053
│ │  @set VERSION .3.0.57741-0e3053
│ ├── guix-1.3.0.57741-0e3053/doc/version-ru.texi
│ │ @@ -1,4 +1,4 @@
│ │ -@set UPDATED 1 Februar 2024
│ │ -@set UPDATED-MONTH Februar 2024
│ │ +@set UPDATED 1 February 2024
│ │ +@set UPDATED-MONTH February 2024
│ │  @set EDITION .3.0.57741-0e3053
│ │  @set VERSION .3.0.57741-0e3053

I have used an de_DE locale where February is Februar.

Regards,
Florian
Janneke Nieuwenhuizen April 10, 2024, 5:25 p.m. UTC | #2 
pelzflorian (Florian Pelz) writes:

Hello Florian,

> Janneke Nieuwenhuizen <janneke@gnu.org> writes:
>> +auto-clean: maintainer-clean-vti doc-clean
>> +	rm -f ABOUT-NLS INSTALL
>> +	rm -f aclocal.m4 configure libtool Makefile.in
>> +	git clean -fdx -- '.am*' build-aux m4 po
>
> Could you enclose the call to git in
> “if test -e .git; then … fi” as well?

I've changed it to

	if test -e .git; then				\
	    git clean -fdx -- '.am*' build-aux m4 po;	\
	else						\
	    rm -rf .am*;				\
	    $(MAKE) -C po/guix maintainer-clean;	\
	    $(MAKE) -C po/packages maintainer-clean;	\
	fi

> I’ve switched to wip-tarball, but the hash is different.
> It is still quite similar.

> florian@floriandesktop ~/src/guix [env]$ diffoscope
> guix-1.3.0.57741-0e3053.tar.gz /tmp/guix-1.3.0.57741-0e3053.tar.gz
> --- guix-1.3.0.57741-0e3053.tar.gz
> +++ /tmp/guix-1.3.0.57741-0e3053.tar.gz
> ├── guix-1.3.0.57741-0e3053.tar
> │ ├── file list
[..]
> │ │ @@ -215,25 +215,25 @@

> │ │ --rw-r--r-- 0 0 0 123 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-2
> │ │ --rw-r--r-- 0 0 0 123 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-3
> │ │ +-rw-r--r-- 0 0 0 125 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-2
> │ │ +-rw-r--r-- 0 0 0 125 2024-04-08 18:08:07.000000 guix-1.3.0.57741-0e3053/doc/stamp-3
[..]

> guix-1.3.0.57741-0e3053/etc/completion/bash/guix
> │ ├── guix-1.3.0.57741-0e3053/doc/stamp-2
> │ │ @@ -1,4 +1,4 @@
> │ │ -@set UPDATED 29 Februar 2024
> │ │ -@set UPDATED-MONTH Februar 2024
> │ │ +@set UPDATED 29 February 2024
> │ │ +@set UPDATED-MONTH February 2024
> │ │  @set EDITION .3.0.57741-0e3053
> │ │  @set VERSION .3.0.57741-0e3053

[..]

> I have used an de_DE locale where February is Februar.

Wow, that's amazing news, and thanks for testing!  (I could have said:
well, using de_DE is /asking/ for such differences ;)

I tested with

    export LANG=de_DE LANGUAGE=fr_FR LC_TIME=pt_BR

and found similar errors, and only those.  In this case LC_TIME
prevailed.  So I made this change

--8<---------------cut here---------------start------------->8---
@@ -264,8 +264,8 @@ endif
 # Git rather than using metadata from the filesystem.
 define version.texi-from-git
 $(srcdir)/doc/stamp-$(1): $(srcdir)/$(2) $(top_srcdir)/configure
-	$$(AM_V_GEN)set -e						\
-	export LC_ALL=C;						\
+	$$(AM_V_GEN)set -e;						\
+	export LANG=C LANGUAGE=C LC_ALL=C LC_TIME=C; 			\
 	export TZ=UTC0;							\
 	timestamp="$$$$(git log --pretty=format:%ct -n1 -- "$$<"	\
 		2>/dev/null						\
--8<---------------cut here---------------end--------------->8---

and checked against a clean/LC_ALL=C locale.

Not sure if I should send a V4 for this?  Let's see if there are more
comments.

Greetings,
Janneke
pelzflorian (Florian Pelz) April 11, 2024, 11:32 a.m. UTC | #3 
Janneke Nieuwenhuizen <janneke@gnu.org> writes:
> @@ -264,8 +264,8 @@ endif
>  # Git rather than using metadata from the filesystem.
>  define version.texi-from-git
>  $(srcdir)/doc/stamp-$(1): $(srcdir)/$(2) $(top_srcdir)/configure
> -	$$(AM_V_GEN)set -e						\
> -	export LC_ALL=C;						\
> +	$$(AM_V_GEN)set -e;						\
> +	export LANG=C LANGUAGE=C LC_ALL=C LC_TIME=C; 			\
>  	export TZ=UTC0;							\
>  	timestamp="$$$$(git log --pretty=format:%ct -n1 -- "$$<"	\
>  		2>/dev/null						\

LGTM.  No v4 needed in my opinion.

Regards,
Florian
Janneke Nieuwenhuizen April 14, 2024, 9:24 a.m. UTC | #4 
pelzflorian (Florian Pelz) writes:

> Janneke Nieuwenhuizen <janneke@gnu.org> writes:
>> @@ -264,8 +264,8 @@ endif
>>  # Git rather than using metadata from the filesystem.
>>  define version.texi-from-git
>>  $(srcdir)/doc/stamp-$(1): $(srcdir)/$(2) $(top_srcdir)/configure
>> -	$$(AM_V_GEN)set -e						\
>> -	export LC_ALL=C;						\
>> +	$$(AM_V_GEN)set -e;						\
>> +	export LANG=C LANGUAGE=C LC_ALL=C LC_TIME=C; 			\
>>  	export TZ=UTC0;							\
>>  	timestamp="$$$$(git log --pretty=format:%ct -n1 -- "$$<"	\
>>  		2>/dev/null						\
>
> LGTM.  No v4 needed in my opinion.

Great, pushed to master as 416f11f1d4b2e12d8db2687e753d760f148cfc2d
diff mbox series

Patch

diff --git a/Makefile.am b/Makefile.am
index 7af88b2988..5f0809ee00 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -954,9 +954,31 @@  guix-binary.%.tar.xz:
 	cp "$$tarball" "$@.tmp" ; mv "$@.tmp" "$@"
 
 
-# The dependency on doc-pot-update is to generate the .pot files, which are
-# not checked in.
-dist: doc-pot-update
+# Assert that Autotools cache is up to date with Git, by checking
+# PACKAGE_VERSION against HEAD.  Indented to get past Automake.
+ ifeq ($(MAKECMDGOALS),dist)
+ git_version = $(shell build-aux/git-version-gen .tarball-version)
+ ifneq ($(PACKAGE_VERSION),$(git_version))
+ $(warning Autotools cache out of date.)
+ $(info Autotools cache version: $(PACKAGE_VERSION).)
+ $(info Git version: $(git_version).)
+ $(info Please run ./bootstrap && ./configure $(DIST_CONFIGURE_FLAGS))
+ ifneq ($(GUIX_ALLOW_IRREPRODUCIBLE_TARBALL),yes)
+ $(error Cannot create reproducible tarball)
+ else
+ $(warning Tarball will be irreproducible; distdir will not get removed!)
+ endif
+ endif
+ endif
+
+# The dependency on dist-doc-pot-update is to clean possibly stale doc and po
+# files and only then generate the .pot files, which are not checked in.
+dist: dist-doc-pot-update
+dist-doc-pot-update: auto-clean
+	$(MAKE) guile$(EXEEXT)
+	$(MAKE) -C po/guix all
+	$(MAKE) -C po/packages all
+	$(MAKE) doc-pot-update
 
 dist-hook: gen-ChangeLog gen-AUTHORS gen-tarball-version
 dist-hook: assert-no-store-file-names
diff --git a/doc/local.mk b/doc/local.mk
index 72f80bb511..f6d6640546 100644
--- a/doc/local.mk
+++ b/doc/local.mk
@@ -291,3 +291,32 @@  $(eval $(call version.texi-from-git,vti,doc/guix.texi,))
 $(foreach lang, $(MANUAL_LANGUAGES),							\
   $(eval i=$(shell echo $$(($(i)+1))))							\
   $(eval $(call version.texi-from-git,$(i),po/doc/guix-manual.$(lang).po,-$(lang))))
+
+DIST_CONFIGURE_FLAGS =				\
+  --localstatedir=/var				\
+  --sysconfdir=/etc
+
+# Delete all Autotools-generated files and rerun configure to ensure
+# a clean cache and distributing reproducible versions.
+auto-clean: maintainer-clean-vti doc-clean
+	rm -f ABOUT-NLS INSTALL
+	rm -f aclocal.m4 configure libtool Makefile.in
+	git clean -fdx -- '.am*' build-aux m4 po
+	rm -f guile
+	rm -f guix-daemon nix/nix-daemon/guix_daemon-guix-daemon.o
+# Automake fails if guix-cookbook-LANG.texi stubs are missing; running
+# autoreconf -vif is not enough.
+	./bootstrap
+# The dependency chain for the guix-cookbook-LANG.texi was cut on purpose;
+# they must be deleted to ensure a rebuild.
+	rm -f $(filter-out %D%/guix.texi %D%/guix-cookbook.texi, $(info_TEXINFOS))
+	./configure $(DIST_CONFIGURE_FLAGS)
+
+# Delete all generated doc files to ensure a clean cache and distributing
+# reproducible versions.
+doc-clean:
+	rm -f $(srcdir)/doc/*.1
+	rm -f $(srcdir)/doc/stamp*
+	rm -f $(DOT_FILES:%.dot=%.png)
+	rm -f $(DOT_VECTOR_GRAPHICS)
+	rm -f doc/images/coreutils-size-map.eps