build: separate stdlib manifests by version (#47596)

We install these by version, so it helps a lot if they reflect that in the
manifest name. Also some other mild cleanup.

Author: vtjnash

Make.inc

@@ -135,23 +135,6 @@ endif
 export BUILDROOT
 unexport O
 
-# Make sure the user didn't try to specify a path that will confuse the shell / make
-METACHARACTERS := ][?*{}() $$%:;&|!\#,\\`\":
-ifneq (,$(findstring ',$(value BUILDROOT)))
-$(error cowardly refusing to build into directory with a single-quote in the path)
-endif
-ifneq (,$(findstring ',$(value JULIAHOME)))
-$(error cowardly refusing to build from source directory with a single-quote in the path)
-endif
-ifneq (,$(shell echo '$(value BUILDROOT)' | grep '[$(METACHARACTERS)]'))
-$(error cowardly refusing to build into directory with a shell-metacharacter in the path\
-    (got: $(value BUILDROOT)))
-endif
-ifneq (,$(shell echo '$(value JULIAHOME)' | grep '[$(METACHARACTERS)]'))
-$(error cowardly refusing to build from source directory with a shell-metacharacter in the path\
-    (got: $(value JULIAHOME)))
-endif
-
 # we include twice to pickup user definitions better
 # include from JULIAHOME first so that BUILDROOT can override
 MAYBE_HOST :=

Makefile

@@ -3,6 +3,23 @@ include $(JULIAHOME)/Make.inc
 # import LLVM_SHARED_LIB_NAME
 include $(JULIAHOME)/deps/llvm-ver.make
 
+# Make sure the user didn't try to build in a path that will confuse the shell or make
+METACHARACTERS := [][?*{}() $$%:;&|!\#,\\`\":]\|/\./\|/\.\./
+ifneq (,$(findstring ',$(value BUILDROOT)))
+$(error cowardly refusing to build into directory with a single-quote in the path)
+endif
+ifneq (,$(findstring ',$(value JULIAHOME)))
+$(error cowardly refusing to build from source directory with a single-quote in the path)
+endif
+ifneq (,$(shell echo '$(value BUILDROOT)/' | grep '$(METACHARACTERS)'))
+$(error cowardly refusing to build into directory with a shell-metacharacter in the path\
+    (got: $(value BUILDROOT)))
+endif
+ifneq (,$(shell echo '$(value JULIAHOME)/' | grep '$(METACHARACTERS)'))
+$(error cowardly refusing to build from source directory with a shell-metacharacter in the path\
+    (got: $(value JULIAHOME)))
+endif
+
 VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION`
 
 default: $(JULIA_BUILD_MODE) # contains either "debug" or "release"
@@ -13,7 +30,7 @@ DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_priva
 ifneq ($(BUILDROOT),$(JULIAHOME))
 BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src src/flisp src/support src/clangsa cli doc deps stdlib test test/clangsa test/embedding test/llvmpasses)
 BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) $(BUILDROOT)/sysimage.mk
-DIRS := $(DIRS) $(BUILDDIRS)
+DIRS += $(BUILDDIRS)
 $(BUILDDIRMAKE): | $(BUILDDIRS)
 	@# add Makefiles to the build directories for convenience (pointing back to the source location of each)
 	@echo '# -- This file is automatically generated in julia/Makefile -- #' > $@
@@ -44,10 +61,6 @@ $(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$$(bu
 julia_flisp.boot.inc.phony: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony
 
-# Build the HTML docs (skipped if already exists, notably in tarballs)
-$(BUILDROOT)/doc/_build/html/en/index.html: $(shell find $(BUILDROOT)/base $(BUILDROOT)/doc \( -path $(BUILDROOT)/doc/_build -o -path $(BUILDROOT)/doc/deps -o -name *_constants.jl -o -name *_h.jl -o -name version_git.jl \) -prune -o -type f -print)
-	@$(MAKE) docs
-
 julia-symlink: julia-cli-$(JULIA_BUILD_MODE)
 ifeq ($(OS),WINNT)
 	echo '@"%~dp0/'"$$(echo '$(call rel_path,$(BUILDROOT),$(JULIA_EXECUTABLE))')"'" %*' | tr / '\\' > $(BUILDROOT)/julia.bat
@@ -236,7 +249,7 @@ define stringreplace
 endef
 
 
-install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html
+install: $(build_depsbindir)/stringreplace docs
 	@$(MAKE) $(QUIET_MAKE) $(JULIA_BUILD_MODE)
 	@for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir) $(libexecdir); do \
 		mkdir -p $(DESTDIR)$$subdir; \
@@ -484,7 +497,7 @@ app:
 darwinframework:
 	$(MAKE) -C $(JULIAHOME)/contrib/mac/framework
 
-light-source-dist.tmp: $(BUILDROOT)/doc/_build/html/en/index.html
+light-source-dist.tmp: docs
 ifneq ($(BUILDROOT),$(JULIAHOME))
 	$(error make light-source-dist does not work in out-of-tree builds)
 endif
@@ -561,12 +574,13 @@ distcleanall: cleanall
 	@-$(MAKE) -C $(BUILDROOT)/deps distcleanall
 	@-$(MAKE) -C $(BUILDROOT)/doc cleanall
 
-.PHONY: default debug release check-whitespace release-candidate \
+.FORCE:
+.PHONY: .FORCE default debug release check-whitespace release-candidate \
 	julia-debug julia-release julia-stdlib julia-deps julia-deps-libs \
 	julia-cli-release julia-cli-debug julia-src-release julia-src-debug \
 	julia-symlink julia-base julia-sysimg julia-sysimg-ji julia-sysimg-release julia-sysimg-debug \
-	test testall testall1 test test-* test-revise-* \
-	clean distcleanall cleanall clean-* \
+	test testall testall1 test \
+	clean distcleanall cleanall $(CLEAN_TARGETS) \
 	run-julia run-julia-debug run-julia-release run \
 	install binary-dist light-source-dist.tmp light-source-dist \
 	dist full-source-dist source-dist
@@ -583,12 +597,12 @@ testall: check-whitespace $(JULIA_BUILD_MODE)
 testall1: check-whitespace $(JULIA_BUILD_MODE)
 	@env JULIA_CPU_THREADS=1 $(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)
 
-test-%: check-whitespace $(JULIA_BUILD_MODE)
+test-%: check-whitespace $(JULIA_BUILD_MODE) .FORCE
 	@([ $$(( $$(date +%s) - $$(date -r $(build_private_libdir)/sys.$(SHLIB_EXT) +%s) )) -le 100 ] && \
 		printf '\033[93m    HINT The system image was recently rebuilt. Are you aware of the test-revise-* targets? See CONTRIBUTING.md. \033[0m\n') || true
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test $* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)
 
-test-revise-%:
+test-revise-%: .FORCE
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test revise-$* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)
 
 # download target for some hardcoded windows dependencies

cli/Makefile

@@ -70,8 +70,7 @@ dump-trampolines: $(SRCDIR)/trampolines/trampolines_$(ARCH).S
 	$(CC) $(SHIPFLAGS) $(LOADER_CFLAGS) $< -S | sed -E 's/ ((%%)|;) /\n/g' | sed -E 's/.global/\n.global/g'
 
 DIRS = $(build_bindir) $(build_libdir)
-$(DIRS):
-	@mkdir -p $@
+$(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
 
 ifeq ($(OS),WINNT)
 $(BUILDDIR)/julia_res.o: $(JULIAHOME)/contrib/windows/julia.rc $(JULIAHOME)/VERSION

deps/tools/common.mk

@@ -107,8 +107,8 @@ endif
 DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_includedir) $(build_sysconfdir) $(build_datarootdir) $(build_staging) $(build_prefix)/manifest)
 
 $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
-
 $(build_prefix): | $(DIRS)
+
 $(eval $(call dir_target,$(SRCCACHE)))
 
 
@@ -174,6 +174,7 @@ $$(build_prefix)/manifest/$(strip $1): $$(build_staging)/$2.tar | $(build_prefix
 	$(UNTAR) $$< -C $$(build_prefix)
 	$6
 	echo '$$(UNINSTALL_$(strip $1))' > $$@
+.PHONY: $(addsuffix -$(strip $1),stage install distclean uninstall reinstall)
 endef
 
 define staged-uninstaller
@@ -192,14 +193,18 @@ endef
 define symlink_install # (target-name, rel-from, abs-to)
 clean-$1: uninstall-$1
 install-$1: $$(build_prefix)/manifest/$1
-reinstall-$1: install-$1
+reinstall-$1:
+	+$$(MAKE) uninstall-$1
+	+$$(MAKE) stage-$1
+	+$$(MAKE) install-$1
+.PHONY: $(addsuffix -$1,clean install reinstall)
 
 UNINSTALL_$(strip $1) := $2 symlink-uninstaller $3
 
-$$(build_prefix)/manifest/$1: $$(BUILDDIR)/$2/build-compiled | $3 $$(build_prefix)/manifest
+$$(build_prefix)/manifest/$1: $$(BUILDDIR)/$2/build-compiled | $$(abspath $$(dir $3/$1)) $$(abspath $$(dir $$(build_prefix)/manifest/$1))
 	-+[ ! \( -e $3/$1 -o -h $3/$1 \) ] || $$(MAKE) uninstall-$1
 ifeq ($$(BUILD_OS), WINNT)
-	cmd //C mklink //J $$(call mingw_to_dos,$3/$1,cd $3 &&) $$(call mingw_to_dos,$$(BUILDDIR)/$2,)
+	cmd //C mklink //J $$(call mingw_to_dos,$3/$1,cd $3/$(dir $1) &&) $$(call mingw_to_dos,$$(BUILDDIR)/$2,)
 else ifneq (,$$(findstring CYGWIN,$$(BUILD_OS)))
 	cmd /C mklink /J $$(call cygpath_w,$3/$1) $$(call cygpath_w,$$(BUILDDIR)/$2)
 else ifdef JULIA_VAGRANT_BUILD
@@ -213,7 +218,7 @@ endef
 define symlink-uninstaller
 uninstall-$1:
 ifeq ($$(BUILD_OS), WINNT)
-	-cmd //C rmdir $$(call mingw_to_dos,$3/$1,cd $3 &&)
+	-cmd //C rmdir $$(call mingw_to_dos,$3/$1,cd $3/$(dir $1) &&)
 else
 	rm -rf $3/$1
 endif

deps/tools/git-external.mk

@@ -68,11 +68,12 @@ $5/$$($2_SRC_DIR)/source-extracted: $$($2_SRC_FILE)
 	$(TAR) -C $$(dir $$@) --strip-components 1 -xf $$<
 	echo 1 > $$@
 
-checksum-$(1): $$($2_SRC_FILE)
+checksum-$1: $$($2_SRC_FILE)
 	$$(JLCHECKSUM) $$<
 endif # DEPS_GIT
 
 $$(build_prefix)/manifest/$1: $$(SRCDIR)/$1.version # make the manifest stale if the version file is touched (causing re-install for compliant targets)
 distclean-$1:
 	rm -rf $5/$$($2_SRC_DIR) $$($2_SRC_FILE) $$(BUILDDIR)/$$($2_SRC_DIR)
+.PHONY: $(addsuffix -$1,checksum distclean)
 endef

deps/tools/stdlib-external.mk

@@ -16,12 +16,17 @@ $$(eval $$(call git-external,$1,$2,,,$$(BUILDDIR)))
 $$(BUILDDIR)/$$($2_SRC_DIR)/build-compiled: $$(BUILDDIR)/$$($2_SRC_DIR)/source-extracted
 	@# no build steps
 	echo 1 > $$@
-$$(eval $$(call symlink_install,$1,$$$$($2_SRC_DIR),$$$$(build_datarootdir)/julia/stdlib/$$$$(VERSDIR)))
+$$(eval $$(call symlink_install,$$$$(VERSDIR)/$1,$$$$($2_SRC_DIR),$$$$(build_datarootdir)/julia/stdlib))
 clean-$1:
 	-rm -f $$(BUILDDIR)/$$($2_SRC_DIR)/build-compiled
 get-$1: $$($2_SRC_FILE)
 extract-$1: $$(BUILDDIR)/$$($2_SRC_DIR)/source-extracted
 configure-$1: extract-$1
 compile-$1: $$(BUILDDIR)/$$($2_SRC_DIR)/build-compiled
-
+install-$1: install-$$(VERSDIR)/$1
+uninstall-$1: uninstall-$$(VERSDIR)/$1
+reinstall-$1: reinstall-$$(VERSDIR)/$1
+version-check-$1: version-check-$$(VERSDIR)/$1
+clean-$1: clean-$$(VERSDIR)/$1
+.PHONY: $(addsuffix -$1,get extract configure compile install uninstall reinstall clean)
 endef

deps/tools/uninstallers.mk

@@ -17,6 +17,7 @@ else
 uninstall-$1:
 	@echo "skipping uninstall: $1 not installed"
 endif
+.PHONY: uninstall-$1
 endef
 $(foreach dep,$(DEP_LIBS_STAGED_ALL),$(eval $(call define-uninstaller,$(dep))))
 

stdlib/Makefile

@@ -14,9 +14,8 @@ include $(JULIAHOME)/deps/*.version
 
 
 VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION)
-
-$(build_datarootdir)/julia/stdlib/$(VERSDIR):
-	mkdir -p $@
+DIRS := $(build_datarootdir)/julia/stdlib/$(VERSDIR) $(build_prefix)/manifest/$(VERSDIR)
+$(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
 
 JLLS = DSFMT GMP CURL LIBGIT2 LLVM LIBSSH2 LIBUV MBEDTLS MPFR NGHTTP2 \
        BLASTRAMPOLINE OPENBLAS OPENLIBM P7ZIP PCRE LIBSUITESPARSE ZLIB \
@@ -43,14 +42,19 @@ $(foreach jll,$(JLLS),$(eval $(call download-artifacts-toml,$(jll))))
 
 STDLIBS = Artifacts Base64 CRC32c Dates Distributed FileWatching \
           Future InteractiveUtils LazyArtifacts Libdl LibGit2 LinearAlgebra Logging \
-          Markdown Mmap Printf Profile Random REPL Serialization SHA \
-          SharedArrays Sockets SparseArrays SuiteSparse Test TOML Unicode UUIDs \
+          Markdown Mmap Printf Profile Random REPL Serialization \
+          SharedArrays Sockets Test TOML Unicode UUIDs \
           $(JLL_NAMES)
 
 STDLIBS_EXT = Pkg Statistics LibCURL Downloads ArgTools Tar NetworkOptions SuiteSparse SparseArrays SHA
 
 $(foreach module, $(STDLIBS_EXT), $(eval $(call stdlib-external,$(module),$(shell echo $(module) | tr a-z A-Z))))
 
+ifneq ($(filter $(STDLIBS),$(STDLIBS_EXT)),)
+$(error ERROR duplicated STDLIBS in list)
+endif
+
+
 # Generate symlinks to all stdlibs at usr/share/julia/stdlib/vX.Y/
 $(foreach module, $(STDLIBS), $(eval $(call symlink_target,$$(JULIAHOME)/stdlib/$(module),$$(build_datarootdir)/julia/stdlib/$$(VERSDIR),$(module))))
 
@@ -68,5 +72,5 @@ clean: $(addprefix clean-, $(STDLIBS_EXT)) $(CLEAN_TARGETS) extstdlibclean
 distclean: $(addprefix distclean-, $(STDLIBS_EXT)) clean
 checksumall: $(addprefix checksum-, $(STDLIBS_EXT))
 
-DEP_LIBS_STAGED_ALL := $(STDLIBS_EXT)
+DEP_LIBS_STAGED_ALL := $(addprefix $(VERSDIR)/,$(STDLIBS_EXT))
 include $(JULIAHOME)/deps/tools/uninstallers.mk

sysimage.mk

@@ -10,7 +10,7 @@ sysimg-bc: $(build_private_libdir)/sys-bc.a
 sysimg-release: $(build_private_libdir)/sys.$(SHLIB_EXT)
 sysimg-debug: $(build_private_libdir)/sys-debug.$(SHLIB_EXT)
 
-VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION`
+VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION)
 
 $(build_private_libdir)/%.$(SHLIB_EXT): $(build_private_libdir)/%-o.a
 	@$(call PRINT_LINK, $(CXX) $(LDFLAGS) -shared $(fPIC) -L$(build_private_libdir) -L$(build_libdir) -L$(build_shlibdir) -o $@ \
@@ -54,7 +54,7 @@ COMPILER_SRCS += $(shell find $(JULIAHOME)/base/compiler -name \*.jl)
 BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \
                     $(shell find $(BUILDROOT)/base -name \*.jl  -and -not -name sysimg.jl))
 STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(build_datarootdir)/julia/stdlib/$(VERSDIR)/*/src -name \*.jl) \
-                    $(build_prefix)/manifest/Pkg
+                    $(wildcard $(build_prefix)/manifest/$(VERSDIR)/*)
 RELBUILDROOT := $(call rel_path,$(JULIAHOME)/base,$(BUILDROOT)/base)/ # <-- make sure this always has a trailing slash
 
 $(build_private_libdir)/corecompiler.ji: $(COMPILER_SRCS)