From 29efbb24d992564db4bbb808597719934ed9ac9f Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 24 Sep 2019 16:29:58 -0700 Subject: docs: Use make invocation's -j argument for parallelism While sphinx 1.7 and later supports "-jauto" for parallelism, this effectively ignores the "-j" flag used in the "make" invocation, which may cause confusion for build systems. Instead, extract the available parallelism from "make"'s job server (since it is not exposed in any special variables) and use that for the "sphinx-build" run. Now things work correctly for builds where -j is specified at the top-level: make -j16 htmldocs If -j is not specified, continue to fallback to "-jauto" if available. Signed-off-by: Kees Cook Signed-off-by: Jonathan Corbet --- Documentation/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Documentation/Makefile') diff --git a/Documentation/Makefile b/Documentation/Makefile index e145e4db508b..c6e564656a5b 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -33,7 +33,7 @@ ifeq ($(HAVE_SPHINX),0) else # HAVE_SPHINX -export SPHINXOPTS = $(shell perl -e 'open IN,"sphinx-build --version 2>&1 |"; while () { if (m/([\d\.]+)/) { print "-jauto" if ($$1 >= "1.7") } ;} close IN') +export SPHINX_PARALLEL = $(shell perl -e 'open IN,"sphinx-build --version 2>&1 |"; while () { if (m/([\d\.]+)/) { print "auto" if ($$1 >= "1.7") } ;} close IN') # User-friendly check for pdflatex and latexmk HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi) @@ -68,6 +68,7 @@ quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(BUILDDIR)/$3/$4) PYTHONDONTWRITEBYTECODE=1 \ BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(srctree)/$(src)/$5/$(SPHINX_CONF)) \ $(SPHINXBUILD) \ + -j $(shell python $(srctree)/scripts/jobserver-count $(SPHINX_PARALLEL)) \ -b $2 \ -c $(abspath $(srctree)/$(src)) \ -d $(abspath $(BUILDDIR)/.doctrees/$3) \ -- cgit v1.2.3 From 61d221b735e80819814dbff3f014b27a457d297b Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 7 Oct 2019 09:38:58 -0600 Subject: docs: Fix "make help" suggestion for SPHINXDIR Commit 9fc3a18a942f ("docs: remove extra conf.py files") broke the setting of _SPHINXDIRS in Documentation/Makefile. Let's just have it look for an index.rst file instead. Fixes: 9fc3a18a942f ("docs: remove extra conf.py files") Reported-by: Randy Dunlap Tested-by: Randy Dunlap Signed-off-by: Jonathan Corbet --- Documentation/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Documentation/Makefile') diff --git a/Documentation/Makefile b/Documentation/Makefile index c6e564656a5b..ce8eb63b523a 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -13,7 +13,7 @@ endif SPHINXBUILD = sphinx-build SPHINXOPTS = SPHINXDIRS = . -_SPHINXDIRS = $(patsubst $(srctree)/Documentation/%/conf.py,%,$(wildcard $(srctree)/Documentation/*/conf.py)) +_SPHINXDIRS = $(patsubst $(srctree)/Documentation/%/index.rst,%,$(wildcard $(srctree)/Documentation/*/index.rst)) SPHINX_CONF = conf.py PAPER = BUILDDIR = $(obj)/output -- cgit v1.2.3 From a64c0440dda1fff1fb5723140828983d0ca821d4 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 25 Oct 2019 13:52:32 +0200 Subject: kbuild: Wrap long "make help" text lines Some "make help" text lines extend beyond 80 characters. Wrap them before an opening parenthesis, or before 80 characters. Signed-off-by: Geert Uytterhoeven Signed-off-by: Masahiro Yamada --- Documentation/Makefile | 6 ++++-- Makefile | 3 ++- scripts/Makefile.package | 3 ++- scripts/kconfig/Makefile | 3 ++- 4 files changed, 10 insertions(+), 5 deletions(-) (limited to 'Documentation/Makefile') diff --git a/Documentation/Makefile b/Documentation/Makefile index e145e4db508b..0c5185187dad 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -128,8 +128,10 @@ dochelp: @echo ' pdfdocs - PDF' @echo ' epubdocs - EPUB' @echo ' xmldocs - XML' - @echo ' linkcheckdocs - check for broken external links (will connect to external hosts)' - @echo ' refcheckdocs - check for references to non-existing files under Documentation' + @echo ' linkcheckdocs - check for broken external links' + @echo ' (will connect to external hosts)' + @echo ' refcheckdocs - check for references to non-existing files under' + @echo ' Documentation' @echo ' cleandocs - clean all generated files' @echo @echo ' make SPHINXDIRS="s1 s2" [target] Generate only docs of folder s1, s2' diff --git a/Makefile b/Makefile index 765a2b5bdada..6cd5af1e7fd4 100644 --- a/Makefile +++ b/Makefile @@ -1523,7 +1523,8 @@ help: @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build' @echo ' make V=2 [targets] 2 => give reason for rebuild of target' @echo ' make O=dir [targets] Locate all output files in "dir", including .config' - @echo ' make C=1 [targets] Check re-compiled c source with $$CHECK (sparse by default)' + @echo ' make C=1 [targets] Check re-compiled c source with $$CHECK' + @echo ' (sparse by default)' @echo ' make C=2 [targets] Force check of all c source with $$CHECK' @echo ' make RECORDMCOUNT_WARN=1 [targets] Warn about ignored mcount sections' @echo ' make W=n [targets] Enable extra build checks, n=1,2,3 where' diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 56eadcc48d46..ee9b368dfcf3 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -146,7 +146,8 @@ help: @echo ' binrpm-pkg - Build only the binary kernel RPM package' @echo ' deb-pkg - Build both source and binary deb kernel packages' @echo ' bindeb-pkg - Build only the binary kernel deb package' - @echo ' snap-pkg - Build only the binary kernel snap package (will connect to external hosts)' + @echo ' snap-pkg - Build only the binary kernel snap package' + @echo ' (will connect to external hosts)' @echo ' tar-pkg - Build the kernel as an uncompressed tarball' @echo ' targz-pkg - Build the kernel as a gzip compressed tarball' @echo ' tarbz2-pkg - Build the kernel as a bzip2 compressed tarball' diff --git a/scripts/kconfig/Makefile b/scripts/kconfig/Makefile index 1ce83269a5dc..588b55f9f618 100644 --- a/scripts/kconfig/Makefile +++ b/scripts/kconfig/Makefile @@ -137,7 +137,8 @@ help: @echo ' olddefconfig - Same as oldconfig but sets new symbols to their' @echo ' default value without prompting' @echo ' kvmconfig - Enable additional options for kvm guest kernel support' - @echo ' xenconfig - Enable additional options for xen dom0 and guest kernel support' + @echo ' xenconfig - Enable additional options for xen dom0 and guest kernel' + @echo ' support' @echo ' tinyconfig - Configure the tiniest possible kernel' @echo ' testconfig - Run Kconfig unit tests (requires python3 and pytest)' -- cgit v1.2.3 From 51e46c7a4007d271b2d42dbc2df953ab968577a7 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 21 Nov 2019 12:59:29 -0800 Subject: docs, parallelism: Rearrange how jobserver reservations are made Rasmus correctly observed that the existing jobserver reservation only worked if no other build targets were specified. The correct approach is to hold the jobserver slots until sphinx has finished. To fix this, the following changes are made: - refactor (and rename) scripts/jobserver-exec to set an environment variable for the maximally reserved jobserver slots and exec a child, to release the slots on exit. - create Documentation/scripts/parallel-wrapper.sh which examines both $PARALLELISM and the detected "-jauto" logic from Documentation/Makefile to decide sphinx's final -j argument. - chain these together in Documentation/Makefile Suggested-by: Rasmus Villemoes Link: https://lore.kernel.org/lkml/eb25959a-9ec4-3530-2031-d9d716b40b20@rasmusvillemoes.dk Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20191121205929.40371-4-keescook@chromium.org Signed-off-by: Jonathan Corbet --- Documentation/Makefile | 5 +-- Documentation/sphinx/parallel-wrapper.sh | 33 ++++++++++++++++ scripts/jobserver-count | 59 ---------------------------- scripts/jobserver-exec | 66 ++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 62 deletions(-) create mode 100644 Documentation/sphinx/parallel-wrapper.sh delete mode 100755 scripts/jobserver-count create mode 100755 scripts/jobserver-exec (limited to 'Documentation/Makefile') diff --git a/Documentation/Makefile b/Documentation/Makefile index ce8eb63b523a..30554a2fbdd7 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -33,8 +33,6 @@ ifeq ($(HAVE_SPHINX),0) else # HAVE_SPHINX -export SPHINX_PARALLEL = $(shell perl -e 'open IN,"sphinx-build --version 2>&1 |"; while () { if (m/([\d\.]+)/) { print "auto" if ($$1 >= "1.7") } ;} close IN') - # User-friendly check for pdflatex and latexmk HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi) HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi) @@ -67,8 +65,9 @@ quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(BUILDDIR)/$3/$4) cmd_sphinx = $(MAKE) BUILDDIR=$(abspath $(BUILDDIR)) $(build)=Documentation/media $2 && \ PYTHONDONTWRITEBYTECODE=1 \ BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(srctree)/$(src)/$5/$(SPHINX_CONF)) \ + $(PYTHON) $(srctree)/scripts/jobserver-exec \ + $(SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \ $(SPHINXBUILD) \ - -j $(shell python $(srctree)/scripts/jobserver-count $(SPHINX_PARALLEL)) \ -b $2 \ -c $(abspath $(srctree)/$(src)) \ -d $(abspath $(BUILDDIR)/.doctrees/$3) \ diff --git a/Documentation/sphinx/parallel-wrapper.sh b/Documentation/sphinx/parallel-wrapper.sh new file mode 100644 index 000000000000..7daf5133bdd3 --- /dev/null +++ b/Documentation/sphinx/parallel-wrapper.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0+ +# +# Figure out if we should follow a specific parallelism from the make +# environment (as exported by scripts/jobserver-exec), or fall back to +# the "auto" parallelism when "-jN" is not specified at the top-level +# "make" invocation. + +sphinx="$1" +shift || true + +parallel="$PARALLELISM" +if [ -z "$parallel" ] ; then + # If no parallelism is specified at the top-level make, then + # fall back to the expected "-jauto" mode that the "htmldocs" + # target has had. + auto=$(perl -e 'open IN,"'"$sphinx"' --version 2>&1 |"; + while () { + if (m/([\d\.]+)/) { + print "auto" if ($1 >= "1.7") + } + } + close IN') + if [ -n "$auto" ] ; then + parallel="$auto" + fi +fi +# Only if some parallelism has been determined do we add the -jN option. +if [ -n "$parallel" ] ; then + parallel="-j$parallel" +fi + +exec "$sphinx" "$parallel" "$@" diff --git a/scripts/jobserver-count b/scripts/jobserver-count deleted file mode 100755 index 7807bfa7dafa..000000000000 --- a/scripts/jobserver-count +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# SPDX-License-Identifier: GPL-2.0+ -# -# This determines how many parallel tasks "make" is expecting, as it is -# not exposed via an special variables. -# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver -from __future__ import print_function -import os, sys, fcntl, errno - -# Default parallelism is "1" unless overridden on the command-line. -default="1" -if len(sys.argv) > 1: - default=sys.argv[1] - -# Extract and prepare jobserver file descriptors from envirnoment. -try: - # Fetch the make environment options. - flags = os.environ['MAKEFLAGS'] - - # Look for "--jobserver=R,W" - # Note that GNU Make has used --jobserver-fds and --jobserver-auth - # so this handles all of them. - opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] - - # Parse out R,W file descriptor numbers and set them nonblocking. - fds = opts[0].split("=", 1)[1] - reader, writer = [int(x) for x in fds.split(",", 1)] - # Open a private copy of reader to avoid setting nonblocking - # on an unexpecting process with the same reader fd. - reader = os.open("/proc/self/fd/%d" % (reader), - os.O_RDONLY | os.O_NONBLOCK) -except (KeyError, IndexError, ValueError, IOError, OSError) as e: - print(e, file=sys.stderr) - # Any missing environment strings or bad fds should result in just - # using the default specified parallelism. - print(default) - sys.exit(0) - -# Read out as many jobserver slots as possible. -jobs = b"" -while True: - try: - slot = os.read(reader, 1) - jobs += slot - except (OSError, IOError) as e: - if e.errno == errno.EWOULDBLOCK: - # Stop when reach the end of the jobserver queue. - break - raise e -# Return all the reserved slots. -os.write(writer, jobs) - -# If the jobserver was (impossibly) full or communication failed, use default. -if len(jobs) < 1: - print(default) - sys.exit(0) - -# Report available slots (with a bump for our caller's reserveration). -print(len(jobs) + 1) diff --git a/scripts/jobserver-exec b/scripts/jobserver-exec new file mode 100755 index 000000000000..0fdb31a790a8 --- /dev/null +++ b/scripts/jobserver-exec @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0+ +# +# This determines how many parallel tasks "make" is expecting, as it is +# not exposed via an special variables, reserves them all, runs a subprocess +# with PARALLELISM environment variable set, and releases the jobs back again. +# +# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver +from __future__ import print_function +import os, sys, errno +import subprocess + +# Extract and prepare jobserver file descriptors from envirnoment. +claim = 0 +jobs = b"" +try: + # Fetch the make environment options. + flags = os.environ['MAKEFLAGS'] + + # Look for "--jobserver=R,W" + # Note that GNU Make has used --jobserver-fds and --jobserver-auth + # so this handles all of them. + opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] + + # Parse out R,W file descriptor numbers and set them nonblocking. + fds = opts[0].split("=", 1)[1] + reader, writer = [int(x) for x in fds.split(",", 1)] + # Open a private copy of reader to avoid setting nonblocking + # on an unexpecting process with the same reader fd. + reader = os.open("/proc/self/fd/%d" % (reader), + os.O_RDONLY | os.O_NONBLOCK) + + # Read out as many jobserver slots as possible. + while True: + try: + slot = os.read(reader, 8) + jobs += slot + except (OSError, IOError) as e: + if e.errno == errno.EWOULDBLOCK: + # Stop at the end of the jobserver queue. + break + # If something went wrong, give back the jobs. + if len(jobs): + os.write(writer, jobs) + raise e + # Add a bump for our caller's reserveration, since we're just going + # to sit here blocked on our child. + claim = len(jobs) + 1 +except (KeyError, IndexError, ValueError, OSError, IOError) as e: + # Any missing environment strings or bad fds should result in just + # not being parallel. + pass + +# We can only claim parallelism if there was a jobserver (i.e. a top-level +# "-jN" argument) and there were no other failures. Otherwise leave out the +# environment variable and let the child figure out what is best. +if claim > 0: + os.environ['PARALLELISM'] = '%d' % (claim) + +rc = subprocess.call(sys.argv[1:]) + +# Return all the reserved slots. +if len(jobs): + os.write(writer, jobs) + +sys.exit(rc) -- cgit v1.2.3