meson: allow WatchdogSec= in services to be configured

As discussed on systemd-devel [1], in Fedora we get lots of abrt reports
about the watchdog firing [2], but 100% of them seem to be caused by resource
starvation in the machine, and never actual deadlocks in the services being
monitored. Killing the services not only does not improve anything, but it
makes the resource starvation worse, because the service needs cycles to restart,
and coredump processing is also fairly expensive. This adds a configuration option
to allow the value to be changed. If the setting is not set, there is no change.

My plan is to set it to some ridiculusly high value, maybe 1h, to catch cases
where a service is actually hanging.

[1] https://lists.freedesktop.org/archives/systemd-devel/2019-October/043618.html
[2] https://bugzilla.redhat.com/show_bug.cgi?id=1300212

17 files changed, 23 insertions, 16 deletions

diff --git a/meson.build b/meson.build
index 1e27be7837..dc1fde60ee 100644
--- a/meson.build
+++ b/meson.build
@@ -795,6 +795,10 @@ conf.set_quoted('SYSTEMD_DEFAULT_LOCALE', default_locale)
 
 conf.set_quoted('GETTEXT_PACKAGE', meson.project_name())
 
+service_watchdog = get_option('service-watchdog')
+substs.set('SERVICE_WATCHDOG',
+           service_watchdog == '' ? '' : 'WatchdogSec=' + service_watchdog)
+
 substs.set('SUSHELL', get_option('debug-shell'))
 substs.set('DEBUGTTY', get_option('debug-tty'))
 conf.set_quoted('DEBUGTTY', get_option('debug-tty'))
@@ -3113,7 +3117,8 @@ status = [
         'default cgroup hierarchy:          @0@'.format(default_hierarchy),
         'default net.naming-scheme setting: @0@'.format(default_net_naming_scheme),
         'default KillUserProcesses setting: @0@'.format(kill_user_processes),
-        'default locale:                    @0@'.format(default_locale)]
+        'default locale:                    @0@'.format(default_locale),
+        'systemd service watchdog:          @0@'.format(service_watchdog == '' ? 'disabled' : service_watchdog)]
 
 alt_dns_servers = '\n                                            '.join(dns_servers.split(' '))
 alt_ntp_servers = '\n                                            '.join(ntp_servers.split(' '))
diff --git a/meson_options.txt b/meson_options.txt
index 5dc898eb80..0919577fd7 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -207,6 +207,8 @@ option('gshadow', type : 'boolean',
        description : 'support for shadow group')
 option('default-locale', type : 'string', value : '',
        description : 'default locale used when /etc/locale.conf does not exist')
+option('service-watchdog', type : 'string', value : '3min',
+       description : 'default watchdog setting for systemd services')
 
 option('default-dnssec', type : 'combo',
        description : 'default DNSSEC mode',
diff --git a/units/systemd-hostnamed.service.in b/units/systemd-hostnamed.service.in
index b4f606cf78..1fbbafdd6f 100644
--- a/units/systemd-hostnamed.service.in
+++ b/units/systemd-hostnamed.service.in
@@ -36,4 +36,4 @@ RestrictSUIDSGID=yes
 SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service sethostname
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
diff --git a/units/systemd-importd.service.in b/units/systemd-importd.service.in
index 38b7d7e94b..1a6fae4b69 100644
--- a/units/systemd-importd.service.in
+++ b/units/systemd-importd.service.in
@@ -15,7 +15,6 @@ Documentation=https://www.freedesktop.org/wiki/Software/systemd/importd
 [Service]
 ExecStart=@rootlibexecdir@/systemd-importd
 BusName=org.freedesktop.import1
-WatchdogSec=3min
 KillMode=mixed
 CapabilityBoundingSet=CAP_CHOWN CAP_FOWNER CAP_FSETID CAP_MKNOD CAP_SETFCAP CAP_SYS_ADMIN CAP_SETPCAP CAP_DAC_OVERRIDE
 NoNewPrivileges=yes
@@ -28,3 +27,4 @@ SystemCallFilter=@system-service @mount
 SystemCallErrorNumber=EPERM
 SystemCallArchitectures=native
 LockPersonality=yes
+@SERVICE_WATCHDOG@
diff --git a/units/systemd-journal-remote.service.in b/units/systemd-journal-remote.service.in
index dd6322e62c..7f5238802f 100644
--- a/units/systemd-journal-remote.service.in
+++ b/units/systemd-journal-remote.service.in
@@ -33,7 +33,7 @@ RestrictRealtime=yes
 RestrictSUIDSGID=yes
 SystemCallArchitectures=native
 User=systemd-journal-remote
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 # If there are many split up journal files we need a lot of fds to access them
 # all in parallel.
diff --git a/units/systemd-journal-upload.service.in b/units/systemd-journal-upload.service.in
index e3800473ec..33ef3b8dca 100644
--- a/units/systemd-journal-upload.service.in
+++ b/units/systemd-journal-upload.service.in
@@ -31,7 +31,7 @@ StateDirectory=systemd/journal-upload
 SupplementaryGroups=systemd-journal
 SystemCallArchitectures=native
 User=systemd-journal-upload
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 # If there are many split up journal files we need a lot of fds to access them
 # all in parallel.
diff --git a/units/systemd-journald.service.in b/units/systemd-journald.service.in
index 089bc38f59..303d5a4826 100644
--- a/units/systemd-journald.service.in
+++ b/units/systemd-journald.service.in
@@ -37,7 +37,7 @@ SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service
 Type=notify
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 # If there are many split up journal files we need a lot of fds to access them
 # all in parallel.
diff --git a/units/systemd-localed.service.in b/units/systemd-localed.service.in
index 7bca34409a..f9a81fa8dd 100644
--- a/units/systemd-localed.service.in
+++ b/units/systemd-localed.service.in
@@ -37,4 +37,4 @@ RestrictSUIDSGID=yes
 SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
diff --git a/units/systemd-logind.service.in b/units/systemd-logind.service.in
index c6f5b81c1d..ef802a4e6f 100644
--- a/units/systemd-logind.service.in
+++ b/units/systemd-logind.service.in
@@ -55,7 +55,7 @@ StateDirectory=systemd/linger
 SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 # Increase the default a bit in order to allow many simultaneous logins since
 # we keep one fd open per session.
diff --git a/units/systemd-machined.service.in b/units/systemd-machined.service.in
index d6deefea08..3db0281f81 100644
--- a/units/systemd-machined.service.in
+++ b/units/systemd-machined.service.in
@@ -29,7 +29,7 @@ RestrictRealtime=yes
 SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service @mount
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 # Note that machined cannot be placed in a mount namespace, since it
 # needs access to the host's mount namespace in order to implement the
diff --git a/units/systemd-networkd.service.in b/units/systemd-networkd.service.in
index 5c6275e5b3..ed985f64fa 100644
--- a/units/systemd-networkd.service.in
+++ b/units/systemd-networkd.service.in
@@ -44,7 +44,7 @@ SystemCallFilter=@system-service
 Type=notify
 RestartKillSignal=SIGUSR2
 User=systemd-network
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 [Install]
 WantedBy=multi-user.target
diff --git a/units/systemd-nspawn@.service.in b/units/systemd-nspawn@.service.in
index 2473a730b4..669fea3c12 100644
--- a/units/systemd-nspawn@.service.in
+++ b/units/systemd-nspawn@.service.in
@@ -23,10 +23,10 @@ KillMode=mixed
 Type=notify
 RestartForceExitStatus=133
 SuccessExitStatus=133
-WatchdogSec=3min
 Slice=machine.slice
 Delegate=yes
 TasksMax=16384
+@SERVICE_WATCHDOG@
 
 # Enforce a strict device policy, similar to the one nspawn configures when it
 # allocates its own scope unit. Make sure to keep these policies in sync if you
diff --git a/units/systemd-portabled.service.in b/units/systemd-portabled.service.in
index c88d3597b7..fb79f454fd 100644
--- a/units/systemd-portabled.service.in
+++ b/units/systemd-portabled.service.in
@@ -15,7 +15,6 @@ RequiresMountsFor=/var/lib/portables
 [Service]
 ExecStart=@rootlibexecdir@/systemd-portabled
 BusName=org.freedesktop.portable1
-WatchdogSec=3min
 CapabilityBoundingSet=CAP_KILL CAP_SYS_PTRACE CAP_SYS_ADMIN CAP_SETGID CAP_SYS_CHROOT CAP_DAC_READ_SEARCH CAP_DAC_OVERRIDE CAP_CHOWN CAP_FOWNER CAP_FSETID CAP_MKNOD
 MemoryDenyWriteExecute=yes
 ProtectHostname=yes
@@ -26,3 +25,4 @@ SystemCallErrorNumber=EPERM
 SystemCallArchitectures=native
 LockPersonality=yes
 IPAddressDeny=any
+@SERVICE_WATCHDOG@
diff --git a/units/systemd-resolved.service.in b/units/systemd-resolved.service.in
index eee5d5ea8f..22cb202363 100644
--- a/units/systemd-resolved.service.in
+++ b/units/systemd-resolved.service.in
@@ -46,7 +46,7 @@ SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service
 Type=notify
 User=systemd-resolve
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 [Install]
 WantedBy=multi-user.target
diff --git a/units/systemd-timedated.service.in b/units/systemd-timedated.service.in
index d430ee2017..819cb4dba2 100644
--- a/units/systemd-timedated.service.in
+++ b/units/systemd-timedated.service.in
@@ -36,4 +36,4 @@ RestrictSUIDSGID=yes
 SystemCallArchitectures=native
 SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service @clock
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
diff --git a/units/systemd-timesyncd.service.in b/units/systemd-timesyncd.service.in
index 2d8d14f6de..1a866fcc7a 100644
--- a/units/systemd-timesyncd.service.in
+++ b/units/systemd-timesyncd.service.in
@@ -46,7 +46,7 @@ SystemCallErrorNumber=EPERM
 SystemCallFilter=@system-service @clock
 Type=notify
 User=systemd-timesync
-WatchdogSec=3min
+@SERVICE_WATCHDOG@
 
 [Install]
 WantedBy=sysinit.target
diff --git a/units/systemd-udevd.service.in b/units/systemd-udevd.service.in
index c257af0efa..8b1dd0efc7 100644
--- a/units/systemd-udevd.service.in
+++ b/units/systemd-udevd.service.in
@@ -25,7 +25,6 @@ RestartSec=0
 ExecStart=@rootlibexecdir@/systemd-udevd
 ExecReload=@rootbindir@/udevadm control --reload --timeout 0
 KillMode=mixed
-WatchdogSec=3min
 TasksMax=infinity
 PrivateMounts=yes
 ProtectHostname=yes
@@ -38,3 +37,4 @@ SystemCallErrorNumber=EPERM
 SystemCallArchitectures=native
 LockPersonality=yes
 IPAddressDeny=any
+@SERVICE_WATCHDOG@