core: let user define start-/stop-timeout behaviour

jkloetzke · poettering · commit bf760801804e · 2020-06-09T10:04:57.000+02:00
The usual behaviour when a timeout expires is to terminate/kill the
service. This is what user usually want in production systems. To debug
services that fail to start/stop (especially sporadic failures) it
might be necessary to trigger the watchdog machinery and write core
dumps, though. Likewise, it is usually just a waste of time to
gracefully stop a stuck service. Instead it might save time to go
directly into kill mode.

This commit adds two new options to services: TimeoutStartFailureMode=
and TimeoutStopFailureMode=. Both take the same values and tweak the
behavior of systemd when a start/stop timeout expires:

 * 'terminate': is the default behaviour as it has always been,
 * 'abort': triggers the watchdog machinery and will send SIGABRT
   (unless WatchdogSignal was changed) and
 * 'kill' will directly send SIGKILL.

To handle the stop failure mode in stop-post state too a new
final-watchdog state needs to be introduced.
diff --git a/man/systemd.service.xml b/man/systemd.service.xml
@@ -560,16 +560,12 @@
 
       <varlistentry>
         <term><varname>TimeoutStartSec=</varname></term>
-        <listitem><para>Configures the time to wait for start-up. If a
-        daemon service does not signal start-up completion within the
-        configured time, the service will be considered failed and
-        will be shut down again. Takes a unit-less value in seconds,
-        or a time span value such as "5min 20s". Pass
-        <literal>infinity</literal> to disable the timeout logic. Defaults to
-        <varname>DefaultTimeoutStartSec=</varname> from the manager
-        configuration file, except when
-        <varname>Type=oneshot</varname> is used, in which case the
-        timeout is disabled by default (see
+        <listitem><para>Configures the time to wait for start-up. If a daemon service does not signal start-up
+        completion within the configured time, the service will be considered failed and will be shut down again. The
+        precise action depends on the <varname>TimeoutStartFailureMode=</varname> option. Takes a unit-less value in
+        seconds, or a time span value such as "5min 20s". Pass <literal>infinity</literal> to disable the timeout logic.
+        Defaults to <varname>DefaultTimeoutStartSec=</varname> from the manager configuration file, except when
+        <varname>Type=oneshot</varname> is used, in which case the timeout is disabled by default (see
         <citerefentry><refentrytitle>systemd-system.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>).
         </para>
 
@@ -588,7 +584,8 @@
         <listitem><para>This option serves two purposes. First, it configures the time to wait for each
         <varname>ExecStop=</varname> command. If any of them times out, subsequent <varname>ExecStop=</varname> commands
         are skipped and the service will be terminated by <constant>SIGTERM</constant>. If no <varname>ExecStop=</varname>
-        commands are specified, the service gets the <constant>SIGTERM</constant> immediately. Second, it configures the time
+        commands are specified, the service gets the <constant>SIGTERM</constant> immediately. This default behavior
+        can be changed by the <varname>TimeoutStopFailureMode=</varname> option. Second, it configures the time
         to wait for the service itself to stop. If it doesn't terminate in the specified time, it will be forcibly terminated
         by <constant>SIGKILL</constant> (see <varname>KillMode=</varname> in
         <citerefentry><refentrytitle>systemd.kill</refentrytitle><manvolnum>5</manvolnum></citerefentry>).
@@ -646,6 +643,28 @@
         </para></listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><varname>TimeoutStartFailureMode=</varname></term>
+        <term><varname>TimeoutStopFailureMode=</varname></term>
+
+        <listitem><para>These options configure the action that is taken in case a daemon service does not signal
+        start-up within its configured <varname>TimeoutStartSec=</varname>, respectively if it does not stop within
+        <varname>TimeoutStopSec=</varname>. Takes one of <option>terminate</option>, <option>abort</option> and
+        <option>kill</option>. Both options default to <option>terminate</option>.</para>
+
+        <para>If <option>terminate</option> is set the service will be gracefully terminated by sending the signal
+        specified in <varname>KillSignal=</varname> (defaults to <constant>SIGTERM</constant>, see
+        <citerefentry><refentrytitle>systemd.kill</refentrytitle><manvolnum>5</manvolnum></citerefentry>). If the
+        service does not terminate the <varname>FinalKillSignal=</varname> is sent after
+        <varname>TimeoutStopSec=</varname>. If <option>abort</option> is set, <varname>WatchdogSignal=</varname> is sent
+        instead and <varname>TimeoutAbortSec=</varname> applies before sending <varname>FinalKillSignal=</varname>.
+        This setting may be used to analyze services that fail to start-up or shut-down intermittently.
+        By using <option>kill</option> the service is immediately terminated by sending
+        <varname>FinalKillSignal=</varname> without any further timeout. This setting can be used to expedite the
+        shutdown of failing services.
+        </para></listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><varname>RuntimeMaxSec=</varname></term>
 
diff --git a/src/basic/unit-def.c b/src/basic/unit-def.c
@@ -185,6 +185,7 @@ static const char* const service_state_table[_SERVICE_STATE_MAX] = {
         [SERVICE_STOP_SIGTERM] = "stop-sigterm",
         [SERVICE_STOP_SIGKILL] = "stop-sigkill",
         [SERVICE_STOP_POST] = "stop-post",
+        [SERVICE_FINAL_WATCHDOG] = "final-watchdog",
         [SERVICE_FINAL_SIGTERM] = "final-sigterm",
         [SERVICE_FINAL_SIGKILL] = "final-sigkill",
         [SERVICE_FAILED] = "failed",
diff --git a/src/basic/unit-def.h b/src/basic/unit-def.h
@@ -127,6 +127,7 @@ typedef enum ServiceState {
         SERVICE_STOP_SIGTERM,
         SERVICE_STOP_SIGKILL,
         SERVICE_STOP_POST,
+        SERVICE_FINAL_WATCHDOG,    /* In case the STOP_POST executable needs to be aborted. */
         SERVICE_FINAL_SIGTERM,     /* In case the STOP_POST executable hangs, we shoot that down, too */
         SERVICE_FINAL_SIGKILL,
         SERVICE_FAILED,
diff --git a/src/core/dbus-service.c b/src/core/dbus-service.c
@@ -29,6 +29,7 @@ static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_notify_access, notify_access, N
 static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_emergency_action, emergency_action, EmergencyAction);
 static BUS_DEFINE_PROPERTY_GET(property_get_timeout_abort_usec, "t", Service, service_timeout_abort_usec);
 static BUS_DEFINE_PROPERTY_GET(property_get_watchdog_usec, "t", Service, service_get_watchdog_usec);
+static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_timeout_failure_mode, service_timeout_failure_mode, ServiceTimeoutFailureMode);
 
 static int property_get_exit_status_set(
                 sd_bus *bus,
@@ -101,6 +102,8 @@ const sd_bus_vtable bus_service_vtable[] = {
         SD_BUS_PROPERTY("TimeoutStartUSec", "t", bus_property_get_usec, offsetof(Service, timeout_start_usec), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("TimeoutStopUSec", "t", bus_property_get_usec, offsetof(Service, timeout_stop_usec), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("TimeoutAbortUSec", "t", property_get_timeout_abort_usec, 0, 0),
+        SD_BUS_PROPERTY("TimeoutStartFailureMode", "s", property_get_timeout_failure_mode, offsetof(Service, timeout_start_failure_mode), SD_BUS_VTABLE_PROPERTY_CONST),
+        SD_BUS_PROPERTY("TimeoutStopFailureMode", "s", property_get_timeout_failure_mode, offsetof(Service, timeout_stop_failure_mode), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("RuntimeMaxUSec", "t", bus_property_get_usec, offsetof(Service, runtime_max_usec), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("WatchdogUSec", "t", property_get_watchdog_usec, 0, 0),
         BUS_PROPERTY_DUAL_TIMESTAMP("WatchdogTimestamp", offsetof(Service, watchdog_timestamp), 0),
@@ -259,6 +262,7 @@ static BUS_DEFINE_SET_TRANSIENT_PARSE(service_type, ServiceType, service_type_fr
 static BUS_DEFINE_SET_TRANSIENT_PARSE(service_restart, ServiceRestart, service_restart_from_string);
 static BUS_DEFINE_SET_TRANSIENT_PARSE(oom_policy, OOMPolicy, oom_policy_from_string);
 static BUS_DEFINE_SET_TRANSIENT_STRING_WITH_CHECK(bus_name, sd_bus_service_name_is_valid);
+static BUS_DEFINE_SET_TRANSIENT_PARSE(timeout_failure_mode, ServiceTimeoutFailureMode, service_timeout_failure_mode_from_string);
 
 static int bus_service_set_transient_property(
                 Service *s,
@@ -316,6 +320,12 @@ static int bus_service_set_transient_property(
                 return r;
         }
 
+        if (streq(name, "TimeoutStartFailureMode"))
+                return bus_set_transient_timeout_failure_mode(u, name, &s->timeout_start_failure_mode, message, flags, error);
+
+        if (streq(name, "TimeoutStopFailureMode"))
+                return bus_set_transient_timeout_failure_mode(u, name, &s->timeout_stop_failure_mode, message, flags, error);
+
         if (streq(name, "RuntimeMaxUSec"))
                 return bus_set_transient_usec(u, name, &s->runtime_max_usec, message, flags, error);
 
diff --git a/src/core/load-fragment-gperf.gperf.m4 b/src/core/load-fragment-gperf.gperf.m4
@@ -322,6 +322,8 @@ Service.TimeoutSec,              config_parse_service_timeout,       0,
 Service.TimeoutStartSec,         config_parse_service_timeout,       0,                             0
 Service.TimeoutStopSec,          config_parse_sec_fix_0,             0,                             offsetof(Service, timeout_stop_usec)
 Service.TimeoutAbortSec,         config_parse_service_timeout_abort, 0,                             0
+Service.TimeoutStartFailureMode, config_parse_service_timeout_failure_mode,  0,                     offsetof(Service, timeout_start_failure_mode)
+Service.TimeoutStopFailureMode,  config_parse_service_timeout_failure_mode,  0,                     offsetof(Service, timeout_stop_failure_mode)
 Service.RuntimeMaxSec,           config_parse_sec,                   0,                             offsetof(Service, runtime_max_usec)
 Service.WatchdogSec,             config_parse_sec,                   0,                             offsetof(Service, watchdog_usec)
 m4_dnl The following five only exist for compatibility, they moved into Unit, see above
diff --git a/src/core/load-fragment.c b/src/core/load-fragment.c
@@ -123,6 +123,7 @@ DEFINE_CONFIG_PARSE_ENUM(config_parse_protect_system, protect_system, ProtectSys
 DEFINE_CONFIG_PARSE_ENUM(config_parse_runtime_preserve_mode, exec_preserve_mode, ExecPreserveMode, "Failed to parse runtime directory preserve mode");
 DEFINE_CONFIG_PARSE_ENUM(config_parse_service_type, service_type, ServiceType, "Failed to parse service type");
 DEFINE_CONFIG_PARSE_ENUM(config_parse_service_restart, service_restart, ServiceRestart, "Failed to parse service restart specifier");
+DEFINE_CONFIG_PARSE_ENUM(config_parse_service_timeout_failure_mode, service_timeout_failure_mode, ServiceTimeoutFailureMode, "Failed to parse timeout failure mode");
 DEFINE_CONFIG_PARSE_ENUM(config_parse_socket_bind, socket_address_bind_ipv6_only_or_bool, SocketAddressBindIPv6Only, "Failed to parse bind IPv6 only value");
 DEFINE_CONFIG_PARSE_ENUM(config_parse_oom_policy, oom_policy, OOMPolicy, "Failed to parse OOM policy");
 DEFINE_CONFIG_PARSE_ENUM_WITH_DEFAULT(config_parse_ip_tos, ip_tos, int, -1, "Failed to parse IP TOS value");
@@ -4941,6 +4942,7 @@ void unit_dump_config_items(FILE *f) {
                 { config_parse_exec,                  "PATH [ARGUMENT [...]]" },
                 { config_parse_service_type,          "SERVICETYPE" },
                 { config_parse_service_restart,       "SERVICERESTART" },
+                { config_parse_service_timeout_failure_mode, "TIMEOUTMODE" },
                 { config_parse_kill_mode,             "KILLMODE" },
                 { config_parse_signal,                "SIGNAL" },
                 { config_parse_socket_listen,         "SOCKET [...]" },
diff --git a/src/core/load-fragment.h b/src/core/load-fragment.h
@@ -30,6 +30,7 @@ CONFIG_PARSER_PROTOTYPE(config_parse_exec_coredump_filter);
 CONFIG_PARSER_PROTOTYPE(config_parse_exec);
 CONFIG_PARSER_PROTOTYPE(config_parse_service_timeout);
 CONFIG_PARSER_PROTOTYPE(config_parse_service_timeout_abort);
+CONFIG_PARSER_PROTOTYPE(config_parse_service_timeout_failure_mode);
 CONFIG_PARSER_PROTOTYPE(config_parse_service_type);
 CONFIG_PARSER_PROTOTYPE(config_parse_service_restart);
 CONFIG_PARSER_PROTOTYPE(config_parse_socket_bindtodevice);
diff --git a/src/core/service.c b/src/core/service.c
diff --git a/src/core/service.h b/src/core/service.h
diff --git a/src/shared/bus-unit-util.c b/src/shared/bus-unit-util.c
