# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Backend spec for the Client API execution modes (V1-internal).
Design: docs/design/client_api_execution_modes.md ("Overview", "Execution Modes",
"Client API Backends"). One ClientAPIExecutor delegates to one mode-specific backend:
- in_process: trainer runs inside the Client Job (CJ) process over DataBus
- external_process: NVFlare launches and owns the trainer process tree over Cell
- attach: externally owned trainer rendezvous over Cell
This module is internal to NVFlare. It is not a user extension point; users configure
``ClientAPIExecutor(execution_mode=...)`` only.
"""
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional
from nvflare.apis.fl_context import FLContext
from nvflare.apis.shareable import Shareable
from nvflare.apis.signal import Signal
from nvflare.app_common.app_constant import AppConstants
from nvflare.client.config import ExchangeFormat, TransferType
from nvflare.utils.argv_utils import CommandArg
if TYPE_CHECKING:
# Import for typing only; avoids a runtime import cycle
# (client_api_executor imports this module to build the context).
from nvflare.app_common.executors.client_api_executor import ClientAPIExecutor
[docs]
@dataclass(frozen=True)
class ClientAPIBackendContext:
"""Immutable config a ClientAPIExecutor hands to its backend at ``initialize()``.
Rationale: the executor's frozen constructor args live in private attributes and the backend
factories are zero-arg, so a backend previously had no clean way to read the heartbeat,
timeout, model-state, and task-name config it needs, nor a supported reference back to the
executor's analytics hook. This frozen snapshot is that supported channel - a backend reads its config from here
rather than reaching into ``ClientAPIExecutor`` private attributes.
The fields contain only settings consumed by the implemented backends. ``executor`` is a
back-reference so a backend can:
- call ``executor.fire_log_analytics(fl_ctx, dxo)`` for every trainer LOG message (the single
LOG-to-analytics ownership point; see design "Configuration Surface"), and
- select the federation-scoped analytics path when appropriate by calling
``executor.set_analytics_fire_fed_event(True)`` in ``initialize()`` (Cell backends do this
when no ConvertToFedEvent widget is configured), and
- use the executor's FLComponent logging helpers.
"""
executor: "ClientAPIExecutor"
# in_process entry point
task_script_path: Optional[str] = None
task_script_args: Optional[CommandArg] = ""
# external_process launch
command: Optional[CommandArg] = None
launch_once: bool = True
launch_timeout: Optional[float] = 300.0
shutdown_timeout: Optional[float] = None
stop_grace_period: float = 30.0
# session / protocol (out-of-process)
heartbeat_interval: float = 5.0
heartbeat_timeout: float = 30.0
task_wait_timeout: Optional[float] = None
result_wait_timeout: Optional[float] = None
# attach rendezvous / lifecycle
attach_id: Optional[str] = None
attach_timeout: Optional[float] = None
allow_reconnect: bool = False
allow_insecure_attach: bool = False
# Declarative API-boundary representation contract. Backends transport these values in
# TASK_EXCHANGE; conversion happens in the trainer-side Client API, never in the executor.
params_exchange_format: ExchangeFormat = ExchangeFormat.RAW
server_expected_format: ExchangeFormat = ExchangeFormat.NUMPY
# FULL/DIFF is model state owned by the trainer-side Client API.
params_transfer_type: TransferType = TransferType.FULL
# task-name / rank contract (all modes)
train_task_name: str = AppConstants.TASK_TRAIN
evaluate_task_name: str = AppConstants.TASK_VALIDATION
submit_model_task_name: str = AppConstants.TASK_SUBMIT_MODEL
train_with_evaluation: bool = False
# memory management (all modes)
memory_gc_rounds: int = 0
cuda_empty_cache: bool = False
[docs]
class ClientAPIBackendSpec(ABC):
"""The narrow lifecycle contract that ClientAPIExecutor drives on its backend.
Lifecycle ownership per execution mode:
- in_process: the backend runs the trainer inside the CJ process and owns its thread.
- external_process: the backend launches and owns the external trainer process tree; it must
not stop the trainer before the payload transfer of a pending result reaches terminal state.
- attach: the backend owns only its protocol session. The trainer process is externally
owned and must never be signalled, terminated, or waited on by the backend.
"""
[docs]
@abstractmethod
def initialize(self, context: ClientAPIBackendContext, fl_ctx: FLContext) -> None:
"""Prepares the backend for the run. Called once when the executor handles START_RUN.
``context`` is the frozen snapshot of the executor's configuration plus a back-reference to
the executor (for ``fire_log_analytics`` and logging). A backend should read all of its
config from ``context`` rather than from executor private attributes, and should retain what
it needs for ``execute``/``finalize``.
The backend sets up its control plane here (DataBus wiring for in_process; Cell session
machinery, bootstrap config, and - per launch_once policy - trainer launch for
external_process).
Contract: raise an exception on any setup failure. The executor converts the exception
into system_panic so the job fails cleanly instead of hanging while tasks wait on a
backend that never became ready.
Cleanup-on-failure contract: ``initialize()`` must be exception-safe and self-unwinding - if
it raises, it must first release any partial setup it already made (threads started,
processes launched, listeners/tokens registered, files written). The executor does NOT call
``finalize()`` on a backend whose ``initialize()`` raised, because ``finalize()`` cannot
assume a consistently half-initialized backend. Own your own rollback.
Args:
context: the frozen backend configuration and executor back-reference.
fl_ctx: the FLContext of the START_RUN event.
"""
[docs]
@abstractmethod
def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable:
"""Executes one task on the trainer and returns its result.
The backend delivers the task to the trainer (TASK_READY over Cell, or DataBus for
in_process), waits for the result within the executor-configured task/result bounds, and
returns the result Shareable.
Contract: this method must always return a Shareable and must not hang past abort:
when abort_signal is triggered, the backend notifies/stops the trainer per its mode's
lifecycle ownership and returns ``make_reply(ReturnCode.TASK_ABORTED)``. On failure it
should return an error reply (e.g. ReturnCode.EXECUTION_EXCEPTION) rather than raise;
exceptions that do escape are converted to EXECUTION_EXCEPTION replies by the executor,
except UnsafeJobError which the executor lets propagate so ClientRunner can apply its
dedicated UNSAFE_JOB handling.
Args:
task_name: name of the task.
shareable: the task data.
fl_ctx: the FLContext of the task.
abort_signal: checked during execution; triggered means the task is aborted.
Returns:
The result Shareable (an error reply on failure/abort - never None).
"""
[docs]
def abort(self, fl_ctx: FLContext) -> None:
"""Notifies the backend that the run's active task is being aborted.
Most backends observe the task's ``abort_signal`` directly and need no
additional action. Backends that retain resources after ``execute()``
returns can override this hook to make their later cleanup abort-aware.
"""
[docs]
@abstractmethod
def finalize(self, fl_ctx: FLContext) -> None:
"""Releases backend resources. Called when the executor handles END_RUN.
The backend tears down per its mode's lifecycle ownership: stop the in-process trainer
thread; send SHUTDOWN and stop the owned process tree (honoring the executor's
shutdown_timeout and stop_grace_period, and pending payload terminal state) for
external_process.
Contract: must be idempotent and must not raise. Not called if ``initialize()`` raised
(see the cleanup-on-failure contract on ``initialize()``).
Args:
fl_ctx: the FLContext of the END_RUN event.
"""