Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions api/machina/v1alpha3/machine_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,11 @@ const (
// For OnDelete updates this remains True until the operator deletes
// the Node and the agent repaves with the desired version.
MachineConditionRepavePending = "RepavePending"

// MachineConditionNVIDIAIMEXPrepared indicates whether the agent validated
// the prequalified host and exposed its IMEX devices to nspawn.
// It does not assert that a DRA ComputeDomain is ready.
MachineConditionNVIDIAIMEXPrepared = "NVIDIAIMEXPrepared"
)

// Annotation keys.
Expand Down
74 changes: 72 additions & 2 deletions api/machina/v1alpha3/machineconfiguration_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,14 +136,84 @@ type MachineConfigurationKubernetes struct {
type MachineConfigurationAgent struct {
// Image is the OCI image reference used for provisioning the
// nspawn machine (e.g. "ghcr.io/org/repo:tag").
// +kubebuilder:validation:Required
Image string `json:"image"`
// +optional
Image string `json:"image,omitempty"`

// LocalDNS configures the optional CoreDNS cache for this version.
// +optional
LocalDNS *LocalDNSSpec `json:"localDNS,omitempty"`

// NVIDIA configures optional NVIDIA host integration.
// +optional
NVIDIA *MachineConfigurationNVIDIA `json:"nvidia,omitempty"`
}

// MachineConfigurationNVIDIA configures NVIDIA host integration.
type MachineConfigurationNVIDIA struct {
// IMEX configures validation and exposure for multi-node NVLink.
// +optional
IMEX *MachineConfigurationNVIDIAIMEX `json:"imex,omitempty"`
}

// MachineConfigurationNVIDIAIMEX configures NVIDIA IMEX integration.
// +kubebuilder:validation:XValidation:rule="has(self.mode) == has(self.hostProfile)",message="hostProfile must be set exactly when mode DRA is set"
// +kubebuilder:validation:XValidation:rule="has(self.mode) == has(self.hostProfileVersion)",message="hostProfileVersion must be set exactly when mode DRA is set"
// +kubebuilder:validation:XValidation:rule="has(self.mode) == has(self.hostContractVersion)",message="hostContractVersion must be set exactly when mode DRA is set"
// +kubebuilder:validation:XValidation:rule="has(self.mode) == has(self.hostArtifactPath)",message="hostArtifactPath must be set exactly when mode DRA is set"
// +kubebuilder:validation:XValidation:rule="has(self.mode) == has(self.hostProfileDigest)",message="hostProfileDigest must be set exactly when mode DRA is set"
type MachineConfigurationNVIDIAIMEX struct {
// Mode selects the IMEX lifecycle owner. DRA exposes a prequalified node to
// NVIDIA DRA ComputeDomains; an empty value disables IMEX exposure.
// +optional
// +kubebuilder:validation:Enum=DRA
Mode NVIDIAIMEXMode `json:"mode,omitempty"`

// HostProfile is the immutable host qualification profile recorded by the
// provisioned image.
// +optional
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=63
// +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
HostProfile string `json:"hostProfile,omitempty"`

// HostProfileVersion is the expected version of HostProfile.
// +optional
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=63
// +kubebuilder:validation:Pattern=`^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$`
HostProfileVersion string `json:"hostProfileVersion,omitempty"`

// HostContractVersion is the exact qualification contract selected for the
// profile and version.
// +optional
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=128
HostContractVersion string `json:"hostContractVersion,omitempty"`

// HostArtifactPath is the exact canonical qualification artifact path
// selected for the profile contract.
// +optional
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=4096
// +kubebuilder:validation:Pattern=`^/.*$`
HostArtifactPath string `json:"hostArtifactPath,omitempty"`

// HostProfileDigest is the sha256 digest of the RFC 8785 canonical
// qualification document produced with the host image.
// +optional
// +kubebuilder:validation:Pattern=`^sha256:[a-f0-9]{64}$`
HostProfileDigest string `json:"hostProfileDigest,omitempty"`
}

// NVIDIAIMEXMode identifies the supported NVIDIA IMEX lifecycle owner.
type NVIDIAIMEXMode string

const (
// NVIDIAIMEXModeDRA delegates IMEX daemon and channel lifecycle to NVIDIA
// DRA ComputeDomains.
NVIDIAIMEXModeDRA NVIDIAIMEXMode = "DRA"
)

// MachineConfigurationUpdateStrategyType defines how configuration
// changes are applied to machines.
// +kubebuilder:validation:Enum=OnDelete;RollingUpdate
Expand Down
12 changes: 12 additions & 0 deletions api/machina/v1alpha3/machineoperation_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,18 @@ type MachineOperationTargetInput struct {
// than reselecting intent from a Machine edited during execution.
// +optional
ProvisioningFormat ProvisioningFormat `json:"provisioningFormat,omitempty"`

// ConfigurationVersionName is the exact MachineConfigurationVersion
// selected with HostImage for HostReplace. Retries reuse it so host and
// agent configuration cannot drift across version publication.
// +optional
ConfigurationVersionName string `json:"configurationVersionName,omitempty"`

// ConfigurationTemplate is the immutable versioned agent intent selected
// with HostImage. Snapshotting the data keeps HostReplace retries independent
// of Machine edits and MachineConfigurationVersion garbage collection.
// +optional
ConfigurationTemplate *MachineConfigurationTemplate `json:"configurationTemplate,omitempty"`
}

// ProviderMachineSnapshot identifies the exact provider-owned resource
Expand Down
45 changes: 45 additions & 0 deletions api/machina/v1alpha3/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

18 changes: 17 additions & 1 deletion cmd/agent/internal/cmd/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package cmd
import (
"context"
"encoding/json"
"errors"
"log/slog"
"net/url"
"strings"
Expand Down Expand Up @@ -69,7 +70,13 @@ func bootstrapIdentity(cfg *provision.UnboundedAgentConfig) (bootstrap.Identity,
KubernetesVersion string
OCIImage string
APIServer string
}{strings.TrimPrefix(cfg.Cluster.Version, "v"), canonicalImageIdentity(cfg.OCIImage), cfg.Kubelet.ApiServer})
NVIDIA *provision.NVIDIAConfig `json:",omitempty"`
}{
strings.TrimPrefix(cfg.Cluster.Version, "v"),
canonicalImageIdentity(cfg.OCIImage),
cfg.Kubelet.ApiServer,
cfg.NVIDIA,
})
if err != nil {
return bootstrap.Identity{}, err
}
Expand All @@ -82,6 +89,11 @@ func (s *agentStages) EnsureHostClean(ctx context.Context) error {
}

func (s *agentStages) ResolveInputs(ctx context.Context) error {
if s.reporter == nil && strings.TrimSpace(s.cfg.Kubelet.Auth.BootstrapToken) != "" {
s.reporter = daemon.NewBootstrapStatusReporter(ctx, s.log, &s.cfg.AgentConfig)
s.reporter.Running(ctx)
}

downloads, archives, err := provision.ResolveDownloadOverridesWithOfflineArtifacts(ctx, s.cfg)
if err != nil {
return err
Expand Down Expand Up @@ -259,6 +271,10 @@ func (s *agentStages) StageStarted(_ context.Context, stage bootstrap.Stage) {

func (s *agentStages) StageFailed(ctx context.Context, stage bootstrap.Stage, err error) {
reason := "Failed"
if errors.Is(err, goalstates.ErrNVIDIAHostUnqualified) {
reason = "HostQualificationFailed"
}

if stage == bootstrap.StagePrepareRootFS {
reason = "RootFSFailed"
}
Expand Down
7 changes: 7 additions & 0 deletions cmd/agent/internal/cmd/bootstrap_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,13 @@ func TestBootstrapFingerprintAllowsCredentialAndDownloadRefresh(t *testing.T) {

cfg.Cluster.Version, cfg.OCIImage, cfg.Kubelet.ApiServer = version, image, endpoint
}

cfg.NVIDIA = &provision.NVIDIAConfig{
IMEX: &provision.NVIDIAIMEXConfig{Mode: provision.NVIDIAIMEXModeDRA},
}
changed, err := bootstrapIdentity(cfg)
require.NoError(t, err)
require.NotEqual(t, original.ConfigFingerprint, changed.ConfigFingerprint)
}

func TestBootstrapFingerprintIgnoresSignedImageQuery(t *testing.T) {
Expand Down
71 changes: 71 additions & 0 deletions cmd/agent/internal/daemon/bootstrap_status.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"
"fmt"
"log/slog"
"slices"
"strings"

apimeta "k8s.io/apimachinery/pkg/api/meta"
Expand All @@ -29,6 +30,7 @@ type BootstrapStatusReporter struct {
client client.Client
machine string
ready bool
imexDRA bool
}

// NewBootstrapStatusReporter builds a best-effort Machine status reporter from
Expand All @@ -38,6 +40,7 @@ func NewBootstrapStatusReporter(ctx context.Context, log *slog.Logger, cfg *prov
reporter := &BootstrapStatusReporter{
log: log,
machine: cfg.MachineName,
imexDRA: cfg.IMEXDRAEnabled(),
}

if strings.TrimSpace(cfg.Kubelet.Auth.BootstrapToken) == "" {
Expand Down Expand Up @@ -71,6 +74,7 @@ func NewBootstrapStatusReporter(ctx context.Context, log *slog.Logger, cfg *prov
// Running reports that initial bootstrap is in progress.
func (r *BootstrapStatusReporter) Running(ctx context.Context) {
r.set(ctx, metav1.ConditionFalse, "Running", "unbounded-agent bootstrap is running")
r.setIMEX(ctx, metav1.ConditionFalse, "Validating", "NVIDIA IMEX DRA host qualification is being validated")
}

// Failed reports that initial bootstrap failed.
Expand All @@ -80,11 +84,78 @@ func (r *BootstrapStatusReporter) Failed(ctx context.Context, reason string, err
}

r.set(ctx, metav1.ConditionFalse, reason, truncateBootstrapConditionMessage(err.Error()))
r.setIMEX(ctx, metav1.ConditionFalse, reason, truncateBootstrapConditionMessage(err.Error()))
}

// Succeeded reports that initial bootstrap completed successfully.
func (r *BootstrapStatusReporter) Succeeded(ctx context.Context) {
r.set(ctx, metav1.ConditionTrue, "Succeeded", "unbounded-agent bootstrap completed successfully")
r.setIMEX(
ctx,
metav1.ConditionTrue,
"HostQualified",
"prequalified host evidence is valid and the nspawn boundary exposes NVIDIA DRA IMEX; this does not assert ComputeDomain readiness",
)
}

func (r *BootstrapStatusReporter) setIMEX(
ctx context.Context,
status metav1.ConditionStatus,
reason string,
message string,
) {
if r == nil || !r.ready {
return
}

if !r.imexDRA {
r.clearIMEX(ctx)

return
}

err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
var machine v1alpha3.Machine
if err := r.client.Get(ctx, client.ObjectKey{Name: r.machine}, &machine); err != nil {
return fmt.Errorf("get Machine %q: %w", r.machine, err)
}

apimeta.SetStatusCondition(&machine.Status.Conditions, metav1.Condition{
Type: v1alpha3.MachineConditionNVIDIAIMEXPrepared,
Status: status,
Reason: reason,
Message: message,
ObservedGeneration: machine.Generation,
})

return r.client.Status().Update(ctx, &machine)
})
if err != nil {
r.log.Warn("failed to update NVIDIAIMEXPrepared condition", "machine", r.machine, "reason", reason, "error", err)
}
}

func (r *BootstrapStatusReporter) clearIMEX(ctx context.Context) {
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
var machine v1alpha3.Machine
if err := r.client.Get(ctx, client.ObjectKey{Name: r.machine}, &machine); err != nil {
return fmt.Errorf("get Machine %q: %w", r.machine, err)
}

conditions := slices.DeleteFunc(machine.Status.Conditions, func(condition metav1.Condition) bool {
return condition.Type == v1alpha3.MachineConditionNVIDIAIMEXPrepared
})
if len(conditions) == len(machine.Status.Conditions) {
return nil
}

machine.Status.Conditions = conditions

return r.client.Status().Update(ctx, &machine)
})
if err != nil {
r.log.Warn("failed to clear NVIDIAIMEXPrepared condition", "machine", r.machine, "error", err)
}
}

func (r *BootstrapStatusReporter) set(ctx context.Context, status metav1.ConditionStatus, reason, message string) {
Expand Down
Loading
Loading