From 827f6bce8f07d64681b86c1c852149b90a9e7c11 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Thu, 21 Nov 2024 18:36:59 +0100 Subject: [PATCH 01/27] go/oasis-test-runner/scenario/e2e/runtime: Add additional logging --- .../scenario/e2e/runtime/helpers_runtime.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/go/oasis-test-runner/scenario/e2e/runtime/helpers_runtime.go b/go/oasis-test-runner/scenario/e2e/runtime/helpers_runtime.go index a842228c3f2..ee90d642fe2 100644 --- a/go/oasis-test-runner/scenario/e2e/runtime/helpers_runtime.go +++ b/go/oasis-test-runner/scenario/e2e/runtime/helpers_runtime.go @@ -174,6 +174,12 @@ func (sc *Scenario) EnsureActiveVersionForComputeWorker(ctx context.Context, nod } // Retry if not yet activated. if cs.ActiveVersion.ToU64() < v.ToU64() { + sc.Logger.Warn("active version mismatch, waiting", + "node", node.Name, + "version", v, + "active_version", cs.ActiveVersion, + ) + time.Sleep(1 * time.Second) continue } @@ -181,6 +187,11 @@ func (sc *Scenario) EnsureActiveVersionForComputeWorker(ctx context.Context, nod return fmt.Errorf("%s: unexpected active version (expected: %s got: %s)", node.Name, v, cs.ActiveVersion) } if cs.Status != commonWorker.StatusStateReady { + sc.Logger.Warn("common worker not ready, waiting", + "node", node.Name, + "status", cs.Status, + ) + time.Sleep(1 * time.Second) continue } From c98c21ceb7a01032654e2502efeb300db7a76f66 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Sun, 17 Nov 2024 15:44:51 +0100 Subject: [PATCH 02/27] go/worker/common/committee/group: Use runtime ID instead of runtime --- go/worker/common/committee/group.go | 16 ++++++++-------- go/worker/common/committee/node.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go/worker/common/committee/group.go b/go/worker/common/committee/group.go index 4279cb85ddb..130eb2d2785 100644 --- a/go/worker/common/committee/group.go +++ b/go/worker/common/committee/group.go @@ -6,6 +6,7 @@ import ( "sync" beacon "github.com/oasisprotocol/oasis-core/go/beacon/api" + "github.com/oasisprotocol/oasis-core/go/common" "github.com/oasisprotocol/oasis-core/go/common/crypto/signature" "github.com/oasisprotocol/oasis-core/go/common/identity" "github.com/oasisprotocol/oasis-core/go/common/logging" @@ -13,7 +14,6 @@ import ( consensus "github.com/oasisprotocol/oasis-core/go/consensus/api" registry "github.com/oasisprotocol/oasis-core/go/registry/api" "github.com/oasisprotocol/oasis-core/go/runtime/nodes" - runtimeRegistry "github.com/oasisprotocol/oasis-core/go/runtime/registry" scheduler "github.com/oasisprotocol/oasis-core/go/scheduler/api" ) @@ -149,8 +149,8 @@ func (e *EpochSnapshot) Node(_ context.Context, id signature.PublicKey) (*node.N type Group struct { sync.RWMutex - identity *identity.Identity - runtime runtimeRegistry.Runtime + runtimeID common.Namespace + identity *identity.Identity consensus consensus.Backend @@ -207,7 +207,7 @@ func (g *Group) EpochTransition(ctx context.Context, height int64) error { // Request committees from scheduler. committees, err := g.consensus.Scheduler().GetCommittees(ctx, &scheduler.GetCommitteesRequest{ - RuntimeID: g.runtime.ID(), + RuntimeID: g.runtimeID, Height: height, }) if err != nil { @@ -268,7 +268,7 @@ func (g *Group) EpochTransition(ctx context.Context, height int64) error { } // Fetch current runtime descriptor. - runtime, err := g.consensus.Registry().GetRuntime(ctx, ®istry.GetRuntimeQuery{ID: g.runtime.ID(), Height: height}) + runtime, err := g.consensus.Registry().GetRuntime(ctx, ®istry.GetRuntimeQuery{ID: g.runtimeID, Height: height}) if err != nil { return err } @@ -321,8 +321,8 @@ func (g *Group) Start() error { // NewGroup creates a new group. func NewGroup( ctx context.Context, + runtimeID common.Namespace, identity *identity.Identity, - runtime runtimeRegistry.Runtime, consensus consensus.Backend, ) (*Group, error) { nw, err := nodes.NewVersionedNodeDescriptorWatcher(ctx, consensus) @@ -331,10 +331,10 @@ func NewGroup( } return &Group{ + runtimeID: runtimeID, identity: identity, - runtime: runtime, consensus: consensus, nodes: nw, - logger: logging.GetLogger("worker/common/committee/group").With("runtime_id", runtime.ID()), + logger: logging.GetLogger("worker/common/committee/group").With("runtime_id", runtimeID), }, nil } diff --git a/go/worker/common/committee/node.go b/go/worker/common/committee/node.go index 48fc62af25d..a35fa85db67 100644 --- a/go/worker/common/committee/node.go +++ b/go/worker/common/committee/node.go @@ -843,7 +843,7 @@ func NewNode( ctx, cancel := context.WithCancel(context.Background()) // Prepare committee group services. - group, err := NewGroup(ctx, identity, runtime, consensus) + group, err := NewGroup(ctx, runtime.ID(), identity, consensus) if err != nil { cancel() return nil, err From 549d54ff684f7b52241a328813d7b171f3c320cf Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Sun, 17 Nov 2024 22:05:54 +0100 Subject: [PATCH 03/27] go/runtime/history: Shorten name for has local storage worker flag --- go/runtime/history/history.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/go/runtime/history/history.go b/go/runtime/history/history.go index b9e01d0d608..076f6d1086a 100644 --- a/go/runtime/history/history.go +++ b/go/runtime/history/history.go @@ -126,7 +126,7 @@ type runtimeHistory struct { syncRoundLock sync.RWMutex lastStorageSyncedRound uint64 - haveLocalStorageWorker bool + hasLocalStorage bool pruner Pruner pruneCh *channels.RingChannel @@ -149,7 +149,7 @@ func (h *runtimeHistory) Commit(blk *roothash.AnnotatedBlock, roundResults *root // If no local storage worker, notify the block watcher that new block is committed, // otherwise the storage-sync-checkpoint will do the notification. - if h.haveLocalStorageWorker || !notify { + if h.hasLocalStorage || !notify { return nil } h.blocksNotifier.Broadcast(blk) @@ -167,7 +167,7 @@ func (h *runtimeHistory) StorageSyncCheckpoint(round uint64) error { return nil } - if !h.haveLocalStorageWorker { + if !h.hasLocalStorage { panic("received storage sync checkpoint when local storage worker is disabled") } @@ -249,7 +249,7 @@ func (h *runtimeHistory) resolveRound(round uint64, includeStorage bool) (uint64 h.syncRoundLock.RLock() defer h.syncRoundLock.RUnlock() // Also take storage sync state into account. - if includeStorage && h.haveLocalStorageWorker && h.lastStorageSyncedRound < meta.LastRound { + if includeStorage && h.hasLocalStorage && h.lastStorageSyncedRound < meta.LastRound { return h.lastStorageSyncedRound, nil } return meta.LastRound, nil @@ -257,7 +257,7 @@ func (h *runtimeHistory) resolveRound(round uint64, includeStorage bool) (uint64 h.syncRoundLock.RLock() defer h.syncRoundLock.RUnlock() // Ensure round exists. - if includeStorage && h.haveLocalStorageWorker && h.lastStorageSyncedRound < round { + if includeStorage && h.hasLocalStorage && h.lastStorageSyncedRound < round { return roothash.RoundInvalid, roothash.ErrNotFound } return round, nil @@ -368,7 +368,7 @@ func (h *runtimeHistory) pruneWorker() { } // New creates a new runtime history keeper. -func New(runtimeID common.Namespace, dataDir string, prunerFactory PrunerFactory, haveLocalStorageWorker bool) (History, error) { +func New(runtimeID common.Namespace, dataDir string, prunerFactory PrunerFactory, hasLocalStorage bool) (History, error) { db, err := newDB(filepath.Join(dataDir, DbFilename), runtimeID) if err != nil { return nil, err @@ -382,17 +382,17 @@ func New(runtimeID common.Namespace, dataDir string, prunerFactory PrunerFactory ctx, cancelCtx := context.WithCancel(context.Background()) h := &runtimeHistory{ - runtimeID: runtimeID, - logger: logging.GetLogger("runtime/history").With("runtime_id", runtimeID), - ctx: ctx, - cancelCtx: cancelCtx, - db: db, - haveLocalStorageWorker: haveLocalStorageWorker, - blocksNotifier: pubsub.NewBroker(true), - pruner: pruner, - pruneCh: channels.NewRingChannel(1), - stopCh: make(chan struct{}), - quitCh: make(chan struct{}), + runtimeID: runtimeID, + logger: logging.GetLogger("runtime/history").With("runtime_id", runtimeID), + ctx: ctx, + cancelCtx: cancelCtx, + db: db, + hasLocalStorage: hasLocalStorage, + blocksNotifier: pubsub.NewBroker(true), + pruner: pruner, + pruneCh: channels.NewRingChannel(1), + stopCh: make(chan struct{}), + quitCh: make(chan struct{}), } go h.pruneWorker() From e52eed0b965cadd945e90ff3f524913dad277d95 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Sun, 17 Nov 2024 12:18:58 +0100 Subject: [PATCH 04/27] go/runtime/registry/host: Simplify runtime host node --- go/runtime/registry/host.go | 141 +++++++++++------------------ go/runtime/registry/registry.go | 9 +- go/runtime/txpool/txpool.go | 21 +++-- go/worker/common/committee/node.go | 19 ++-- go/worker/keymanager/worker.go | 19 ++-- 5 files changed, 95 insertions(+), 114 deletions(-) diff --git a/go/runtime/registry/host.go b/go/runtime/registry/host.go index f2efda9ab52..b806e275b9f 100644 --- a/go/runtime/registry/host.go +++ b/go/runtime/registry/host.go @@ -27,98 +27,91 @@ const ( // RuntimeHostNode provides methods for nodes that need to host runtimes. type RuntimeHostNode struct { - sync.Mutex + agg *multi.Aggregate + rr host.RichRuntime + + runtime Runtime - factory RuntimeHostHandlerFactory notifier protocol.Notifier + handler host.RuntimeHandler + + provisioner host.Provisioner - agg *multi.Aggregate - runtime host.RichRuntime - runtimeNotify chan struct{} + runtimeNotify chan struct{} + runtimeNotifyOnce sync.Once } // NewRuntimeHostNode creates a new runtime host node. func NewRuntimeHostNode(factory RuntimeHostHandlerFactory) (*RuntimeHostNode, error) { + runtime := factory.GetRuntime() + agg := multi.New(runtime.ID()) + rr := host.NewRichRuntime(agg) + + notifier := factory.NewRuntimeHostNotifier(agg) + handler := factory.NewRuntimeHostHandler() + provisioner := runtime.HostProvisioner() + return &RuntimeHostNode{ - factory: factory, + agg: agg, + rr: rr, + runtime: runtime, + notifier: notifier, + handler: handler, + provisioner: provisioner, runtimeNotify: make(chan struct{}), }, nil } -// ProvisionHostedRuntime provisions the configured runtime. -// -// This method may return before the runtime is fully provisioned. The returned runtime will not be -// started automatically, you must call Start explicitly. -func (n *RuntimeHostNode) ProvisionHostedRuntime() (host.RichRuntime, protocol.Notifier, error) { - runtime := n.factory.GetRuntime() - cfgs := runtime.HostConfig() - provisioner := runtime.HostProvisioner() - if cfgs == nil || provisioner == nil { - return nil, nil, fmt.Errorf("runtime provisioner is not available") +// ProvisionHostedRuntimeVersion provisions the configured runtime version. +func (n *RuntimeHostNode) ProvisionHostedRuntimeVersion(version version.Version) error { + cfg := n.runtime.HostConfig(version) + if cfg == nil { + return fmt.Errorf("runtime version %s not found", version) } - agg := multi.New(runtime.ID()) - rr := host.NewRichRuntime(agg) - - notifier := n.factory.NewRuntimeHostNotifier(agg) - handler := n.factory.NewRuntimeHostHandler() - - for version, cfg := range cfgs { - rtCfg := *cfg - rtCfg.MessageHandler = handler + rtCfg := *cfg + rtCfg.MessageHandler = n.handler - // Provision the runtime. - rt, err := composite.NewHost(rtCfg, provisioner) - if err != nil { - return nil, nil, fmt.Errorf("failed to provision runtime version %s: %w", version, err) - } - - if err := agg.AddVersion(rt, version); err != nil { - return nil, nil, fmt.Errorf("failed to add runtime version to aggregate %s: %w", version, err) - } + rt, err := composite.NewHost(rtCfg, n.provisioner) + if err != nil { + return fmt.Errorf("failed to provision runtime version %s: %w", version, err) } - n.Lock() - n.agg = agg - n.runtime = rr - n.notifier = notifier - n.Unlock() + if err := n.agg.AddVersion(rt, version); err != nil { + return fmt.Errorf("failed to add runtime version to aggregate %s: %w", version, err) + } - close(n.runtimeNotify) + n.runtimeNotifyOnce.Do(func() { + close(n.runtimeNotify) + }) - return rr, notifier, nil + return nil } -// GetHostedRuntime returns the provisioned hosted runtime (if any). +// GetHostedRuntime returns the hosted runtime. func (n *RuntimeHostNode) GetHostedRuntime() host.RichRuntime { - n.Lock() - defer n.Unlock() + return n.rr +} - return n.runtime +// GetRuntimeHostNotifier returns the runtime host notifier. +func (n *RuntimeHostNode) GetRuntimeHostNotifier() protocol.Notifier { + return n.notifier } -// WaitHostedRuntime waits for the hosted runtime to be provisioned and returns it. -func (n *RuntimeHostNode) WaitHostedRuntime(ctx context.Context) (host.RichRuntime, error) { +// WaitHostedRuntime waits for the hosted runtime to be provisioned. +func (n *RuntimeHostNode) WaitHostedRuntime(ctx context.Context) error { select { case <-ctx.Done(): - return nil, ctx.Err() + return ctx.Err() case <-n.runtimeNotify: } - return n.GetHostedRuntime(), nil + return nil } // GetHostedRuntimeActiveVersion returns the version of the active runtime. func (n *RuntimeHostNode) GetHostedRuntimeActiveVersion() (*version.Version, error) { - n.Lock() - agg := n.agg - n.Unlock() - - if agg == nil { - return nil, fmt.Errorf("runtime not available") - } - - return agg.GetActiveVersion() + return n.agg.GetActiveVersion() } // GetHostedRuntimeCapabilityTEE returns the CapabilityTEE for the active runtime version. @@ -126,15 +119,7 @@ func (n *RuntimeHostNode) GetHostedRuntimeActiveVersion() (*version.Version, err // It may be nil in case the CapabilityTEE is not available or if the runtime is not running // inside a TEE. func (n *RuntimeHostNode) GetHostedRuntimeCapabilityTEE() (*node.CapabilityTEE, error) { - n.Lock() - agg := n.agg - n.Unlock() - - if agg == nil { - return nil, fmt.Errorf("runtime not available") - } - - return agg.GetCapabilityTEE() + return n.agg.GetCapabilityTEE() } // GetHostedRuntimeCapabilityTEEForVersion returns the CapabilityTEE for a specific runtime version. @@ -142,15 +127,7 @@ func (n *RuntimeHostNode) GetHostedRuntimeCapabilityTEE() (*node.CapabilityTEE, // It may be nil in case the CapabilityTEE is not available or if the runtime is not running // inside a TEE. func (n *RuntimeHostNode) GetHostedRuntimeCapabilityTEEForVersion(version version.Version) (*node.CapabilityTEE, error) { - n.Lock() - agg := n.agg - n.Unlock() - - if agg == nil { - return nil, fmt.Errorf("runtime not available") - } - - rt, err := agg.GetVersion(version) + rt, err := n.agg.GetVersion(version) if err != nil { return nil, err } @@ -158,16 +135,6 @@ func (n *RuntimeHostNode) GetHostedRuntimeCapabilityTEEForVersion(version versio } // SetHostedRuntimeVersion sets the currently active and next versions for the hosted runtime. -func (n *RuntimeHostNode) SetHostedRuntimeVersion(active *version.Version, next *version.Version) error { - n.Lock() - agg := n.agg - n.Unlock() - - if agg == nil { - return fmt.Errorf("runtime not available") - } - +func (n *RuntimeHostNode) SetHostedRuntimeVersion(active *version.Version, next *version.Version) { n.agg.SetVersion(active, next) - - return nil } diff --git a/go/runtime/registry/registry.go b/go/runtime/registry/registry.go index 2d35e3955fc..b616cda81df 100644 --- a/go/runtime/registry/registry.go +++ b/go/runtime/registry/registry.go @@ -100,8 +100,9 @@ type Runtime interface { // LocalStorage returns the per-runtime local storage. LocalStorage() localstorage.LocalStorage - // HostConfig returns the runtime host configuration when available. Otherwise returns nil. - HostConfig() map[version.Version]*runtimeHost.Config + // HostConfig returns the runtime host configuration for the given version + // when available. Otherwise returns nil. + HostConfig(version version.Version) *runtimeHost.Config // HostProvisioner returns the runtime host provisioner when available. Otherwise returns nil. HostProvisioner() runtimeHost.Provisioner @@ -218,8 +219,8 @@ func (r *runtime) LocalStorage() localstorage.LocalStorage { return r.localStorage } -func (r *runtime) HostConfig() map[version.Version]*runtimeHost.Config { - return r.hostConfig +func (r *runtime) HostConfig(version version.Version) *runtimeHost.Config { + return r.hostConfig[version] } func (r *runtime) HostProvisioner() runtimeHost.Provisioner { diff --git a/go/runtime/txpool/txpool.go b/go/runtime/txpool/txpool.go index ef02eb7a986..f354e69df87 100644 --- a/go/runtime/txpool/txpool.go +++ b/go/runtime/txpool/txpool.go @@ -136,8 +136,11 @@ type TransactionPool interface { // RuntimeHostProvisioner is a runtime host provisioner. type RuntimeHostProvisioner interface { - // WaitHostedRuntime waits for the hosted runtime to be provisioned and returns it. - WaitHostedRuntime(ctx context.Context) (host.RichRuntime, error) + // GetHostedRuntime returns the hosted runtime. + GetHostedRuntime() host.RichRuntime + + // WaitHostedRuntime waits for the hosted runtime to be provisioned. + WaitHostedRuntime(ctx context.Context) error } // TransactionPublisher is an interface representing a mechanism for publishing transactions. @@ -664,19 +667,19 @@ func (t *txPool) checkWorker() { cancel() }() + // Wait for initialization. + if err := t.ensureInitialized(); err != nil { + return + } + // Wait for the hosted runtime to be available. - rr, err := t.host.WaitHostedRuntime(ctx) - if err != nil { + if err := t.host.WaitHostedRuntime(ctx); err != nil { t.logger.Error("failed waiting for hosted runtime to become available", "err", err, ) return } - - // Wait for initialization. - if err = t.ensureInitialized(); err != nil { - return - } + rr := t.host.GetHostedRuntime() for { select { diff --git a/go/worker/common/committee/node.go b/go/worker/common/committee/node.go index a35fa85db67..31420edc339 100644 --- a/go/worker/common/committee/node.go +++ b/go/worker/common/committee/node.go @@ -443,7 +443,7 @@ func (n *Node) updateHostedRuntimeVersionLocked() { } } - _ = n.SetHostedRuntimeVersion(activeVersion, nextVersion) + n.SetHostedRuntimeVersion(activeVersion, nextVersion) if _, err := n.GetHostedRuntimeActiveVersion(); err != nil { n.logger.Error("failed to activate runtime version(s)", @@ -681,14 +681,19 @@ func (n *Node) worker() { defer blocksSub.Close() // Provision the hosted runtime. - hrt, hrtNotifier, err := n.ProvisionHostedRuntime() - if err != nil { - n.logger.Error("failed to provision hosted runtime", - "err", err, - ) - return + for _, version := range n.GetRuntime().HostVersions() { + if err := n.ProvisionHostedRuntimeVersion(version); err != nil { + n.logger.Error("failed to provision hosted runtime", + "err", err, + "version", version, + ) + return + } } + hrt := n.GetHostedRuntime() + hrtNotifier := n.GetRuntimeHostNotifier() + hrtEventCh, hrtSub := hrt.WatchEvents() defer hrtSub.Close() diff --git a/go/worker/keymanager/worker.go b/go/worker/keymanager/worker.go index 553c09932b6..e3ad3bdfd57 100644 --- a/go/worker/keymanager/worker.go +++ b/go/worker/keymanager/worker.go @@ -394,14 +394,19 @@ func (w *Worker) worker() { // Provision the hosted runtime. w.logger.Info("provisioning key manager runtime") - hrt, hrtNotifier, err := w.ProvisionHostedRuntime() - if err != nil { - w.logger.Error("failed to provision key manager runtime", - "err", err, - ) - return + for _, version := range w.GetRuntime().HostVersions() { + if err := w.ProvisionHostedRuntimeVersion(version); err != nil { + w.logger.Error("failed to provision key manager runtime", + "err", err, + "version", version, + ) + return + } } + hrt := w.GetHostedRuntime() + hrtNotifier := w.GetRuntimeHostNotifier() + hrtEventCh, hrtSub := hrt.WatchEvents() defer hrtSub.Close() @@ -414,7 +419,7 @@ func (w *Worker) worker() { // Key managers always need to use the enclave version given to them in the bundle // as they need to make sure that replication is possible during upgrades. activeVersion := w.runtime.HostVersions()[0] // Init made sure we have exactly one. - _ = w.SetHostedRuntimeVersion(&activeVersion, nil) + w.SetHostedRuntimeVersion(&activeVersion, nil) if _, err := w.GetHostedRuntimeActiveVersion(); err != nil { w.logger.Error("failed to activate key manager runtime version", From d0bb785cab589abf31b7b9b0026eedb88cf868cc Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Sun, 17 Nov 2024 13:33:13 +0100 Subject: [PATCH 05/27] go/runtime/registry/host: Remove function WaitHostedRuntime Since runtime versions can be configured dynamically, and none of them may be active at a given moment, this function has no meaningful purpose in its current form and how it is currently used. --- go/runtime/registry/host.go | 33 +++--------- go/runtime/txpool/txpool.go | 82 ++++++++++++++---------------- go/worker/common/committee/node.go | 2 +- 3 files changed, 46 insertions(+), 71 deletions(-) diff --git a/go/runtime/registry/host.go b/go/runtime/registry/host.go index b806e275b9f..5dbf6a4d848 100644 --- a/go/runtime/registry/host.go +++ b/go/runtime/registry/host.go @@ -1,9 +1,7 @@ package registry import ( - "context" "fmt" - "sync" "time" "github.com/oasisprotocol/oasis-core/go/common/node" @@ -36,9 +34,6 @@ type RuntimeHostNode struct { handler host.RuntimeHandler provisioner host.Provisioner - - runtimeNotify chan struct{} - runtimeNotifyOnce sync.Once } // NewRuntimeHostNode creates a new runtime host node. @@ -52,13 +47,12 @@ func NewRuntimeHostNode(factory RuntimeHostHandlerFactory) (*RuntimeHostNode, er provisioner := runtime.HostProvisioner() return &RuntimeHostNode{ - agg: agg, - rr: rr, - runtime: runtime, - notifier: notifier, - handler: handler, - provisioner: provisioner, - runtimeNotify: make(chan struct{}), + agg: agg, + rr: rr, + runtime: runtime, + notifier: notifier, + handler: handler, + provisioner: provisioner, }, nil } @@ -81,10 +75,6 @@ func (n *RuntimeHostNode) ProvisionHostedRuntimeVersion(version version.Version) return fmt.Errorf("failed to add runtime version to aggregate %s: %w", version, err) } - n.runtimeNotifyOnce.Do(func() { - close(n.runtimeNotify) - }) - return nil } @@ -98,17 +88,6 @@ func (n *RuntimeHostNode) GetRuntimeHostNotifier() protocol.Notifier { return n.notifier } -// WaitHostedRuntime waits for the hosted runtime to be provisioned. -func (n *RuntimeHostNode) WaitHostedRuntime(ctx context.Context) error { - select { - case <-ctx.Done(): - return ctx.Err() - case <-n.runtimeNotify: - } - - return nil -} - // GetHostedRuntimeActiveVersion returns the version of the active runtime. func (n *RuntimeHostNode) GetHostedRuntimeActiveVersion() (*version.Version, error) { return n.agg.GetActiveVersion() diff --git a/go/runtime/txpool/txpool.go b/go/runtime/txpool/txpool.go index f354e69df87..aa8fa8d919b 100644 --- a/go/runtime/txpool/txpool.go +++ b/go/runtime/txpool/txpool.go @@ -134,15 +134,6 @@ type TransactionPool interface { GetTxs() []*TxQueueMeta } -// RuntimeHostProvisioner is a runtime host provisioner. -type RuntimeHostProvisioner interface { - // GetHostedRuntime returns the hosted runtime. - GetHostedRuntime() host.RichRuntime - - // WaitHostedRuntime waits for the hosted runtime to be provisioned. - WaitHostedRuntime(ctx context.Context) error -} - // TransactionPublisher is an interface representing a mechanism for publishing transactions. type TransactionPublisher interface { // PublishTx publishes a transaction to remote peers. @@ -163,7 +154,7 @@ type txPool struct { runtimeID common.Namespace cfg config.Config - host RuntimeHostProvisioner + runtime host.RichRuntime txPublisher TransactionPublisher history history.History @@ -457,13 +448,16 @@ func (t *txPool) getCurrentBlockInfo() (*runtime.BlockInfo, time.Time, error) { // checkTxBatch requests the runtime to check the validity of a transaction batch. // Transactions that pass the check are queued for scheduling. -func (t *txPool) checkTxBatch(ctx context.Context, rr host.RichRuntime) { +func (t *txPool) checkTxBatch(ctx context.Context) error { + // Ensure the runtime is available. + if _, err := t.runtime.GetActiveVersion(); err != nil { + return fmt.Errorf("runtime is not available") + } + + // Get the current block info. bi, lastBlockProcessed, err := t.getCurrentBlockInfo() if err != nil { - t.logger.Warn("failed to get current block info, unable to check transactions", - "err", err, - ) - return + return fmt.Errorf("failed to get current block info: %w", err) } // Ensure block round is synced to storage. @@ -479,13 +473,13 @@ func (t *txPool) checkTxBatch(ctx context.Context, rr host.RichRuntime) { "err", err, ) t.checkTxCh.In() <- struct{}{} - return + return nil } // Pop the next batch from the queue, check it, and notify submitters. batch := t.checkTxQueue.pop() if len(batch) == 0 { - return + return nil } results, err := func() ([]protocol.CheckTxResult, error) { @@ -497,7 +491,7 @@ func (t *txPool) checkTxBatch(ctx context.Context, rr host.RichRuntime) { for _, pct := range batch { rawTxBatch = append(rawTxBatch, pct.Raw()) } - return rr.CheckTx(checkCtx, bi.RuntimeBlock, bi.ConsensusBlock, bi.Epoch, bi.ActiveDescriptor.Executor.MaxMessages, rawTxBatch) + return t.runtime.CheckTx(checkCtx, bi.RuntimeBlock, bi.ConsensusBlock, bi.Epoch, bi.ActiveDescriptor.Executor.MaxMessages, rawTxBatch) }() switch { case err == nil: @@ -509,7 +503,7 @@ func (t *txPool) checkTxBatch(ctx context.Context, rr host.RichRuntime) { abortCtx, cancel := context.WithTimeout(ctx, abortTimeout) defer cancel() - if err = rr.Abort(abortCtx, false); err != nil { + if err = t.runtime.Abort(abortCtx, false); err != nil { t.logger.Error("failed to abort the runtime", "err", err, ) @@ -517,19 +511,10 @@ func (t *txPool) checkTxBatch(ctx context.Context, rr host.RichRuntime) { fallthrough default: - t.logger.Warn("transaction batch check failed", - "err", err, - ) - // Return transaction batch back to the check queue. t.checkTxQueue.retryBatch(batch) - // Make sure that the batch check is retried later. - go func() { - time.Sleep(checkTxRetryDelay) - t.checkTxCh.In() <- struct{}{} - }() - return + return err } pendingCheckSize.With(t.getMetricLabels()).Set(float64(t.PendingCheckSize())) @@ -586,7 +571,7 @@ func (t *txPool) checkTxBatch(ctx context.Context, rr host.RichRuntime) { } if len(goodPcts) == 0 { - return + return nil } t.logger.Debug("checked new transactions", @@ -645,6 +630,8 @@ func (t *txPool) checkTxBatch(ctx context.Context, rr host.RichRuntime) { mainQueueSize.With(t.getMetricLabels()).Set(float64(t.mainQueue.inner.size())) localQueueSize.With(t.getMetricLabels()).Set(float64(t.localQueue.size())) + + return nil } func (t *txPool) ensureInitialized() error { @@ -672,25 +659,34 @@ func (t *txPool) checkWorker() { return } - // Wait for the hosted runtime to be available. - if err := t.host.WaitHostedRuntime(ctx); err != nil { - t.logger.Error("failed waiting for hosted runtime to become available", - "err", err, - ) - return - } - rr := t.host.GetHostedRuntime() + // Create a timer for retrying if the active version of the runtime + // is not available. + retryTimer := time.NewTimer(0) + defer retryTimer.Stop() + + retryTimer.Stop() for { select { case <-t.stopCh: return case <-t.checkTxCh.Out(): - t.logger.Debug("checking queued transactions") + case <-retryTimer.C: + } + + // Check if there are any transactions to check and run the checks. + t.logger.Debug("checking queued transactions") + + if err := t.checkTxBatch(ctx); err != nil { + t.logger.Warn("transaction batch check failed", + "err", err, + ) - // Check if there are any transactions to check and run the checks. - t.checkTxBatch(ctx, rr) + retryTimer.Reset(checkTxRetryDelay) + continue } + + retryTimer.Stop() } } @@ -882,7 +878,7 @@ func (t *txPool) recheck() { func New( runtimeID common.Namespace, cfg config.Config, - host RuntimeHostProvisioner, + runtime host.RichRuntime, history history.History, txPublisher TransactionPublisher, ) TransactionPool { @@ -905,7 +901,7 @@ func New( initCh: make(chan struct{}), runtimeID: runtimeID, cfg: cfg, - host: host, + runtime: runtime, history: history, txPublisher: txPublisher, seenCache: seenCache, diff --git a/go/worker/common/committee/node.go b/go/worker/common/committee/node.go index 31420edc339..f109cd60055 100644 --- a/go/worker/common/committee/node.go +++ b/go/worker/common/committee/node.go @@ -887,7 +887,7 @@ func NewNode( n.RuntimeHostNode = rhn // Prepare transaction pool. - n.TxPool = txpool.New(runtime.ID(), txPoolCfg, n, runtime.History(), n) + n.TxPool = txpool.New(runtime.ID(), txPoolCfg, rhn.GetHostedRuntime(), runtime.History(), n) // Register transaction message handler as that is something that all workers must handle. p2pHost.RegisterHandler(txTopic, &txMsgHandler{n}) From 12a2b5c3e539883d239b94f3e8a01cd1fa3334f4 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Tue, 19 Nov 2024 02:30:17 +0100 Subject: [PATCH 06/27] go/runtime/registry/host: Provision every version only once --- go/runtime/host/multi/multi.go | 9 +++++++++ go/runtime/registry/host.go | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/go/runtime/host/multi/multi.go b/go/runtime/host/multi/multi.go index d06f1486002..7044fa0e1aa 100644 --- a/go/runtime/host/multi/multi.go +++ b/go/runtime/host/multi/multi.go @@ -359,6 +359,15 @@ func (agg *Aggregate) GetVersion(version version.Version) (host.Runtime, error) return host.host, nil } +// HasVersion checks if runtime host exists for the specified version. +func (agg *Aggregate) HasVersion(version version.Version) bool { + agg.l.RLock() + defer agg.l.RUnlock() + + _, ok := agg.hosts[version] + return ok +} + // AddVersion adds a new runtime version to the aggregate. // // If the newly added version matches the active or the next version, diff --git a/go/runtime/registry/host.go b/go/runtime/registry/host.go index 5dbf6a4d848..3e6fffa73ef 100644 --- a/go/runtime/registry/host.go +++ b/go/runtime/registry/host.go @@ -58,6 +58,10 @@ func NewRuntimeHostNode(factory RuntimeHostHandlerFactory) (*RuntimeHostNode, er // ProvisionHostedRuntimeVersion provisions the configured runtime version. func (n *RuntimeHostNode) ProvisionHostedRuntimeVersion(version version.Version) error { + if n.agg.HasVersion(version) { + return nil + } + cfg := n.runtime.HostConfig(version) if cfg == nil { return fmt.Errorf("runtime version %s not found", version) From 312f123fec3a84c7bfbc43a0d40ac8f204f24eb0 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Sun, 17 Nov 2024 17:59:30 +0100 Subject: [PATCH 07/27] go/runtime/registry: Add runtime constructor --- go/runtime/registry/registry.go | 313 ++++++++++++++++++-------------- go/worker/keymanager/init.go | 2 +- 2 files changed, 174 insertions(+), 141 deletions(-) diff --git a/go/runtime/registry/registry.go b/go/runtime/registry/registry.go index b616cda81df..6c6825f8634 100644 --- a/go/runtime/registry/registry.go +++ b/go/runtime/registry/registry.go @@ -15,6 +15,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/logging" "github.com/oasisprotocol/oasis-core/go/common/persistent" "github.com/oasisprotocol/oasis-core/go/common/pubsub" + cmSync "github.com/oasisprotocol/oasis-core/go/common/sync" "github.com/oasisprotocol/oasis-core/go/common/version" consensus "github.com/oasisprotocol/oasis-core/go/consensus/api" ias "github.com/oasisprotocol/oasis-core/go/ias/api" @@ -42,15 +43,15 @@ var ErrRuntimeHostNotConfigured = errors.New("runtime/registry: runtime host not // Registry is the running node's runtime registry interface. type Registry interface { - // GetRuntime returns the per-runtime interface if the runtime is supported. + // GetRuntime returns the per-runtime interface if the runtime is managed. GetRuntime(runtimeID common.Namespace) (Runtime, error) - // Runtimes returns a list of all supported runtimes. + // Runtimes returns a list of all managed runtimes. Runtimes() []Runtime - // NewUnmanagedRuntime creates a new runtime that is not managed by this - // registry. - NewUnmanagedRuntime(ctx context.Context, runtimeID common.Namespace) (Runtime, error) + // NewRuntime creates a new runtime that may or may not be managed + // by this registry. + NewRuntime(ctx context.Context, runtimeID common.Namespace, managed bool) (Runtime, error) // RegisterClient registers a runtime client service. If the service has already been registered // this method returns an error. @@ -113,6 +114,7 @@ type Runtime interface { type runtime struct { // nolint: maligned sync.RWMutex + startOne cmSync.One id common.Namespace dataDir string @@ -127,7 +129,6 @@ type runtime struct { // nolint: maligned history history.History - cancelCtx context.CancelFunc registryDescriptorCh chan struct{} registryDescriptorNotifier *pubsub.Broker activeDescriptorCh chan struct{} @@ -139,14 +140,56 @@ type runtime struct { // nolint: maligned logger *logging.Logger } +func newRuntime( + runtimeID common.Namespace, + managed bool, + dataDir string, + consensus consensus.Backend, + provisioner runtimeHost.Provisioner, + cfg map[version.Version]*runtimeHost.Config, +) (*runtime, error) { + logger := logging.GetLogger("runtime/registry").With("runtime_id", runtimeID) + + // Ensure runtime state directory exists. + rtDataDir, err := EnsureRuntimeStateDir(dataDir, runtimeID) + if err != nil { + return nil, err + } + + // Create runtime-specific local storage backend. + localStorage, err := localstorage.New(rtDataDir, LocalStorageFile, runtimeID) + if err != nil { + return nil, fmt.Errorf("runtime/registry: cannot create local storage for runtime %s: %w", runtimeID, err) + } + + return &runtime{ + startOne: cmSync.NewOne(), + id: runtimeID, + dataDir: rtDataDir, + managed: managed, + consensus: consensus, + localStorage: localStorage, + registryDescriptorCh: make(chan struct{}), + registryDescriptorNotifier: pubsub.NewBroker(true), + activeDescriptorCh: make(chan struct{}), + activeDescriptorNotifier: pubsub.NewBroker(true), + hostProvisioner: provisioner, + hostConfig: cfg, + logger: logger, + }, nil +} + +// ID implements Runtime. func (r *runtime) ID() common.Namespace { return r.id } +// DataDir implements Runtime. func (r *runtime) DataDir() string { return r.dataDir } +// RegistryDescriptor implements Runtime. func (r *runtime) RegistryDescriptor(ctx context.Context) (*registry.Runtime, error) { // Wait for the descriptor to be ready. select { @@ -161,6 +204,7 @@ func (r *runtime) RegistryDescriptor(ctx context.Context) (*registry.Runtime, er return d, nil } +// ActiveDescriptor implements Runtime. func (r *runtime) ActiveDescriptor(ctx context.Context) (*registry.Runtime, error) { // Wait for the descriptor to be ready. select { @@ -175,6 +219,7 @@ func (r *runtime) ActiveDescriptor(ctx context.Context) (*registry.Runtime, erro return d, nil } +// WatchActiveDescriptor implements Runtime. func (r *runtime) WatchActiveDescriptor() (<-chan *registry.Runtime, pubsub.ClosableSubscription, error) { sub := r.activeDescriptorNotifier.Subscribe() ch := make(chan *registry.Runtime) @@ -183,6 +228,7 @@ func (r *runtime) WatchActiveDescriptor() (<-chan *registry.Runtime, pubsub.Clos return ch, sub, nil } +// WatchRegistryDescriptor implements Runtime. func (r *runtime) WatchRegistryDescriptor() (<-chan *registry.Runtime, pubsub.ClosableSubscription, error) { sub := r.registryDescriptorNotifier.Subscribe() ch := make(chan *registry.Runtime) @@ -191,6 +237,7 @@ func (r *runtime) WatchRegistryDescriptor() (<-chan *registry.Runtime, pubsub.Cl return ch, sub, nil } +// RegisterStorage implements Runtime. func (r *runtime) RegisterStorage(storage storageAPI.Backend) { r.Lock() defer r.Unlock() @@ -201,10 +248,12 @@ func (r *runtime) RegisterStorage(storage storageAPI.Backend) { r.storage = storage } +// History implements Runtime. func (r *runtime) History() history.History { return r.history } +// Storage implements Runtime. func (r *runtime) Storage() storageAPI.Backend { r.RLock() defer r.RUnlock() @@ -215,18 +264,22 @@ func (r *runtime) Storage() storageAPI.Backend { return r.storage } +// LocalStorage implements Runtime. func (r *runtime) LocalStorage() localstorage.LocalStorage { return r.localStorage } +// HostConfig implements Runtime. func (r *runtime) HostConfig(version version.Version) *runtimeHost.Config { return r.hostConfig[version] } +// HostProvisioner implements Runtime. func (r *runtime) HostProvisioner() runtimeHost.Provisioner { return r.hostProvisioner } +// HostVersions implements Runtime. func (r *runtime) HostVersions() []version.Version { var versions []version.Version for v := range r.hostConfig { @@ -235,59 +288,31 @@ func (r *runtime) HostVersions() []version.Version { return versions } +// start starts the runtime worker. +func (r *runtime) start() { + r.startOne.TryStart(r.run) +} + +// stop halts the runtime worker. func (r *runtime) stop() { // Stop watching runtime updates. - r.cancelCtx() + r.startOne.TryStop() + // Close local storage backend. r.localStorage.Stop() + // Close storage backend. if r.storage != nil { r.storage.Cleanup() } + // Close history keeper. if r.history != nil { r.history.Close() } } -func (r *runtime) updateActiveDescriptor(ctx context.Context) bool { - state, err := r.consensus.RootHash().GetRuntimeState(ctx, &roothash.RuntimeRequest{ - RuntimeID: r.id, - Height: consensus.HeightLatest, - }) - if err != nil { - r.logger.Error("querying roothash state", - "err", err, - ) - return false - } - - h := hash.NewFrom(state.Runtime) - // This is only called from the watchUpdates thread and activeDescriptorHash - // is only mutated bellow, so no need for a lock here. - if h.Equal(&r.activeDescriptorHash) { - r.logger.Debug("active runtime descriptor didn't change", - "runtime", state.Runtime, - "hash", h, - ) - return false - } - - r.logger.Debug("updating active runtime descriptor", - "runtime", state.Runtime, - "hash", h, - ) - r.Lock() - r.activeDescriptor = state.Runtime - r.activeDescriptorHash = h - r.Unlock() - - r.activeDescriptorNotifier.Broadcast(state.Runtime) - - return true -} - -func (r *runtime) watchUpdates(ctx context.Context) { +func (r *runtime) run(ctx context.Context) { r.logger.Debug("waiting consensus sync") select { case <-ctx.Done(): @@ -358,11 +383,48 @@ func (r *runtime) watchUpdates(ctx context.Context) { } } +func (r *runtime) updateActiveDescriptor(ctx context.Context) bool { + state, err := r.consensus.RootHash().GetRuntimeState(ctx, &roothash.RuntimeRequest{ + RuntimeID: r.id, + Height: consensus.HeightLatest, + }) + if err != nil { + r.logger.Error("querying roothash state", + "err", err, + ) + return false + } + + h := hash.NewFrom(state.Runtime) + // This is only called from the `run` thread and `activeDescriptorHash` + // is only mutated bellow, so no need for a lock here. + if h.Equal(&r.activeDescriptorHash) { + r.logger.Debug("active runtime descriptor didn't change", + "runtime", state.Runtime, + "hash", h, + ) + return false + } + + r.logger.Debug("updating active runtime descriptor", + "runtime", state.Runtime, + "hash", h, + ) + r.Lock() + r.activeDescriptor = state.Runtime + r.activeDescriptorHash = h + r.Unlock() + + r.activeDescriptorNotifier.Broadcast(state.Runtime) + + return true +} + func (r *runtime) finishInitialization() error { r.Lock() defer r.Unlock() - if r.storage == nil { + if r.storage == nil && r.managed { return fmt.Errorf("runtime/registry: nobody provided a storage backend for runtime %s", r.id) } @@ -385,32 +447,82 @@ type runtimeRegistry struct { historyFactory history.Factory } +// GetRuntime implements Registry. func (r *runtimeRegistry) GetRuntime(runtimeID common.Namespace) (Runtime, error) { r.RLock() defer r.RUnlock() - rt := r.runtimes[runtimeID] - if rt == nil { - return nil, fmt.Errorf("runtime/registry: runtime %s is not supported", runtimeID) + rt, ok := r.runtimes[runtimeID] + if !ok || !rt.managed { + return nil, fmt.Errorf("runtime/registry: runtime %s not found", runtimeID) } return rt, nil } +// Runtimes implements Registry. func (r *runtimeRegistry) Runtimes() []Runtime { r.RLock() defer r.RUnlock() - var rts []Runtime + rts := make([]Runtime, 0, len(r.runtimes)) for _, rt := range r.runtimes { - rts = append(rts, rt) + if rt.managed { + rts = append(rts, rt) + } } return rts } -func (r *runtimeRegistry) NewUnmanagedRuntime(ctx context.Context, runtimeID common.Namespace) (Runtime, error) { - return r.newRuntime(ctx, runtimeID) +// NewRuntime implements Registry. +func (r *runtimeRegistry) NewRuntime(ctx context.Context, runtimeID common.Namespace, managed bool) (Runtime, error) { + r.Lock() + defer r.Unlock() + + r.logger.Info("adding runtime", + "id", runtimeID, + "managed", managed, + ) + + if len(r.runtimes) >= MaxRuntimeCount { + return nil, fmt.Errorf("runtime/registry: too many registered runtimes") + } + + if _, ok := r.runtimes[runtimeID]; ok { + return nil, fmt.Errorf("runtime/registry: runtime already registered: %s", runtimeID) + } + + rt, err := newRuntime(runtimeID, managed, r.dataDir, r.consensus, r.cfg.Provisioner, r.cfg.Runtimes[runtimeID]) + if err != nil { + return nil, err + } + + if managed { + // Create runtime history keeper. + history, err := r.historyFactory(runtimeID, rt.dataDir) + if err != nil { + return nil, fmt.Errorf("runtime/registry: cannot create block history for runtime %s: %w", runtimeID, err) + } + rt.history = history + + // Start tracking this runtime. + if err = r.consensus.RootHash().TrackRuntime(ctx, history); err != nil { + return nil, fmt.Errorf("runtime/registry: cannot track runtime %s: %w", runtimeID, err) + } + } + + r.runtimes[runtimeID] = rt + + rt.start() + + r.logger.Info("runtime added", + "id", runtimeID, + "managed", managed, + ) + + return rt, nil } +// RegisterClient implements Registry. func (r *runtimeRegistry) RegisterClient(rc runtimeClient.RuntimeClient) error { r.Lock() defer r.Unlock() @@ -422,6 +534,7 @@ func (r *runtimeRegistry) RegisterClient(rc runtimeClient.RuntimeClient) error { return nil } +// Client implements Registry. func (r *runtimeRegistry) Client() (runtimeClient.RuntimeClient, error) { r.RLock() defer r.RUnlock() @@ -432,6 +545,7 @@ func (r *runtimeRegistry) Client() (runtimeClient.RuntimeClient, error) { return r.client, nil } +// Cleanup implements Registry. func (r *runtimeRegistry) Cleanup() { r.Lock() defer r.Unlock() @@ -441,6 +555,7 @@ func (r *runtimeRegistry) Cleanup() { } } +// FinishInitialization implements Registry. func (r *runtimeRegistry) FinishInitialization() error { r.RLock() defer r.RUnlock() @@ -453,84 +568,6 @@ func (r *runtimeRegistry) FinishInitialization() error { return nil } -func (r *runtimeRegistry) addSupportedRuntime(ctx context.Context, runtimeID common.Namespace) (rerr error) { - r.Lock() - defer r.Unlock() - - if len(r.runtimes) >= MaxRuntimeCount { - return fmt.Errorf("runtime/registry: too many registered runtimes") - } - - if _, ok := r.runtimes[runtimeID]; ok { - return fmt.Errorf("runtime/registry: runtime already registered: %s", runtimeID) - } - - rt, err := r.newRuntime(ctx, runtimeID) - if err != nil { - return err - } - defer func() { - if rerr != nil { - rt.stop() - } - }() - rt.managed = true - - // Create runtime history keeper. - history, err := r.historyFactory(runtimeID, rt.dataDir) - if err != nil { - return fmt.Errorf("runtime/registry: cannot create block history for runtime %s: %w", runtimeID, err) - } - - // Start tracking this runtime. - if err = r.consensus.RootHash().TrackRuntime(ctx, history); err != nil { - return fmt.Errorf("runtime/registry: cannot track runtime %s: %w", runtimeID, err) - } - - rt.history = history - r.runtimes[runtimeID] = rt - - return nil -} - -func (r *runtimeRegistry) newRuntime(ctx context.Context, runtimeID common.Namespace) (*runtime, error) { - // Ensure runtime state directory exists. - rtDataDir, err := EnsureRuntimeStateDir(r.dataDir, runtimeID) - if err != nil { - return nil, err - } - - // Create runtime-specific local storage backend. - localStorage, err := localstorage.New(rtDataDir, LocalStorageFile, runtimeID) - if err != nil { - return nil, fmt.Errorf("runtime/registry: cannot create local storage for runtime %s: %w", runtimeID, err) - } - - watchCtx, cancel := context.WithCancel(ctx) - - rt := &runtime{ - id: runtimeID, - dataDir: rtDataDir, - consensus: r.consensus, - localStorage: localStorage, - cancelCtx: cancel, - registryDescriptorCh: make(chan struct{}), - registryDescriptorNotifier: pubsub.NewBroker(true), - activeDescriptorCh: make(chan struct{}), - activeDescriptorNotifier: pubsub.NewBroker(true), - logger: r.logger.With("runtime_id", runtimeID), - } - go rt.watchUpdates(watchCtx) - - // Configure runtime host if needed. - if r.cfg.Provisioner != nil { - rt.hostProvisioner = r.cfg.Provisioner - rt.hostConfig = r.cfg.Runtimes[runtimeID] - } - - return rt, nil -} - // New creates a new runtime registry. func New( ctx context.Context, @@ -559,17 +596,13 @@ func New( historyFactory: historyFactory, } - for _, id := range cfg.RuntimeIDs() { - r.logger.Info("adding supported runtime", - "id", id, - ) - - if err := r.addSupportedRuntime(ctx, id); err != nil { - r.logger.Error("failed to add supported runtime", + for _, runtimeID := range cfg.RuntimeIDs() { + if _, err := r.NewRuntime(ctx, runtimeID, true); err != nil { + r.logger.Error("failed to add runtime", "err", err, - "id", id, + "id", runtimeID, ) - return nil, fmt.Errorf("failed to add runtime %s: %w", id, err) + return nil, fmt.Errorf("failed to add runtime %s: %w", runtimeID, err) } } diff --git a/go/worker/keymanager/init.go b/go/worker/keymanager/init.go index 31aeabb6ca1..c82f8fcbb06 100644 --- a/go/worker/keymanager/init.go +++ b/go/worker/keymanager/init.go @@ -65,7 +65,7 @@ func New( return nil, fmt.Errorf("worker/keymanager: failed to create role provider: %w", err) } - w.runtime, err = commonWorker.RuntimeRegistry.NewUnmanagedRuntime(ctx, w.runtimeID) + w.runtime, err = commonWorker.RuntimeRegistry.NewRuntime(ctx, w.runtimeID, false) if err != nil { return nil, fmt.Errorf("worker/keymanager: failed to create runtime registry entry: %w", err) } From 52314913a49e4032b528f9c86c655a7b7cc4832a Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Sun, 17 Nov 2024 18:00:57 +0100 Subject: [PATCH 08/27] go/runtime/registry/config: Refactor function newRuntimeConfig Extracted the creation of the provisioner, host info and caching quote service into a dedicated helper function to improve code readability. --- go/runtime/registry/config.go | 358 +++++++++++++++----------------- go/runtime/registry/registry.go | 38 +++- 2 files changed, 204 insertions(+), 192 deletions(-) diff --git a/go/runtime/registry/config.go b/go/runtime/registry/config.go index 57fa40220d3..e52d9087ddd 100644 --- a/go/runtime/registry/config.go +++ b/go/runtime/registry/config.go @@ -3,9 +3,7 @@ package registry import ( "context" "fmt" - "maps" "os" - "slices" "strings" "time" @@ -44,42 +42,8 @@ const ( // Flags has the configuration flags. var Flags = flag.NewFlagSet("", flag.ContinueOnError) -// RuntimeConfig is the node runtime configuration. -type RuntimeConfig struct { - // Runtimes contains per-runtime provisioning configuration. - // - // Some fields may be omitted as they are provided when the runtime - // is provisioned. - Runtimes map[common.Namespace]map[version.Version]*runtimeHost.Config - - // Provisioner is the runtime provisioner to use. - // - // It may be nil if no runtimes are configured. - Provisioner runtimeHost.Provisioner - - // History is the runtime history factory to use. - History history.Factory -} - -// RuntimeIDs returns a list of configured runtime IDs. -func (cfg *RuntimeConfig) RuntimeIDs() []common.Namespace { - if config.GlobalConfig.Mode == config.ModeKeyManager { - return nil - } - - return slices.Collect(maps.Keys(cfg.Runtimes)) -} - // newRuntimeConfig creates a new node runtime configuration. -func newRuntimeConfig( //nolint: gocyclo - dataDir string, - commonStore *persistent.CommonStore, - identity *identity.Identity, - consensus consensus.Backend, - ias []ias.Endpoint, -) (*RuntimeConfig, error) { - var cfg RuntimeConfig - +func newRuntimeConfig(dataDir string) (map[common.Namespace]map[version.Version]*runtimeHost.Config, error) { //nolint: gocyclo haveSetRuntimes := len(config.GlobalConfig.Runtime.Paths) > 0 // Validate configured runtimes based on the runtime mode. @@ -99,6 +63,8 @@ func newRuntimeConfig( //nolint: gocyclo } // Check if any runtimes are configured to be hosted. + runtimes := make(map[common.Namespace]map[version.Version]*runtimeHost.Config) + if haveSetRuntimes || (cmdFlags.DebugDontBlameOasis() && viper.IsSet(CfgDebugMockIDs)) { // By default start with the environment specified in configuration. runtimeEnv := config.GlobalConfig.Runtime.Environment @@ -156,152 +122,13 @@ func newRuntimeConfig( //nolint: gocyclo } } - isEnvSGX := runtimeEnv == rtConfig.RuntimeEnvironmentSGX || runtimeEnv == rtConfig.RuntimeEnvironmentSGXMock - forceNoSGX := (config.GlobalConfig.Mode.IsClientOnly() && !isEnvSGX) || - (cmdFlags.DebugDontBlameOasis() && runtimeEnv == rtConfig.RuntimeEnvironmentELF) - - // Configure host environment information. - cs, err := consensus.GetStatus(context.Background()) - if err != nil { - return nil, fmt.Errorf("failed to get consensus layer status: %w", err) - } - chainCtx, err := consensus.GetChainContext(context.Background()) - if err != nil { - return nil, fmt.Errorf("failed to get chain context: %w", err) - } - hostInfo := &hostProtocol.HostInfo{ - ConsensusBackend: cs.Backend, - ConsensusProtocolVersion: cs.Version, - ConsensusChainContext: chainCtx, - } - - // Create the PCS client and quote service. - pc, err := pcs.NewHTTPClient(&pcs.HTTPClientConfig{ - // TODO: Support configuring the API key. - }) - if err != nil { - return nil, fmt.Errorf("failed to create PCS HTTP client: %w", err) - } - qs := pcs.NewCachingQuoteService(pc, commonStore) - - // Register provisioners based on the configured provisioner. - var insecureNoSandbox bool - sandboxBinary := config.GlobalConfig.Runtime.SandboxBinary - attestInterval := config.GlobalConfig.Runtime.AttestInterval - provisioners := make(map[component.TEEKind]runtimeHost.Provisioner) - switch p := config.GlobalConfig.Runtime.Provisioner; p { - case rtConfig.RuntimeProvisionerMock: - // Mock provisioner, only supported when the runtime requires no TEE hardware. - if !cmdFlags.DebugDontBlameOasis() { - return nil, fmt.Errorf("mock provisioner requires use of unsafe debug flags") - } - - provisioners[component.TEEKindNone] = hostMock.New() - case rtConfig.RuntimeProvisionerUnconfined: - // Unconfined provisioner, can be used with no TEE or with Intel SGX. - if !cmdFlags.DebugDontBlameOasis() { - return nil, fmt.Errorf("unconfined provisioner requires use of unsafe debug flags") - } - - insecureNoSandbox = true - - fallthrough - case rtConfig.RuntimeProvisionerSandboxed: - // Sandboxed provisioner, can be used with no TEE or with Intel SGX. - if !insecureNoSandbox { - if _, err = os.Stat(sandboxBinary); err != nil { - return nil, fmt.Errorf("failed to stat sandbox binary: %w", err) - } - } - - // Configure the non-TEE provisioner. - provisioners[component.TEEKindNone], err = hostSandbox.New(hostSandbox.Config{ - HostInfo: hostInfo, - InsecureNoSandbox: insecureNoSandbox, - SandboxBinaryPath: sandboxBinary, - }) - if err != nil { - return nil, fmt.Errorf("failed to create runtime provisioner: %w", err) - } - - // Configure the Intel SGX provisioner. - switch sgxLoader := config.GlobalConfig.Runtime.SGXLoader; { - case forceNoSGX: - // Remap SGX to non-SGX when forced to do so. - provisioners[component.TEEKindSGX], err = hostSandbox.New(hostSandbox.Config{ - HostInfo: hostInfo, - InsecureNoSandbox: insecureNoSandbox, - SandboxBinaryPath: sandboxBinary, - }) - if err != nil { - return nil, fmt.Errorf("failed to create runtime provisioner: %w", err) - } - case sgxLoader == "" && runtimeEnv == rtConfig.RuntimeEnvironmentSGX: - // SGX environment is forced, but we don't have the needed loader. - return nil, fmt.Errorf("SGX runtime environment requires setting the SGX loader") - case sgxLoader == "" && runtimeEnv != rtConfig.RuntimeEnvironmentSGXMock: - // SGX may be needed, but we don't have a loader configured. - break - default: - // Configure mock SGX if configured and we are in a debug mode. - insecureMock := runtimeEnv == rtConfig.RuntimeEnvironmentSGXMock - if insecureMock && !cmdFlags.DebugDontBlameOasis() { - return nil, fmt.Errorf("mock SGX requires use of unsafe debug flags") - } - - provisioners[component.TEEKindSGX], err = hostSgx.New(hostSgx.Config{ - HostInfo: hostInfo, - CommonStore: commonStore, - LoaderPath: sgxLoader, - IAS: ias, - PCS: qs, - Consensus: consensus, - Identity: identity, - SandboxBinaryPath: sandboxBinary, - InsecureNoSandbox: insecureNoSandbox, - InsecureMock: insecureMock, - RuntimeAttestInterval: attestInterval, - }) - if err != nil { - return nil, fmt.Errorf("failed to create SGX runtime provisioner: %w", err) - } - } - default: - return nil, fmt.Errorf("unsupported runtime provisioner: %s", p) - } - - // Configure TDX provisioner. - // TODO: Allow provisioner selection in the future, currently we only have QEMU. - provisioners[component.TEEKindTDX], err = hostTdx.NewQemu(hostTdx.QemuConfig{ - HostInfo: hostInfo, - CommonStore: commonStore, - PCS: qs, - Consensus: consensus, - Identity: identity, - RuntimeAttestInterval: attestInterval, - }) - if err != nil { - return nil, fmt.Errorf("failed to create TDX runtime provisioner: %w", err) - } - - // Configure optional load balancing. - for tee, rp := range provisioners { - provisioners[tee] = hostLoadBalance.New(rp, hostLoadBalance.Config{ - NumInstances: int(config.GlobalConfig.Runtime.LoadBalancer.NumInstances), - }) - } - - // Create a composite provisioner to provision the individual components. - cfg.Provisioner = hostComposite.NewProvisioner(provisioners) - // Configure runtimes. - cfg.Runtimes = make(map[common.Namespace]map[version.Version]*runtimeHost.Config) for _, bnd := range regularBundles { id := bnd.Manifest.ID - if cfg.Runtimes[id] == nil { - cfg.Runtimes[id] = make(map[version.Version]*runtimeHost.Config) + if runtimes[id] == nil { + runtimes[id] = make(map[version.Version]*runtimeHost.Config) } - if _, ok := cfg.Runtimes[id][bnd.Manifest.Version]; ok { + if _, ok := runtimes[id][bnd.Manifest.Version]; ok { return nil, fmt.Errorf("duplicate runtime '%s' version '%s'", id, bnd.Manifest.Version) } @@ -363,12 +190,13 @@ func newRuntimeConfig( //nolint: gocyclo wantedComponents = append(wantedComponents, comp.ID()) } - cfg.Runtimes[id][bnd.Manifest.Version] = &runtimeHost.Config{ + runtimes[id][bnd.Manifest.Version] = &runtimeHost.Config{ Bundle: rtBnd, Components: wantedComponents, LocalConfig: localConfig, } } + if cmdFlags.DebugDontBlameOasis() { // This is to allow the mock provisioner to function, as it does // not use an actual runtime, thus is missing a bundle. This is @@ -397,23 +225,181 @@ func newRuntimeConfig( //nolint: gocyclo component.ID_RONL, }, } - cfg.Runtimes[id] = map[version.Version]*runtimeHost.Config{ + runtimes[id] = map[version.Version]*runtimeHost.Config{ {}: runtimeHostCfg, } } } - if len(cfg.Runtimes) == 0 { + + if len(runtimes) == 0 { return nil, fmt.Errorf("no runtimes configured") } } - history, err := createHistoryFactory() + return runtimes, nil +} + +func createHostInfo(consensus consensus.Backend) (*hostProtocol.HostInfo, error) { + cs, err := consensus.GetStatus(context.Background()) + if err != nil { + return nil, fmt.Errorf("failed to get consensus layer status: %w", err) + } + + chainCtx, err := consensus.GetChainContext(context.Background()) + if err != nil { + return nil, fmt.Errorf("failed to get chain context: %w", err) + } + + return &hostProtocol.HostInfo{ + ConsensusBackend: cs.Backend, + ConsensusProtocolVersion: cs.Version, + ConsensusChainContext: chainCtx, + }, nil +} + +func createProvisioner( + commonStore *persistent.CommonStore, + identity *identity.Identity, + consensus consensus.Backend, + hostInfo *hostProtocol.HostInfo, + ias []ias.Endpoint, + qs pcs.QuoteService, +) (runtimeHost.Provisioner, error) { + var err error + + // By default start with the environment specified in configuration. + runtimeEnv := config.GlobalConfig.Runtime.Environment + + // TODO: isEnvSGX should also be true if runtimeEnv is auto and at least + // one component requires SGX. + isEnvSGX := runtimeEnv == rtConfig.RuntimeEnvironmentSGX || runtimeEnv == rtConfig.RuntimeEnvironmentSGXMock + forceNoSGX := (config.GlobalConfig.Mode.IsClientOnly() && !isEnvSGX) || + (cmdFlags.DebugDontBlameOasis() && runtimeEnv == rtConfig.RuntimeEnvironmentELF) + + // Register provisioners based on the configured provisioner. + var insecureNoSandbox bool + sandboxBinary := config.GlobalConfig.Runtime.SandboxBinary + attestInterval := config.GlobalConfig.Runtime.AttestInterval + provisioners := make(map[component.TEEKind]runtimeHost.Provisioner) + switch p := config.GlobalConfig.Runtime.Provisioner; p { + case rtConfig.RuntimeProvisionerMock: + // Mock provisioner, only supported when the runtime requires no TEE hardware. + if !cmdFlags.DebugDontBlameOasis() { + return nil, fmt.Errorf("mock provisioner requires use of unsafe debug flags") + } + + provisioners[component.TEEKindNone] = hostMock.New() + case rtConfig.RuntimeProvisionerUnconfined: + // Unconfined provisioner, can be used with no TEE or with Intel SGX. + if !cmdFlags.DebugDontBlameOasis() { + return nil, fmt.Errorf("unconfined provisioner requires use of unsafe debug flags") + } + + insecureNoSandbox = true + + fallthrough + case rtConfig.RuntimeProvisionerSandboxed: + // Sandboxed provisioner, can be used with no TEE or with Intel SGX. + if !insecureNoSandbox { + if _, err = os.Stat(sandboxBinary); err != nil { + return nil, fmt.Errorf("failed to stat sandbox binary: %w", err) + } + } + + // Configure the non-TEE provisioner. + provisioners[component.TEEKindNone], err = hostSandbox.New(hostSandbox.Config{ + HostInfo: hostInfo, + InsecureNoSandbox: insecureNoSandbox, + SandboxBinaryPath: sandboxBinary, + }) + if err != nil { + return nil, fmt.Errorf("failed to create runtime provisioner: %w", err) + } + + // Configure the Intel SGX provisioner. + switch sgxLoader := config.GlobalConfig.Runtime.SGXLoader; { + case forceNoSGX: + // Remap SGX to non-SGX when forced to do so. + provisioners[component.TEEKindSGX], err = hostSandbox.New(hostSandbox.Config{ + HostInfo: hostInfo, + InsecureNoSandbox: insecureNoSandbox, + SandboxBinaryPath: sandboxBinary, + }) + if err != nil { + return nil, fmt.Errorf("failed to create runtime provisioner: %w", err) + } + case sgxLoader == "" && runtimeEnv == rtConfig.RuntimeEnvironmentSGX: + // SGX environment is forced, but we don't have the needed loader. + return nil, fmt.Errorf("SGX runtime environment requires setting the SGX loader") + case sgxLoader == "" && runtimeEnv != rtConfig.RuntimeEnvironmentSGXMock: + // SGX may be needed, but we don't have a loader configured. + break + default: + // Configure mock SGX if configured and we are in a debug mode. + insecureMock := runtimeEnv == rtConfig.RuntimeEnvironmentSGXMock + if insecureMock && !cmdFlags.DebugDontBlameOasis() { + return nil, fmt.Errorf("mock SGX requires use of unsafe debug flags") + } + + provisioners[component.TEEKindSGX], err = hostSgx.New(hostSgx.Config{ + HostInfo: hostInfo, + CommonStore: commonStore, + LoaderPath: sgxLoader, + IAS: ias, + PCS: qs, + Consensus: consensus, + Identity: identity, + SandboxBinaryPath: sandboxBinary, + InsecureNoSandbox: insecureNoSandbox, + InsecureMock: insecureMock, + RuntimeAttestInterval: attestInterval, + }) + if err != nil { + return nil, fmt.Errorf("failed to create SGX runtime provisioner: %w", err) + } + } + default: + return nil, fmt.Errorf("unsupported runtime provisioner: %s", p) + } + + // Configure TDX provisioner. + // TODO: Allow provisioner selection in the future, currently we only have QEMU. + provisioners[component.TEEKindTDX], err = hostTdx.NewQemu(hostTdx.QemuConfig{ + HostInfo: hostInfo, + CommonStore: commonStore, + PCS: qs, + Consensus: consensus, + Identity: identity, + RuntimeAttestInterval: attestInterval, + }) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create TDX runtime provisioner: %w", err) } - cfg.History = history - return &cfg, nil + // Configure optional load balancing. + for tee, rp := range provisioners { + provisioners[tee] = hostLoadBalance.New(rp, hostLoadBalance.Config{ + NumInstances: int(config.GlobalConfig.Runtime.LoadBalancer.NumInstances), + }) + } + + // Create a composite provisioner to provision the individual components. + provisioner := hostComposite.NewProvisioner(provisioners) + + return provisioner, nil +} + +func createCachingQuoteService(commonStore *persistent.CommonStore) (pcs.QuoteService, error) { + pc, err := pcs.NewHTTPClient(&pcs.HTTPClientConfig{ + // TODO: Support configuring the API key. + }) + if err != nil { + return nil, fmt.Errorf("failed to create PCS HTTP client: %w", err) + } + + qs := pcs.NewCachingQuoteService(pc, commonStore) + + return qs, nil } func createHistoryFactory() (history.Factory, error) { diff --git a/go/runtime/registry/registry.go b/go/runtime/registry/registry.go index 6c6825f8634..74a9b48c2d9 100644 --- a/go/runtime/registry/registry.go +++ b/go/runtime/registry/registry.go @@ -17,6 +17,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/pubsub" cmSync "github.com/oasisprotocol/oasis-core/go/common/sync" "github.com/oasisprotocol/oasis-core/go/common/version" + "github.com/oasisprotocol/oasis-core/go/config" consensus "github.com/oasisprotocol/oasis-core/go/consensus/api" ias "github.com/oasisprotocol/oasis-core/go/ias/api" registry "github.com/oasisprotocol/oasis-core/go/registry/api" @@ -437,13 +438,14 @@ type runtimeRegistry struct { logger *logging.Logger dataDir string - cfg *RuntimeConfig consensus consensus.Backend client runtimeClient.RuntimeClient - runtimes map[common.Namespace]*runtime + runtimes map[common.Namespace]*runtime + runtimesCfg map[common.Namespace]map[version.Version]*runtimeHost.Config + provisioner runtimeHost.Provisioner historyFactory history.Factory } @@ -491,7 +493,7 @@ func (r *runtimeRegistry) NewRuntime(ctx context.Context, runtimeID common.Names return nil, fmt.Errorf("runtime/registry: runtime already registered: %s", runtimeID) } - rt, err := newRuntime(runtimeID, managed, r.dataDir, r.consensus, r.cfg.Provisioner, r.cfg.Runtimes[runtimeID]) + rt, err := newRuntime(runtimeID, managed, r.dataDir, r.consensus, r.provisioner, r.runtimesCfg[runtimeID]) if err != nil { return nil, err } @@ -577,12 +579,31 @@ func New( consensus consensus.Backend, ias []ias.Endpoint, ) (Registry, error) { + // Create history keeper factory. historyFactory, err := createHistoryFactory() if err != nil { return nil, err } - cfg, err := newRuntimeConfig(dataDir, commonStore, identity, consensus, ias) + // Configure host environment information. + hostInfo, err := createHostInfo(consensus) + if err != nil { + return nil, err + } + + // Create the PCS client and quote service. + qs, err := createCachingQuoteService(commonStore) + if err != nil { + return nil, err + } + + // Create runtime provisioner. + provisioner, err := createProvisioner(commonStore, identity, consensus, hostInfo, ias, qs) + if err != nil { + return nil, err + } + + runtimesCfg, err := newRuntimeConfig(dataDir) if err != nil { return nil, err } @@ -590,13 +611,18 @@ func New( r := &runtimeRegistry{ logger: logging.GetLogger("runtime/registry"), dataDir: dataDir, - cfg: cfg, consensus: consensus, runtimes: make(map[common.Namespace]*runtime), + runtimesCfg: runtimesCfg, + provisioner: provisioner, historyFactory: historyFactory, } - for _, runtimeID := range cfg.RuntimeIDs() { + if config.GlobalConfig.Mode == config.ModeKeyManager { + return r, nil + } + + for runtimeID := range runtimesCfg { if _, err := r.NewRuntime(ctx, runtimeID, true); err != nil { r.logger.Error("failed to add runtime", "err", err, From aac8259ffba02e4ba6fa5eeb18238fe0c1176def Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Tue, 19 Nov 2024 05:26:22 +0100 Subject: [PATCH 09/27] go/runtime/registry: Add support for adding new runtime versions --- go/oasis-node/cmd/node/node_control.go | 4 ++ go/runtime/registry/registry.go | 85 +++++++++++++++++++++----- go/worker/common/worker.go | 5 +- go/worker/keymanager/init.go | 4 +- go/worker/registration/worker.go | 4 ++ 5 files changed, 83 insertions(+), 19 deletions(-) diff --git a/go/oasis-node/cmd/node/node_control.go b/go/oasis-node/cmd/node/node_control.go index 90df07933df..9a654c0dc31 100644 --- a/go/oasis-node/cmd/node/node_control.go +++ b/go/oasis-node/cmd/node/node_control.go @@ -199,6 +199,10 @@ func (n *Node) getRuntimeStatus(ctx context.Context) (map[common.Namespace]contr runtimes := make(map[common.Namespace]control.RuntimeStatus) for _, rt := range n.RuntimeRegistry.Runtimes() { + if !rt.IsManaged() { + continue + } + var status control.RuntimeStatus // Fetch runtime registry descriptor. Do not wait too long for the descriptor to become diff --git a/go/runtime/registry/registry.go b/go/runtime/registry/registry.go index 74a9b48c2d9..142773c0b48 100644 --- a/go/runtime/registry/registry.go +++ b/go/runtime/registry/registry.go @@ -44,10 +44,10 @@ var ErrRuntimeHostNotConfigured = errors.New("runtime/registry: runtime host not // Registry is the running node's runtime registry interface. type Registry interface { - // GetRuntime returns the per-runtime interface if the runtime is managed. + // GetRuntime returns the per-runtime interface. GetRuntime(runtimeID common.Namespace) (Runtime, error) - // Runtimes returns a list of all managed runtimes. + // Runtimes returns a list of all runtimes. Runtimes() []Runtime // NewRuntime creates a new runtime that may or may not be managed @@ -76,6 +76,9 @@ type Runtime interface { // DataDir returns the runtime-specific data directory. DataDir() string + // IsManaged returns true iff the runtime is managed by the registry. + IsManaged() bool + // RegistryDescriptor waits for the runtime to be registered and // then returns its registry descriptor. RegistryDescriptor(ctx context.Context) (*registry.Runtime, error) @@ -111,6 +114,10 @@ type Runtime interface { // HostVersions returns a list of supported runtime versions. HostVersions() []version.Version + + // WatchHostVersions returns a channel that produces a stream of versions + // as they are added to the runtime. + WatchHostVersions() (<-chan version.Version, *pubsub.Subscription) } type runtime struct { // nolint: maligned @@ -134,6 +141,7 @@ type runtime struct { // nolint: maligned registryDescriptorNotifier *pubsub.Broker activeDescriptorCh chan struct{} activeDescriptorNotifier *pubsub.Broker + versionNotifier *pubsub.Broker hostProvisioner runtimeHost.Provisioner hostConfig map[version.Version]*runtimeHost.Config @@ -147,7 +155,6 @@ func newRuntime( dataDir string, consensus consensus.Backend, provisioner runtimeHost.Provisioner, - cfg map[version.Version]*runtimeHost.Config, ) (*runtime, error) { logger := logging.GetLogger("runtime/registry").With("runtime_id", runtimeID) @@ -174,8 +181,9 @@ func newRuntime( registryDescriptorNotifier: pubsub.NewBroker(true), activeDescriptorCh: make(chan struct{}), activeDescriptorNotifier: pubsub.NewBroker(true), + versionNotifier: pubsub.NewBroker(false), hostProvisioner: provisioner, - hostConfig: cfg, + hostConfig: make(map[version.Version]*runtimeHost.Config), logger: logger, }, nil } @@ -190,6 +198,11 @@ func (r *runtime) DataDir() string { return r.dataDir } +// IsManaged implements Runtime. +func (r *runtime) IsManaged() bool { + return r.managed +} + // RegistryDescriptor implements Runtime. func (r *runtime) RegistryDescriptor(ctx context.Context) (*registry.Runtime, error) { // Wait for the descriptor to be ready. @@ -289,6 +302,30 @@ func (r *runtime) HostVersions() []version.Version { return versions } +// HostVersions implements Runtime. +func (r *runtime) WatchHostVersions() (<-chan version.Version, *pubsub.Subscription) { + sub := r.versionNotifier.Subscribe() + ch := make(chan version.Version) + sub.Unwrap(ch) + + return ch, sub +} + +// addVersion adds the given version configuration to the runtime. +func (r *runtime) addVersion(version version.Version, cfg *runtimeHost.Config) error { + r.Lock() + defer r.Unlock() + + if _, ok := r.hostConfig[version]; ok { + return fmt.Errorf("runtime/registry: duplicate runtime version %s", version) + } + + r.hostConfig[version] = cfg + r.versionNotifier.Broadcast(version) + + return nil +} + // start starts the runtime worker. func (r *runtime) start() { r.startOne.TryStart(r.run) @@ -442,8 +479,7 @@ type runtimeRegistry struct { consensus consensus.Backend client runtimeClient.RuntimeClient - runtimes map[common.Namespace]*runtime - runtimesCfg map[common.Namespace]map[version.Version]*runtimeHost.Config + runtimes map[common.Namespace]*runtime provisioner runtimeHost.Provisioner historyFactory history.Factory @@ -451,11 +487,15 @@ type runtimeRegistry struct { // GetRuntime implements Registry. func (r *runtimeRegistry) GetRuntime(runtimeID common.Namespace) (Runtime, error) { + return r.getRuntime(runtimeID) +} + +func (r *runtimeRegistry) getRuntime(runtimeID common.Namespace) (*runtime, error) { r.RLock() defer r.RUnlock() rt, ok := r.runtimes[runtimeID] - if !ok || !rt.managed { + if !ok { return nil, fmt.Errorf("runtime/registry: runtime %s not found", runtimeID) } return rt, nil @@ -468,9 +508,7 @@ func (r *runtimeRegistry) Runtimes() []Runtime { rts := make([]Runtime, 0, len(r.runtimes)) for _, rt := range r.runtimes { - if rt.managed { - rts = append(rts, rt) - } + rts = append(rts, rt) } return rts } @@ -493,7 +531,7 @@ func (r *runtimeRegistry) NewRuntime(ctx context.Context, runtimeID common.Names return nil, fmt.Errorf("runtime/registry: runtime already registered: %s", runtimeID) } - rt, err := newRuntime(runtimeID, managed, r.dataDir, r.consensus, r.provisioner, r.runtimesCfg[runtimeID]) + rt, err := newRuntime(runtimeID, managed, r.dataDir, r.consensus, r.provisioner) if err != nil { return nil, err } @@ -524,6 +562,16 @@ func (r *runtimeRegistry) NewRuntime(ctx context.Context, runtimeID common.Names return rt, nil } +// addRuntimeVersion adds the given version configuration to the given runtime. +func (r *runtimeRegistry) addRuntimeVersion(version version.Version, cfg *runtimeHost.Config) error { + rt, err := r.getRuntime(cfg.Bundle.Manifest.ID) + if err != nil { + return err + } + + return rt.addVersion(version, cfg) +} + // RegisterClient implements Registry. func (r *runtimeRegistry) RegisterClient(rc runtimeClient.RuntimeClient) error { r.Lock() @@ -613,17 +661,14 @@ func New( dataDir: dataDir, consensus: consensus, runtimes: make(map[common.Namespace]*runtime), - runtimesCfg: runtimesCfg, provisioner: provisioner, historyFactory: historyFactory, } - if config.GlobalConfig.Mode == config.ModeKeyManager { - return r, nil - } + managed := config.GlobalConfig.Mode != config.ModeKeyManager for runtimeID := range runtimesCfg { - if _, err := r.NewRuntime(ctx, runtimeID, true); err != nil { + if _, err := r.NewRuntime(ctx, runtimeID, managed); err != nil { r.logger.Error("failed to add runtime", "err", err, "id", runtimeID, @@ -632,5 +677,13 @@ func New( } } + for _, cfgs := range runtimesCfg { + for version, cfg := range cfgs { + if err := r.addRuntimeVersion(version, cfg); err != nil { + return nil, err + } + } + } + return r, nil } diff --git a/go/worker/common/worker.go b/go/worker/common/worker.go index 3ac06761faa..39917f46fb2 100644 --- a/go/worker/common/worker.go +++ b/go/worker/common/worker.go @@ -235,8 +235,11 @@ func New( return w, nil } - // Register all configured runtimes. + // Register all configured managed runtimes. for _, rt := range runtimeRegistry.Runtimes() { + if !rt.IsManaged() { + continue + } if err := w.registerRuntime(rt); err != nil { return nil, err } diff --git a/go/worker/keymanager/init.go b/go/worker/keymanager/init.go index c82f8fcbb06..df83bd64af9 100644 --- a/go/worker/keymanager/init.go +++ b/go/worker/keymanager/init.go @@ -65,9 +65,9 @@ func New( return nil, fmt.Errorf("worker/keymanager: failed to create role provider: %w", err) } - w.runtime, err = commonWorker.RuntimeRegistry.NewRuntime(ctx, w.runtimeID, false) + w.runtime, err = commonWorker.RuntimeRegistry.GetRuntime(w.runtimeID) if err != nil { - return nil, fmt.Errorf("worker/keymanager: failed to create runtime registry entry: %w", err) + return nil, fmt.Errorf("worker/keymanager: failed to get runtime: %w", err) } if numVers := len(w.runtime.HostVersions()); numVers != 1 { return nil, fmt.Errorf("worker/keymanager: expected a single runtime version (got %d)", numVers) diff --git a/go/worker/registration/worker.go b/go/worker/registration/worker.go index 2cb23844896..0225250742e 100644 --- a/go/worker/registration/worker.go +++ b/go/worker/registration/worker.go @@ -601,6 +601,10 @@ func (w *Worker) metricsWorker() { // Runtime metrics. for _, rt := range w.runtimeRegistry.Runtimes() { + if !rt.IsManaged() { + continue + } + rtLabel := rt.ID().String() faults := nodeStatus.Faults[rt.ID()] From 6080a4c22fb6e9b41941d1a8477af58a10ea799d Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Tue, 19 Nov 2024 05:25:44 +0100 Subject: [PATCH 10/27] go/worker/common/committee: Provision newly discovered runtime versions --- go/worker/common/committee/node.go | 29 ++++++++++---- go/worker/keymanager/init.go | 3 -- go/worker/keymanager/worker.go | 62 ++++++++++++++++++++++-------- 3 files changed, 69 insertions(+), 25 deletions(-) diff --git a/go/worker/common/committee/node.go b/go/worker/common/committee/node.go index f109cd60055..8c484c1f644 100644 --- a/go/worker/common/committee/node.go +++ b/go/worker/common/committee/node.go @@ -680,7 +680,12 @@ func (n *Node) worker() { } defer blocksSub.Close() - // Provision the hosted runtime. + // Start watching runtime versions so that we can provision new versions + // once they are discovered. + versionCh, versionSub := n.GetRuntime().WatchHostVersions() + defer versionSub.Close() + + // Provision all known versions. for _, version := range n.GetRuntime().HostVersions() { if err := n.ProvisionHostedRuntimeVersion(version); err != nil { n.logger.Error("failed to provision hosted runtime", @@ -691,6 +696,13 @@ func (n *Node) worker() { } } + // Perform initial hosted runtime version update to ensure we have something even in cases where + // initial block processing fails for any reason. + n.CrossNode.Lock() + n.updateHostedRuntimeVersionLocked() + n.CrossNode.Unlock() + + // Start the runtime and its notifier. hrt := n.GetHostedRuntime() hrtNotifier := n.GetRuntimeHostNotifier() @@ -703,12 +715,7 @@ func (n *Node) worker() { hrtNotifier.Start() defer hrtNotifier.Stop() - // Perform initial hosted runtime version update to ensure we have something even in cases where - // initial block processing fails for any reason. - n.CrossNode.Lock() - n.updateHostedRuntimeVersionLocked() - n.CrossNode.Unlock() - + // Enter the main processing loop. initialized := false for { select { @@ -752,6 +759,14 @@ func (n *Node) worker() { defer n.CrossNode.Unlock() n.handleRuntimeHostEventLocked(ev) }() + case version := <-versionCh: + // Received a new runtime version. + if err := n.ProvisionHostedRuntimeVersion(version); err != nil { + n.logger.Error("failed to provision hosted runtime", + "err", err, + "version", version, + ) + } } } } diff --git a/go/worker/keymanager/init.go b/go/worker/keymanager/init.go index df83bd64af9..175d081ab27 100644 --- a/go/worker/keymanager/init.go +++ b/go/worker/keymanager/init.go @@ -69,9 +69,6 @@ func New( if err != nil { return nil, fmt.Errorf("worker/keymanager: failed to get runtime: %w", err) } - if numVers := len(w.runtime.HostVersions()); numVers != 1 { - return nil, fmt.Errorf("worker/keymanager: expected a single runtime version (got %d)", numVers) - } // Prepare the runtime host node helpers. w.RuntimeHostNode, err = runtimeRegistry.NewRuntimeHostNode(w) diff --git a/go/worker/keymanager/worker.go b/go/worker/keymanager/worker.go index e3ad3bdfd57..f050aa953f0 100644 --- a/go/worker/keymanager/worker.go +++ b/go/worker/keymanager/worker.go @@ -391,19 +391,55 @@ func (w *Worker) worker() { } w.logger.Info("consensus has finished initial synchronization") - // Provision the hosted runtime. - w.logger.Info("provisioning key manager runtime") + // Key managers always need to use the enclave version given to them in the bundle + // as they need to make sure that replication is possible during upgrades. + var version version.Version - for _, version := range w.GetRuntime().HostVersions() { - if err := w.ProvisionHostedRuntimeVersion(version); err != nil { - w.logger.Error("failed to provision key manager runtime", - "err", err, - "version", version, - ) - return + if ok := func() bool { + // Start watching runtime versions so that we can wait for the runtime + // to be discovered. + versionCh, versionSub := w.GetRuntime().WatchHostVersions() + defer versionSub.Close() + + // Make sure we have one version before proceeding. + versions := w.runtime.HostVersions() + + switch numVers := len(versions); numVers { + case 0: + w.logger.Info("waiting runtime version to be discovered") + + select { + case version = <-versionCh: + case <-w.ctx.Done(): + return false + } + case 1: + version = versions[0] + default: + w.logger.Error("expected a single runtime version (got %d)", numVers) + return false } + + return true + }(); !ok { + return } + // Provision the specified runtime version. + w.logger.Info("provisioning key manager runtime") + + if err := w.ProvisionHostedRuntimeVersion(version); err != nil { + w.logger.Error("failed to provision key manager runtime", + "err", err, + "version", version, + ) + return + } + + // Set the runtime to the specified version. + w.SetHostedRuntimeVersion(&version, nil) + + // Start the runtime and its notifier. hrt := w.GetHostedRuntime() hrtNotifier := w.GetRuntimeHostNotifier() @@ -416,15 +452,11 @@ func (w *Worker) worker() { hrtNotifier.Start() defer hrtNotifier.Stop() - // Key managers always need to use the enclave version given to them in the bundle - // as they need to make sure that replication is possible during upgrades. - activeVersion := w.runtime.HostVersions()[0] // Init made sure we have exactly one. - w.SetHostedRuntimeVersion(&activeVersion, nil) - + // Ensure that the runtime version is active. if _, err := w.GetHostedRuntimeActiveVersion(); err != nil { w.logger.Error("failed to activate key manager runtime version", "err", err, - "version", activeVersion, + "version", version, ) return } From 2bda3797efc4e4e11ef0f33984e662a3d586a957 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Mon, 2 Dec 2024 00:10:59 +0100 Subject: [PATCH 11/27] go/runtime/bundle/manifest: Add version to component --- go/oasis-test-runner/oasis/runtime.go | 6 +++--- go/runtime/bundle/bundle.go | 10 ++++++++++ go/runtime/bundle/manifest.go | 5 +++++ go/runtime/host/composite/composite.go | 11 ++++++----- go/runtime/host/sandbox/sandbox.go | 11 ++++++----- go/runtime/host/sgx/common/common.go | 2 +- go/runtime/registry/config.go | 5 +++-- 7 files changed, 34 insertions(+), 16 deletions(-) diff --git a/go/oasis-test-runner/oasis/runtime.go b/go/oasis-test-runner/oasis/runtime.go index 6dfe78e06f6..673f88ce519 100644 --- a/go/oasis-test-runner/oasis/runtime.go +++ b/go/oasis-test-runner/oasis/runtime.go @@ -259,9 +259,8 @@ func (rt *Runtime) toRuntimeBundle(deploymentIndex int) (*bundle.Bundle, error) // Prepare bundle. bnd := &bundle.Bundle{ Manifest: &bundle.Manifest{ - Name: "test-runtime", - ID: rt.cfgSave.id, - Version: deployCfg.version, + Name: "test-runtime", + ID: rt.cfgSave.id, }, } @@ -278,6 +277,7 @@ func (rt *Runtime) toRuntimeBundle(deploymentIndex int) (*bundle.Bundle, error) comp := &bundle.Component{ Kind: compCfg.Kind, + Version: deployCfg.version, Executable: elfBin, } diff --git a/go/runtime/bundle/bundle.go b/go/runtime/bundle/bundle.go index 397f6500765..f6d259d59c4 100644 --- a/go/runtime/bundle/bundle.go +++ b/go/runtime/bundle/bundle.go @@ -561,6 +561,16 @@ func Open(fn string) (*Bundle, error) { return nil, err } + // Support legacy manifests where the runtime version is defined at the top level. + if bnd.Manifest.Version.ToU64() > 0 { + for _, comp := range bnd.Manifest.Components { + if comp.ID().IsRONL() { + comp.Version = bnd.Manifest.Version + break + } + } + } + return bnd, nil } diff --git a/go/runtime/bundle/manifest.go b/go/runtime/bundle/manifest.go index 73ca1540f18..ffa765db4aa 100644 --- a/go/runtime/bundle/manifest.go +++ b/go/runtime/bundle/manifest.go @@ -24,6 +24,7 @@ type Manifest struct { ID common.Namespace `json:"id"` // Version is the runtime version. + // NOTE: This may go away in the future, use `Component.Version` instead. Version version.Version `json:"version,omitempty"` // Executable is the name of the runtime ELF executable file. @@ -115,6 +116,7 @@ func (m *Manifest) GetComponentByID(id component.ID) *Component { return &Component{ Kind: component.RONL, Executable: m.Executable, + Version: m.Version, SGX: m.SGX, } } @@ -233,6 +235,9 @@ type Component struct { // provided by a runtime. Name string `json:"name,omitempty"` + // Version is the component version. + Version version.Version + // Executable is the name of the runtime ELF executable file if any. Executable string `json:"executable,omitempty"` diff --git a/go/runtime/host/composite/composite.go b/go/runtime/host/composite/composite.go index 727c5183fbf..80ee3e7878f 100644 --- a/go/runtime/host/composite/composite.go +++ b/go/runtime/host/composite/composite.go @@ -31,11 +31,10 @@ type rhost struct { // NewHost creates a new composite runtime host. func NewHost(cfg host.Config, provisioner host.Provisioner) (host.Runtime, error) { h := &rhost{ - id: cfg.Bundle.Manifest.ID, - version: cfg.Bundle.Manifest.Version, - comps: make(map[component.ID]host.Runtime), - stopCh: make(chan struct{}), - logger: logging.GetLogger("runtime/host/composite").With("runtime_id", cfg.Bundle.Manifest.ID), + id: cfg.Bundle.Manifest.ID, + comps: make(map[component.ID]host.Runtime), + stopCh: make(chan struct{}), + logger: logging.GetLogger("runtime/host/composite").With("runtime_id", cfg.Bundle.Manifest.ID), } // Collect available components. @@ -51,6 +50,8 @@ func NewHost(cfg host.Config, provisioner host.Provisioner) (host.Runtime, error compCfg := cfg compCfg.Components = []component.ID{id} switch id.Kind { + case component.RONL: + h.version = c.Version case component.ROFL: // Wrap message handler for ROFL component. var err error diff --git a/go/runtime/host/sandbox/sandbox.go b/go/runtime/host/sandbox/sandbox.go index c25f753e529..2a07d9f5313 100644 --- a/go/runtime/host/sandbox/sandbox.go +++ b/go/runtime/host/sandbox/sandbox.go @@ -369,18 +369,19 @@ func (r *sandboxedRuntime) startProcess() (err error) { hi.LocalConfig = r.rtCfg.LocalConfig // Perform common host initialization. - var rtVersion *version.Version initCtx, cancelInit := context.WithTimeout(ctx, runtimeInitTimeout) defer cancelInit() - if rtVersion, err = pc.InitHost(initCtx, conn, hi); err != nil { + + rtVersion, err := pc.InitHost(initCtx, conn, hi) + if err != nil { return fmt.Errorf("failed to initialize connection: %w", err) } - if r.rtCfg.Components[0].IsRONL() { + if id := r.rtCfg.Components[0]; id.IsRONL() { // Make sure the version matches what is configured in the bundle. This check is skipped for // non-RONL components to support detached bundles. - if bndVersion := r.rtCfg.Bundle.Manifest.Version; *rtVersion != bndVersion { - return fmt.Errorf("version mismatch (runtime reported: %s bundle: %s)", *rtVersion, bndVersion) + if comp := r.rtCfg.Bundle.Manifest.GetComponentByID(id); comp.Version != *rtVersion { + return fmt.Errorf("version mismatch (runtime reported: %s bundle: %s)", *rtVersion, comp.Version) } } diff --git a/go/runtime/host/sgx/common/common.go b/go/runtime/host/sgx/common/common.go index 7ab8f6367e8..984087e5c88 100644 --- a/go/runtime/host/sgx/common/common.go +++ b/go/runtime/host/sgx/common/common.go @@ -45,7 +45,7 @@ func GetQuotePolicy( if err != nil { return nil, fmt.Errorf("failed to query runtime descriptor: %w", err) } - if d := rt.DeploymentForVersion(rtCfg.Bundle.Manifest.Version); d != nil { + if d := rt.DeploymentForVersion(comp.Version); d != nil { var sc node.SGXConstraints if err = cbor.Unmarshal(d.TEE, &sc); err != nil { return nil, fmt.Errorf("malformed runtime SGX constraints: %w", err) diff --git a/go/runtime/registry/config.go b/go/runtime/registry/config.go index e52d9087ddd..50ba5a1942c 100644 --- a/go/runtime/registry/config.go +++ b/go/runtime/registry/config.go @@ -128,7 +128,8 @@ func newRuntimeConfig(dataDir string) (map[common.Namespace]map[version.Version] if runtimes[id] == nil { runtimes[id] = make(map[version.Version]*runtimeHost.Config) } - if _, ok := runtimes[id][bnd.Manifest.Version]; ok { + version := bnd.Manifest.GetComponentByID(component.ID_RONL).Version + if _, ok := runtimes[id][version]; ok { return nil, fmt.Errorf("duplicate runtime '%s' version '%s'", id, bnd.Manifest.Version) } @@ -190,7 +191,7 @@ func newRuntimeConfig(dataDir string) (map[common.Namespace]map[version.Version] wantedComponents = append(wantedComponents, comp.ID()) } - runtimes[id][bnd.Manifest.Version] = &runtimeHost.Config{ + runtimes[id][version] = &runtimeHost.Config{ Bundle: rtBnd, Components: wantedComponents, LocalConfig: localConfig, From 15f22f5adbcfa9e688fe06d3f24168b7ff9c546d Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Tue, 19 Nov 2024 11:45:11 +0100 Subject: [PATCH 12/27] go/runtime/host: Replace runtime bundle with exploded components --- go/runtime/bundle/manifest.go | 21 +++++++- go/runtime/host/composite/composite.go | 30 +++++------ go/runtime/host/host.go | 47 +++++++---------- go/runtime/host/loadbalance/loadbalance.go | 6 +-- go/runtime/host/mock/mock.go | 2 +- go/runtime/host/sandbox/sandbox.go | 20 ++++---- go/runtime/host/sandbox/sandbox_test.go | 15 ++++-- go/runtime/host/sgx/common/common.go | 6 +-- go/runtime/host/sgx/epid.go | 2 +- go/runtime/host/sgx/sgx.go | 16 +++--- go/runtime/host/sgx/sgx_test.go | 20 ++++++-- go/runtime/host/tdx/qemu.go | 17 +++--- go/runtime/host/tests/tester.go | 3 ++ go/runtime/registry/config.go | 60 ++++++++++++---------- go/runtime/registry/registry.go | 2 +- 15 files changed, 147 insertions(+), 120 deletions(-) diff --git a/go/runtime/bundle/manifest.go b/go/runtime/bundle/manifest.go index ffa765db4aa..b82ecf05322 100644 --- a/go/runtime/bundle/manifest.go +++ b/go/runtime/bundle/manifest.go @@ -2,6 +2,7 @@ package bundle import ( "fmt" + "path/filepath" "github.com/oasisprotocol/oasis-core/go/common" "github.com/oasisprotocol/oasis-core/go/common/crypto/hash" @@ -20,7 +21,7 @@ type Manifest struct { // Name is the optional human readable runtime name. Name string `json:"name,omitempty"` - // ID is the runtime ID. + // ID is the runtime identifier. ID common.Namespace `json:"id"` // Version is the runtime version. @@ -226,6 +227,24 @@ type Identity struct { Enclave sgx.EnclaveIdentity `json:"enclave"` } +// ExplodedComponent is an exploded runtime component ready for execution. +type ExplodedComponent struct { + *Component + + // Detached returns true iff the bundle containing the component does not + // include a RONL component. + Detached bool + + // ExplodedDataDir is the path to the data directory where the bundle + // containing the component has been extracted. + ExplodedDataDir string +} + +// ExplodedPath returns the path that the corresponding asset will be written to via WriteExploded. +func (c *ExplodedComponent) ExplodedPath(fn string) string { + return filepath.Join(c.ExplodedDataDir, fn) +} + // Component is a runtime component. type Component struct { // Kind is the component kind. diff --git a/go/runtime/host/composite/composite.go b/go/runtime/host/composite/composite.go index 80ee3e7878f..c4b01124156 100644 --- a/go/runtime/host/composite/composite.go +++ b/go/runtime/host/composite/composite.go @@ -13,6 +13,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/node" "github.com/oasisprotocol/oasis-core/go/common/pubsub" "github.com/oasisprotocol/oasis-core/go/common/version" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle" "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" "github.com/oasisprotocol/oasis-core/go/runtime/host" "github.com/oasisprotocol/oasis-core/go/runtime/host/protocol" @@ -31,31 +32,23 @@ type rhost struct { // NewHost creates a new composite runtime host. func NewHost(cfg host.Config, provisioner host.Provisioner) (host.Runtime, error) { h := &rhost{ - id: cfg.Bundle.Manifest.ID, + id: cfg.ID, comps: make(map[component.ID]host.Runtime), stopCh: make(chan struct{}), - logger: logging.GetLogger("runtime/host/composite").With("runtime_id", cfg.Bundle.Manifest.ID), + logger: logging.GetLogger("runtime/host/composite").With("runtime_id", cfg.ID), } - // Collect available components. - availableComps := cfg.Bundle.Manifest.GetAvailableComponents() - // Iterate over all wanted components and provision the individual runtimes. - for _, id := range cfg.Components { - c, ok := availableComps[id] - if !ok { - continue // Skip any components that we want but don't have. - } - + for _, comp := range cfg.Components { compCfg := cfg - compCfg.Components = []component.ID{id} - switch id.Kind { + compCfg.Components = []*bundle.ExplodedComponent{comp} + switch comp.Kind { case component.RONL: - h.version = c.Version + h.version = comp.Version case component.ROFL: // Wrap message handler for ROFL component. var err error - compCfg.MessageHandler, err = compCfg.MessageHandler.NewSubHandler(h, c) + compCfg.MessageHandler, err = compCfg.MessageHandler.NewSubHandler(h, comp.Component) if err != nil { return nil, fmt.Errorf("host/composite: failed to create sub-handler: %w", err) } @@ -64,11 +57,12 @@ func NewHost(cfg host.Config, provisioner host.Provisioner) (host.Runtime, error compRt, err := provisioner.NewRuntime(compCfg) if err != nil { - return nil, fmt.Errorf("host/composite: failed to provision runtime component '%s': %w", id, err) + return nil, fmt.Errorf("host/composite: failed to provision runtime component '%s': %w", comp.ID(), err) } - h.comps[id] = compRt + h.comps[comp.ID()] = compRt } + if _, ronlExists := h.comps[component.ID_RONL]; !ronlExists { return nil, fmt.Errorf("host/composite: required RONL component not available") } @@ -205,7 +199,7 @@ func (p *provisioner) NewRuntime(cfg host.Config) (host.Runtime, error) { return nil, fmt.Errorf("host/composite: exactly one component should be selected") } - comp := cfg.Bundle.Manifest.GetComponentByID(cfg.Components[0]) + comp := cfg.Components[0] if comp == nil { return nil, fmt.Errorf("host/composite: component not available") } diff --git a/go/runtime/host/host.go b/go/runtime/host/host.go index d3ac72cda05..adb4b6d925d 100644 --- a/go/runtime/host/host.go +++ b/go/runtime/host/host.go @@ -4,7 +4,6 @@ package host import ( "context" "fmt" - "path/filepath" "github.com/oasisprotocol/oasis-core/go/common" "github.com/oasisprotocol/oasis-core/go/common/node" @@ -17,12 +16,14 @@ import ( // Config contains common configuration for the provisioned runtime. type Config struct { - // Bundle is the runtime bundle. - Bundle *RuntimeBundle + // Name is the optional human readable runtime name. + Name string - // Components are optional component ids that should be provisioned in case the runtime has - // multiple components. - Components []component.ID + // ID is the runtime identifier. + ID common.Namespace + + // Components are components that should be provisioned. + Components []*bundle.ExplodedComponent // Extra is an optional provisioner-specific configuration. Extra interface{} @@ -34,36 +35,24 @@ type Config struct { LocalConfig map[string]interface{} } -// GetComponent ensures that only a single component is configured for this runtime and returns it. -func (cfg *Config) GetComponent() (*bundle.Component, error) { +// GetExplodedComponent ensures that only a single exploded component is configured for this runtime +// and returns it. +func (cfg *Config) GetExplodedComponent() (*bundle.ExplodedComponent, error) { if numComps := len(cfg.Components); numComps != 1 { return nil, fmt.Errorf("expected a single component (got %d)", numComps) } - comp := cfg.Bundle.Manifest.GetComponentByID(cfg.Components[0]) - if comp == nil { - return nil, fmt.Errorf("component '%s' not available", cfg.Components[0]) - } - return comp, nil -} -// RuntimeBundle is a exploded runtime bundle ready for execution. -type RuntimeBundle struct { - *bundle.Bundle - - // ExplodedDataDir is the path to the data directory under which the bundle has been exploded. - ExplodedDataDir string - - // ExplodedDetachedDirs are the paths to the data directories of detached components. - ExplodedDetachedDirs map[component.ID]string + return cfg.Components[0], nil } -// ExplodedPath returns the path where the given asset for the given component can be found. -func (bnd *RuntimeBundle) ExplodedPath(comp component.ID, fn string) string { - if detachedDir, ok := bnd.ExplodedDetachedDirs[comp]; ok { - return filepath.Join(detachedDir, fn) +// GetComponent ensures that only a single component is configured for this runtime and returns it. +func (cfg *Config) GetComponent() (*bundle.Component, error) { + comp, err := cfg.GetExplodedComponent() + if err != nil { + return nil, err } - // Default to the exploded bundle. - return bnd.Bundle.ExplodedPath(bnd.ExplodedDataDir, fn) + + return comp.Component, nil } // Provisioner is the runtime provisioner interface. diff --git a/go/runtime/host/loadbalance/loadbalance.go b/go/runtime/host/loadbalance/loadbalance.go index bbee436f36b..d6ed63cd04d 100644 --- a/go/runtime/host/loadbalance/loadbalance.go +++ b/go/runtime/host/loadbalance/loadbalance.go @@ -262,7 +262,7 @@ func (lb *lbProvisioner) NewRuntime(cfg host.Config) (host.Runtime, error) { if len(cfg.Components) != 1 { return nil, fmt.Errorf("host/loadbalance: must specify a single component") } - if cfg.Components[0] != component.ID_RONL { + if cfg.Components[0].ID() != component.ID_RONL { return lb.inner.NewRuntime(cfg) } @@ -278,11 +278,11 @@ func (lb *lbProvisioner) NewRuntime(cfg host.Config) (host.Runtime, error) { } return &lbRuntime{ - id: cfg.Bundle.Manifest.ID, + id: cfg.ID, instances: instances, healthyInstances: make(map[int]struct{}), stopCh: make(chan struct{}), - logger: logging.GetLogger("runtime/host/loadbalance").With("runtime_id", cfg.Bundle.Manifest.ID), + logger: logging.GetLogger("runtime/host/loadbalance").With("runtime_id", cfg.ID), }, nil } diff --git a/go/runtime/host/mock/mock.go b/go/runtime/host/mock/mock.go index 8dde4ea82d0..bb9ab5a33f9 100644 --- a/go/runtime/host/mock/mock.go +++ b/go/runtime/host/mock/mock.go @@ -28,7 +28,7 @@ var CheckTxFailInput = []byte("checktx-mock-fail") // Implements host.Provisioner. func (p *provisioner) NewRuntime(cfg host.Config) (host.Runtime, error) { r := &runtime{ - runtimeID: cfg.Bundle.Manifest.ID, + runtimeID: cfg.ID, notifier: pubsub.NewBroker(false), } return r, nil diff --git a/go/runtime/host/sandbox/sandbox.go b/go/runtime/host/sandbox/sandbox.go index 2a07d9f5313..92e34865b53 100644 --- a/go/runtime/host/sandbox/sandbox.go +++ b/go/runtime/host/sandbox/sandbox.go @@ -81,17 +81,15 @@ type provisioner struct { // Implements host.Provisioner. func (p *provisioner) NewRuntime(cfg host.Config) (host.Runtime, error) { - id := cfg.Bundle.Manifest.ID - r := &sandboxedRuntime{ cfg: p.cfg, rtCfg: cfg, - id: id, + id: cfg.ID, stopCh: make(chan struct{}), ctrlCh: make(chan interface{}, ctrlChannelBufferSize), notifier: pubsub.NewBroker(false), notifyUpdateCapabilityTEECh: make(chan struct{}, 1), - logger: p.cfg.Logger.With("runtime_id", id), + logger: p.cfg.Logger.With("runtime_id", cfg.ID), } err := cfg.MessageHandler.AttachRuntime(r) @@ -377,11 +375,11 @@ func (r *sandboxedRuntime) startProcess() (err error) { return fmt.Errorf("failed to initialize connection: %w", err) } - if id := r.rtCfg.Components[0]; id.IsRONL() { + if comp := r.rtCfg.Components[0]; comp.ID().IsRONL() { // Make sure the version matches what is configured in the bundle. This check is skipped for // non-RONL components to support detached bundles. - if comp := r.rtCfg.Bundle.Manifest.GetComponentByID(id); comp.Version != *rtVersion { - return fmt.Errorf("version mismatch (runtime reported: %s bundle: %s)", *rtVersion, comp.Version) + if bndVersion := comp.Version; *rtVersion != bndVersion { + return fmt.Errorf("version mismatch (runtime reported: %s bundle: %s)", *rtVersion, bndVersion) } } @@ -605,21 +603,21 @@ func (r *sandboxedRuntime) manager() { // DefaultGetSandboxConfig is the default function for generating sandbox configuration. func DefaultGetSandboxConfig(logger *logging.Logger, sandboxBinaryPath string) GetSandboxConfigFunc { return func(hostCfg host.Config, _ Connector, _ string) (process.Config, error) { - comp, err := hostCfg.GetComponent() + comp, err := hostCfg.GetExplodedComponent() if err != nil { return process.Config{}, err } logWrapper := host.NewRuntimeLogWrapper( logger, - "runtime_id", hostCfg.Bundle.Manifest.ID, - "runtime_name", hostCfg.Bundle.Manifest.Name, + "runtime_id", hostCfg.ID, + "runtime_name", hostCfg.Name, "component", comp.ID(), "provisioner", "sandbox", ) return process.Config{ - Path: hostCfg.Bundle.ExplodedPath(comp.ID(), comp.Executable), + Path: comp.ExplodedPath(comp.Executable), SandboxBinaryPath: sandboxBinaryPath, Stdout: logWrapper, Stderr: logWrapper, diff --git a/go/runtime/host/sandbox/sandbox_test.go b/go/runtime/host/sandbox/sandbox_test.go index 972a3424a2c..8ec4e0f7b31 100644 --- a/go/runtime/host/sandbox/sandbox_test.go +++ b/go/runtime/host/sandbox/sandbox_test.go @@ -31,11 +31,18 @@ func TestProvisionerSandbox(t *testing.T) { _, err = bnd.WriteExploded(tmpDir) require.NoError(t, err, "bnd.WriteExploded") + explodedDataDir := bnd.ExplodedPath(tmpDir, "") + cfg := host.Config{ - Bundle: &host.RuntimeBundle{ - Bundle: bnd, - ExplodedDataDir: tmpDir, - }, + Name: bnd.Manifest.Name, + ID: bnd.Manifest.ID, + } + + for _, comp := range bnd.Manifest.Components { + cfg.Components = append(cfg.Components, &bundle.ExplodedComponent{ + Component: comp, + ExplodedDataDir: explodedDataDir, + }) } t.Run("Naked", func(t *testing.T) { diff --git a/go/runtime/host/sgx/common/common.go b/go/runtime/host/sgx/common/common.go index 984087e5c88..09a59e462f4 100644 --- a/go/runtime/host/sgx/common/common.go +++ b/go/runtime/host/sgx/common/common.go @@ -21,7 +21,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/runtime/host/sandbox" ) -// GetQuotePolicy fetches the quote policy for the given manifest/component. In case the policy is +// GetQuotePolicy fetches the quote policy for the given component. In case the policy is // not available, return the fallback policy. func GetQuotePolicy( ctx context.Context, @@ -29,7 +29,7 @@ func GetQuotePolicy( cb consensus.Backend, fallbackPolicy *sgxQuote.Policy, ) (*sgxQuote.Policy, error) { - comp, err := rtCfg.GetComponent() + comp, err := rtCfg.GetExplodedComponent() if err != nil { return nil, err } @@ -39,7 +39,7 @@ func GetQuotePolicy( // Load RONL policy from the consensus layer. rt, err := cb.Registry().GetRuntime(ctx, ®istry.GetRuntimeQuery{ Height: consensus.HeightLatest, - ID: rtCfg.Bundle.Manifest.ID, + ID: rtCfg.ID, IncludeSuspended: true, }) if err != nil { diff --git a/go/runtime/host/sgx/epid.go b/go/runtime/host/sgx/epid.go index 4fcad54c80e..c8b493f8077 100644 --- a/go/runtime/host/sgx/epid.go +++ b/go/runtime/host/sgx/epid.go @@ -117,7 +117,7 @@ func (ep *teeStateEPID) update( } evidence := ias.Evidence{ - RuntimeID: ep.cfg.Bundle.Manifest.ID, + RuntimeID: ep.cfg.ID, Quote: quote, Nonce: nonce, EarlyTCBUpdate: true, diff --git a/go/runtime/host/sgx/sgx.go b/go/runtime/host/sgx/sgx.go index 5546972b4bd..e57091ffe3b 100644 --- a/go/runtime/host/sgx/sgx.go +++ b/go/runtime/host/sgx/sgx.go @@ -149,7 +149,7 @@ func (ts *teeState) update(ctx context.Context, sp *sgxProvisioner, conn protoco attestation, err := ts.impl.Update(ctx, sp, conn, report, nonce) - sgxCommon.UpdateAttestationMetrics(ts.cfg.Bundle.Manifest.ID, component.TEEKindSGX, err) + sgxCommon.UpdateAttestationMetrics(ts.cfg.ID, component.TEEKindSGX, err) return attestation, err } @@ -170,11 +170,11 @@ type sgxProvisioner struct { logger *logging.Logger } -func (s *sgxProvisioner) loadEnclaveBinaries(rtCfg host.Config, comp *bundle.Component) ([]byte, []byte, error) { +func (s *sgxProvisioner) loadEnclaveBinaries(comp *bundle.ExplodedComponent) ([]byte, []byte, error) { if comp.SGX == nil || comp.SGX.Executable == "" { return nil, nil, fmt.Errorf("SGX executable not available in bundle") } - sgxExecutablePath := rtCfg.Bundle.ExplodedPath(comp.ID(), comp.SGX.Executable) + sgxExecutablePath := comp.ExplodedPath(comp.SGX.Executable) var ( sig, sgxs []byte @@ -190,7 +190,7 @@ func (s *sgxProvisioner) loadEnclaveBinaries(rtCfg host.Config, comp *bundle.Com } if comp.SGX.Signature != "" { - sgxSignaturePath := rtCfg.Bundle.ExplodedPath(comp.ID(), comp.SGX.Signature) + sgxSignaturePath := comp.ExplodedPath(comp.SGX.Signature) sig, err = os.ReadFile(sgxSignaturePath) if err != nil { return nil, nil, fmt.Errorf("failed to load SIGSTRUCT: %w", err) @@ -235,7 +235,7 @@ func (s *sgxProvisioner) discoverSGXDevice() (string, error) { } func (s *sgxProvisioner) getSandboxConfig(rtCfg host.Config, conn sandbox.Connector, runtimeDir string) (process.Config, error) { - comp, err := rtCfg.GetComponent() + comp, err := rtCfg.GetExplodedComponent() if err != nil { return process.Config{}, err } @@ -255,7 +255,7 @@ func (s *sgxProvisioner) getSandboxConfig(rtCfg host.Config, conn sandbox.Connec signaturePath = filepath.Join(runtimeDir, signaturePath) } - sgxs, sig, err := s.loadEnclaveBinaries(rtCfg, comp) + sgxs, sig, err := s.loadEnclaveBinaries(comp) if err != nil { return process.Config{}, fmt.Errorf("host/sgx: failed to load enclave/signature: %w", err) } @@ -296,8 +296,8 @@ func (s *sgxProvisioner) getSandboxConfig(rtCfg host.Config, conn sandbox.Connec logWrapper := host.NewRuntimeLogWrapper( s.logger, - "runtime_id", rtCfg.Bundle.Manifest.ID, - "runtime_name", rtCfg.Bundle.Manifest.Name, + "runtime_id", rtCfg.ID, + "runtime_name", rtCfg.Name, "component", comp.ID(), "provisioner", s.Name(), ) diff --git a/go/runtime/host/sgx/sgx_test.go b/go/runtime/host/sgx/sgx_test.go index 15d6818b2d0..a3be2a814a7 100644 --- a/go/runtime/host/sgx/sgx_test.go +++ b/go/runtime/host/sgx/sgx_test.go @@ -54,11 +54,18 @@ func TestProvisionerSGX(t *testing.T) { _, err = bnd.WriteExploded(tmpDir) require.NoError(err, "bnd.WriteExploded") + explodedDataDir := bnd.ExplodedPath(tmpDir, "") + cfg := host.Config{ - Bundle: &host.RuntimeBundle{ - Bundle: bnd, - ExplodedDataDir: tmpDir, - }, + Name: "test-runtime", + ID: bnd.Manifest.ID, + } + + for _, comp := range bnd.Manifest.Components { + cfg.Components = append(cfg.Components, &bundle.ExplodedComponent{ + Component: comp, + ExplodedDataDir: explodedDataDir, + }) } ias, err := iasHttp.New(&iasHttp.Config{ @@ -69,7 +76,10 @@ func TestProvisionerSGX(t *testing.T) { require.NoError(err, "iasHttp.New") extraTests := []tests.TestCase{ - {"AttestationWorker", testAttestationWorker}, // nolint: govet + { + Name: "AttestationWorker", + Fn: testAttestationWorker, + }, } t.Run("Naked", func(t *testing.T) { diff --git a/go/runtime/host/tdx/qemu.go b/go/runtime/host/tdx/qemu.go index 4cb59261fac..6a6d04d3d36 100644 --- a/go/runtime/host/tdx/qemu.go +++ b/go/runtime/host/tdx/qemu.go @@ -105,7 +105,7 @@ func (q *qemuProvisioner) Name() string { } func (q *qemuProvisioner) getSandboxConfig(rtCfg host.Config, _ sandbox.Connector, _ string) (process.Config, error) { - comp, err := rtCfg.GetComponent() + comp, err := rtCfg.GetExplodedComponent() if err != nil { return process.Config{}, err } @@ -114,10 +114,9 @@ func (q *qemuProvisioner) getSandboxConfig(rtCfg host.Config, _ sandbox.Connecto } cid := rtCfg.Extra.(*QemuExtraConfig).CID // Ensured above. - bnd := rtCfg.Bundle tdxCfg := comp.TDX resources := tdxCfg.Resources - firmware := bnd.ExplodedPath(comp.ID(), tdxCfg.Firmware) + firmware := comp.ExplodedPath(tdxCfg.Firmware) cfg := process.Config{ Path: defaultQemuSystemPath, @@ -125,7 +124,7 @@ func (q *qemuProvisioner) getSandboxConfig(rtCfg host.Config, _ sandbox.Connecto "-accel", "kvm", "-m", fmt.Sprintf("%d", resources.Memory), "-smp", fmt.Sprintf("%d", resources.CPUCount), - "-name", fmt.Sprintf("oasis-%s-%s", bnd.Manifest.ID, comp.ID()), + "-name", fmt.Sprintf("oasis-%s-%s", rtCfg.ID, comp.ID()), "-cpu", "host", "-machine", "q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off", "-bios", firmware, @@ -144,18 +143,18 @@ func (q *qemuProvisioner) getSandboxConfig(rtCfg host.Config, _ sandbox.Connecto // Configure kernel when one is available. We can set up TDs that only include the virtual // firmware for special-purpose locked down TDs. if tdxCfg.HasKernel() { - kernelImage := bnd.ExplodedPath(comp.ID(), tdxCfg.Kernel) + kernelImage := comp.ExplodedPath(tdxCfg.Kernel) cfg.Args = append(cfg.Args, "-kernel", kernelImage) if tdxCfg.HasInitRD() { - initrdImage := bnd.ExplodedPath(comp.ID(), tdxCfg.InitRD) + initrdImage := comp.ExplodedPath(tdxCfg.InitRD) cfg.Args = append(cfg.Args, "-initrd", initrdImage) } // Configure stage 2 image. if tdxCfg.HasStage2() { - stage2Image := bnd.ExplodedPath(comp.ID(), tdxCfg.Stage2Image) + stage2Image := comp.ExplodedPath(tdxCfg.Stage2Image) cfg.Args = append(cfg.Args, // Stage 2 drive. @@ -191,8 +190,8 @@ func (q *qemuProvisioner) getSandboxConfig(rtCfg host.Config, _ sandbox.Connecto // Logging. logWrapper := host.NewRuntimeLogWrapper( q.logger, - "runtime_id", rtCfg.Bundle.Manifest.ID, - "runtime_name", rtCfg.Bundle.Manifest.Name, + "runtime_id", rtCfg.ID, + "runtime_name", rtCfg.Name, "component", comp.ID(), "provisioner", q.Name(), ) diff --git a/go/runtime/host/tests/tester.go b/go/runtime/host/tests/tester.go index c55a843c78b..b89cef7ac3d 100644 --- a/go/runtime/host/tests/tester.go +++ b/go/runtime/host/tests/tester.go @@ -157,6 +157,8 @@ func testBasic(t *testing.T, cfg host.Config, p host.Provisioner) { require.NoError(err, "Call") require.NotNil(rsp.Empty, "runtime response to RuntimePingRequest should return an Empty body") + // FIXME: Remove or replace this call, as the mockMessageHandler cannot + // handle HostFetchConsensusBlockRequest. req, err := mockRuntimeKeyManagerStatusUpdateRequest() require.NoError(err, "mockKeyManagerStatusRequest") @@ -196,6 +198,7 @@ func testRestart(t *testing.T, cfg host.Config, p host.Provisioner) { } // Trigger a force abort (should restart the runtime). + // FIXME: Runtimes do not support abort requests anymore. err = r.Abort(context.Background(), true) require.NoError(err, "Abort(force=true)") diff --git a/go/runtime/registry/config.go b/go/runtime/registry/config.go index 50ba5a1942c..20203d68006 100644 --- a/go/runtime/registry/config.go +++ b/go/runtime/registry/config.go @@ -139,30 +139,43 @@ func newRuntimeConfig(dataDir string) (map[common.Namespace]map[version.Version] localConfig = lc } - rtBnd := &runtimeHost.RuntimeBundle{ - Bundle: bnd, - ExplodedDataDir: dataDir, - ExplodedDetachedDirs: make(map[component.ID]string), + // Gather all components. + components := make(map[component.ID]*bundle.ExplodedComponent) + + // Add bundle components. + explodedDir := bnd.ExplodedPath(dataDir, "") + + for _, comp := range bnd.Manifest.Components { + components[comp.ID()] = &bundle.ExplodedComponent{ + Component: comp, + Detached: false, + ExplodedDataDir: explodedDir, + } } // Merge in detached components. for _, detachedBnd := range detachedBundles[id] { + explodedDir := detachedBnd.ExplodedPath(dataDir, "") + for _, detachedComp := range detachedBnd.Manifest.Components { // Skip components that already exist in the bundle itself. - if bnd.Manifest.GetComponentByID(detachedComp.ID()) != nil { + if _, ok := components[detachedComp.ID()]; ok { continue } - bnd.Manifest.Components = append(bnd.Manifest.Components, detachedComp) - rtBnd.ExplodedDetachedDirs[detachedComp.ID()] = detachedBnd.ExplodedPath(dataDir, "") + components[detachedComp.ID()] = &bundle.ExplodedComponent{ + Component: detachedComp, + Detached: true, + ExplodedDataDir: explodedDir, + } } } // Determine what kind of components we want. - wantedComponents := []component.ID{ - component.ID_RONL, + wantedComponents := []*bundle.ExplodedComponent{ + components[component.ID_RONL], } - for _, comp := range bnd.Manifest.Components { + for _, comp := range components { if comp.ID().IsRONL() { continue // Always enabled above. } @@ -174,7 +187,7 @@ func newRuntimeConfig(dataDir string) (map[common.Namespace]map[version.Version] enabled = false } // Detached components are explicit and they should be enabled by default. - if _, ok := rtBnd.ExplodedDetachedDirs[comp.ID()]; ok { + if comp.Detached { enabled = true } @@ -188,11 +201,12 @@ func newRuntimeConfig(dataDir string) (map[common.Namespace]map[version.Version] continue } - wantedComponents = append(wantedComponents, comp.ID()) + wantedComponents = append(wantedComponents, comp) } runtimes[id][version] = &runtimeHost.Config{ - Bundle: rtBnd, + Name: bnd.Manifest.Name, + ID: bnd.Manifest.ID, Components: wantedComponents, LocalConfig: localConfig, } @@ -209,22 +223,16 @@ func newRuntimeConfig(dataDir string) (map[common.Namespace]map[version.Version] } runtimeHostCfg := &runtimeHost.Config{ - Bundle: &runtimeHost.RuntimeBundle{ - Bundle: &bundle.Bundle{ - Manifest: &bundle.Manifest{ - ID: id, - Components: []*bundle.Component{ - { - Kind: component.RONL, - Executable: "mock", - }, - }, + ID: id, + Components: []*bundle.ExplodedComponent{ + { + Component: &bundle.Component{ + Kind: component.RONL, + Executable: "mock", }, + Detached: false, }, }, - Components: []component.ID{ - component.ID_RONL, - }, } runtimes[id] = map[version.Version]*runtimeHost.Config{ {}: runtimeHostCfg, diff --git a/go/runtime/registry/registry.go b/go/runtime/registry/registry.go index 142773c0b48..ae5d225b072 100644 --- a/go/runtime/registry/registry.go +++ b/go/runtime/registry/registry.go @@ -564,7 +564,7 @@ func (r *runtimeRegistry) NewRuntime(ctx context.Context, runtimeID common.Names // addRuntimeVersion adds the given version configuration to the given runtime. func (r *runtimeRegistry) addRuntimeVersion(version version.Version, cfg *runtimeHost.Config) error { - rt, err := r.getRuntime(cfg.Bundle.Manifest.ID) + rt, err := r.getRuntime(cfg.ID) if err != nil { return err } From cd092b7c0c2eadd13e6d8e146ea96dac3b98312c Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Mon, 2 Dec 2024 11:57:35 +0100 Subject: [PATCH 13/27] go/runtime/host/sandbox: Clear cmds before notifying of runtime start Fixes a bug that occurs when attempting to abort the runtime immediately after it starts. --- go/runtime/host/sandbox/sandbox.go | 36 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/go/runtime/host/sandbox/sandbox.go b/go/runtime/host/sandbox/sandbox.go index 92e34865b53..3893895c31d 100644 --- a/go/runtime/host/sandbox/sandbox.go +++ b/go/runtime/host/sandbox/sandbox.go @@ -408,6 +408,24 @@ func (r *sandboxedRuntime) startProcess() (err error) { r.rtVersion = rtVersion r.Unlock() + // Ensure the command queue is empty to avoid processing any stale requests after the + // runtime restarts. +drainLoop: + for { + select { + case grq := <-r.ctrlCh: + switch rq := grq.(type) { + case *abortRequest: + rq.ch <- fmt.Errorf("runtime restarted") + close(rq.ch) + default: + // Ignore unknown requests. + } + default: + break drainLoop + } + } + // Notify subscribers that a runtime has been started. r.notifier.Broadcast(&host.Event{Started: ev}) @@ -530,24 +548,6 @@ func (r *sandboxedRuntime) manager() { continue } - - // Ensure the command queue is empty to avoid processing any stale requests after the - // runtime restarts. - drainLoop: - for { - select { - case grq := <-r.ctrlCh: - switch rq := grq.(type) { - case *abortRequest: - rq.ch <- fmt.Errorf("runtime restarted") - close(rq.ch) - default: - // Ignore unknown requests. - } - default: - break drainLoop - } - } } // Wait for either the runtime or the runtime manager to terminate. From 7aeb4221e6497aa8555f1cacb8e2a7d11b2685af Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Mon, 2 Dec 2024 17:31:36 +0100 Subject: [PATCH 14/27] go/runtime/bundle: Add recommended filename to bundle --- go/runtime/bundle/bundle.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/go/runtime/bundle/bundle.go b/go/runtime/bundle/bundle.go index f6d259d59c4..44ba756263e 100644 --- a/go/runtime/bundle/bundle.go +++ b/go/runtime/bundle/bundle.go @@ -17,6 +17,15 @@ import ( "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" ) +const ( + // bundleFileExtension is the file extension used for storing the bundle. + bundleFileExtension = ".orc" + + // bundleFilenameRegexp is the regular expression pattern used for + // validating bundle filenames. + bundleFilenameRegexp = `^[a-f0-9]{64}\.orc$` +) + // Bundle is a runtime bundle instance. type Bundle struct { Manifest *Manifest @@ -28,6 +37,11 @@ type Bundle struct { manifestHash hash.Hash } +// GenerateFilename returns the recommended filename for storing the bundle. +func (bnd *Bundle) GenerateFilename() string { + return fmt.Sprintf("%s%s", bnd.manifestHash.Hex(), bundleFileExtension) +} + // Validate validates the runtime bundle for well-formedness. func (bnd *Bundle) Validate() error { // Ensure the manifest is valid. From 9276539cecf3847919e2caa4f9bf7f7875a8ee38 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Sat, 23 Nov 2024 03:03:10 +0100 Subject: [PATCH 15/27] go/runtime: Support hot-loading of runtime bundles The node can now fetch and verify runtime bundles from remote repositories and automatically update to new versions. --- .changelog/5962.cfg.md | 26 + go/oasis-net-runner/fixtures/default.go | 4 +- go/oasis-node/cmd/node/flags.go | 4 +- go/oasis-node/cmd/node/node.go | 7 +- go/oasis-node/cmd/storage/storage.go | 5 +- go/oasis-node/node_test.go | 4 +- go/oasis-test-runner/oasis/keymanager.go | 8 + go/oasis-test-runner/oasis/network.go | 30 +- go/oasis-test-runner/oasis/oasis.go | 29 +- go/oasis-test-runner/oasis/runtime.go | 47 +- .../scenario/e2e/runtime/helpers_runtime.go | 46 +- .../e2e/runtime/keymanager_upgrade.go | 29 +- .../scenario/e2e/runtime/runtime_upgrade.go | 166 ++++++- go/runtime/bundle/bundle.go | 12 +- go/runtime/bundle/discovery.go | 452 ++++++++++++++++++ go/runtime/bundle/discovery_test.go | 112 +++++ go/runtime/bundle/registry.go | 346 ++++++++++++++ go/runtime/bundle/registry_test.go | 148 ++++++ go/runtime/config/config.go | 73 ++- go/runtime/config/config_test.go | 36 +- go/runtime/registry/config.go | 259 ++-------- go/runtime/registry/registry.go | 239 +++++---- go/worker/common/committee/node.go | 2 +- go/worker/keymanager/worker.go | 4 + 24 files changed, 1686 insertions(+), 402 deletions(-) create mode 100644 .changelog/5962.cfg.md create mode 100644 go/runtime/bundle/discovery.go create mode 100644 go/runtime/bundle/discovery_test.go create mode 100644 go/runtime/bundle/registry.go create mode 100644 go/runtime/bundle/registry_test.go diff --git a/.changelog/5962.cfg.md b/.changelog/5962.cfg.md new file mode 100644 index 00000000000..903bf0ebe75 --- /dev/null +++ b/.changelog/5962.cfg.md @@ -0,0 +1,26 @@ +go/runtime: Support hot-loading of runtime bundles + +The node can now fetch and verify runtime bundles from remote repositories +and automatically update to new versions. + +The following configuration option has been removed: + +- `runtime.components` + +The following configuration option has been deprecated: + +- `runtime.config` + +The following configuration options have been added: + +- `runtime.runtimes.id` is the runtime identifier, + +- `runtime.runtimes.components` is the list of components to configure, + +- `runtime.runtimes.config` is the runtime local configuration, + +- `runtime.runtimes.repositories` is the list of runtime specific URLs + used to fetch runtime bundles, + +- `runtime.repositories` is the list of global URLs used to fetch + runtime bundles. diff --git a/go/oasis-net-runner/fixtures/default.go b/go/oasis-net-runner/fixtures/default.go index 182e20c7e4c..46c02c96b51 100644 --- a/go/oasis-net-runner/fixtures/default.go +++ b/go/oasis-net-runner/fixtures/default.go @@ -241,11 +241,11 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) { GovernanceModel: registry.GovernanceEntity, Deployments: []oasis.DeploymentCfg{ { - Version: rtVersion, ValidFrom: 0, Components: []oasis.ComponentCfg{ { - Kind: component.RONL, + Kind: component.RONL, + Version: rtVersion, Binaries: map[node.TEEHardware]string{ tee: rt, }, diff --git a/go/oasis-node/cmd/node/flags.go b/go/oasis-node/cmd/node/flags.go index cae1892b347..f3f03e7c9dd 100644 --- a/go/oasis-node/cmd/node/flags.go +++ b/go/oasis-node/cmd/node/flags.go @@ -9,7 +9,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/flags" cmdGrpc "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/grpc" cmdSigner "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/signer" - runtimeRegistry "github.com/oasisprotocol/oasis-core/go/runtime/registry" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle" workerStorage "github.com/oasisprotocol/oasis-core/go/worker/storage" ) @@ -38,7 +38,7 @@ func init() { for _, v := range []*flag.FlagSet{ cmdGrpc.ServerLocalFlags, cmdSigner.Flags, - runtimeRegistry.Flags, + bundle.Flags, workerStorage.Flags, crash.InitFlags(), } { diff --git a/go/oasis-node/cmd/node/node.go b/go/oasis-node/cmd/node/node.go index fcb999c767f..677389cffa8 100644 --- a/go/oasis-node/cmd/node/node.go +++ b/go/oasis-node/cmd/node/node.go @@ -232,7 +232,7 @@ func (n *Node) initRuntimeWorkers() error { if err != nil { return err } - n.svcMgr.RegisterCleanupOnly(n.RuntimeRegistry, "runtime registry") + n.svcMgr.Register(n.RuntimeRegistry) // Initialize the common worker. n.CommonWorker, err = workerCommon.New( @@ -353,6 +353,11 @@ func (n *Node) initRuntimeWorkers() error { } func (n *Node) startRuntimeWorkers() error { + // Start the runtime registry. + if err := n.RuntimeRegistry.Start(); err != nil { + return err + } + // Start the common worker. if err := n.CommonWorker.Start(); err != nil { return err diff --git a/go/oasis-node/cmd/storage/storage.go b/go/oasis-node/cmd/storage/storage.go index f0c820b4eb5..0f57586da10 100644 --- a/go/oasis-node/cmd/storage/storage.go +++ b/go/oasis-node/cmd/storage/storage.go @@ -17,6 +17,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/config" cmdCommon "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common" roothash "github.com/oasisprotocol/oasis-core/go/roothash/api" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle" "github.com/oasisprotocol/oasis-core/go/runtime/history" "github.com/oasisprotocol/oasis-core/go/runtime/registry" db "github.com/oasisprotocol/oasis-core/go/storage/mkvs/db/api" @@ -284,8 +285,8 @@ func doRenameNs(_ *cobra.Command, args []string) error { // Register registers the client sub-command and all of its children. func Register(parentCmd *cobra.Command) { - storageMigrateCmd.Flags().AddFlagSet(registry.Flags) - storageCheckCmd.Flags().AddFlagSet(registry.Flags) + storageMigrateCmd.Flags().AddFlagSet(bundle.Flags) + storageCheckCmd.Flags().AddFlagSet(bundle.Flags) storageCmd.AddCommand(storageMigrateCmd) storageCmd.AddCommand(storageCheckCmd) storageCmd.AddCommand(storageRenameNsCmd) diff --git a/go/oasis-node/node_test.go b/go/oasis-node/node_test.go index 0d7c332209d..f906ad24a9b 100644 --- a/go/oasis-node/node_test.go +++ b/go/oasis-node/node_test.go @@ -35,10 +35,10 @@ import ( registryTests "github.com/oasisprotocol/oasis-core/go/registry/tests" roothash "github.com/oasisprotocol/oasis-core/go/roothash/api" roothashTests "github.com/oasisprotocol/oasis-core/go/roothash/tests" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle" runtimeClient "github.com/oasisprotocol/oasis-core/go/runtime/client/api" clientTests "github.com/oasisprotocol/oasis-core/go/runtime/client/tests" runtimeConfig "github.com/oasisprotocol/oasis-core/go/runtime/config" - runtimeRegistry "github.com/oasisprotocol/oasis-core/go/runtime/registry" scheduler "github.com/oasisprotocol/oasis-core/go/scheduler/api" schedulerTests "github.com/oasisprotocol/oasis-core/go/scheduler/tests" staking "github.com/oasisprotocol/oasis-core/go/staking/api" @@ -164,7 +164,7 @@ func newTestNode(t *testing.T) *testNode { config.GlobalConfig.Common.DataDir = dataDir config.GlobalConfig.Common.Log.File = filepath.Join(dataDir, "test-node.log") - viper.Set(runtimeRegistry.CfgDebugMockIDs, []string{ + viper.Set(bundle.CfgDebugMockIDs, []string{ testRuntimeID.String(), }) for _, kv := range testNodeStaticConfig { diff --git a/go/oasis-test-runner/oasis/keymanager.go b/go/oasis-test-runner/oasis/keymanager.go index baf30c4baa8..0c2035ee984 100644 --- a/go/oasis-test-runner/oasis/keymanager.go +++ b/go/oasis-test-runner/oasis/keymanager.go @@ -17,6 +17,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env" registry "github.com/oasisprotocol/oasis-core/go/registry/api" "github.com/oasisprotocol/oasis-core/go/runtime/bundle" + runtimeCfg "github.com/oasisprotocol/oasis-core/go/runtime/config" runtimeConfig "github.com/oasisprotocol/oasis-core/go/runtime/config" keymanagerConfig "github.com/oasisprotocol/oasis-core/go/worker/keymanager/config" ) @@ -296,7 +297,14 @@ func (km *Keymanager) ModifyConfig() error { km.Config.Runtime.Provisioner = km.runtimeProvisioner km.Config.Runtime.SGXLoader = km.net.cfg.RuntimeSGXLoaderBinary km.Config.Runtime.AttestInterval = km.net.cfg.RuntimeAttestInterval + + rtCfg := runtimeCfg.RuntimeConfig{ + ID: km.runtime.cfgSave.id, + } + + km.Config.Runtime.Runtimes = append(km.Config.Runtime.Runtimes, rtCfg) km.Config.Runtime.Paths = append(km.Config.Runtime.Paths, km.runtime.BundlePaths()...) + km.Config.Runtime.Repositories = []string{fmt.Sprintf("http://127.0.0.1:%d", km.net.getProvisionedPort(netPortRepository))} km.Config.Keymanager.RuntimeID = km.runtime.ID().String() km.Config.Keymanager.PrivatePeerPubKeys = km.privatePeerPubKeys diff --git a/go/oasis-test-runner/oasis/network.go b/go/oasis-test-runner/oasis/network.go index cbb97d27eaf..d558c0d02c4 100644 --- a/go/oasis-test-runner/oasis/network.go +++ b/go/oasis-test-runner/oasis/network.go @@ -69,8 +69,9 @@ type Network struct { // nolint: maligned iasProxy *iasProxy - cfg *NetworkCfg - nextNodePort uint16 + cfg *NetworkCfg + nextPort uint16 + ports map[string]uint16 logWatchers []*log.Watcher @@ -296,7 +297,7 @@ func (net *Network) GetNamedNode(defaultName string, cfg *NodeCfg) (*Node, error Name: name, net: net, dir: nodeDir, - assignedPorts: map[string]uint16{}, + ports: map[string]uint16{}, hostedRuntimes: map[common.Namespace]*hostedRuntime{}, } @@ -986,6 +987,16 @@ func (net *Network) provisionNodeIdentity(dataDir *env.Dir, seed string) (signat return nodeIdentity.NodeSigner.Public(), nodeIdentity.P2PSigner.Public(), sentryCert, nil } +func (net *Network) getProvisionedPort(portName string) uint16 { + port, ok := net.ports[portName] + if !ok { + port = net.nextPort + net.nextPort++ + net.ports[portName] = port + } + return port +} + // New creates a new test Oasis network. func New(env *env.Env, cfg *NetworkCfg) (*Network, error) { baseDir, err := env.NewSubDir("network") @@ -1034,12 +1045,13 @@ func New(env *env.Env, cfg *NetworkCfg) (*Network, error) { } net := &Network{ - logger: logging.GetLogger("oasis/" + env.Name()), - env: env, - baseDir: baseDir, - cfg: &cfgCopy, - nextNodePort: baseNodePort, - errCh: make(chan error, maxNodes), + logger: logging.GetLogger("oasis/" + env.Name()), + env: env, + baseDir: baseDir, + cfg: &cfgCopy, + nextPort: basePort, + ports: make(map[string]uint16), + errCh: make(chan error, maxNodes), } // Pre-provision node objects if they were listed in the top-level network fixture. diff --git a/go/oasis-test-runner/oasis/oasis.go b/go/oasis-test-runner/oasis/oasis.go index db70a99aaba..163f8e37a7c 100644 --- a/go/oasis-test-runner/oasis/oasis.go +++ b/go/oasis-test-runner/oasis/oasis.go @@ -23,11 +23,12 @@ import ( cmdCommon "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/log" + runtimeCfg "github.com/oasisprotocol/oasis-core/go/runtime/config" "github.com/oasisprotocol/oasis-core/go/storage/database" ) const ( - baseNodePort = 20000 + basePort = 20000 validatorStartDelay = 3 * time.Second @@ -55,6 +56,7 @@ const ( nodePortP2P = "p2p" nodePortP2PSeed = "p2p-seed" nodePortPprof = "pprof" + netPortRepository = "repository" allInterfacesAddr = "tcp://0.0.0.0" localhostAddr = "tcp://127.0.0.1" @@ -97,7 +99,7 @@ type Node struct { // nolint: maligned extraArgs []Argument features []Feature hasValidators bool - assignedPorts map[string]uint16 + ports map[string]uint16 hostedRuntimes map[common.Namespace]*hostedRuntime exitCh chan error @@ -132,11 +134,11 @@ func (n *Node) SetArchiveMode(archive bool) { } func (n *Node) getProvisionedPort(portName string) uint16 { - port, ok := n.assignedPorts[portName] + port, ok := n.ports[portName] if !ok { - port = n.net.nextNodePort - n.net.nextNodePort++ - n.assignedPorts[portName] = port + port = n.net.nextPort + n.net.nextPort++ + n.ports[portName] = port } return port } @@ -292,16 +294,17 @@ func (n *Node) Start() error { n.Config.Runtime.Prune.NumKept = hosted.runtime.pruner.NumKept } - n.Config.Runtime.Paths = append(n.Config.Runtime.Paths, hosted.runtime.BundlePaths()...) - - if hosted.localConfig != nil { - if n.Config.Runtime.RuntimeConfig == nil { - n.Config.Runtime.RuntimeConfig = make(map[string]map[string]interface{}) - } - n.Config.Runtime.RuntimeConfig[hosted.runtime.ID().String()] = hosted.localConfig + rtCfg := runtimeCfg.RuntimeConfig{ + ID: hosted.runtime.cfgSave.id, + Config: hosted.localConfig, } + + n.Config.Runtime.Runtimes = append(n.Config.Runtime.Runtimes, rtCfg) + n.Config.Runtime.Paths = append(n.Config.Runtime.Paths, hosted.runtime.BundlePaths()...) } + n.Config.Runtime.Repositories = []string{fmt.Sprintf("http://127.0.0.1:%d", n.net.getProvisionedPort(netPortRepository))} + if n.consensus.EnableArchiveMode { n.Config.Mode = config.ModeArchive } diff --git a/go/oasis-test-runner/oasis/runtime.go b/go/oasis-test-runner/oasis/runtime.go index 673f88ce519..4230475701b 100644 --- a/go/oasis-test-runner/oasis/runtime.go +++ b/go/oasis-test-runner/oasis/runtime.go @@ -30,10 +30,10 @@ type runtimeCfgSave struct { } type deploymentCfg struct { - version version.Version - components []ComponentCfg - mrEnclave *sgx.MrEnclave - bundle *bundle.Bundle + components []ComponentCfg + mrEnclave *sgx.MrEnclave + bundle *bundle.Bundle + excludeBundle bool } // Runtime is an Oasis runtime. @@ -87,9 +87,9 @@ type RuntimeCfg struct { // nolint: maligned // DeploymentCfg is a deployment configuration. type DeploymentCfg struct { - Version version.Version `json:"version"` - ValidFrom beacon.EpochTime `json:"valid_from"` - Components []ComponentCfg `json:"components"` + ValidFrom beacon.EpochTime `json:"valid_from"` + Components []ComponentCfg `json:"components"` + ExcludeBundle bool `json:"exclude_bundle"` // DeprecatedBinaries is deprecated, use Components.Binaries instead. DeprecatedBinaries map[node.TEEHardware]string `json:"binaries"` @@ -98,6 +98,7 @@ type DeploymentCfg struct { // ComponentCfg is a runtime component configuration. type ComponentCfg struct { Kind component.Kind `json:"kind"` + Version version.Version `json:"version"` Binaries map[node.TEEHardware]string `json:"binaries"` } @@ -147,13 +148,16 @@ func (rt *Runtime) ToRuntimeDescriptor() registry.Runtime { } func (rt *Runtime) bundlePath(index int) string { - return filepath.Join(rt.dir.String(), fmt.Sprintf("bundle-%d.orc", index)) + return filepath.Join(rt.dir.String(), fmt.Sprintf("bundle-%d%s", index, bundle.FileExtension)) } // BundlePaths returns the paths to the dynamically generated bundles. func (rt *Runtime) BundlePaths() []string { var paths []string - for i := range rt.cfgSave.deployments { + for i, dpl := range rt.cfgSave.deployments { + if dpl.excludeBundle { + continue + } paths = append(paths, rt.bundlePath(i)) } return paths @@ -277,7 +281,7 @@ func (rt *Runtime) toRuntimeBundle(deploymentIndex int) (*bundle.Bundle, error) comp := &bundle.Component{ Kind: compCfg.Kind, - Version: deployCfg.version, + Version: compCfg.Version, Executable: elfBin, } @@ -305,6 +309,10 @@ func (rt *Runtime) toRuntimeBundle(deploymentIndex int) (*bundle.Bundle, error) return nil, err } + // Update bundle's manifest checksum. + manifestHash := bnd.Manifest.Hash() + rt.descriptor.Deployments[deploymentIndex].BundleChecksum = manifestHash[:] + return bnd, nil } @@ -371,14 +379,21 @@ func (net *Network) NewRuntime(cfg *RuntimeCfg) (*Runtime, error) { } components = append(components, deployCfg.Components...) - rt.cfgSave.deployments = append(rt.cfgSave.deployments, &deploymentCfg{ - version: deployCfg.Version, - components: components, - }) - rt.descriptor.Deployments = append(rt.descriptor.Deployments, ®istry.VersionInfo{ - Version: deployCfg.Version, + versionInfo := registry.VersionInfo{ ValidFrom: deployCfg.ValidFrom, + } + for _, comp := range components { + if comp.Kind == component.RONL { + versionInfo.Version = comp.Version + break + } + } + + rt.cfgSave.deployments = append(rt.cfgSave.deployments, &deploymentCfg{ + components: components, + excludeBundle: deployCfg.ExcludeBundle, }) + rt.descriptor.Deployments = append(rt.descriptor.Deployments, &versionInfo) } if _, err := rt.ToRuntimeBundles(); err != nil { diff --git a/go/oasis-test-runner/scenario/e2e/runtime/helpers_runtime.go b/go/oasis-test-runner/scenario/e2e/runtime/helpers_runtime.go index ee90d642fe2..07cc5b9dbc2 100644 --- a/go/oasis-test-runner/scenario/e2e/runtime/helpers_runtime.go +++ b/go/oasis-test-runner/scenario/e2e/runtime/helpers_runtime.go @@ -137,13 +137,13 @@ func (sc *Scenario) BuildAllRuntimes(childEnv *env.Env, trustRoot *e2e.TrustRoot // EnsureActiveVersionForComputeWorker ensures that the specified compute worker // has the correct active version of the given runtime. -func (sc *Scenario) EnsureActiveVersionForComputeWorker(ctx context.Context, node *oasis.Compute, rt *oasis.Runtime, v version.Version) error { +func (sc *Scenario) EnsureActiveVersionForComputeWorker(ctx context.Context, node *oasis.Compute, runtimeID common.Namespace, v version.Version) error { ctx, cancel := context.WithTimeout(ctx, versionActivationTimeout) defer cancel() sc.Logger.Info("ensuring that the compute worker has the correct active version", "node", node.Name, - "runtime_id", rt.ID(), + "runtime_id", runtimeID, "version", v, ) @@ -159,14 +159,14 @@ func (sc *Scenario) EnsureActiveVersionForComputeWorker(ctx context.Context, nod return fmt.Errorf("%s: failed to query status: %w", node.Name, err) } - provisioner := status.Runtimes[rt.ID()].Provisioner + provisioner := status.Runtimes[runtimeID].Provisioner if provisioner == "none" { - return fmt.Errorf("%s: unexpected runtime provisioner for runtime '%s': %s", node.Name, rt.ID(), provisioner) + return fmt.Errorf("%s: unexpected runtime provisioner for runtime '%s': %s", node.Name, runtimeID, provisioner) } - cs := status.Runtimes[rt.ID()].Committee + cs := status.Runtimes[runtimeID].Committee if cs == nil { - return fmt.Errorf("%s: missing status for runtime '%s'", node.Name, rt.ID()) + return fmt.Errorf("%s: missing status for runtime '%s'", node.Name, runtimeID) } if cs.ActiveVersion == nil { @@ -203,14 +203,14 @@ func (sc *Scenario) EnsureActiveVersionForComputeWorker(ctx context.Context, nod // EnsureActiveVersionForComputeWorkers ensures that all compute workers // have the correct active version of the given runtime. -func (sc *Scenario) EnsureActiveVersionForComputeWorkers(ctx context.Context, rt *oasis.Runtime, v version.Version) error { +func (sc *Scenario) EnsureActiveVersionForComputeWorkers(ctx context.Context, runtimeID common.Namespace, v version.Version) error { sc.Logger.Info("ensuring that all compute workers have the correct active version", - "runtime_id", rt.ID(), + "runtime_id", runtimeID, "version", v, ) for _, node := range sc.Net.ComputeWorkers() { - if err := sc.EnsureActiveVersionForComputeWorker(ctx, node, rt, v); err != nil { + if err := sc.EnsureActiveVersionForComputeWorker(ctx, node, runtimeID, v); err != nil { return err } } @@ -219,13 +219,13 @@ func (sc *Scenario) EnsureActiveVersionForComputeWorkers(ctx context.Context, rt // EnsureActiveVersionForKeyManager ensures that the specified key manager // has the correct active version of the given runtime. -func (sc *Scenario) EnsureActiveVersionForKeyManager(ctx context.Context, node *oasis.Keymanager, id common.Namespace, v version.Version) error { +func (sc *Scenario) EnsureActiveVersionForKeyManager(ctx context.Context, node *oasis.Keymanager, runtimeID common.Namespace, v version.Version) error { ctx, cancel := context.WithTimeout(ctx, versionActivationTimeout) defer cancel() sc.Logger.Info("ensuring that the key manager has the correct active version", "node", node.Name, - "runtime_id", id, + "runtime_id", runtimeID, "version", v, ) @@ -246,8 +246,8 @@ func (sc *Scenario) EnsureActiveVersionForKeyManager(ctx context.Context, node * } ws := status.Keymanager - if !id.Equal(ws.RuntimeID) { - return fmt.Errorf("%s: unsupported runtime (expected: %s got: %s)", node.Name, ws.RuntimeID, id) + if !runtimeID.Equal(ws.RuntimeID) { + return fmt.Errorf("%s: unsupported runtime (expected: %s got: %s)", node.Name, ws.RuntimeID, runtimeID) } if ws.ActiveVersion == nil { @@ -269,14 +269,14 @@ func (sc *Scenario) EnsureActiveVersionForKeyManager(ctx context.Context, node * // EnsureActiveVersionForKeyManagers ensures that all key managers // have the correct active version of the given runtime. -func (sc *Scenario) EnsureActiveVersionForKeyManagers(ctx context.Context, id common.Namespace, v version.Version) error { +func (sc *Scenario) EnsureActiveVersionForKeyManagers(ctx context.Context, runtimeID common.Namespace, v version.Version) error { sc.Logger.Info("ensuring that all key managers have the correct active version", - "runtime_id", id, + "runtime_id", runtimeID, "version", v, ) for _, node := range sc.Net.Keymanagers() { - if err := sc.EnsureActiveVersionForKeyManager(ctx, node, id, v); err != nil { + if err := sc.EnsureActiveVersionForKeyManager(ctx, node, runtimeID, v); err != nil { return err } } @@ -362,7 +362,7 @@ func (sc *Scenario) EnableRuntimeDeployment(ctx context.Context, childEnv *env.E } // UpgradeComputeRuntimeFixture select the first compute runtime and prepares it for the upgrade. -func (sc *Scenario) UpgradeComputeRuntimeFixture(f *oasis.NetworkFixture) (int, error) { +func (sc *Scenario) UpgradeComputeRuntimeFixture(f *oasis.NetworkFixture, excludeBundle bool) (int, error) { // Select the first compute runtime for upgrade. idx := -1 for i := range f.Runtimes { @@ -386,13 +386,14 @@ func (sc *Scenario) UpgradeComputeRuntimeFixture(f *oasis.NetworkFixture) (int, // they will be retained. f.Runtimes[idx].ExcludeFromGenesis = true f.Runtimes[idx].Deployments = append(f.Runtimes[idx].Deployments, oasis.DeploymentCfg{ - Version: version.Version{Major: 0, Minor: 1, Patch: 0}, Components: []oasis.ComponentCfg{ { Kind: component.RONL, + Version: version.Version{Major: 0, Minor: 1, Patch: 0}, Binaries: newRuntimeBinaries, }, }, + ExcludeBundle: excludeBundle, }) return idx, nil @@ -403,7 +404,7 @@ func (sc *Scenario) UpgradeComputeRuntime(ctx context.Context, childEnv *env.Env newRt := sc.Net.Runtimes()[idx] // Make sure the old version is active on all compute nodes. - if err := sc.EnsureActiveVersionForComputeWorkers(ctx, newRt, version.MustFromString("0.0.0")); err != nil { + if err := sc.EnsureActiveVersionForComputeWorkers(ctx, newRt.ID(), version.MustFromString("0.0.0")); err != nil { return err } @@ -413,11 +414,11 @@ func (sc *Scenario) UpgradeComputeRuntime(ctx context.Context, childEnv *env.Env } // Make sure the new version is active. - return sc.EnsureActiveVersionForComputeWorkers(ctx, newRt, version.MustFromString("0.1.0")) + return sc.EnsureActiveVersionForComputeWorkers(ctx, newRt.ID(), version.MustFromString("0.1.0")) } // UpgradeKeyManagerFixture select the first key manager runtime and prepares it for the upgrade. -func (sc *Scenario) UpgradeKeyManagerFixture(f *oasis.NetworkFixture) (int, error) { +func (sc *Scenario) UpgradeKeyManagerFixture(f *oasis.NetworkFixture, excludeBundle bool) (int, error) { // Select the first key manager for upgrade. idx := -1 for i := range f.Runtimes { @@ -438,13 +439,14 @@ func (sc *Scenario) UpgradeKeyManagerFixture(f *oasis.NetworkFixture) (int, erro newRt.ExcludeFromGenesis = true newRt.Deployments = []oasis.DeploymentCfg{ { - Version: version.Version{Major: 0, Minor: 1, Patch: 0}, Components: []oasis.ComponentCfg{ { Kind: component.RONL, + Version: version.Version{Major: 0, Minor: 1, Patch: 0}, Binaries: newRuntimeBinaries, }, }, + ExcludeBundle: excludeBundle, }, } f.Runtimes = append(f.Runtimes, newRt) diff --git a/go/oasis-test-runner/scenario/e2e/runtime/keymanager_upgrade.go b/go/oasis-test-runner/scenario/e2e/runtime/keymanager_upgrade.go index 21da4cd4321..3bb58bc0a25 100644 --- a/go/oasis-test-runner/scenario/e2e/runtime/keymanager_upgrade.go +++ b/go/oasis-test-runner/scenario/e2e/runtime/keymanager_upgrade.go @@ -2,6 +2,8 @@ package runtime import ( "context" + "fmt" + "net/url" "time" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env" @@ -36,7 +38,7 @@ func (sc *KmUpgradeImpl) Fixture() (*oasis.NetworkFixture, error) { return nil, err } - if sc.upgradedKeyManagerIndex, err = sc.UpgradeKeyManagerFixture(f); err != nil { + if sc.upgradedKeyManagerIndex, err = sc.UpgradeKeyManagerFixture(f, true); err != nil { return nil, err } @@ -63,11 +65,36 @@ func (sc *KmUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error { return err } + // Discover bundles. + bundles, err := findBundles(sc.Net.BasePath()) + if err != nil { + return err + } + + // Determine the port on which the nodes are trying to fetch bundles. + rawURL := sc.Net.Clients()[0].Config.Runtime.Repositories[0] + parsedURL, err := url.Parse(rawURL) + if err != nil { + return err + } + port := parsedURL.Port() + + // Start serving bundles. + server := newBundleServer(port, bundles) + server.Start() + defer server.Stop() + // Upgrade the key manager runtime. if err := sc.UpgradeKeyManager(ctx, childEnv, cli, sc.upgradedKeyManagerIndex, 0); err != nil { return err } + // Verify that all key manager nodes requested bundle from the server. + n := len(sc.Net.Keymanagers()) + if m := server.getRequestCount(); m != n { + return fmt.Errorf("invalid number of bundle requests (got: %d, expected: %d)", m, n) + } + // Run client again. sc.Logger.Info("starting a second client to check if key manager works") sc.Scenario.TestClient = NewTestClient().WithSeed("seed2").WithScenario(InsertRemoveEncWithSecretsScenarioV2) diff --git a/go/oasis-test-runner/scenario/e2e/runtime/runtime_upgrade.go b/go/oasis-test-runner/scenario/e2e/runtime/runtime_upgrade.go index d5ae536f094..d9bc8687a5c 100644 --- a/go/oasis-test-runner/scenario/e2e/runtime/runtime_upgrade.go +++ b/go/oasis-test-runner/scenario/e2e/runtime/runtime_upgrade.go @@ -2,12 +2,24 @@ package runtime import ( "context" + "fmt" + "maps" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "sync" + "sync/atomic" "time" + cmSync "github.com/oasisprotocol/oasis-core/go/common/sync" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/oasis" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/oasis/cli" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/scenario" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle" ) // RuntimeUpgrade is the runtime upgrade scenario. @@ -36,7 +48,7 @@ func (sc *runtimeUpgradeImpl) Fixture() (*oasis.NetworkFixture, error) { return nil, err } - if sc.upgradedRuntimeIndex, err = sc.UpgradeComputeRuntimeFixture(f); err != nil { + if sc.upgradedRuntimeIndex, err = sc.UpgradeComputeRuntimeFixture(f, true); err != nil { return nil, err } @@ -60,13 +72,165 @@ func (sc *runtimeUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error return err } + // Discover bundles. + bundles, err := findBundles(sc.Net.BasePath()) + if err != nil { + return err + } + + // Determine the port on which the nodes are trying to fetch bundles. + rawURL := sc.Net.Clients()[0].Config.Runtime.Repositories[0] + parsedURL, err := url.Parse(rawURL) + if err != nil { + return err + } + port := parsedURL.Port() + + // Start serving bundles. + server := newBundleServer(port, bundles) + server.Start() + defer server.Stop() + // Upgrade the compute runtime. if err := sc.UpgradeComputeRuntime(ctx, childEnv, cli, sc.upgradedRuntimeIndex, 0); err != nil { return err } + // Verify that all client and compute nodes requested bundle from the server. + n := len(sc.Net.Clients()) + len(sc.Net.ComputeWorkers()) + if m := server.getRequestCount(); m != n { + return fmt.Errorf("invalid number of bundle requests (got: %d, expected: %d)", m, n) + } + // Run client again. sc.Logger.Info("starting a second client to check if runtime works") sc.Scenario.TestClient = NewTestClient().WithSeed("seed2").WithScenario(InsertRemoveEncWithSecretsScenarioV2) return sc.RunTestClientAndCheckLogs(ctx, childEnv) } + +type bundleServer struct { + startOne cmSync.One + + port string + server *http.Server + + bundles map[string]string + + requestCount uint64 +} + +func newBundleServer(port string, bundles map[string]string) *bundleServer { + return &bundleServer{ + startOne: cmSync.NewOne(), + port: port, + bundles: bundles, + } +} + +func (s *bundleServer) Start() { + s.startOne.TryStart(s.run) +} + +func (s *bundleServer) Stop() { + s.startOne.TryStop() +} + +func (s *bundleServer) run(ctx context.Context) { + mux := http.NewServeMux() + mux.HandleFunc("/", s.handleRequest) + + s.server = &http.Server{ + Addr: ":" + s.port, + Handler: mux, + ReadHeaderTimeout: time.Minute, + } + + var wg sync.WaitGroup + defer wg.Wait() + + wg.Add(1) + go func() { + defer wg.Done() + _ = s.server.ListenAndServe() + }() + + <-ctx.Done() + + s.server.Close() +} + +func (s *bundleServer) handleRequest(w http.ResponseWriter, r *http.Request) { + filename := path.Base(r.URL.Path) + + path, ok := s.bundles[filename] + if !ok { + http.Error(w, "Bundle not found", http.StatusNotFound) + return + } + + content, err := os.ReadFile(path) + if err != nil { + http.Error(w, "Error reading bundle", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(content) + + atomic.AddUint64(&s.requestCount, 1) +} + +func (s *bundleServer) getRequestCount() int { + return int(atomic.LoadUint64(&s.requestCount)) +} + +func findBundles(dir string) (map[string]string, error) { + bundles := make(map[string]string) + + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + + for _, entry := range entries { + if entry.IsDir() && strings.HasPrefix(entry.Name(), "runtime-") { + subDir := filepath.Join(dir, entry.Name()) + + runtimeBundles, err := findBundlesIn(subDir) + if err != nil { + return nil, err + } + + maps.Insert(bundles, maps.All(runtimeBundles)) + } + } + + return bundles, nil +} + +func findBundlesIn(dir string) (map[string]string, error) { + bundles := make(map[string]string) + + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), bundle.FileExtension) { + continue + } + + path := filepath.Join(dir, entry.Name()) + + bnd, err := bundle.Open(path) + if err != nil { + return nil, err + } + + bundles[bnd.GenerateFilename()] = path + } + + return bundles, nil +} diff --git a/go/runtime/bundle/bundle.go b/go/runtime/bundle/bundle.go index 44ba756263e..633b5de15b2 100644 --- a/go/runtime/bundle/bundle.go +++ b/go/runtime/bundle/bundle.go @@ -17,14 +17,8 @@ import ( "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" ) -const ( - // bundleFileExtension is the file extension used for storing the bundle. - bundleFileExtension = ".orc" - - // bundleFilenameRegexp is the regular expression pattern used for - // validating bundle filenames. - bundleFilenameRegexp = `^[a-f0-9]{64}\.orc$` -) +// FileExtension is the file extension used for storing the bundle. +const FileExtension = ".orc" // Bundle is a runtime bundle instance. type Bundle struct { @@ -39,7 +33,7 @@ type Bundle struct { // GenerateFilename returns the recommended filename for storing the bundle. func (bnd *Bundle) GenerateFilename() string { - return fmt.Sprintf("%s%s", bnd.manifestHash.Hex(), bundleFileExtension) + return fmt.Sprintf("%s%s", bnd.manifestHash.Hex(), FileExtension) } // Validate validates the runtime bundle for well-formedness. diff --git a/go/runtime/bundle/discovery.go b/go/runtime/bundle/discovery.go new file mode 100644 index 00000000000..6e63be91d85 --- /dev/null +++ b/go/runtime/bundle/discovery.go @@ -0,0 +1,452 @@ +package bundle + +import ( + "context" + "errors" + "fmt" + "io" + "maps" + "net/http" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "time" + + "github.com/oasisprotocol/oasis-core/go/common" + "github.com/oasisprotocol/oasis-core/go/common/crypto/hash" + "github.com/oasisprotocol/oasis-core/go/common/logging" + cmSync "github.com/oasisprotocol/oasis-core/go/common/sync" + "github.com/oasisprotocol/oasis-core/go/config" +) + +// discoveryInterval is the time interval between (failed) bundle discoveries. +const discoveryInterval = 15 * time.Minute + +// Discovery is responsible for discovering new bundles. +type Discovery struct { + mu sync.RWMutex + + startOne cmSync.One + discoverCh chan struct{} + + bundleDir string + manifestHashes map[common.Namespace][]hash.Hash + + globalBaseURLs []string + runtimeBaseURLs map[common.Namespace][]string + + registry Registry + + logger logging.Logger +} + +// NewDiscovery creates a new bundle discovery. +func NewDiscovery(dataDir string, registry Registry) *Discovery { + logger := logging.GetLogger("runtime/bundle/discovery") + + return &Discovery{ + startOne: cmSync.NewOne(), + bundleDir: ExplodedPath(dataDir), + manifestHashes: make(map[common.Namespace][]hash.Hash), + discoverCh: make(chan struct{}, 1), + registry: registry, + logger: *logger, + } +} + +// Init sets up bundle discovery using node configuration and adds configured +// and cached bundles to the registry. +func (d *Discovery) Init() error { + // Consolidate all bundles in one place, which could be useful + // if we implement P2P sharing in the future. + if err := d.copyBundles(); err != nil { + return err + } + + // Add copied and cached bundles to the registry. + if err := d.Discover(); err != nil { + return err + } + + d.mu.Lock() + defer d.mu.Unlock() + + // Validate global repository URLs. + globalBaseURLs, err := validateAndNormalizeURLs(config.GlobalConfig.Runtime.Repositories) + if err != nil { + return err + } + + // Validate each runtime's repository URLs. + runtimeBaseURLs := make(map[common.Namespace][]string) + + for _, runtime := range config.GlobalConfig.Runtime.Runtimes { + urls, err := validateAndNormalizeURLs(runtime.Repositories) + if err != nil { + return err + } + if len(urls) == 0 { + continue + } + runtimeBaseURLs[runtime.ID] = urls + } + + // Update discovery. + d.globalBaseURLs = globalBaseURLs + d.runtimeBaseURLs = runtimeBaseURLs + + return nil +} + +// Start starts the bundle discovery. +func (d *Discovery) Start() { + d.startOne.TryStart(d.run) +} + +// Stop halts the bundle discovery. +func (d *Discovery) Stop() { + d.startOne.TryStop() +} + +func (d *Discovery) run(ctx context.Context) { + d.logger.Info("starting discovery", + "dir", d.bundleDir, + ) + + ticker := time.NewTicker(discoveryInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + case <-d.discoverCh: + case <-ctx.Done(): + d.logger.Info("stopping discovery") + return + } + + _ = d.Discover() + d.Download() + } +} + +// Discover searches for new bundles in the bundle directory and adds them +// to the bundle registry. +func (d *Discovery) Discover() error { + d.logger.Debug("discovering bundles") + + entries, err := os.ReadDir(d.bundleDir) + if err != nil { + d.logger.Error("failed to read bundle directory", + "err", err, + "dir", d.bundleDir, + ) + return fmt.Errorf("failed to read bundle directory: %w", err) + } + + for _, entry := range entries { + filename := entry.Name() + if entry.IsDir() || filepath.Ext(filename) != FileExtension { + continue + } + + baseFilename := strings.TrimSuffix(filename, FileExtension) + if len(baseFilename) != 2*hash.Size { + continue + } + + var manifestHash hash.Hash + if err = manifestHash.UnmarshalHex(baseFilename); err != nil { + continue + } + + if d.registry.HasBundle(manifestHash) { + continue + } + + d.logger.Info("found new bundle", + "file", filename, + ) + + path := filepath.Join(d.bundleDir, filename) + if err = d.registry.AddBundle(path, manifestHash); err != nil { + d.logger.Error("failed to add bundle to registry", + "err", err, + "path", path, + ) + return fmt.Errorf("failed to add bundle to registry: %w", err) + } + } + + return nil +} + +// Queue updates the checksums of bundles that need to be downloaded +// for the given runtime. +func (d *Discovery) Queue(runtimeID common.Namespace, manifestHashes []hash.Hash) { + d.mu.Lock() + defer d.mu.Unlock() + + // Download bundles only if at least one endpoint is configured. + if len(d.globalBaseURLs) == 0 && len(d.runtimeBaseURLs[runtimeID]) == 0 { + return + } + + // Filter out bundles that have already been fetched. + var hashes []hash.Hash + for _, hash := range manifestHashes { + if d.registry.HasBundle(hash) { + continue + } + hashes = append(hashes, hash) + } + + // Update the queue with the new hashes. + if len(hashes) == 0 { + delete(d.manifestHashes, runtimeID) + return + } + d.manifestHashes[runtimeID] = hashes + + // Trigger immediate discovery or download of new bundles. + select { + case d.discoverCh <- struct{}{}: + default: + } +} + +// Download tries to download bundles in the queue. +func (d *Discovery) Download() { + d.mu.RLock() + runtimeIDs := slices.Collect(maps.Keys(d.manifestHashes)) + d.mu.RUnlock() + + for _, runtimeID := range runtimeIDs { + d.downloadBundles(runtimeID) + } +} + +func (d *Discovery) downloadBundles(runtimeID common.Namespace) { + // Try to download queued bundles. + d.mu.RLock() + hashes := d.manifestHashes[runtimeID] + d.mu.RUnlock() + + downloaded := make(map[hash.Hash]struct{}) + for _, hash := range hashes { + if err := d.downloadBundle(runtimeID, hash); err != nil { + d.logger.Error("failed to download bundle", + "err", err, + "runtime_id", runtimeID, + "manifest_hash", hash.Hex(), + ) + continue + } + downloaded[hash] = struct{}{} + } + + // Remove downloaded bundles from the queue. + d.mu.Lock() + defer d.mu.Unlock() + + var pending []hash.Hash + for _, hash := range d.manifestHashes[runtimeID] { + if _, ok := downloaded[hash]; ok { + continue + } + pending = append(pending, hash) + } + if len(pending) == 0 { + delete(d.manifestHashes, runtimeID) + return + } + d.manifestHashes[runtimeID] = pending +} + +func (d *Discovery) downloadBundle(runtimeID common.Namespace, manifestHash hash.Hash) error { + var errs error + + for _, baseURLs := range [][]string{d.runtimeBaseURLs[runtimeID], d.globalBaseURLs} { + for _, baseURL := range baseURLs { + if err := d.tryDownloadBundle(runtimeID, manifestHash, baseURL); err != nil { + errs = errors.Join(errs, err) + continue + } + + return nil + } + } + + return errs +} + +func (d *Discovery) tryDownloadBundle(runtimeID common.Namespace, manifestHash hash.Hash, baseURL string) error { + filename := fmt.Sprintf("%s%s", manifestHash.Hex(), FileExtension) + + d.logger.Debug("downloading bundle", + "runtime_id", runtimeID, + "base_url", baseURL, + "filename", filename, + ) + + url, err := url.JoinPath(baseURL, filename) + if err != nil { + d.logger.Error("failed to construct URL", + "err", err, + "base_url", baseURL, + "filename", filename, + ) + return fmt.Errorf("failed to construct URL: %w", err) + } + + src, err := d.fetchBundle(url) + if err != nil { + d.logger.Error("failed to download bundle", + "err", err, + "url", url, + ) + return fmt.Errorf("failed to download bundle: %w", err) + } + defer os.Remove(src) + + d.logger.Info("bundle downloaded", + "runtime_id", runtimeID, + "base_url", baseURL, + "filename", filename, + ) + + if err := d.registry.AddBundle(src, manifestHash); err != nil { + d.logger.Error("failed to add bundle to registry", + "err", err, + ) + return fmt.Errorf("failed to add bundle: %w", err) + } + + dst := filepath.Join(d.bundleDir, filename) + if err = os.Rename(src, dst); err != nil { + d.logger.Error("failed to move bundle", + "err", err, + "src", src, + "dst", dst, + ) + } + return nil +} + +func (d *Discovery) fetchBundle(url string) (string, error) { + resp, err := http.Get(url) // nolint: gosec + if err != nil { + return "", fmt.Errorf("failed to fetch bundle: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to fetch bundle: invalid status code %d", resp.StatusCode) + } + + // Copy to a temporary file. as downloaded bundles are unverified. + file, err := os.CreateTemp("", fmt.Sprintf("oasis-bundle-*%s", FileExtension)) + if err != nil { + return "", fmt.Errorf("failed to create temporary file: %w", err) + } + defer func() { + file.Close() + if err != nil { + _ = os.Remove(file.Name()) + } + }() + + if _, err = io.Copy(file, resp.Body); err != nil { + return "", fmt.Errorf("failed to save bundle: %w", err) + } + + return file.Name(), nil +} + +func (d *Discovery) copyBundles() error { + if err := common.Mkdir(d.bundleDir); err != nil { + return err + } + + for _, path := range config.GlobalConfig.Runtime.Paths { + if err := d.copyBundle(path); err != nil { + return err + } + } + + return nil +} + +func (d *Discovery) copyBundle(src string) error { + d.logger.Info("copying bundle", + "src", src, + ) + + filename, err := func() (string, error) { + bnd, err := Open(src) + if err != nil { + d.logger.Error("failed to open bundle", + "err", err, + "src", src, + ) + return "", fmt.Errorf("failed to open bundle: %w", err) + } + defer bnd.Close() + + return bnd.GenerateFilename(), nil + }() + if err != nil { + return err + } + + dst := filepath.Join(d.bundleDir, filename) + switch _, err := os.Stat(dst); err { + case nil: + d.logger.Info("bundle already exists", + "src", src, + "dst", dst, + ) + return nil + default: + if !os.IsNotExist(err) { + d.logger.Error("failed to stat bundle", + "err", err, + "dst", dst, + ) + return fmt.Errorf("failed to stat bundle %w", err) + } + } + + if err := common.CopyFile(src, dst); err != nil { + d.logger.Error("failed to copy bundle", + "err", err, + "src", src, + "dst", dst, + ) + return fmt.Errorf("failed to open bundle: %w", err) + } + + d.logger.Info("bundle copied", + "src", src, + "dst", dst, + ) + + return nil +} + +func validateAndNormalizeURLs(rawURLs []string) ([]string, error) { + var normalizedURLs []string + + for _, rawURL := range rawURLs { + parsedURL, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("invalid URL '%s': %w", rawURL, err) + } + normalizedURLs = append(normalizedURLs, parsedURL.String()) + } + + return normalizedURLs, nil +} diff --git a/go/runtime/bundle/discovery_test.go b/go/runtime/bundle/discovery_test.go new file mode 100644 index 00000000000..30ae82d63b2 --- /dev/null +++ b/go/runtime/bundle/discovery_test.go @@ -0,0 +1,112 @@ +package bundle + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/oasisprotocol/oasis-core/go/common" + "github.com/oasisprotocol/oasis-core/go/common/crypto/hash" + "github.com/oasisprotocol/oasis-core/go/common/pubsub" + "github.com/oasisprotocol/oasis-core/go/common/version" +) + +var _ Registry = (*mockRegistry)(nil) + +type mockRegistry struct { + manifestHashes map[hash.Hash]struct{} +} + +// HasBundle implements Registry. +func (r *mockRegistry) HasBundle(manifestHash hash.Hash) bool { + _, ok := r.manifestHashes[manifestHash] + return ok +} + +// AddBundle implements Registry. +func (r *mockRegistry) AddBundle(_ string, manifestHash hash.Hash) error { + r.manifestHashes[manifestHash] = struct{}{} + return nil +} + +// GetVersions implements Registry. +func (r *mockRegistry) GetVersions(common.Namespace) []version.Version { + panic("unimplemented") +} + +// WatchVersions implements Registry. +func (r *mockRegistry) WatchVersions(common.Namespace) (<-chan version.Version, *pubsub.Subscription) { + panic("unimplemented") +} + +// GetManifests implements Registry. +func (r *mockRegistry) GetManifests() []*Manifest { + panic("unimplemented") +} + +// GetName implements Registry. +func (r *mockRegistry) GetName(common.Namespace, version.Version) (string, error) { + panic("unimplemented") +} + +// GetComponents implements Registry. +func (r *mockRegistry) GetComponents(common.Namespace, version.Version) ([]*ExplodedComponent, error) { + panic("unimplemented") +} + +func newMockListener() *mockRegistry { + return &mockRegistry{ + manifestHashes: make(map[hash.Hash]struct{}), + } +} + +func TestBundleDiscovery(t *testing.T) { + // Prepare a temporary directory for storing bundles. + dataDir := t.TempDir() + + // Create discovery. + registry := newMockListener() + discovery := NewDiscovery(dataDir, registry) + + // Get bundle directory. + dir := ExplodedPath(dataDir) + err := common.Mkdir(dir) + require.NoError(t, err) + + // Create an empty file, which should be ignored by the discovery. + file, err := os.Create(filepath.Join(dir, fmt.Sprintf("bundle%s", FileExtension))) + require.NoError(t, err) + file.Close() + + // Discovery should not find any bundles at this point. + err = discovery.Discover() + require.NoError(t, err) + require.Equal(t, 0, len(registry.manifestHashes)) + + // Test multiple rounds of discovery. + total := 0 + for r := 0; r < 3; r++ { + // Add new bundle files for this round. + for i := 0; i < r+2; i++ { + manifestHash := hash.Hash{byte(total)} + fn := fmt.Sprintf("%s%s", manifestHash.Hex(), FileExtension) + path := filepath.Join(dir, fn) + + file, err := os.Create(path) + require.NoError(t, err) + + err = file.Close() + require.NoError(t, err) + + total++ + } + + // Discovery should find the newly added bundles. + err = discovery.Discover() + require.NoError(t, err) + require.Equal(t, total, len(registry.manifestHashes)) + } +} diff --git a/go/runtime/bundle/registry.go b/go/runtime/bundle/registry.go new file mode 100644 index 00000000000..d8efd1a08ca --- /dev/null +++ b/go/runtime/bundle/registry.go @@ -0,0 +1,346 @@ +package bundle + +import ( + "fmt" + "maps" + "slices" + "sync" + + flag "github.com/spf13/pflag" + "github.com/spf13/viper" + + "github.com/oasisprotocol/oasis-core/go/common" + "github.com/oasisprotocol/oasis-core/go/common/crypto/hash" + "github.com/oasisprotocol/oasis-core/go/common/logging" + "github.com/oasisprotocol/oasis-core/go/common/pubsub" + "github.com/oasisprotocol/oasis-core/go/common/version" + "github.com/oasisprotocol/oasis-core/go/config" + cmdFlags "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/flags" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" +) + +// CfgDebugMockIDs configures mock runtime IDs for the purpose of testing. +const CfgDebugMockIDs = "runtime.debug.mock_ids" + +// Flags has the configuration flags. +var Flags = flag.NewFlagSet("", flag.ContinueOnError) + +// Registry is an interface for handling newly discovered bundles. +type Registry interface { + // HasBundle returns true iff the registry has the bundle. + HasBundle(manifestHash hash.Hash) bool + + // AddBundle adds a bundle from the given path. + AddBundle(path string, manifestHash hash.Hash) error + + // GetVersions returns versions for the given runtime. + GetVersions(runtimeID common.Namespace) []version.Version + + // WatchVersions provides a channel that streams runtime versions as they + // are added to the registry. + WatchVersions(runtimeID common.Namespace) (<-chan version.Version, *pubsub.Subscription) + + // GetManifests returns all known manifests that contain RONL component. + GetManifests() []*Manifest + + // GetName returns optional human readable runtime name. + GetName(runtimeID common.Namespace, version version.Version) (string, error) + + // GetComponents returns components for the given runtime and version. + GetComponents(runtimeID common.Namespace, version version.Version) ([]*ExplodedComponent, error) +} + +// registry is a registry of runtime bundle manifests and components. +type registry struct { + mu sync.RWMutex + + dataDir string + + bundles map[hash.Hash]struct{} + manifests map[common.Namespace]map[version.Version]*Manifest + components map[common.Namespace]map[component.ID]map[version.Version]*ExplodedComponent + notifiers map[common.Namespace]*pubsub.Broker + + logger *logging.Logger +} + +// NewRegistry creates a new bundle registry, using the given data directory +// to store the extracted bundle files. +func NewRegistry(dataDir string) Registry { + logger := logging.GetLogger("runtime/bundle/registry") + + return ®istry{ + dataDir: dataDir, + bundles: make(map[hash.Hash]struct{}), + manifests: make(map[common.Namespace]map[version.Version]*Manifest), + components: make(map[common.Namespace]map[component.ID]map[version.Version]*ExplodedComponent), + notifiers: make(map[common.Namespace]*pubsub.Broker), + logger: logger, + } +} + +// HasBundle implements Registry. +func (r *registry) HasBundle(manifestHash hash.Hash) bool { + r.mu.RLock() + defer r.mu.RUnlock() + + _, ok := r.bundles[manifestHash] + return ok +} + +// AddBundle implements Registry. +func (r *registry) AddBundle(path string, manifestHash hash.Hash) error { + r.mu.Lock() + defer r.mu.Unlock() + + r.logger.Info("adding bundle", + "path", path, + "manifest_hash", manifestHash, + ) + + // Open the bundle and release resources when done. + bnd, err := Open(path) + if err != nil { + return fmt.Errorf("failed to open bundle '%s': %w", path, err) + } + defer bnd.Close() + + // Verify manifest hash. + if !bnd.manifestHash.Equal(&manifestHash) { + return fmt.Errorf("invalid manifest hash (got: '%s', expected: '%s')", + bnd.manifestHash.Hex(), + manifestHash.Hex(), + ) + } + + // Skip already processed bundles. + if _, ok := r.bundles[manifestHash]; ok { + return nil + } + + // Verify that components are unique. + components := bnd.Manifest.GetAvailableComponents() + + for compID, comp := range components { + if _, ok := r.components[bnd.Manifest.ID][compID][comp.Version]; ok { + return fmt.Errorf("duplicate component '%s', version '%s', for runtime '%s'", + compID, + comp.Version, + bnd.Manifest.ID, + ) + } + } + + // Explode the bundle. + explodedDataDir, err := bnd.WriteExploded(r.dataDir) + if err != nil { + return fmt.Errorf("failed to explode bundle '%s': %w", path, err) + } + + // Add manifests containing RONL component to the registry. + detached := true + if ronl, ok := components[component.ID_RONL]; ok { + detached = false + + rtManifests, ok := r.manifests[bnd.Manifest.ID] + if !ok { + rtManifests = make(map[version.Version]*Manifest) + r.manifests[bnd.Manifest.ID] = rtManifests + } + + rtManifests[ronl.Version] = bnd.Manifest + + if notifier, ok := r.notifiers[bnd.Manifest.ID]; ok { + notifier.Broadcast(ronl.Version) + } + } + + // Add components to the registry. + for compID, comp := range components { + runtimeComponents, ok := r.components[bnd.Manifest.ID] + if !ok { + runtimeComponents = make(map[component.ID]map[version.Version]*ExplodedComponent) + r.components[bnd.Manifest.ID] = runtimeComponents + } + + componentVersions, ok := runtimeComponents[compID] + if !ok { + componentVersions = make(map[version.Version]*ExplodedComponent) + runtimeComponents[compID] = componentVersions + } + + componentVersions[comp.Version] = &ExplodedComponent{ + Component: comp, + Detached: detached, + ExplodedDataDir: explodedDataDir, + } + } + + // Remember which bundles were added. + r.bundles[manifestHash] = struct{}{} + + r.logger.Info("bundle added", + "path", path, + "runtime_id", bnd.Manifest.ID, + "manifest_hash", bnd.manifestHash, + ) + + return nil +} + +// GetVersions implements Registry. +func (r *registry) GetVersions(runtimeID common.Namespace) []version.Version { + r.mu.RLock() + defer r.mu.RUnlock() + + if cmdFlags.DebugDontBlameOasis() && viper.IsSet(CfgDebugMockIDs) { + // Allow the mock provisioner to function, as it does not use an actual + // runtime. This is only used for the basic node tests. + return []version.Version{ + {Major: 0, Minor: 0, Patch: 0}, + } + } + + return slices.Collect(maps.Keys(r.manifests[runtimeID])) +} + +// WatchVersions implements Registry. +func (r *registry) WatchVersions(runtimeID common.Namespace) (<-chan version.Version, *pubsub.Subscription) { + r.mu.Lock() + defer r.mu.Unlock() + + notifier, ok := r.notifiers[runtimeID] + if !ok { + notifier = pubsub.NewBroker(false) + r.notifiers[runtimeID] = notifier + } + + sub := notifier.Subscribe() + ch := make(chan version.Version) + sub.Unwrap(ch) + + return ch, sub +} + +// GetManifests implements Registry. +func (r *registry) GetManifests() []*Manifest { + r.mu.RLock() + defer r.mu.RUnlock() + + manifests := make([]*Manifest, 0) + for _, manifest := range r.manifests { + manifests = slices.AppendSeq(manifests, maps.Values(manifest)) + } + + return manifests +} + +// GetName implements Registry. +func (r *registry) GetName(runtimeID common.Namespace, version version.Version) (string, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + if cmdFlags.DebugDontBlameOasis() && viper.IsSet(CfgDebugMockIDs) { + // Allow the mock provisioner to function, as it does not use an actual + // runtime. This is only used for the basic node tests. + return "mock-runtime", nil + } + + manifest, ok := r.manifests[runtimeID][version] + if !ok { + return "", fmt.Errorf("manifest for runtime '%s', version '%s' not found", runtimeID, version) + } + + return manifest.Name, nil +} + +// GetComponents implements Registry. +func (r *registry) GetComponents(runtimeID common.Namespace, version version.Version) ([]*ExplodedComponent, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + if cmdFlags.DebugDontBlameOasis() && viper.IsSet(CfgDebugMockIDs) { + // Allow the mock provisioner to function, as it does not use an actual + // runtime. This is only used for the basic node tests. + return []*ExplodedComponent{ + { + Component: &Component{ + Kind: component.RONL, + Executable: "mock", + }, + Detached: false, + }, + }, nil + } + + // Prepare function to determine what kind of components we want. + isComponentWanted := func(compID component.ID, comp *ExplodedComponent) bool { + // Skip the RONL component, as the exact version is added manually. + if compID.IsRONL() { + return false + } + + // Node configuration overrides all other settings. + if compCfg, ok := config.GlobalConfig.Runtime.GetComponent(runtimeID, compID); ok { + return !compCfg.Disabled + } + + // Detached components are explicit and they should be enabled by default. + if comp.Detached { + return true + } + + // On non-compute nodes, assume all components are disabled by default. + if config.GlobalConfig.Mode != config.ModeCompute { + return false + } + + // By default honor the status of the component itself. + return !comp.Disabled + } + + // Collect all components into a slice. + components := make([]*ExplodedComponent, 0, 1) + + // Add the specified version of the RONL component. + ronl, ok := r.components[runtimeID][component.ID_RONL][version] + if !ok { + return nil, fmt.Errorf("component '%s', version '%s', for runtime '%s' not found", component.RONL, version, runtimeID) + } + components = append(components, ronl) + + // Add the latest version of the remaining components. + for compID, runtimeComponents := range r.components[runtimeID] { + var latestVersion uint64 + var latestComp *ExplodedComponent + + for version, comp := range runtimeComponents { + // Skip if the version is not the highest. + if version.ToU64() < latestVersion { + continue + } + + // Skip if the component is not wanted. + if !isComponentWanted(compID, comp) { + continue + } + + latestVersion = version.ToU64() + latestComp = comp + } + + if latestComp != nil { + components = append(components, latestComp) + } + + } + + return components, nil +} + +func init() { + Flags.StringSlice(CfgDebugMockIDs, nil, "Mock runtime IDs (format: ,,...)") + _ = Flags.MarkHidden(CfgDebugMockIDs) + + _ = viper.BindPFlags(Flags) +} diff --git a/go/runtime/bundle/registry_test.go b/go/runtime/bundle/registry_test.go new file mode 100644 index 00000000000..503a1fe8bdd --- /dev/null +++ b/go/runtime/bundle/registry_test.go @@ -0,0 +1,148 @@ +package bundle + +import ( + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/oasisprotocol/oasis-core/go/common" + "github.com/oasisprotocol/oasis-core/go/common/crypto/hash" + "github.com/oasisprotocol/oasis-core/go/common/version" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" +) + +func createSyntheticBundle(dir string, runtimeID common.Namespace, version version.Version, components []component.Kind) (string, error) { + manifest := &Manifest{ + Name: "test-runtime", + ID: runtimeID, + Components: make([]*Component, 0), + } + + for _, comp := range components { + switch comp { + case component.RONL: + manifest.Components = append(manifest.Components, &Component{ + Kind: component.RONL, + Version: version, + Executable: "runtime.bin", + }) + case component.ROFL: + manifest.Components = append(manifest.Components, &Component{ + Kind: component.ROFL, + Version: version, + }) + default: + } + } + + bnd := &Bundle{ + Manifest: manifest, + } + + if slices.Contains(components, component.RONL) { + if err := bnd.Add(manifest.Components[0].Executable, NewBytesData([]byte{1})); err != nil { + return "", err + } + } + + bnd.manifestHash = manifest.Hash() + path := filepath.Join(dir, bnd.GenerateFilename()) + + if err := bnd.Write(path); err != nil { + return "", err + } + + return path, nil +} + +func TestBundleRegistry(t *testing.T) { + // Prepare a temporary directory for storing bundles. + dir := t.TempDir() + + // Initialize runtimes. + var runtimeID1 common.Namespace + err := runtimeID1.UnmarshalHex("8000000000000000000000000000000000000000000000000000000000000001") + require.NoError(t, err) + + var runtimeID2 common.Namespace + err = runtimeID2.UnmarshalHex("8000000000000000000000000000000000000000000000000000000000000002") + require.NoError(t, err) + + // Define versions. + version1 := version.Version{Major: 1} + version2 := version.Version{Major: 2} + version3 := version.Version{Major: 3} + + // Generate synthetic bundles. + path0, err := createSyntheticBundle(dir, runtimeID1, version1, []component.Kind{component.RONL, component.ROFL}) + require.NoError(t, err) + + path1, err := createSyntheticBundle(dir, runtimeID1, version2, []component.Kind{component.ROFL}) + require.NoError(t, err) + + path2, err := createSyntheticBundle(dir, runtimeID2, version3, []component.Kind{component.RONL}) + require.NoError(t, err) + + path3, err := createSyntheticBundle(dir, runtimeID1, version1, []component.Kind{component.RONL}) + require.NoError(t, err) + + paths := []string{path0, path1, path2, path3} + + // Compute manifest hashes. + var hashes []hash.Hash + for _, path := range paths { + var hash hash.Hash + err = hash.UnmarshalHex(strings.TrimSuffix(filepath.Base(path), FileExtension)) + require.NoError(t, err) + + hashes = append(hashes, hash) + } + + // Create registry instance. + registry := NewRegistry(dir) + + // Add bundles to the registry + for i := 0; i < 3; i++ { + err = registry.AddBundle(paths[i], hashes[i]) + require.NoError(t, err) + } + + // Attempt to add the first bundle again (duplicate manifest hash). + err = registry.AddBundle(paths[0], hashes[0]) + require.NoError(t, err) + + // Attempt to add the fourth bundle (duplicate RONL component). + err = registry.AddBundle(paths[3], hashes[3]) + require.Error(t, err) + require.ErrorContains(t, err, "duplicate component 'ronl', version '1.0.0', for runtime '8000000000000000000000000000000000000000000000000000000000000001'") + + // Fetch manifests. + manifests := registry.GetManifests() + require.Equal(t, 2, len(manifests)) + + // Fetch components for runtime 1, version 1. + comps, err := registry.GetComponents(runtimeID1, version1) + require.NoError(t, err) + require.Equal(t, 2, len(comps)) + require.Equal(t, version1, comps[0].Version) + require.Equal(t, version2, comps[1].Version) + + // Fetch components for runtime 2, version 3. + comps, err = registry.GetComponents(runtimeID2, version3) + require.NoError(t, err) + require.Equal(t, 1, len(comps)) + require.Equal(t, version3, comps[0].Version) + + // Attempt to fetch components for runtime 1, version 2 (no RONL component). + _, err = registry.GetComponents(runtimeID1, version2) + require.Error(t, err) + require.ErrorContains(t, err, "component 'ronl', version '2.0.0', for runtime '8000000000000000000000000000000000000000000000000000000000000001' not found") + + // Attempt to fetch components for runtime 1, version 3 (no components). + _, err = registry.GetComponents(runtimeID1, version3) + require.Error(t, err) + require.ErrorContains(t, err, "component 'ronl', version '3.0.0', for runtime '8000000000000000000000000000000000000000000000000000000000000001' not found") +} diff --git a/go/runtime/config/config.go b/go/runtime/config/config.go index a621d6000f8..6c6d32f1bb1 100644 --- a/go/runtime/config/config.go +++ b/go/runtime/config/config.go @@ -7,6 +7,7 @@ import ( "gopkg.in/yaml.v3" + "github.com/oasisprotocol/oasis-core/go/common" "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" tpConfig "github.com/oasisprotocol/oasis-core/go/runtime/txpool/config" ) @@ -71,21 +72,29 @@ const ( // Config is the runtime registry configuration structure. type Config struct { + // Runtimes is the list of runtimes to configure. + Runtimes []RuntimeConfig `yaml:"runtimes,omitempty"` + + // Paths to runtime bundles. + Paths []string `yaml:"paths,omitempty"` + // Runtime provisioner to use (mock, unconfined, sandboxed). Provisioner RuntimeProvisioner `yaml:"provisioner"` - // Paths to runtime bundles. - Paths []string `yaml:"paths"` + // Path to the sandbox binary (bubblewrap). - SandboxBinary string `yaml:"sandbox_binary"` + SandboxBinary string `yaml:"sandbox_binary,omitempty"` + // Path to SGXS runtime loader binary (for SGX runtimes). - SGXLoader string `yaml:"sgx_loader"` + SGXLoader string `yaml:"sgx_loader,omitempty"` + // The runtime environment (sgx, elf, auto). - Environment RuntimeEnvironment `yaml:"environment"` + Environment RuntimeEnvironment `yaml:"environment,omitempty"` // History pruner configuration. Prune PruneConfig `yaml:"prune,omitempty"` // RuntimeConfig maps runtime IDs to their respective local configurations. + // NOTE: This may go away in the future, use `RuntimeConfig.Config` instead. RuntimeConfig map[string]map[string]interface{} `yaml:"config,omitempty"` // Address(es) of sentry node(s) to connect to of the form [PubKey@]ip:port @@ -106,20 +115,56 @@ type Config struct { // LoadBalancer is the load balancer configuration. LoadBalancer LoadBalancerConfig `yaml:"load_balancer,omitempty"` - // Components is the list of components to configure. - Components []ComponentConfig `yaml:"components,omitempty"` + // Repositories is the list of URLs used to fetch runtime bundles. + Repositories []string `yaml:"repositories,omitempty"` } -// GetComponent returns configuration for the given component if it exists. -func (c *Config) GetComponent(id component.ID) (ComponentConfig, bool) { - for _, comp := range c.Components { - if comp.ID == id { - return comp, true +// GetComponent returns the configuration for the given component +// of the specified runtime, if it exists. +func (c *Config) GetComponent(runtimeID common.Namespace, compID component.ID) (ComponentConfig, bool) { + for _, rt := range c.Runtimes { + if rt.ID != runtimeID { + continue + } + for _, comp := range rt.Components { + if comp.ID == compID { + return comp, true + } } } + return ComponentConfig{}, false } +// GetLocalConfig returns the local configuration for the given runtime, +// if it exists. +func (c *Config) GetLocalConfig(runtimeID common.Namespace) map[string]interface{} { + for _, rt := range c.Runtimes { + if rt.ID == runtimeID { + return rt.Config + } + } + + // Support legacy configuration where the runtime configuration is defined + // at the top level. + return c.RuntimeConfig[runtimeID.String()] +} + +// RuntimeConfig is the runtime configuration. +type RuntimeConfig struct { + // ID is the runtime identifier. + ID common.Namespace `yaml:"id"` + + // Components is the list of components to configure. + Components []ComponentConfig `yaml:"components,omitempty"` + + // Config contains runtime local configuration. + Config map[string]interface{} `yaml:"config,omitempty"` + + // Repositories is the list of URLs used to fetch runtime bundles. + Repositories []string `yaml:"repositories,omitempty"` +} + // ComponentConfig is the component configuration. type ComponentConfig struct { // ID is the component identifier. @@ -207,8 +252,9 @@ func (c *Config) Validate() error { // DefaultConfig returns the default configuration settings. func DefaultConfig() Config { return Config{ - Provisioner: RuntimeProvisionerSandboxed, + Runtimes: make([]RuntimeConfig, 0), Paths: []string{}, + Provisioner: RuntimeProvisionerSandboxed, SandboxBinary: "/usr/bin/bwrap", SGXLoader: "", Environment: RuntimeEnvironmentAuto, @@ -217,7 +263,6 @@ func DefaultConfig() Config { Interval: 2 * time.Minute, NumKept: 600, }, - RuntimeConfig: nil, SentryAddresses: []string{}, TxPool: tpConfig.Config{ MaxPoolSize: 50_000, diff --git a/go/runtime/config/config_test.go b/go/runtime/config/config_test.go index f278451c924..9e44fe0e35e 100644 --- a/go/runtime/config/config_test.go +++ b/go/runtime/config/config_test.go @@ -6,49 +6,61 @@ import ( "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" + "github.com/oasisprotocol/oasis-core/go/common" "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" ) func TestComponentConfig(t *testing.T) { require := require.New(t) + var runtimeID common.Namespace + err := runtimeID.UnmarshalHex("8000000000000000000000000000000000000000000000000000000000000000") + require.NoError(err) + cfg := Config{ - Components: []ComponentConfig{ + Runtimes: []RuntimeConfig{ { - ID: component.ID{Kind: component.ROFL, Name: "foo-test"}, - Disabled: false, + ID: runtimeID, + Components: []ComponentConfig{ + { + ID: component.ID{Kind: component.ROFL, Name: "foo-test"}, + Disabled: false, + }, + }, }, }, } - compCfg, ok := cfg.GetComponent(component.ID{Kind: component.ROFL, Name: "foo-test"}) + compCfg, ok := cfg.GetComponent(runtimeID, component.ID{Kind: component.ROFL, Name: "foo-test"}) require.True(ok) require.EqualValues(compCfg.ID.Kind, component.ROFL) require.EqualValues(compCfg.ID.Name, "foo-test") require.False(compCfg.Disabled) - compCfg, ok = cfg.GetComponent(component.ID{Kind: component.ROFL, Name: "does-not-exist"}) + compCfg, ok = cfg.GetComponent(runtimeID, component.ID{Kind: component.ROFL, Name: "does-not-exist"}) require.False(ok) require.EqualValues(compCfg, ComponentConfig{}) // Deserialization. yamlCfg := ` -components: - - rofl.foo-test - - id: rofl.another - disabled: true +runtimes: + - id: 8000000000000000000000000000000000000000000000000000000000000000 + components: + - rofl.foo-test + - id: rofl.another + disabled: true ` var decCfg Config - err := yaml.Unmarshal([]byte(yamlCfg), &decCfg) + err = yaml.Unmarshal([]byte(yamlCfg), &decCfg) require.NoError(err, "yaml.Unmarshal") - compCfg, ok = decCfg.GetComponent(component.ID{Kind: component.ROFL, Name: "foo-test"}) + compCfg, ok = decCfg.GetComponent(runtimeID, component.ID{Kind: component.ROFL, Name: "foo-test"}) require.True(ok) require.EqualValues(compCfg.ID.Kind, component.ROFL) require.EqualValues(compCfg.ID.Name, "foo-test") require.False(compCfg.Disabled) - compCfg, ok = decCfg.GetComponent(component.ID{Kind: component.ROFL, Name: "another"}) + compCfg, ok = decCfg.GetComponent(runtimeID, component.ID{Kind: component.ROFL, Name: "another"}) require.True(ok) require.EqualValues(compCfg.ID.Kind, component.ROFL) require.EqualValues(compCfg.ID.Name, "another") diff --git a/go/runtime/registry/config.go b/go/runtime/registry/config.go index 20203d68006..c74aaafa398 100644 --- a/go/runtime/registry/config.go +++ b/go/runtime/registry/config.go @@ -3,18 +3,18 @@ package registry import ( "context" "fmt" + "maps" "os" + "slices" "strings" "time" - flag "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/oasisprotocol/oasis-core/go/common" "github.com/oasisprotocol/oasis-core/go/common/identity" "github.com/oasisprotocol/oasis-core/go/common/persistent" "github.com/oasisprotocol/oasis-core/go/common/sgx/pcs" - "github.com/oasisprotocol/oasis-core/go/common/version" "github.com/oasisprotocol/oasis-core/go/config" consensus "github.com/oasisprotocol/oasis-core/go/consensus/api" ias "github.com/oasisprotocol/oasis-core/go/ias/api" @@ -33,219 +33,55 @@ import ( hostTdx "github.com/oasisprotocol/oasis-core/go/runtime/host/tdx" ) -const ( - // CfgDebugMockIDs configures mock runtime IDs for the purpose - // of testing. - CfgDebugMockIDs = "runtime.debug.mock_ids" -) +func getLocalConfig(runtimeID common.Namespace) map[string]interface{} { + return config.GlobalConfig.Runtime.GetLocalConfig(runtimeID) +} + +func getConfiguredRuntimeIDs(registry bundle.Registry) ([]common.Namespace, error) { + // Check if any runtimes are configured to be hosted. + runtimes := make(map[common.Namespace]struct{}) + for _, cfg := range config.GlobalConfig.Runtime.Runtimes { + runtimes[cfg.ID] = struct{}{} + } -// Flags has the configuration flags. -var Flags = flag.NewFlagSet("", flag.ContinueOnError) + // Support legacy configurations where runtimes are specified within + // configured bundles. + for _, manifest := range registry.GetManifests() { + runtimes[manifest.ID] = struct{}{} + } + + if cmdFlags.DebugDontBlameOasis() && viper.IsSet(bundle.CfgDebugMockIDs) { + // Allow the mock provisioner to function, as it does not use an actual + // runtime. This is only used for the basic node tests. + for _, str := range viper.GetStringSlice(bundle.CfgDebugMockIDs) { + var runtimeID common.Namespace + if err := runtimeID.UnmarshalText([]byte(str)); err != nil { + return nil, fmt.Errorf("failed to deserialize runtime ID: %w", err) + } + runtimes[runtimeID] = struct{}{} + } -// newRuntimeConfig creates a new node runtime configuration. -func newRuntimeConfig(dataDir string) (map[common.Namespace]map[version.Version]*runtimeHost.Config, error) { //nolint: gocyclo - haveSetRuntimes := len(config.GlobalConfig.Runtime.Paths) > 0 + // Skip validation + return slices.Collect(maps.Keys(runtimes)), nil + } // Validate configured runtimes based on the runtime mode. switch config.GlobalConfig.Mode { case config.ModeValidator, config.ModeSeed: // No runtimes should be configured. - if haveSetRuntimes && !cmdFlags.DebugDontBlameOasis() { + if len(runtimes) > 0 && !cmdFlags.DebugDontBlameOasis() { return nil, fmt.Errorf("no runtimes should be configured when in validator or seed modes") } case config.ModeCompute, config.ModeKeyManager, config.ModeStatelessClient: // At least one runtime should be configured. - if !haveSetRuntimes && !cmdFlags.DebugDontBlameOasis() { + if len(runtimes) == 0 && !cmdFlags.DebugDontBlameOasis() { return nil, fmt.Errorf("at least one runtime must be configured when in compute, keymanager, or client-stateless modes") } default: // In any other mode, runtimes can be optionally configured. } - // Check if any runtimes are configured to be hosted. - runtimes := make(map[common.Namespace]map[version.Version]*runtimeHost.Config) - - if haveSetRuntimes || (cmdFlags.DebugDontBlameOasis() && viper.IsSet(CfgDebugMockIDs)) { - // By default start with the environment specified in configuration. - runtimeEnv := config.GlobalConfig.Runtime.Environment - - // Preprocess runtimes to separate detached from non-detached. - type nameKey struct { - runtime common.Namespace - comp component.ID - } - - var ( - regularBundles []*bundle.Bundle - err error - ) - detachedBundles := make(map[common.Namespace][]*bundle.Bundle) - existingNames := make(map[nameKey]struct{}) - for _, path := range config.GlobalConfig.Runtime.Paths { - var bnd *bundle.Bundle - if bnd, err = bundle.Open(path); err != nil { - return nil, fmt.Errorf("failed to load runtime bundle '%s': %w", path, err) - } - if _, err = bnd.WriteExploded(dataDir); err != nil { - return nil, fmt.Errorf("failed to explode runtime bundle '%s': %w", path, err) - } - // Release resources as the bundle has been exploded anyway. - bnd.Data = nil - - switch bnd.Manifest.IsDetached() { - case false: - // A regular non-detached bundle that has the RONL component. - regularBundles = append(regularBundles, bnd) - case true: - // A detached bundle without the RONL component that needs to be attached. - detachedBundles[bnd.Manifest.ID] = append(detachedBundles[bnd.Manifest.ID], bnd) - - // Ensure there are no name conflicts among the components. - for compID := range bnd.Manifest.GetAvailableComponents() { - nk := nameKey{bnd.Manifest.ID, compID} - if _, ok := existingNames[nk]; ok { - return nil, fmt.Errorf("duplicate component '%s' for runtime '%s'", compID, bnd.Manifest.ID) - } - existingNames[nk] = struct{}{} - } - } - - // If the runtime environment is set to automatic selection and a bundle has a component - // that requires the use of a TEE, force a TEE environment to simplify configuration. - if runtimeEnv == rtConfig.RuntimeEnvironmentAuto { - for _, comp := range bnd.Manifest.GetAvailableComponents() { - if comp.IsTEERequired() { - runtimeEnv = rtConfig.RuntimeEnvironmentSGX - break - } - } - } - } - - // Configure runtimes. - for _, bnd := range regularBundles { - id := bnd.Manifest.ID - if runtimes[id] == nil { - runtimes[id] = make(map[version.Version]*runtimeHost.Config) - } - version := bnd.Manifest.GetComponentByID(component.ID_RONL).Version - if _, ok := runtimes[id][version]; ok { - return nil, fmt.Errorf("duplicate runtime '%s' version '%s'", id, bnd.Manifest.Version) - } - - // Get any local runtime configuration. - var localConfig map[string]interface{} - if lc, ok := config.GlobalConfig.Runtime.RuntimeConfig[id.String()]; ok { - localConfig = lc - } - - // Gather all components. - components := make(map[component.ID]*bundle.ExplodedComponent) - - // Add bundle components. - explodedDir := bnd.ExplodedPath(dataDir, "") - - for _, comp := range bnd.Manifest.Components { - components[comp.ID()] = &bundle.ExplodedComponent{ - Component: comp, - Detached: false, - ExplodedDataDir: explodedDir, - } - } - - // Merge in detached components. - for _, detachedBnd := range detachedBundles[id] { - explodedDir := detachedBnd.ExplodedPath(dataDir, "") - - for _, detachedComp := range detachedBnd.Manifest.Components { - // Skip components that already exist in the bundle itself. - if _, ok := components[detachedComp.ID()]; ok { - continue - } - - components[detachedComp.ID()] = &bundle.ExplodedComponent{ - Component: detachedComp, - Detached: true, - ExplodedDataDir: explodedDir, - } - } - } - - // Determine what kind of components we want. - wantedComponents := []*bundle.ExplodedComponent{ - components[component.ID_RONL], - } - for _, comp := range components { - if comp.ID().IsRONL() { - continue // Always enabled above. - } - - // By default honor the status of the component itself. - enabled := !comp.Disabled - // On non-compute nodes, assume all components are disabled by default. - if config.GlobalConfig.Mode != config.ModeCompute { - enabled = false - } - // Detached components are explicit and they should be enabled by default. - if comp.Detached { - enabled = true - } - - // Check for any overrides in the node configuration. - compCfg, ok := config.GlobalConfig.Runtime.GetComponent(comp.ID()) - if ok { - enabled = !compCfg.Disabled - } - - if !enabled { - continue - } - - wantedComponents = append(wantedComponents, comp) - } - - runtimes[id][version] = &runtimeHost.Config{ - Name: bnd.Manifest.Name, - ID: bnd.Manifest.ID, - Components: wantedComponents, - LocalConfig: localConfig, - } - } - - if cmdFlags.DebugDontBlameOasis() { - // This is to allow the mock provisioner to function, as it does - // not use an actual runtime, thus is missing a bundle. This is - // only used for the basic node tests. - for _, idStr := range viper.GetStringSlice(CfgDebugMockIDs) { - var id common.Namespace - if err = id.UnmarshalText([]byte(idStr)); err != nil { - return nil, fmt.Errorf("failed to deserialize runtime ID: %w", err) - } - - runtimeHostCfg := &runtimeHost.Config{ - ID: id, - Components: []*bundle.ExplodedComponent{ - { - Component: &bundle.Component{ - Kind: component.RONL, - Executable: "mock", - }, - Detached: false, - }, - }, - } - runtimes[id] = map[version.Version]*runtimeHost.Config{ - {}: runtimeHostCfg, - } - } - } - - if len(runtimes) == 0 { - return nil, fmt.Errorf("no runtimes configured") - } - } - - return runtimes, nil + return slices.Collect(maps.Keys(runtimes)), nil } func createHostInfo(consensus consensus.Backend) (*hostProtocol.HostInfo, error) { @@ -271,6 +107,7 @@ func createProvisioner( identity *identity.Identity, consensus consensus.Backend, hostInfo *hostProtocol.HostInfo, + bundleRegistry bundle.Registry, ias []ias.Endpoint, qs pcs.QuoteService, ) (runtimeHost.Provisioner, error) { @@ -279,8 +116,23 @@ func createProvisioner( // By default start with the environment specified in configuration. runtimeEnv := config.GlobalConfig.Runtime.Environment - // TODO: isEnvSGX should also be true if runtimeEnv is auto and at least - // one component requires SGX. + // If the runtime environment is set to automatic selection and at least + // one bundle has a component that requires the use of a TEE, force a TEE + // environment to simplify configuration. + func() { + if runtimeEnv != rtConfig.RuntimeEnvironmentAuto { + return + } + for _, manifest := range bundleRegistry.GetManifests() { + for _, comp := range manifest.GetAvailableComponents() { + if comp.IsTEERequired() { + runtimeEnv = rtConfig.RuntimeEnvironmentSGX + return + } + } + } + }() + isEnvSGX := runtimeEnv == rtConfig.RuntimeEnvironmentSGX || runtimeEnv == rtConfig.RuntimeEnvironmentSGXMock forceNoSGX := (config.GlobalConfig.Mode.IsClientOnly() && !isEnvSGX) || (cmdFlags.DebugDontBlameOasis() && runtimeEnv == rtConfig.RuntimeEnvironmentELF) @@ -434,10 +286,3 @@ func createHistoryFactory() (history.Factory, error) { return historyFactory, nil } - -func init() { - Flags.StringSlice(CfgDebugMockIDs, nil, "Mock runtime IDs (format: ,,...)") - _ = Flags.MarkHidden(CfgDebugMockIDs) - - _ = viper.BindPFlags(Flags) -} diff --git a/go/runtime/registry/registry.go b/go/runtime/registry/registry.go index ae5d225b072..64fd702a011 100644 --- a/go/runtime/registry/registry.go +++ b/go/runtime/registry/registry.go @@ -15,6 +15,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/logging" "github.com/oasisprotocol/oasis-core/go/common/persistent" "github.com/oasisprotocol/oasis-core/go/common/pubsub" + "github.com/oasisprotocol/oasis-core/go/common/service" cmSync "github.com/oasisprotocol/oasis-core/go/common/sync" "github.com/oasisprotocol/oasis-core/go/common/version" "github.com/oasisprotocol/oasis-core/go/config" @@ -22,8 +23,10 @@ import ( ias "github.com/oasisprotocol/oasis-core/go/ias/api" registry "github.com/oasisprotocol/oasis-core/go/registry/api" roothash "github.com/oasisprotocol/oasis-core/go/roothash/api" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle" runtimeClient "github.com/oasisprotocol/oasis-core/go/runtime/client/api" "github.com/oasisprotocol/oasis-core/go/runtime/history" + "github.com/oasisprotocol/oasis-core/go/runtime/host" runtimeHost "github.com/oasisprotocol/oasis-core/go/runtime/host" "github.com/oasisprotocol/oasis-core/go/runtime/localstorage" storageAPI "github.com/oasisprotocol/oasis-core/go/storage/api" @@ -44,6 +47,8 @@ var ErrRuntimeHostNotConfigured = errors.New("runtime/registry: runtime host not // Registry is the running node's runtime registry interface. type Registry interface { + service.BackgroundService + // GetRuntime returns the per-runtime interface. GetRuntime(runtimeID common.Namespace) (Runtime, error) @@ -61,9 +66,6 @@ type Registry interface { // Client returns the runtime client service if available. Client() (runtimeClient.RuntimeClient, error) - // Cleanup performs post-termination cleanup. - Cleanup() - // FinishInitialization finalizes setup for all runtimes. FinishInitialization() error } @@ -141,10 +143,11 @@ type runtime struct { // nolint: maligned registryDescriptorNotifier *pubsub.Broker activeDescriptorCh chan struct{} activeDescriptorNotifier *pubsub.Broker - versionNotifier *pubsub.Broker hostProvisioner runtimeHost.Provisioner - hostConfig map[version.Version]*runtimeHost.Config + + bundleRegistry bundle.Registry + bundleDiscovery *bundle.Discovery logger *logging.Logger } @@ -155,6 +158,8 @@ func newRuntime( dataDir string, consensus consensus.Backend, provisioner runtimeHost.Provisioner, + bundleRegistry bundle.Registry, + bundleDiscovery *bundle.Discovery, ) (*runtime, error) { logger := logging.GetLogger("runtime/registry").With("runtime_id", runtimeID) @@ -181,9 +186,9 @@ func newRuntime( registryDescriptorNotifier: pubsub.NewBroker(true), activeDescriptorCh: make(chan struct{}), activeDescriptorNotifier: pubsub.NewBroker(true), - versionNotifier: pubsub.NewBroker(false), hostProvisioner: provisioner, - hostConfig: make(map[version.Version]*runtimeHost.Config), + bundleRegistry: bundleRegistry, + bundleDiscovery: bundleDiscovery, logger: logger, }, nil } @@ -285,7 +290,26 @@ func (r *runtime) LocalStorage() localstorage.LocalStorage { // HostConfig implements Runtime. func (r *runtime) HostConfig(version version.Version) *runtimeHost.Config { - return r.hostConfig[version] + name, err := r.bundleRegistry.GetName(r.id, version) + if err != nil { + return nil + } + + components, err := r.bundleRegistry.GetComponents(r.id, version) + if err != nil { + return nil + } + + localConfig := getLocalConfig(r.id) + + return &host.Config{ + Name: name, + ID: r.id, + Components: components, + Extra: nil, + MessageHandler: nil, + LocalConfig: localConfig, + } } // HostProvisioner implements Runtime. @@ -295,35 +319,12 @@ func (r *runtime) HostProvisioner() runtimeHost.Provisioner { // HostVersions implements Runtime. func (r *runtime) HostVersions() []version.Version { - var versions []version.Version - for v := range r.hostConfig { - versions = append(versions, v) - } - return versions + return r.bundleRegistry.GetVersions(r.id) } // HostVersions implements Runtime. func (r *runtime) WatchHostVersions() (<-chan version.Version, *pubsub.Subscription) { - sub := r.versionNotifier.Subscribe() - ch := make(chan version.Version) - sub.Unwrap(ch) - - return ch, sub -} - -// addVersion adds the given version configuration to the runtime. -func (r *runtime) addVersion(version version.Version, cfg *runtimeHost.Config) error { - r.Lock() - defer r.Unlock() - - if _, ok := r.hostConfig[version]; ok { - return fmt.Errorf("runtime/registry: duplicate runtime version %s", version) - } - - r.hostConfig[version] = cfg - r.versionNotifier.Broadcast(version) - - return nil + return r.bundleRegistry.WatchVersions(r.id) } // start starts the runtime worker. @@ -417,6 +418,32 @@ func (r *runtime) run(ctx context.Context) { activeInitialized = true } } + + // Download bundles for the active and future versions. + now, err := r.consensus.Beacon().GetEpoch(ctx, consensus.HeightLatest) + if err != nil { + r.logger.Error("failed to get current epoch", + "err", err, + ) + continue + } + + var manifestHashes []hash.Hash + for i := len(rt.Deployments) - 1; i >= 0; i-- { + // Some deployments may lack a bundle manifest checksum, + // as it is optional. + if h := rt.Deployments[i].BundleChecksum; len(h) == hash.Size { + manifestHashes = append(manifestHashes, hash.Hash(h)) + } + + // Stop at the active deployment since versions follow + // chronological order. + if rt.Deployments[i].ValidFrom <= now { + break + } + } + + r.bundleDiscovery.Queue(r.id, manifestHashes) } } } @@ -472,6 +499,8 @@ func (r *runtime) finishInitialization() error { type runtimeRegistry struct { sync.RWMutex + quitCh chan struct{} + logger *logging.Logger dataDir string @@ -483,14 +512,13 @@ type runtimeRegistry struct { provisioner runtimeHost.Provisioner historyFactory history.Factory + + bundleRegistry bundle.Registry + bundleDiscovery *bundle.Discovery } // GetRuntime implements Registry. func (r *runtimeRegistry) GetRuntime(runtimeID common.Namespace) (Runtime, error) { - return r.getRuntime(runtimeID) -} - -func (r *runtimeRegistry) getRuntime(runtimeID common.Namespace) (*runtime, error) { r.RLock() defer r.RUnlock() @@ -531,7 +559,7 @@ func (r *runtimeRegistry) NewRuntime(ctx context.Context, runtimeID common.Names return nil, fmt.Errorf("runtime/registry: runtime already registered: %s", runtimeID) } - rt, err := newRuntime(runtimeID, managed, r.dataDir, r.consensus, r.provisioner) + rt, err := newRuntime(runtimeID, managed, r.dataDir, r.consensus, r.provisioner, r.bundleRegistry, r.bundleDiscovery) if err != nil { return nil, err } @@ -562,16 +590,6 @@ func (r *runtimeRegistry) NewRuntime(ctx context.Context, runtimeID common.Names return rt, nil } -// addRuntimeVersion adds the given version configuration to the given runtime. -func (r *runtimeRegistry) addRuntimeVersion(version version.Version, cfg *runtimeHost.Config) error { - rt, err := r.getRuntime(cfg.ID) - if err != nil { - return err - } - - return rt.addVersion(version, cfg) -} - // RegisterClient implements Registry. func (r *runtimeRegistry) RegisterClient(rc runtimeClient.RuntimeClient) error { r.Lock() @@ -595,7 +613,42 @@ func (r *runtimeRegistry) Client() (runtimeClient.RuntimeClient, error) { return r.client, nil } -// Cleanup implements Registry. +// FinishInitialization implements Registry. +func (r *runtimeRegistry) FinishInitialization() error { + r.RLock() + defer r.RUnlock() + + for _, rt := range r.runtimes { + if err := rt.finishInitialization(); err != nil { + return err + } + } + return nil +} + +// Name implements BackgroundService. +func (r *runtimeRegistry) Name() string { + return "runtime registry" +} + +// Start implements BackgroundService. +func (r *runtimeRegistry) Start() error { + r.bundleDiscovery.Start() + return nil +} + +// Stop implements BackgroundService. +func (r *runtimeRegistry) Stop() { + r.bundleDiscovery.Stop() + close(r.quitCh) +} + +// Quit implements BackgroundService. +func (r *runtimeRegistry) Quit() <-chan struct{} { + return r.quitCh +} + +// Cleanup implements BackgroundService. func (r *runtimeRegistry) Cleanup() { r.Lock() defer r.Unlock() @@ -605,16 +658,26 @@ func (r *runtimeRegistry) Cleanup() { } } -// FinishInitialization implements Registry. -func (r *runtimeRegistry) FinishInitialization() error { - r.RLock() - defer r.RUnlock() +// Init initializes the runtime registry by adding runtimes from the global +// runtime configuration to the registry. +func (r *runtimeRegistry) Init(ctx context.Context) error { + runtimeIDs, err := getConfiguredRuntimeIDs(r.bundleRegistry) + if err != nil { + return err + } - for _, rt := range r.runtimes { - if err := rt.finishInitialization(); err != nil { - return err + managed := config.GlobalConfig.Mode != config.ModeKeyManager + + for _, runtimeID := range runtimeIDs { + if _, err := r.NewRuntime(ctx, runtimeID, managed); err != nil { + r.logger.Error("failed to add runtime", + "err", err, + "id", runtimeID, + ) + return fmt.Errorf("failed to add runtime %s: %w", runtimeID, err) } } + return nil } @@ -627,6 +690,22 @@ func New( consensus consensus.Backend, ias []ias.Endpoint, ) (Registry, error) { + // Create bundle registry. + bundleRegistry := bundle.NewRegistry(dataDir) + + // Fill the registry with local bundles. + // + // This enables the provisioner to determine which runtime environment + // to use when the configuration is set to 'auto'. + // + // FIXME: Handle cases where the configuration is set to 'auto' but + // no bundles are configured. After addressing this, move the + // initialization to the bottom for better organization. + bundleDiscovery := bundle.NewDiscovery(dataDir, bundleRegistry) + if err := bundleDiscovery.Init(); err != nil { + return nil, err + } + // Create history keeper factory. historyFactory, err := createHistoryFactory() if err != nil { @@ -646,43 +725,27 @@ func New( } // Create runtime provisioner. - provisioner, err := createProvisioner(commonStore, identity, consensus, hostInfo, ias, qs) - if err != nil { - return nil, err - } - - runtimesCfg, err := newRuntimeConfig(dataDir) + provisioner, err := createProvisioner(commonStore, identity, consensus, hostInfo, bundleRegistry, ias, qs) if err != nil { return nil, err } + // Create runtime registry. r := &runtimeRegistry{ - logger: logging.GetLogger("runtime/registry"), - dataDir: dataDir, - consensus: consensus, - runtimes: make(map[common.Namespace]*runtime), - provisioner: provisioner, - historyFactory: historyFactory, - } - - managed := config.GlobalConfig.Mode != config.ModeKeyManager - - for runtimeID := range runtimesCfg { - if _, err := r.NewRuntime(ctx, runtimeID, managed); err != nil { - r.logger.Error("failed to add runtime", - "err", err, - "id", runtimeID, - ) - return nil, fmt.Errorf("failed to add runtime %s: %w", runtimeID, err) - } - } - - for _, cfgs := range runtimesCfg { - for version, cfg := range cfgs { - if err := r.addRuntimeVersion(version, cfg); err != nil { - return nil, err - } - } + logger: logging.GetLogger("runtime/registry"), + quitCh: make(chan struct{}), + dataDir: dataDir, + consensus: consensus, + runtimes: make(map[common.Namespace]*runtime), + provisioner: provisioner, + historyFactory: historyFactory, + bundleRegistry: bundleRegistry, + bundleDiscovery: bundleDiscovery, + } + + // Initialize the runtime registry. + if err = r.Init(ctx); err != nil { + return nil, err } return r, nil diff --git a/go/worker/common/committee/node.go b/go/worker/common/committee/node.go index 8c484c1f644..9f06bcd1528 100644 --- a/go/worker/common/committee/node.go +++ b/go/worker/common/committee/node.go @@ -446,7 +446,7 @@ func (n *Node) updateHostedRuntimeVersionLocked() { n.SetHostedRuntimeVersion(activeVersion, nextVersion) if _, err := n.GetHostedRuntimeActiveVersion(); err != nil { - n.logger.Error("failed to activate runtime version(s)", + n.logger.Warn("failed to activate runtime version(s)", "err", err, "version", activeVersion, "next_version", nextVersion, diff --git a/go/worker/keymanager/worker.go b/go/worker/keymanager/worker.go index f050aa953f0..4bc674abebb 100644 --- a/go/worker/keymanager/worker.go +++ b/go/worker/keymanager/worker.go @@ -420,6 +420,10 @@ func (w *Worker) worker() { return false } + w.logger.Info("runtime version discovered", + "version", version, + ) + return true }(); !ok { return From 28f70e29b47cc19fd116dc9ca3021d57ec171f8c Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Thu, 21 Nov 2024 15:32:28 +0100 Subject: [PATCH 16/27] go/control/api: Add bundle status to GetStatus output --- go/control/api/api.go | 36 ++++++++++++++++++++++ go/oasis-node/cmd/node/node_control.go | 41 ++++++++++++++++++++++++++ go/runtime/bundle/manifest.go | 18 +++++++++++ go/runtime/registry/registry.go | 8 +++++ 4 files changed, 103 insertions(+) diff --git a/go/control/api/api.go b/go/control/api/api.go index 6cd27d74e56..341b5dce460 100644 --- a/go/control/api/api.go +++ b/go/control/api/api.go @@ -11,11 +11,13 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/crypto/signature" "github.com/oasisprotocol/oasis-core/go/common/errors" "github.com/oasisprotocol/oasis-core/go/common/node" + "github.com/oasisprotocol/oasis-core/go/common/version" "github.com/oasisprotocol/oasis-core/go/config" consensus "github.com/oasisprotocol/oasis-core/go/consensus/api" p2p "github.com/oasisprotocol/oasis-core/go/p2p/api" registry "github.com/oasisprotocol/oasis-core/go/registry/api" block "github.com/oasisprotocol/oasis-core/go/roothash/api/block" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" storage "github.com/oasisprotocol/oasis-core/go/storage/api" upgrade "github.com/oasisprotocol/oasis-core/go/upgrade/api" commonWorker "github.com/oasisprotocol/oasis-core/go/worker/common/api" @@ -85,6 +87,9 @@ type Status struct { // Runtimes is the status overview for each runtime supported by the node. Runtimes map[common.Namespace]RuntimeStatus `json:"runtimes,omitempty"` + // Bundles is the status overview of known runtime bundles. + Bundles []BundleStatus `json:"bundles,omitempty"` + // Registration is the node's registration status. Registration *RegistrationStatus `json:"registration,omitempty"` @@ -188,6 +193,37 @@ type RuntimeStatus struct { Provisioner string `json:"provisioner,omitempty"` } +// BundleStatus is the per-runtime bundle status overview. +type BundleStatus struct { + // Name is the optional human readable runtime name. + Name string `json:"name,omitempty"` + + // ID is the runtime identifier. + ID common.Namespace `json:"id"` + + // Components contains statuses of the runtime components. + Components []ComponentStatus `json:"components,omitempty"` +} + +// ComponentStatus is the component status overview. +type ComponentStatus struct { + // Kind is the component kind. + Kind component.Kind `json:"kind"` + + // Name is the name of the component. + Name string `json:"name,omitempty"` + + // Version is the component version. + Version version.Version `json:"version,omitempty"` + + // Detached specifies whether the component was in a detached bundled. + Detached bool `json:"detached,omitempty"` + + // Disabled specifies whether the component is disabled by default + // and needs to be explicitly enabled via node configuration to be used. + Disabled bool `json:"disabled,omitempty"` +} + // SeedStatus is the status of the seed node. type SeedStatus struct { // ChainContext is the chain domain separation context. diff --git a/go/oasis-node/cmd/node/node_control.go b/go/oasis-node/cmd/node/node_control.go index 9a654c0dc31..a15e5863010 100644 --- a/go/oasis-node/cmd/node/node_control.go +++ b/go/oasis-node/cmd/node/node_control.go @@ -135,6 +135,11 @@ func (n *Node) GetStatus(ctx context.Context) (*control.Status, error) { return nil, fmt.Errorf("failed to get runtime status: %w", err) } + bundles, err := n.getBundleStatus() + if err != nil { + return nil, fmt.Errorf("failed to get bundle status: %w", err) + } + kms, err := n.getKeymanagerStatus() if err != nil { return nil, fmt.Errorf("failed to get key manager worker status: %w", err) @@ -165,6 +170,7 @@ func (n *Node) GetStatus(ctx context.Context) (*control.Status, error) { Consensus: cs, LightClient: lcs, Runtimes: runtimes, + Bundles: bundles, Keymanager: kms, Registration: rs, PendingUpgrades: pendingUpgrades, @@ -347,6 +353,41 @@ func (n *Node) getRuntimeStatus(ctx context.Context) (map[common.Namespace]contr return runtimes, nil } +func (n *Node) getBundleStatus() ([]control.BundleStatus, error) { + bundleRegistry := n.RuntimeRegistry.GetBundleRegistry() + manifests := bundleRegistry.GetManifests() + bundles := make([]control.BundleStatus, 0, len(manifests)) + + for _, manifest := range manifests { + explodedComponents, err := bundleRegistry.GetComponents(manifest.ID, manifest.GetVersion()) + if err != nil { + return nil, err + } + + components := make([]control.ComponentStatus, 0, len(explodedComponents)) + for _, comp := range explodedComponents { + component := control.ComponentStatus{ + Kind: comp.Kind, + Name: comp.Name, + Version: comp.Version, + Detached: comp.Detached, + Disabled: comp.Disabled, + } + components = append(components, component) + } + + bundle := control.BundleStatus{ + Name: manifest.Name, + ID: manifest.ID, + Components: components, + } + + bundles = append(bundles, bundle) + } + + return bundles, nil +} + func (n *Node) getKeymanagerStatus() (*keymanagerWorker.Status, error) { if n.KeymanagerWorker == nil || !n.KeymanagerWorker.Enabled() { return nil, nil diff --git a/go/runtime/bundle/manifest.go b/go/runtime/bundle/manifest.go index b82ecf05322..95e038e7f2b 100644 --- a/go/runtime/bundle/manifest.go +++ b/go/runtime/bundle/manifest.go @@ -124,6 +124,24 @@ func (m *Manifest) GetComponentByID(id component.ID) *Component { return nil } +// GetVersion returns the runtime version. +func (m *Manifest) GetVersion() version.Version { + // We also support legacy manifests which define version at the top-level. + for _, comp := range m.Components { + if !comp.ID().IsRONL() { + continue + } + + if comp.Version.ToU64() > m.Version.ToU64() { + return comp.Version + } + + break + } + + return m.Version +} + // SGXMetadata is the SGX specific manifest metadata. type SGXMetadata struct { // Executable is the name of the SGX enclave executable file. diff --git a/go/runtime/registry/registry.go b/go/runtime/registry/registry.go index 64fd702a011..147d961f56d 100644 --- a/go/runtime/registry/registry.go +++ b/go/runtime/registry/registry.go @@ -68,6 +68,9 @@ type Registry interface { // FinishInitialization finalizes setup for all runtimes. FinishInitialization() error + + // GetBundleRegistry returns the bundle registry. + GetBundleRegistry() bundle.Registry } // Runtime is the running node's supported runtime interface. @@ -626,6 +629,11 @@ func (r *runtimeRegistry) FinishInitialization() error { return nil } +// GetBundleRegistry implements Registry. +func (r *runtimeRegistry) GetBundleRegistry() bundle.Registry { + return r.bundleRegistry +} + // Name implements BackgroundService. func (r *runtimeRegistry) Name() string { return "runtime registry" From 146aa95bea2886e1f535efa0b9885b9b3fbf552b Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Sun, 8 Dec 2024 12:54:00 +0100 Subject: [PATCH 17/27] go/runtime/bundle/discovery: Add timeout to http requests --- go/runtime/bundle/discovery.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/go/runtime/bundle/discovery.go b/go/runtime/bundle/discovery.go index 6e63be91d85..394a26cea90 100644 --- a/go/runtime/bundle/discovery.go +++ b/go/runtime/bundle/discovery.go @@ -22,8 +22,13 @@ import ( "github.com/oasisprotocol/oasis-core/go/config" ) -// discoveryInterval is the time interval between (failed) bundle discoveries. -const discoveryInterval = 15 * time.Minute +const ( + // discoveryInterval is the time interval between (failed) bundle discoveries. + discoveryInterval = 15 * time.Minute + + // requestTimeout is the time limit for http client requests. + requestTimeout = 10 * time.Second +) // Discovery is responsible for discovering new bundles. type Discovery struct { @@ -37,6 +42,7 @@ type Discovery struct { globalBaseURLs []string runtimeBaseURLs map[common.Namespace][]string + client *http.Client registry Registry @@ -47,11 +53,16 @@ type Discovery struct { func NewDiscovery(dataDir string, registry Registry) *Discovery { logger := logging.GetLogger("runtime/bundle/discovery") + client := http.Client{ + Timeout: requestTimeout, + } + return &Discovery{ startOne: cmSync.NewOne(), + discoverCh: make(chan struct{}, 1), bundleDir: ExplodedPath(dataDir), manifestHashes: make(map[common.Namespace][]hash.Hash), - discoverCh: make(chan struct{}, 1), + client: &client, registry: registry, logger: *logger, } @@ -337,7 +348,7 @@ func (d *Discovery) tryDownloadBundle(runtimeID common.Namespace, manifestHash h } func (d *Discovery) fetchBundle(url string) (string, error) { - resp, err := http.Get(url) // nolint: gosec + resp, err := d.client.Get(url) if err != nil { return "", fmt.Errorf("failed to fetch bundle: %w", err) } From 1802d4fa9ccb6e79f6340d3878bddd2f6f7ece83 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Sun, 8 Dec 2024 13:04:56 +0100 Subject: [PATCH 18/27] go/runtime/bundle/discovery: Add bundle size limit --- go/runtime/bundle/discovery.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/go/runtime/bundle/discovery.go b/go/runtime/bundle/discovery.go index 394a26cea90..4cb54f49351 100644 --- a/go/runtime/bundle/discovery.go +++ b/go/runtime/bundle/discovery.go @@ -28,6 +28,9 @@ const ( // requestTimeout is the time limit for http client requests. requestTimeout = 10 * time.Second + + // maxBundleSizeBytes is the maximum allowed bundle size in bytes. + maxBundleSizeBytes = 20 * 1024 * 1024 // 20 MB ) // Discovery is responsible for discovering new bundles. @@ -370,10 +373,19 @@ func (d *Discovery) fetchBundle(url string) (string, error) { } }() - if _, err = io.Copy(file, resp.Body); err != nil { + limitedReader := io.LimitedReader{ + R: resp.Body, + N: maxBundleSizeBytes, + } + + if _, err = io.Copy(file, &limitedReader); err != nil { return "", fmt.Errorf("failed to save bundle: %w", err) } + if limitedReader.N <= 0 { + return "", fmt.Errorf("bundle exceeds size limit of %d bytes", maxBundleSizeBytes) + } + return file.Name(), nil } From a7bab02ae8dc9258cf393e816550da0f791eed05 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Sun, 8 Dec 2024 13:07:24 +0100 Subject: [PATCH 19/27] go/runtime/bundle/discovery: Lock mutex in Init as late as possible --- go/runtime/bundle/discovery.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/runtime/bundle/discovery.go b/go/runtime/bundle/discovery.go index 4cb54f49351..6251205554a 100644 --- a/go/runtime/bundle/discovery.go +++ b/go/runtime/bundle/discovery.go @@ -85,9 +85,6 @@ func (d *Discovery) Init() error { return err } - d.mu.Lock() - defer d.mu.Unlock() - // Validate global repository URLs. globalBaseURLs, err := validateAndNormalizeURLs(config.GlobalConfig.Runtime.Repositories) if err != nil { @@ -109,6 +106,9 @@ func (d *Discovery) Init() error { } // Update discovery. + d.mu.Lock() + defer d.mu.Unlock() + d.globalBaseURLs = globalBaseURLs d.runtimeBaseURLs = runtimeBaseURLs From 3a2a455f4e0227de874e0e45e34ded5fab6ac191 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Wed, 11 Dec 2024 10:50:42 +0100 Subject: [PATCH 20/27] go/runtime/bundle/registry: Sort versions --- go/common/version/version.go | 12 ++++++++++++ go/runtime/bundle/registry.go | 8 ++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/go/common/version/version.go b/go/common/version/version.go index 80a69670efe..16059c7edfb 100644 --- a/go/common/version/version.go +++ b/go/common/version/version.go @@ -39,6 +39,18 @@ func (v Version) ValidateBasic() error { return nil } +// Cmp returns a negative number when the given version is larger, a positive +// number when smaller, and zero when both versions are equal. +func (v Version) Cmp(other Version) int { + if v.Major != other.Major { + return int(v.Major) - int(other.Major) + } + if v.Minor != other.Minor { + return int(v.Minor) - int(other.Minor) + } + return int(v.Patch) - int(other.Patch) +} + // ToU64 returns the version as platform-dependent uint64. func (v Version) ToU64() uint64 { return (uint64(v.Major) << 32) | (uint64(v.Minor) << 16) | (uint64(v.Patch)) diff --git a/go/runtime/bundle/registry.go b/go/runtime/bundle/registry.go index d8efd1a08ca..b72ff6b25af 100644 --- a/go/runtime/bundle/registry.go +++ b/go/runtime/bundle/registry.go @@ -33,7 +33,8 @@ type Registry interface { // AddBundle adds a bundle from the given path. AddBundle(path string, manifestHash hash.Hash) error - // GetVersions returns versions for the given runtime. + // GetVersions returns versions for the given runtime, sorted in ascending + // order. GetVersions(runtimeID common.Namespace) []version.Version // WatchVersions provides a channel that streams runtime versions as they @@ -201,7 +202,10 @@ func (r *registry) GetVersions(runtimeID common.Namespace) []version.Version { } } - return slices.Collect(maps.Keys(r.manifests[runtimeID])) + versions := slices.Collect(maps.Keys(r.manifests[runtimeID])) + slices.SortFunc(versions, version.Version.Cmp) + + return versions } // WatchVersions implements Registry. From e6e72cf3a106201d63d8327c7a14a02b40ea1d19 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Wed, 11 Dec 2024 10:54:00 +0100 Subject: [PATCH 21/27] go/runtime/bundle: Fix comments and errors --- go/control/api/api.go | 2 +- go/runtime/bundle/discovery.go | 7 ++++--- go/runtime/bundle/manifest.go | 2 +- go/runtime/bundle/registry.go | 9 ++++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/go/control/api/api.go b/go/control/api/api.go index 341b5dce460..852090914ca 100644 --- a/go/control/api/api.go +++ b/go/control/api/api.go @@ -216,7 +216,7 @@ type ComponentStatus struct { // Version is the component version. Version version.Version `json:"version,omitempty"` - // Detached specifies whether the component was in a detached bundled. + // Detached specifies whether the component was in a detached bundle. Detached bool `json:"detached,omitempty"` // Disabled specifies whether the component is disabled by default diff --git a/go/runtime/bundle/discovery.go b/go/runtime/bundle/discovery.go index 6251205554a..5766cc3338c 100644 --- a/go/runtime/bundle/discovery.go +++ b/go/runtime/bundle/discovery.go @@ -72,7 +72,7 @@ func NewDiscovery(dataDir string, registry Registry) *Discovery { } // Init sets up bundle discovery using node configuration and adds configured -// and cached bundles to the registry. +// and cached bundles (that are guaranteed to be exploded) to the registry. func (d *Discovery) Init() error { // Consolidate all bundles in one place, which could be useful // if we implement P2P sharing in the future. @@ -80,7 +80,8 @@ func (d *Discovery) Init() error { return err } - // Add copied and cached bundles to the registry. + // Add copied and cached bundles (that are guaranteed to be exploded) + // to the registry. if err := d.Discover(); err != nil { return err } @@ -449,7 +450,7 @@ func (d *Discovery) copyBundle(src string) error { "src", src, "dst", dst, ) - return fmt.Errorf("failed to open bundle: %w", err) + return fmt.Errorf("failed to copy bundle: %w", err) } d.logger.Info("bundle copied", diff --git a/go/runtime/bundle/manifest.go b/go/runtime/bundle/manifest.go index 95e038e7f2b..a6de1854721 100644 --- a/go/runtime/bundle/manifest.go +++ b/go/runtime/bundle/manifest.go @@ -249,7 +249,7 @@ type Identity struct { type ExplodedComponent struct { *Component - // Detached returns true iff the bundle containing the component does not + // Detached is true iff the bundle containing the component does not // include a RONL component. Detached bool diff --git a/go/runtime/bundle/registry.go b/go/runtime/bundle/registry.go index b72ff6b25af..970e3c1cac6 100644 --- a/go/runtime/bundle/registry.go +++ b/go/runtime/bundle/registry.go @@ -47,7 +47,8 @@ type Registry interface { // GetName returns optional human readable runtime name. GetName(runtimeID common.Namespace, version version.Version) (string, error) - // GetComponents returns components for the given runtime and version. + // GetComponents returns RONL component for the given runtime and version, + // together with latest version of the remaining components. GetComponents(runtimeID common.Namespace, version version.Version) ([]*ExplodedComponent, error) } @@ -106,7 +107,8 @@ func (r *registry) AddBundle(path string, manifestHash hash.Hash) error { } defer bnd.Close() - // Verify manifest hash. + // Verify that the manifest hash belongs to the bundle before checking + // if the bundle is already in the registry. if !bnd.manifestHash.Equal(&manifestHash) { return fmt.Errorf("invalid manifest hash (got: '%s', expected: '%s')", bnd.manifestHash.Hex(), @@ -119,7 +121,8 @@ func (r *registry) AddBundle(path string, manifestHash hash.Hash) error { return nil } - // Verify that components are unique. + // Ensure the manifest doesn't include a component version already + // in the registry. components := bnd.Manifest.GetAvailableComponents() for compID, comp := range components { From efeadb0ce40207ebd9155eea002b58d587572ac2 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Wed, 11 Dec 2024 12:03:21 +0100 Subject: [PATCH 22/27] go/runtime/bundle: Verify manifest hash when opening a bundle --- go/runtime/bundle/bundle.go | 36 +++++++++++++++++-- go/runtime/bundle/bundle_test.go | 60 ++++++++++++++------------------ go/runtime/bundle/registry.go | 14 ++------ 3 files changed, 64 insertions(+), 46 deletions(-) diff --git a/go/runtime/bundle/bundle.go b/go/runtime/bundle/bundle.go index 633b5de15b2..534d9adc58b 100644 --- a/go/runtime/bundle/bundle.go +++ b/go/runtime/bundle/bundle.go @@ -518,7 +518,9 @@ func (bnd *Bundle) Close() error { } // Open opens and validates a runtime bundle instance. -func Open(fn string) (*Bundle, error) { +func Open(fn string, opts ...OpenOption) (*Bundle, error) { + options := NewOpenOptions(opts...) + r, err := zip.OpenReader(fn) if err != nil { return nil, fmt.Errorf("runtime/bundle: failed to open bundle: %w", err) @@ -558,12 +560,18 @@ func Open(fn string) (*Bundle, error) { return nil, fmt.Errorf("runtime/bundle: failed to parse manifest: %w", err) } + // Verify the manifest hash, if requested. + manifestHash := manifest.Hash() + if h := options.manifestHash; h != nil && !manifestHash.Equal(h) { + return nil, fmt.Errorf("runtime/bundle: invalid manifest (got: %s, expected: %s)", manifestHash.Hex(), h.Hex()) + } + // Ensure the bundle is well-formed. bnd := &Bundle{ Manifest: &manifest, Data: data, archive: r, - manifestHash: manifest.Hash(), + manifestHash: manifestHash, } if err = bnd.Validate(); err != nil { return nil, err @@ -631,3 +639,27 @@ func HashAllData(d Data) (hash.Hash, error) { defer f.Close() return hash.NewFromReader(f) } + +// OpenOptions are options for opening bundle files. +type OpenOptions struct { + manifestHash *hash.Hash +} + +// NewOpenOptions creates options using default and given values. +func NewOpenOptions(opts ...OpenOption) *OpenOptions { + var o OpenOptions + for _, opt := range opts { + opt(&o) + } + return &o +} + +// OpenOption is an option used when opening a bundle file. +type OpenOption func(o *OpenOptions) + +// WithManifestHash sets the manifest hash for verification. +func WithManifestHash(manifestHash hash.Hash) OpenOption { + return func(o *OpenOptions) { + o.manifestHash = &manifestHash + } +} diff --git a/go/runtime/bundle/bundle_test.go b/go/runtime/bundle/bundle_test.go index 889b0ffad16..e48b252ee34 100644 --- a/go/runtime/bundle/bundle_test.go +++ b/go/runtime/bundle/bundle_test.go @@ -1,7 +1,6 @@ package bundle import ( - "crypto/rand" "os" "path/filepath" "testing" @@ -9,17 +8,12 @@ import ( "github.com/stretchr/testify/require" "github.com/oasisprotocol/oasis-core/go/common" + "github.com/oasisprotocol/oasis-core/go/common/crypto/hash" "github.com/oasisprotocol/oasis-core/go/common/sgx" "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" ) func TestBundle(t *testing.T) { - execFile := os.Args[0] - execBuf, err := os.ReadFile(execFile) - if err != nil { - t.Fatalf("failed to read test executable %s: %v", execFile, err) - } - // Create a synthetic bundle. // // Assets will be populated during the Add/Write combined test. @@ -60,17 +54,9 @@ func TestBundle(t *testing.T) { require.False(t, manifest.IsDetached(), "manifest with RONL component should not be detached") t.Run("Add_Write", func(t *testing.T) { - // Generate random assets. - randomBuffer := func() []byte { - b := make([]byte, 1024*256) - _, err := rand.Read(b) - require.NoError(t, err, "rand.Read") - return b - } - - err := bundle.Add(manifest.Components[0].Executable, NewBytesData(execBuf)) + err := bundle.Add(manifest.Components[0].Executable, NewBytesData(randBuffer(3252))) require.NoError(t, err, "bundle.Add(elf)") - err = bundle.Add(manifest.Components[0].SGX.Executable, NewBytesData(randomBuffer())) + err = bundle.Add(manifest.Components[0].SGX.Executable, NewBytesData(randBuffer(6541))) require.NoError(t, err, "bundle.Add(sgx)") err = bundle.Write(bundleFn) @@ -96,6 +82,19 @@ func TestBundle(t *testing.T) { require.Equal(t, "0200000000000000000000000000000000000000000000000000000000000000", eids[0].MrEnclave.String()) }) + t.Run("Open_WithManifestHash", func(t *testing.T) { + var manifestHash hash.Hash + err := manifestHash.UnmarshalHex("905e9866eccb967e8991698273f41d20c616cab4f9e9332a2a12d9d3a1c8a486") + require.NoError(t, err, "UnmarshalHex") + + _, err = Open(bundleFn, WithManifestHash(manifestHash)) + require.NoError(t, err, "Open_WithManifestHash") + + _, err = Open(bundleFn, WithManifestHash(hash.Hash{})) + require.Error(t, err, "Open_WithManifestHash") + require.ErrorContains(t, err, "invalid manifest (got: 905e9866eccb967e8991698273f41d20c616cab4f9e9332a2a12d9d3a1c8a486, expected: 0000000000000000000000000000000000000000000000000000000000000000)") + }) + t.Run("ResetManifest", func(t *testing.T) { bundle2, err := Open(bundleFn) require.NoError(t, err, "Open") @@ -121,12 +120,6 @@ func TestBundle(t *testing.T) { } func TestDetachedBundle(t *testing.T) { - execFile := os.Args[0] - execBuf, err := os.ReadFile(execFile) - if err != nil { - t.Fatalf("failed to read test executable %s: %v", execFile, err) - } - // Create a synthetic bundle. // // Assets will be populated during the Add/Write combined test. @@ -159,17 +152,9 @@ func TestDetachedBundle(t *testing.T) { require.True(t, manifest.IsDetached(), "manifest without RONL component should be detached") t.Run("Add_Write", func(t *testing.T) { - // Generate random assets. - randomBuffer := func() []byte { - b := make([]byte, 1024*256) - _, err := rand.Read(b) - require.NoError(t, err, "rand.Read") - return b - } - - err := bundle.Add(manifest.Components[0].Executable, NewBytesData(execBuf)) + err := bundle.Add(manifest.Components[0].Executable, NewBytesData(randBuffer(2231))) require.NoError(t, err, "bundle.Add(elf)") - err = bundle.Add(manifest.Components[0].SGX.Executable, NewBytesData(randomBuffer())) + err = bundle.Add(manifest.Components[0].SGX.Executable, NewBytesData(randBuffer(7627))) require.NoError(t, err, "bundle.Add(sgx)") err = bundle.Write(bundleFn) @@ -246,3 +231,12 @@ func ensureBundlesEqual(t *testing.T, b1, b2 *Bundle, msg string) { require.EqualValues(t, d1, d2, msg) } } + +func randBuffer(seed int) []byte { + buffer := make([]byte, 1024*256) + for i := range buffer { + buffer[i] = byte((seed * 0x3C5C5) ^ 0xDEADBEEF) + seed++ + } + return buffer +} diff --git a/go/runtime/bundle/registry.go b/go/runtime/bundle/registry.go index 970e3c1cac6..c47a1aa8c3d 100644 --- a/go/runtime/bundle/registry.go +++ b/go/runtime/bundle/registry.go @@ -101,22 +101,14 @@ func (r *registry) AddBundle(path string, manifestHash hash.Hash) error { ) // Open the bundle and release resources when done. - bnd, err := Open(path) + bnd, err := Open(path, WithManifestHash(manifestHash)) if err != nil { return fmt.Errorf("failed to open bundle '%s': %w", path, err) } defer bnd.Close() - // Verify that the manifest hash belongs to the bundle before checking - // if the bundle is already in the registry. - if !bnd.manifestHash.Equal(&manifestHash) { - return fmt.Errorf("invalid manifest hash (got: '%s', expected: '%s')", - bnd.manifestHash.Hex(), - manifestHash.Hex(), - ) - } - - // Skip already processed bundles. + // Skip already processed bundles. This check should be performed + // after the bundle is opened and its manifest hash is verified. if _, ok := r.bundles[manifestHash]; ok { return nil } From 3b5008694650536f370c307065c1c5cda7a0b149 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Thu, 12 Dec 2024 11:47:53 +0100 Subject: [PATCH 23/27] go/runtime/bundle: Move component to a new file --- go/runtime/bundle/component.go | 139 +++++++++++++++++++++++++++++++++ go/runtime/bundle/manifest.go | 130 ------------------------------ 2 files changed, 139 insertions(+), 130 deletions(-) create mode 100644 go/runtime/bundle/component.go diff --git a/go/runtime/bundle/component.go b/go/runtime/bundle/component.go new file mode 100644 index 00000000000..b821bc3066a --- /dev/null +++ b/go/runtime/bundle/component.go @@ -0,0 +1,139 @@ +package bundle + +import ( + "fmt" + "path/filepath" + + "github.com/oasisprotocol/oasis-core/go/common" + "github.com/oasisprotocol/oasis-core/go/common/version" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" +) + +// ExplodedComponent is an exploded runtime component ready for execution. +type ExplodedComponent struct { + *Component + + // Detached is true iff the bundle containing the component does not + // include a RONL component. + Detached bool + + // ExplodedDataDir is the path to the data directory where the bundle + // containing the component has been extracted. + ExplodedDataDir string +} + +// ExplodedPath returns the path that the corresponding asset will be written to via WriteExploded. +func (c *ExplodedComponent) ExplodedPath(fn string) string { + return filepath.Join(c.ExplodedDataDir, fn) +} + +// Component is a runtime component. +type Component struct { + // Kind is the component kind. + Kind component.Kind `json:"kind"` + + // Name is the name of the component that can be used to filter components when multiple are + // provided by a runtime. + Name string `json:"name,omitempty"` + + // Version is the component version. + Version version.Version + + // Executable is the name of the runtime ELF executable file if any. + Executable string `json:"executable,omitempty"` + + // SGX is the SGX specific manifest metadata if any. + SGX *SGXMetadata `json:"sgx,omitempty"` + + // TDX is the TDX specific manifest metadata if any. + TDX *TDXMetadata `json:"tdx,omitempty"` + + // Identities are the (optional) expected enclave identities. When not provided, it must be + // computed at runtime. In the future, this field will become required. + // + // Multiple identities may be provided because they can differ across different deployment + // systems (e.g. hypervisors). + Identities []Identity `json:"identity,omitempty"` + + // Disabled specifies whether the component is disabled by default and needs to be explicitly + // enabled via node configuration to be used. + Disabled bool `json:"disabled,omitempty"` +} + +// ID returns this component's identifier. +func (c *Component) ID() component.ID { + return component.ID{Kind: c.Kind, Name: c.Name} +} + +// Matches returns true iff the component matches the given component ID. +func (c *Component) Matches(id component.ID) bool { + return c.ID() == id +} + +// Validate validates the component structure for well-formedness. +func (c *Component) Validate() error { + if !common.AtMostOneTrue( + c.SGX != nil, + c.TDX != nil, + ) { + return fmt.Errorf("each component can only include metadata for a single TEE") + } + if c.SGX != nil { + err := c.SGX.Validate() + if err != nil { + return fmt.Errorf("sgx: %w", err) + } + } + if c.TDX != nil { + err := c.TDX.Validate() + if err != nil { + return fmt.Errorf("tdx: %w", err) + } + } + + switch c.Kind { + case component.RONL: + if c.Name != "" { + return fmt.Errorf("RONL component must have an empty name") + } + if c.Executable == "" { + return fmt.Errorf("RONL component must define an executable") + } + if c.Disabled { + return fmt.Errorf("RONL component cannot be disabled") + } + case component.ROFL: + default: + return fmt.Errorf("unknown component kind: '%s'", c.Kind) + } + return nil +} + +// IsNetworkAllowed returns true if network access should be allowed for the component. +func (c *Component) IsNetworkAllowed() bool { + switch c.Kind { + case component.ROFL: + // Off-chain logic is allowed to access the network. + return true + default: + // Network access is generally not allowed. + return false + } +} + +// IsTEERequired returns true iff the component only provides TEE executables. +func (c *Component) IsTEERequired() bool { + return c.Executable == "" && c.TEEKind() != component.TEEKindNone +} + +// TEEKind returns the kind of TEE supported by the component. +func (c *Component) TEEKind() component.TEEKind { + switch { + case c.TDX != nil: + return component.TEEKindTDX + case c.SGX != nil: + return component.TEEKindSGX + default: + return component.TEEKindNone + } +} diff --git a/go/runtime/bundle/manifest.go b/go/runtime/bundle/manifest.go index a6de1854721..7a1c656c322 100644 --- a/go/runtime/bundle/manifest.go +++ b/go/runtime/bundle/manifest.go @@ -2,7 +2,6 @@ package bundle import ( "fmt" - "path/filepath" "github.com/oasisprotocol/oasis-core/go/common" "github.com/oasisprotocol/oasis-core/go/common/crypto/hash" @@ -244,132 +243,3 @@ type Identity struct { // Enclave is the enclave identity. Enclave sgx.EnclaveIdentity `json:"enclave"` } - -// ExplodedComponent is an exploded runtime component ready for execution. -type ExplodedComponent struct { - *Component - - // Detached is true iff the bundle containing the component does not - // include a RONL component. - Detached bool - - // ExplodedDataDir is the path to the data directory where the bundle - // containing the component has been extracted. - ExplodedDataDir string -} - -// ExplodedPath returns the path that the corresponding asset will be written to via WriteExploded. -func (c *ExplodedComponent) ExplodedPath(fn string) string { - return filepath.Join(c.ExplodedDataDir, fn) -} - -// Component is a runtime component. -type Component struct { - // Kind is the component kind. - Kind component.Kind `json:"kind"` - - // Name is the name of the component that can be used to filter components when multiple are - // provided by a runtime. - Name string `json:"name,omitempty"` - - // Version is the component version. - Version version.Version - - // Executable is the name of the runtime ELF executable file if any. - Executable string `json:"executable,omitempty"` - - // SGX is the SGX specific manifest metadata if any. - SGX *SGXMetadata `json:"sgx,omitempty"` - - // TDX is the TDX specific manifest metadata if any. - TDX *TDXMetadata `json:"tdx,omitempty"` - - // Identities are the (optional) expected enclave identities. When not provided, it must be - // computed at runtime. In the future, this field will become required. - // - // Multiple identities may be provided because they can differ across different deployment - // systems (e.g. hypervisors). - Identities []Identity `json:"identity,omitempty"` - - // Disabled specifies whether the component is disabled by default and needs to be explicitly - // enabled via node configuration to be used. - Disabled bool `json:"disabled,omitempty"` -} - -// ID returns this component's identifier. -func (c *Component) ID() component.ID { - return component.ID{Kind: c.Kind, Name: c.Name} -} - -// Matches returns true iff the component matches the given component ID. -func (c *Component) Matches(id component.ID) bool { - return c.ID() == id -} - -// Validate validates the component structure for well-formedness. -func (c *Component) Validate() error { - if !common.AtMostOneTrue( - c.SGX != nil, - c.TDX != nil, - ) { - return fmt.Errorf("each component can only include metadata for a single TEE") - } - if c.SGX != nil { - err := c.SGX.Validate() - if err != nil { - return fmt.Errorf("sgx: %w", err) - } - } - if c.TDX != nil { - err := c.TDX.Validate() - if err != nil { - return fmt.Errorf("tdx: %w", err) - } - } - - switch c.Kind { - case component.RONL: - if c.Name != "" { - return fmt.Errorf("RONL component must have an empty name") - } - if c.Executable == "" { - return fmt.Errorf("RONL component must define an executable") - } - if c.Disabled { - return fmt.Errorf("RONL component cannot be disabled") - } - case component.ROFL: - default: - return fmt.Errorf("unknown component kind: '%s'", c.Kind) - } - return nil -} - -// IsNetworkAllowed returns true if network access should be allowed for the component. -func (c *Component) IsNetworkAllowed() bool { - switch c.Kind { - case component.ROFL: - // Off-chain logic is allowed to access the network. - return true - default: - // Network access is generally not allowed. - return false - } -} - -// IsTEERequired returns true iff the component only provides TEE executables. -func (c *Component) IsTEERequired() bool { - return c.Executable == "" && c.TEEKind() != component.TEEKindNone -} - -// TEEKind returns the kind of TEE supported by the component. -func (c *Component) TEEKind() component.TEEKind { - switch { - case c.TDX != nil: - return component.TEEKindTDX - case c.SGX != nil: - return component.TEEKindSGX - default: - return component.TEEKindNone - } -} From a9c782047848d7228d01bca9190da34df355b5a2 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Thu, 12 Dec 2024 12:02:41 +0100 Subject: [PATCH 24/27] go/runtime/bundle: Make max bundle size configurable --- .changelog/5962.cfg.md | 4 +++- go/runtime/bundle/discovery.go | 31 ++++++++++++++++++++----------- go/runtime/config/config.go | 5 +++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/.changelog/5962.cfg.md b/.changelog/5962.cfg.md index 903bf0ebe75..6e7c4ba19e8 100644 --- a/.changelog/5962.cfg.md +++ b/.changelog/5962.cfg.md @@ -23,4 +23,6 @@ The following configuration options have been added: used to fetch runtime bundles, - `runtime.repositories` is the list of global URLs used to fetch - runtime bundles. + runtime bundles, + +- `runtime.max_bundle_size` is the maximum allowed bundle size. diff --git a/go/runtime/bundle/discovery.go b/go/runtime/bundle/discovery.go index 5766cc3338c..357f966a838 100644 --- a/go/runtime/bundle/discovery.go +++ b/go/runtime/bundle/discovery.go @@ -29,8 +29,9 @@ const ( // requestTimeout is the time limit for http client requests. requestTimeout = 10 * time.Second - // maxBundleSizeBytes is the maximum allowed bundle size in bytes. - maxBundleSizeBytes = 20 * 1024 * 1024 // 20 MB + // maxDefaultBundleSizeBytes is the maximum allowed default bundle size + // in bytes. + maxDefaultBundleSizeBytes = 20 * 1024 * 1024 // 20 MB ) // Discovery is responsible for discovering new bundles. @@ -47,6 +48,8 @@ type Discovery struct { runtimeBaseURLs map[common.Namespace][]string client *http.Client + maxBundleSizeBytes int64 + registry Registry logger logging.Logger @@ -60,14 +63,20 @@ func NewDiscovery(dataDir string, registry Registry) *Discovery { Timeout: requestTimeout, } + bundleSize := int64(maxDefaultBundleSizeBytes) + if size := config.GlobalConfig.Runtime.MaxBundleSize; size != "" { + bundleSize = int64(config.ParseSizeInBytes(size)) + } + return &Discovery{ - startOne: cmSync.NewOne(), - discoverCh: make(chan struct{}, 1), - bundleDir: ExplodedPath(dataDir), - manifestHashes: make(map[common.Namespace][]hash.Hash), - client: &client, - registry: registry, - logger: *logger, + startOne: cmSync.NewOne(), + discoverCh: make(chan struct{}, 1), + bundleDir: ExplodedPath(dataDir), + manifestHashes: make(map[common.Namespace][]hash.Hash), + client: &client, + maxBundleSizeBytes: bundleSize, + registry: registry, + logger: *logger, } } @@ -376,7 +385,7 @@ func (d *Discovery) fetchBundle(url string) (string, error) { limitedReader := io.LimitedReader{ R: resp.Body, - N: maxBundleSizeBytes, + N: d.maxBundleSizeBytes, } if _, err = io.Copy(file, &limitedReader); err != nil { @@ -384,7 +393,7 @@ func (d *Discovery) fetchBundle(url string) (string, error) { } if limitedReader.N <= 0 { - return "", fmt.Errorf("bundle exceeds size limit of %d bytes", maxBundleSizeBytes) + return "", fmt.Errorf("bundle exceeds size limit of %d bytes", d.maxBundleSizeBytes) } return file.Name(), nil diff --git a/go/runtime/config/config.go b/go/runtime/config/config.go index 6c6d32f1bb1..b44918a777f 100644 --- a/go/runtime/config/config.go +++ b/go/runtime/config/config.go @@ -117,6 +117,11 @@ type Config struct { // Repositories is the list of URLs used to fetch runtime bundles. Repositories []string `yaml:"repositories,omitempty"` + + // MaxBundleSize is the maximum allowed bundle size. + // + // If not specified, a default value is used. + MaxBundleSize string `yaml:"max_bundle_size,omitempty"` } // GetComponent returns the configuration for the given component From 8af27bd732e80dbf39d48790dd26190ac9c632b8 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Thu, 12 Dec 2024 12:09:55 +0100 Subject: [PATCH 25/27] go/runtime/bundle: Close zip file if Open fails --- go/runtime/bundle/bundle.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/go/runtime/bundle/bundle.go b/go/runtime/bundle/bundle.go index 534d9adc58b..4505b0b9ffa 100644 --- a/go/runtime/bundle/bundle.go +++ b/go/runtime/bundle/bundle.go @@ -518,13 +518,19 @@ func (bnd *Bundle) Close() error { } // Open opens and validates a runtime bundle instance. -func Open(fn string, opts ...OpenOption) (*Bundle, error) { +func Open(fn string, opts ...OpenOption) (_ *Bundle, err error) { options := NewOpenOptions(opts...) + // Open the zip file and close it on error. r, err := zip.OpenReader(fn) if err != nil { return nil, fmt.Errorf("runtime/bundle: failed to open bundle: %w", err) } + defer func() { + if err != nil { + r.Close() + } + }() // Read the contents. data := make(map[string]Data) From 3bd356f504560998336c5d04116479e388bb0822 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Mon, 16 Dec 2024 04:48:08 +0100 Subject: [PATCH 26/27] go/runtime/bundle: Use indirection when downloading bundles --- .changelog/5962.cfg.md | 4 +- .../e2e/runtime/keymanager_upgrade.go | 4 +- .../scenario/e2e/runtime/runtime_upgrade.go | 37 +++++++- go/runtime/bundle/discovery.go | 93 +++++++++++++++---- go/runtime/config/config.go | 4 +- 5 files changed, 112 insertions(+), 30 deletions(-) diff --git a/.changelog/5962.cfg.md b/.changelog/5962.cfg.md index 6e7c4ba19e8..67f97e81ae0 100644 --- a/.changelog/5962.cfg.md +++ b/.changelog/5962.cfg.md @@ -20,9 +20,9 @@ The following configuration options have been added: - `runtime.runtimes.config` is the runtime local configuration, - `runtime.runtimes.repositories` is the list of runtime specific URLs - used to fetch runtime bundles, + used to fetch runtime bundle metadata, - `runtime.repositories` is the list of global URLs used to fetch - runtime bundles, + runtime bundle metadata, - `runtime.max_bundle_size` is the maximum allowed bundle size. diff --git a/go/oasis-test-runner/scenario/e2e/runtime/keymanager_upgrade.go b/go/oasis-test-runner/scenario/e2e/runtime/keymanager_upgrade.go index 3bb58bc0a25..6de11db96bd 100644 --- a/go/oasis-test-runner/scenario/e2e/runtime/keymanager_upgrade.go +++ b/go/oasis-test-runner/scenario/e2e/runtime/keymanager_upgrade.go @@ -80,7 +80,7 @@ func (sc *KmUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error { port := parsedURL.Port() // Start serving bundles. - server := newBundleServer(port, bundles) + server := newBundleServer(port, bundles, sc.Logger) server.Start() defer server.Stop() @@ -90,7 +90,7 @@ func (sc *KmUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error { } // Verify that all key manager nodes requested bundle from the server. - n := len(sc.Net.Keymanagers()) + n := 2 * len(sc.Net.Keymanagers()) if m := server.getRequestCount(); m != n { return fmt.Errorf("invalid number of bundle requests (got: %d, expected: %d)", m, n) } diff --git a/go/oasis-test-runner/scenario/e2e/runtime/runtime_upgrade.go b/go/oasis-test-runner/scenario/e2e/runtime/runtime_upgrade.go index d9bc8687a5c..4f52a607fa4 100644 --- a/go/oasis-test-runner/scenario/e2e/runtime/runtime_upgrade.go +++ b/go/oasis-test-runner/scenario/e2e/runtime/runtime_upgrade.go @@ -14,6 +14,7 @@ import ( "sync/atomic" "time" + "github.com/oasisprotocol/oasis-core/go/common/logging" cmSync "github.com/oasisprotocol/oasis-core/go/common/sync" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/oasis" @@ -87,7 +88,7 @@ func (sc *runtimeUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error port := parsedURL.Port() // Start serving bundles. - server := newBundleServer(port, bundles) + server := newBundleServer(port, bundles, sc.Logger) server.Start() defer server.Stop() @@ -97,7 +98,7 @@ func (sc *runtimeUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error } // Verify that all client and compute nodes requested bundle from the server. - n := len(sc.Net.Clients()) + len(sc.Net.ComputeWorkers()) + n := 2 * (len(sc.Net.Clients()) + len(sc.Net.ComputeWorkers())) if m := server.getRequestCount(); m != n { return fmt.Errorf("invalid number of bundle requests (got: %d, expected: %d)", m, n) } @@ -117,13 +118,16 @@ type bundleServer struct { bundles map[string]string requestCount uint64 + + logger *logging.Logger } -func newBundleServer(port string, bundles map[string]string) *bundleServer { +func newBundleServer(port string, bundles map[string]string, logger *logging.Logger) *bundleServer { return &bundleServer{ startOne: cmSync.NewOne(), port: port, bundles: bundles, + logger: logger, } } @@ -160,6 +164,30 @@ func (s *bundleServer) run(ctx context.Context) { } func (s *bundleServer) handleRequest(w http.ResponseWriter, r *http.Request) { + s.logger.Info("handling request", + "path", r.URL.Path, + ) + + if strings.HasSuffix(r.URL.Path, bundle.FileExtension) { + s.handleGetBundle(w, r) + } else { + s.handleGetMetadata(w, r) + } +} + +func (s *bundleServer) handleGetMetadata(w http.ResponseWriter, r *http.Request) { + manifestHash := path.Base(r.URL.Path) + content := []byte(fmt.Sprintf("http://127.0.0.1:%s/%s%s\n", s.port, manifestHash, bundle.FileExtension)) + + w.Header().Set("Content-Disposition", "attachment; filename=metadata.txt") + w.Header().Set("Content-Type", "application/octet-stream") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(content) + + atomic.AddUint64(&s.requestCount, 1) +} + +func (s *bundleServer) handleGetBundle(w http.ResponseWriter, r *http.Request) { filename := path.Base(r.URL.Path) path, ok := s.bundles[filename] @@ -174,7 +202,8 @@ func (s *bundleServer) handleRequest(w http.ResponseWriter, r *http.Request) { return } - w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Disposition", "attachment; filename=bundle.orc") + w.Header().Set("Content-Type", "application/octet-stream") w.WriteHeader(http.StatusOK) _, _ = w.Write(content) diff --git a/go/runtime/bundle/discovery.go b/go/runtime/bundle/discovery.go index 357f966a838..a4319755a14 100644 --- a/go/runtime/bundle/discovery.go +++ b/go/runtime/bundle/discovery.go @@ -1,6 +1,7 @@ package bundle import ( + "bytes" "context" "errors" "fmt" @@ -29,6 +30,9 @@ const ( // requestTimeout is the time limit for http client requests. requestTimeout = 10 * time.Second + // maxMetadataSizeBytes is the maximum allowed metadata size in bytes. + maxMetadataSizeBytes = 2 * 1024 // 2 KB + // maxDefaultBundleSizeBytes is the maximum allowed default bundle size // in bytes. maxDefaultBundleSizeBytes = 20 * 1024 * 1024 // 20 MB @@ -295,7 +299,7 @@ func (d *Discovery) downloadBundle(runtimeID common.Namespace, manifestHash hash for _, baseURLs := range [][]string{d.runtimeBaseURLs[runtimeID], d.globalBaseURLs} { for _, baseURL := range baseURLs { - if err := d.tryDownloadBundle(runtimeID, manifestHash, baseURL); err != nil { + if err := d.tryDownloadBundle(manifestHash, baseURL); err != nil { errs = errors.Join(errs, err) continue } @@ -307,39 +311,49 @@ func (d *Discovery) downloadBundle(runtimeID common.Namespace, manifestHash hash return errs } -func (d *Discovery) tryDownloadBundle(runtimeID common.Namespace, manifestHash hash.Hash, baseURL string) error { - filename := fmt.Sprintf("%s%s", manifestHash.Hex(), FileExtension) +func (d *Discovery) tryDownloadBundle(manifestHash hash.Hash, baseURL string) error { + metaURL, err := url.JoinPath(baseURL, manifestHash.Hex()) + if err != nil { + d.logger.Error("failed to construct metadata URL", + "err", err, + ) + return fmt.Errorf("failed to construct metadata URL: %w", err) + } - d.logger.Debug("downloading bundle", - "runtime_id", runtimeID, - "base_url", baseURL, - "filename", filename, + d.logger.Debug("downloading metadata", + "url", metaURL, ) - url, err := url.JoinPath(baseURL, filename) + bundleURL, err := d.fetchMetadata(metaURL) if err != nil { - d.logger.Error("failed to construct URL", + d.logger.Error("failed to download metadata", "err", err, - "base_url", baseURL, - "filename", filename, + "url", metaURL, ) - return fmt.Errorf("failed to construct URL: %w", err) + return fmt.Errorf("failed to download metadata: %w", err) + } + + bundleURL, err = validateAndNormalizeURL(bundleURL) + if err != nil { + return err } - src, err := d.fetchBundle(url) + d.logger.Debug("downloading bundle", + "url", bundleURL, + ) + + src, err := d.fetchBundle(bundleURL) if err != nil { d.logger.Error("failed to download bundle", "err", err, - "url", url, + "url", metaURL, ) return fmt.Errorf("failed to download bundle: %w", err) } defer os.Remove(src) d.logger.Info("bundle downloaded", - "runtime_id", runtimeID, - "base_url", baseURL, - "filename", filename, + "url", bundleURL, ) if err := d.registry.AddBundle(src, manifestHash); err != nil { @@ -349,6 +363,7 @@ func (d *Discovery) tryDownloadBundle(runtimeID common.Namespace, manifestHash h return fmt.Errorf("failed to add bundle: %w", err) } + filename := fmt.Sprintf("%s%s", manifestHash.Hex(), FileExtension) dst := filepath.Join(d.bundleDir, filename) if err = os.Rename(src, dst); err != nil { d.logger.Error("failed to move bundle", @@ -357,9 +372,39 @@ func (d *Discovery) tryDownloadBundle(runtimeID common.Namespace, manifestHash h "dst", dst, ) } + + d.logger.Debug("bundle stored", + "dst", dst, + ) + return nil } +func (d *Discovery) fetchMetadata(url string) (string, error) { + resp, err := d.client.Get(url) + if err != nil { + return "", fmt.Errorf("failed to fetch metadata: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to fetch metadata: invalid status code %d", resp.StatusCode) + } + + limitedReader := io.LimitedReader{ + R: resp.Body, + N: maxMetadataSizeBytes, + } + + var buffer bytes.Buffer + _, err = buffer.ReadFrom(&limitedReader) + if err != nil && err != io.EOF { + return "", fmt.Errorf("failed to read metadata content: %w", err) + } + + return strings.TrimSpace(buffer.String()), nil +} + func (d *Discovery) fetchBundle(url string) (string, error) { resp, err := d.client.Get(url) if err != nil { @@ -470,15 +515,23 @@ func (d *Discovery) copyBundle(src string) error { return nil } +func validateAndNormalizeURL(rawURL string) (string, error) { + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("invalid URL '%s': %w", rawURL, err) + } + return parsedURL.String(), nil +} + func validateAndNormalizeURLs(rawURLs []string) ([]string, error) { var normalizedURLs []string for _, rawURL := range rawURLs { - parsedURL, err := url.Parse(rawURL) + normalizedURL, err := validateAndNormalizeURL(rawURL) if err != nil { - return nil, fmt.Errorf("invalid URL '%s': %w", rawURL, err) + return nil, err } - normalizedURLs = append(normalizedURLs, parsedURL.String()) + normalizedURLs = append(normalizedURLs, normalizedURL) } return normalizedURLs, nil diff --git a/go/runtime/config/config.go b/go/runtime/config/config.go index b44918a777f..ff8b507e677 100644 --- a/go/runtime/config/config.go +++ b/go/runtime/config/config.go @@ -115,7 +115,7 @@ type Config struct { // LoadBalancer is the load balancer configuration. LoadBalancer LoadBalancerConfig `yaml:"load_balancer,omitempty"` - // Repositories is the list of URLs used to fetch runtime bundles. + // Repositories is the list of URLs used to fetch runtime bundle metadata. Repositories []string `yaml:"repositories,omitempty"` // MaxBundleSize is the maximum allowed bundle size. @@ -166,7 +166,7 @@ type RuntimeConfig struct { // Config contains runtime local configuration. Config map[string]interface{} `yaml:"config,omitempty"` - // Repositories is the list of URLs used to fetch runtime bundles. + // Repositories is the list of URLs used to fetch runtime bundle metadata. Repositories []string `yaml:"repositories,omitempty"` } From 3716fee9b75e557412811126bf5bb547c43dbef6 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Mon, 16 Dec 2024 04:51:09 +0100 Subject: [PATCH 27/27] go/runtime/bundle/discovery: Increase request timeout --- go/runtime/bundle/discovery.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/runtime/bundle/discovery.go b/go/runtime/bundle/discovery.go index a4319755a14..8c49c618b55 100644 --- a/go/runtime/bundle/discovery.go +++ b/go/runtime/bundle/discovery.go @@ -28,7 +28,7 @@ const ( discoveryInterval = 15 * time.Minute // requestTimeout is the time limit for http client requests. - requestTimeout = 10 * time.Second + requestTimeout = time.Minute // maxMetadataSizeBytes is the maximum allowed metadata size in bytes. maxMetadataSizeBytes = 2 * 1024 // 2 KB