-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
collect cpu and memory stats for each container
Collect runc stats and stream them back to the client via a new stream
- Loading branch information
Showing
35 changed files
with
1,513 additions
and
114 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| package runcexecutor | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/binary" | ||
| "encoding/json" | ||
| "io" | ||
| "time" | ||
|
|
||
| runc "github.com/containerd/go-runc" | ||
| "github.com/moby/buildkit/util/bklog" | ||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| // earthly-specific: This entire file is earthly-specific, it is used to collect runc Stats and sends them via a stream to the client | ||
| // | ||
| // the stats protocol is: | ||
| // loop of stats events: | ||
| // <uint8> version (currently 1) | ||
| // <uint32> length of payload (stored as n) | ||
| // <bytes> n bytes (a json-encoded string of the go-runc Stats structure) | ||
|
|
||
| // writeUint32PrefixedBytes writes a uint32 representing the length of the b byte array, followed by the actual | ||
| // byte array. | ||
| func writeUint32PrefixedBytes(w io.Writer, b []byte) error { | ||
| n := len(b) | ||
| err := binary.Write(w, binary.LittleEndian, uint32(n)) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| _, err = w.Write(b) | ||
| return err | ||
| } | ||
|
|
||
| func writeStatsToStream(w io.Writer, stats *runc.Stats) error { | ||
| statsJSON, err := json.Marshal(stats) | ||
| if err != nil { | ||
| return errors.Wrap(err, "failed to encode runc stats") | ||
| } | ||
| err = binary.Write(w, binary.LittleEndian, uint8(1)) // earthly stats stream protocol v1 | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return writeUint32PrefixedBytes(w, statsJSON) | ||
| } | ||
|
|
||
| func (w *runcExecutor) monitorContainerStats(ctx context.Context, id string, sampleFrequency time.Duration, statsWriter io.WriteCloser) { | ||
| numFailuresAllowed := 10 | ||
| for { | ||
| // sleep at the top of the loop to give it time to start | ||
| time.Sleep(sampleFrequency) | ||
|
|
||
| stats, err := w.runc.Stats(ctx, id) | ||
| if err != nil { | ||
| if numFailuresAllowed > 0 { | ||
| // allow the initial calls to runc.Stats to fail, for cases where the program didn't start within the initial | ||
| // sampleFrequency; this should only occur under heavy workloads | ||
| bklog.G(ctx).Warnf("ignoring runc stats collection error: %s", err) | ||
| numFailuresAllowed-- | ||
| continue | ||
| } | ||
| bklog.G(ctx).Errorf("runc stats collection error: %s", err) | ||
| return | ||
| } | ||
|
|
||
| // once runc.Stats has succeeded, don't ignore future errors | ||
| numFailuresAllowed = 0 | ||
|
|
||
| err = writeStatsToStream(statsWriter, stats) | ||
| if err != nil { | ||
| bklog.G(ctx).Errorf("failed to send runc stats to client-stream: %s", err) | ||
| return | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package progress | ||
|
|
||
| import ( | ||
| "context" | ||
| ) | ||
|
|
||
| // this entire file is earthly-specific, it implements a filtered Reader which can be used to skip statsStream data | ||
|
|
||
| type FilteredReaderSkipFn func(ctx context.Context, p *Progress) (bool, error) | ||
|
|
||
| type filteredReader struct { | ||
| r Reader | ||
| skipFn FilteredReaderSkipFn | ||
| } | ||
|
|
||
| func NewFilteredReader(r Reader, skipFn FilteredReaderSkipFn) Reader { | ||
| return &filteredReader{ | ||
| r: r, | ||
| skipFn: skipFn, | ||
| } | ||
| } | ||
|
|
||
| func (fr *filteredReader) Read(ctx context.Context) ([]*Progress, error) { | ||
| progress, err := fr.r.Read(ctx) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| filteredProgress := []*Progress{} | ||
| for _, p := range progress { | ||
| shouldSkip, err := fr.skipFn(ctx, p) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if shouldSkip { | ||
| continue | ||
| } | ||
| filteredProgress = append(filteredProgress, p) | ||
| } | ||
| return filteredProgress, nil | ||
| } |
Oops, something went wrong.