From c93560f3b3602447244a9c98f382b9bb390f29be Mon Sep 17 00:00:00 2001 From: mitchell Date: Tue, 23 Jun 2026 13:40:10 -0400 Subject: [PATCH 1/4] Decrypt and install encrypted private artifacts in the runtime setup path Detect encrypted private-ingredient payloads by envelope magic during the unpack stage, decrypt them under the org key supplied via the options seam, and extract the inner tar.gz archive in place before the artifact is committed to the depot. Decrypted artifacts are marked private so they survive LRU eviction. Extraction routes through the shared untrusted-source sanitizer. A wrong key or corrupt payload fails closed with an error naming the artifact. A missing key (no key service configured, or the key service unreachable) fails open: the artifact is skipped, not committed to the depot, and the runtime hash is left unsaved so the next update retries once a key is available. Skipped artifacts fire ArtifactInstallSkipped so the install progress bar still completes. ENG-1635 Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/artifactcrypto/artifactcrypto.go | 24 ++ .../artifactcrypto/artifactcrypto_test.go | 27 ++ internal/runbits/runtime/progress/progress.go | 14 ++ .../runtime/progress/progress_skip_test.go | 58 +++++ internal/runbits/runtime/runtime.go | 17 +- pkg/runtime/decrypt_test.go | 232 ++++++++++++++++++ pkg/runtime/depot.go | 26 +- pkg/runtime/events/events.go | 9 + pkg/runtime/runtime.go | 4 +- pkg/runtime/setup.go | 194 +++++++++++++++ 10 files changed, 600 insertions(+), 5 deletions(-) create mode 100644 internal/runbits/runtime/progress/progress_skip_test.go create mode 100644 pkg/runtime/decrypt_test.go diff --git a/internal/artifactcrypto/artifactcrypto.go b/internal/artifactcrypto/artifactcrypto.go index 822f508c6a..e531249ec5 100644 --- a/internal/artifactcrypto/artifactcrypto.go +++ b/internal/artifactcrypto/artifactcrypto.go @@ -99,6 +99,30 @@ func (h Header) CheckKey(key []byte) error { return nil } +// IsEncrypted reports whether src begins with the v1 payload marker. It reads +// only the leading length prefix and magic, so a non-payload stream (or one too +// short to be a payload) returns false without error. A stream whose marker +// matches but whose body is malformed still returns true here; that is caught +// when the payload is actually parsed or decrypted, so detection and validation +// stay distinct. +func IsEncrypted(src io.Reader) (bool, error) { + var lenBuf [4]byte + if _, err := io.ReadFull(src, lenBuf[:]); err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return false, nil + } + return false, errs.Wrap(err, "reading payload prefix") + } + magic := make([]byte, len(magicMarker)) + if _, err := io.ReadFull(src, magic); err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return false, nil + } + return false, errs.Wrap(err, "reading payload marker") + } + return string(magic) == magicMarker, nil +} + // ParseHeader reads the header from src, consuming exactly the header bytes and // leaving src positioned at the first chunk. func ParseHeader(src io.Reader) (Header, error) { diff --git a/internal/artifactcrypto/artifactcrypto_test.go b/internal/artifactcrypto/artifactcrypto_test.go index cd124013a8..b7447b7650 100644 --- a/internal/artifactcrypto/artifactcrypto_test.go +++ b/internal/artifactcrypto/artifactcrypto_test.go @@ -121,6 +121,33 @@ func TestRoundTripPreservesZip(t *testing.T) { } } +func TestIsEncrypted(t *testing.T) { + payload := encryptToBytes(t, []byte("secret"), "kid") + + cases := []struct { + name string + data []byte + want bool + }{ + {"encrypted payload", payload, true}, + {"json plaintext", []byte(`{"schema":"x"}`), false}, + {"short input", []byte{0x00, 0x01}, false}, + {"empty", nil, false}, + {"marker present but truncated body", payload[:4+len(magicMarker)], true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := IsEncrypted(bytes.NewReader(tc.data)) + if err != nil { + t.Fatalf("IsEncrypted: %v", err) + } + if got != tc.want { + t.Errorf("IsEncrypted = %v, want %v", got, tc.want) + } + }) + } +} + func TestFingerprint(t *testing.T) { fp := Fingerprint(testKey) if len(fp) != len("sha256:")+64 { diff --git a/internal/runbits/runtime/progress/progress.go b/internal/runbits/runtime/progress/progress.go index 2cd10e8632..b9db6ba8df 100644 --- a/internal/runbits/runtime/progress/progress.go +++ b/internal/runbits/runtime/progress/progress.go @@ -299,6 +299,20 @@ func (p *ProgressDigester) Handle(ev events.Event) error { } p.installBar.Increment() + case events.ArtifactInstallSkipped: + // A skipped artifact fires no Started event, so create the bar on demand, + // then advance it as if installed so it still reaches completion. + if p.installBar == nil { + p.installBar = p.addTotalBar(locale.Tl("progress_installing", "Installing"), int64(len(p.installsExpected)), mpb.BarPriority(StepInstall.priority)) + } + if _, ok := p.installsExpected[v.ArtifactID]; !ok { + return errs.New("ArtifactInstallSkipped called for an artifact that was not expected: %s", v.ArtifactID.String()) + } + if p.installBar.Current() == p.installBar.total { + return errs.New("Install bar is already complete, this should not happen") + } + p.installBar.Increment() + } return nil diff --git a/internal/runbits/runtime/progress/progress_skip_test.go b/internal/runbits/runtime/progress/progress_skip_test.go new file mode 100644 index 0000000000..b990b3b865 --- /dev/null +++ b/internal/runbits/runtime/progress/progress_skip_test.go @@ -0,0 +1,58 @@ +package progress + +import ( + "io" + "testing" + + "github.com/ActiveState/cli/internal/testhelpers/outputhelper" + "github.com/ActiveState/cli/pkg/buildplan" + "github.com/ActiveState/cli/pkg/runtime/events" + "github.com/go-openapi/strfmt" +) + +// A skipped artifact fires only ArtifactInstallSkipped (no Started/Success), yet +// it is counted in the install bar's total. The bar must still reach its total, +// otherwise Close() stalls waiting for it and reports a spurious error. +func TestInstallSkippedCompletesBar(t *testing.T) { + installed := strfmt.UUID("11111111-1111-1111-1111-111111111111") + skipped := strfmt.UUID("22222222-2222-2222-2222-222222222222") + + t.Run("alongside an installed artifact", func(t *testing.T) { + p := newProgressIndicator(io.Discard, outputhelper.NewCatcher()) + defer p.cancelMpb() + + expected := buildplan.ArtifactIDMap{installed: nil, skipped: nil} + handle(t, p, events.Start{ArtifactsToInstall: expected}) + handle(t, p, events.ArtifactInstallStarted{installed}) + handle(t, p, events.ArtifactInstallSuccess{installed}) + handle(t, p, events.ArtifactInstallSkipped{skipped}) + + if got, want := p.installBar.Current(), int64(len(expected)); got != want { + t.Errorf("install bar at %d/%d; want %d (a skipped artifact must still advance the bar)", got, p.installBar.total, want) + } + }) + + t.Run("when every artifact is skipped", func(t *testing.T) { + p := newProgressIndicator(io.Discard, outputhelper.NewCatcher()) + defer p.cancelMpb() + + // No Started event fires, so the skip event itself must create the bar. + expected := buildplan.ArtifactIDMap{skipped: nil} + handle(t, p, events.Start{ArtifactsToInstall: expected}) + handle(t, p, events.ArtifactInstallSkipped{skipped}) + + if p.installBar == nil { + t.Fatal("install bar was never created for an all-skipped install") + } + if got, want := p.installBar.Current(), int64(len(expected)); got != want { + t.Errorf("install bar at %d/%d; want %d", got, p.installBar.total, want) + } + }) +} + +func handle(t *testing.T, p *ProgressDigester, ev events.Event) { + t.Helper() + if err := p.Handle(ev); err != nil { + t.Fatalf("Handle(%T): %v", ev, err) + } +} diff --git a/internal/runbits/runtime/runtime.go b/internal/runbits/runtime/runtime.go index 1b4aa69754..d6c55131b9 100644 --- a/internal/runbits/runtime/runtime.go +++ b/internal/runbits/runtime/runtime.go @@ -1,6 +1,7 @@ package runtime_runbit import ( + "context" "fmt" "net/url" "os" @@ -22,6 +23,7 @@ import ( "github.com/ActiveState/cli/internal/rtutils/ptr" buildscript_runbit "github.com/ActiveState/cli/internal/runbits/buildscript" "github.com/ActiveState/cli/internal/runbits/checkout" + "github.com/ActiveState/cli/internal/runbits/orgkey" "github.com/ActiveState/cli/internal/runbits/rationalize" "github.com/ActiveState/cli/internal/runbits/runtime/progress" "github.com/ActiveState/cli/internal/runbits/runtime/trigger" @@ -36,7 +38,6 @@ import ( "github.com/ActiveState/cli/pkg/runtime_helpers" "github.com/ActiveState/cli/pkg/sysinfo" "github.com/go-openapi/strfmt" - "golang.org/x/net/context" ) func init() { @@ -285,6 +286,20 @@ func Update( } rtOpts = append(rtOpts, runtime.WithCacheSize(prime.Config().GetInt(constants.RuntimeCacheSizeConfigKey))) + // Fetch the organization key for private ingredients, if a key service is configured. + orgKeyProvider := orgkey.New(prime.Config(), proj.Owner()) + if orgKeyProvider.Configured() { + defer orgKeyProvider.Close() + key, keyID, err := orgKeyProvider.Key(context.Background()) + if err != nil { + prime.Output().Notice(locale.Tl("warn_orgkey_unavailable", + "[WARNING]Warning:[/RESET] Could not fetch the organization key: {{.V0}}. Encrypted private ingredients will be skipped and retried once the key service is reachable.", + errs.JoinMessage(err))) + } else { + rtOpts = append(rtOpts, runtime.WithDecryptionKey(key, keyID)) + } + } + if isArmPlatform(buildPlan) { prime.Output().Notice(locale.Tl("warning_arm_unstable", "[WARNING]Warning:[/RESET] You are using an ARM64 architecture, which is currently unstable. While it may work, you might encounter issues.")) } diff --git a/pkg/runtime/decrypt_test.go b/pkg/runtime/decrypt_test.go new file mode 100644 index 0000000000..075726f1c7 --- /dev/null +++ b/pkg/runtime/decrypt_test.go @@ -0,0 +1,232 @@ +package runtime + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "os" + "path/filepath" + goruntime "runtime" + "testing" + + "github.com/ActiveState/cli/internal/artifactcrypto" + "github.com/go-openapi/strfmt" +) + +func testOrgKey() []byte { + k := make([]byte, artifactcrypto.KeySize) + for i := range k { + k[i] = byte(i + 1) + } + return k +} + +func encryptToBytes(t *testing.T, plaintext, key []byte) []byte { + t.Helper() + var buf bytes.Buffer + if err := artifactcrypto.Encrypt(bytes.NewReader(plaintext), &buf, key, "kid"); err != nil { + t.Fatalf("Encrypt: %v", err) + } + return buf.Bytes() +} + +// makeTarGz builds a gzip-compressed tar archive. symlinks maps a link path to +// its (relative) target, exercising the archive format's symlink support. +func makeTarGz(t *testing.T, files, symlinks map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + for name, body := range files { + if err := tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: name, + Mode: 0644, + Size: int64(len(body)), + }); err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(tw, body); err != nil { + t.Fatal(err) + } + } + for name, target := range symlinks { + if err := tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeSymlink, + Name: name, + Mode: 0777, + Linkname: target, + }); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func writeFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.WriteFile(path, data, 0644); err != nil { + t.Fatal(err) + } +} + +func TestFindEncryptedPayload(t *testing.T) { + key := testOrgKey() + + t.Run("plaintext only", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"."}`)) + writeFile(t, filepath.Join(dir, "data.txt"), []byte("ordinary file")) + got, err := findEncryptedPayload(dir) + if err != nil { + t.Fatal(err) + } + if got != "" { + t.Errorf("found a payload in a plaintext dir: %q", got) + } + }) + + t.Run("finds the encrypted file", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"."}`)) + payload := filepath.Join(dir, "anything.bin") + writeFile(t, payload, encryptToBytes(t, []byte("secret"), key)) + got, err := findEncryptedPayload(dir) + if err != nil { + t.Fatal(err) + } + if got != payload { + t.Errorf("got %q, want %q", got, payload) + } + }) +} + +func TestDecryptPayload(t *testing.T) { + key := testOrgKey() + payload := makeTarGz(t, + map[string]string{ + "pkg/__init__.py": "print('private')\n", + "pkg-1.0.dist-info/METADATA": "Name: pkg\n", + }, + map[string]string{ + "pkg/alias.py": "__init__.py", // relative symlink, resolves inside the artifact dir + }, + ) + + t.Run("happy path", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"."}`)) + writeFile(t, filepath.Join(dir, "payload"), encryptToBytes(t, payload, key)) + + s := &setup{opts: &Opts{OrgKey: key}} + outcome, err := s.decryptPayload("pkg", dir) + if err != nil { + t.Fatalf("decryptPayload: %v", err) + } + if outcome != decryptDone { + t.Fatalf("outcome = %v, want decryptDone", outcome) + } + // Ciphertext is removed. + if _, err := os.Stat(filepath.Join(dir, "payload")); !os.IsNotExist(err) { + t.Error("ciphertext was not removed") + } + // Archive contents extracted in place; the cleartext runtime.json survives. + if got, _ := os.ReadFile(filepath.Join(dir, "pkg", "__init__.py")); string(got) != "print('private')\n" { + t.Errorf("payload not extracted: got %q", got) + } + if !exists(filepath.Join(dir, "runtime.json")) { + t.Error("runtime.json was lost") + } + // The symlink survived the tar.gz round-trip as a symlink. + if goruntime.GOOS != "windows" { + info, err := os.Lstat(filepath.Join(dir, "pkg", "alias.py")) + if err != nil { + t.Errorf("symlink not extracted: %v", err) + } else if info.Mode()&os.ModeSymlink == 0 { + t.Errorf("alias.py is not a symlink: mode %v", info.Mode()) + } + } + // 0700 owner-only boundary on the decrypted artifact dir. + if goruntime.GOOS != "windows" { + if info, _ := os.Stat(dir); info.Mode().Perm() != 0700 { + t.Errorf("artifact dir mode = %v, want 0700", info.Mode().Perm()) + } + } + }) + + t.Run("missing key skips", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "payload"), encryptToBytes(t, payload, key)) + + s := &setup{opts: &Opts{}} // no OrgKey + outcome, err := s.decryptPayload("pkg", dir) + if err != nil { + t.Fatalf("decryptPayload: %v", err) + } + if outcome != decryptSkipped { + t.Fatalf("outcome = %v, want decryptSkipped", outcome) + } + }) + + t.Run("wrong key fails closed", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "payload"), encryptToBytes(t, payload, key)) + + wrong := make([]byte, artifactcrypto.KeySize) // all zeros + s := &setup{opts: &Opts{OrgKey: wrong}} + _, err := s.decryptPayload("pkg", dir) + if err == nil { + t.Fatal("expected a wrong-key error, got nil") + } + }) + + t.Run("plaintext artifact is untouched", func(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"."}`)) + s := &setup{opts: &Opts{OrgKey: key}} + outcome, err := s.decryptPayload("pkg", dir) + if err != nil { + t.Fatal(err) + } + if outcome != decryptNotEncrypted { + t.Fatalf("outcome = %v, want decryptNotEncrypted", outcome) + } + }) +} + +func TestPrivateArtifactSurvivesEviction(t *testing.T) { + d := &depot{ + config: depotConfig{ + Deployments: map[strfmt.UUID][]deployment{}, + Cache: map[strfmt.UUID]*artifactInfo{ + strfmt.UUID("private"): {Size: 100 * MB, Private: true, LastAccessTime: 1}, + strfmt.UUID("old-public"): {Size: 100 * MB, LastAccessTime: 1}, + }, + }, + depotPath: t.TempDir(), + artifacts: map[strfmt.UUID]struct{}{}, + cacheSize: 50 * MB, // under pressure: the non-private entry must be evicted + } + + if err := d.removeStaleArtifacts(); err != nil { + t.Fatalf("removeStaleArtifacts: %v", err) + } + if _, ok := d.config.Cache[strfmt.UUID("private")]; !ok { + t.Error("private artifact was evicted under cache pressure") + } + if _, ok := d.config.Cache[strfmt.UUID("old-public")]; ok { + t.Error("non-private artifact was not evicted") + } +} + +func exists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/pkg/runtime/depot.go b/pkg/runtime/depot.go index c67b46eed4..a558400442 100644 --- a/pkg/runtime/depot.go +++ b/pkg/runtime/depot.go @@ -58,6 +58,9 @@ type artifactInfo struct { Name string `json:"name,omitempty"` Version string `json:"version,omitempty"` + // For private decrypted artifacts. + Private bool `json:"private,omitempty"` + id strfmt.UUID // for convenience when removing stale artifacts; should NOT have json tag } @@ -192,6 +195,23 @@ func (d *depot) Put(id strfmt.UUID) error { return nil } +// MarkPrivate flags an artifact as a decrypted private artifact, ensuring a +// cache entry exists for it. Private artifacts are exempt from stale removal. +func (d *depot) MarkPrivate(id strfmt.UUID) error { + d.mapMutex.Lock() + defer d.mapMutex.Unlock() + + if _, exists := d.config.Cache[id]; !exists { + size, err := fileutils.GetDirSize(d.Path(id)) + if err != nil { + return errs.Wrap(err, "Could not get artifact size on disk") + } + d.config.Cache[id] = &artifactInfo{Size: size, id: id} + } + d.config.Cache[id].Private = true + return nil +} + // DeployViaLink will take an artifact from the depot and link it to the target path. // It should return deployment info to be used for tracking the artifact. func (d *depot) DeployViaLink(id strfmt.UUID, relativeSrc, absoluteDest string) (*deployment, error) { @@ -530,14 +550,14 @@ func someFilesExist(filePaths []string, basePath string) bool { return false } -// removeStaleArtifacts iterates over all unused artifacts in the depot, sorts them by last access -// time, and removes them until the size of cached artifacts is under the limit. +// removeStaleArtifacts iterates over all unused, non-private artifacts in the depot, sorts +// them by last access time, and removes them until the size of cached artifacts is under the limit. func (d *depot) removeStaleArtifacts() error { var totalSize int64 unusedArtifacts := make([]*artifactInfo, 0) for id, info := range d.config.Cache { - if !info.InUse { + if !info.InUse && !info.Private { totalSize += info.Size unusedInfo := *info unusedInfo.id = id // id is not set in cache since info is keyed by id diff --git a/pkg/runtime/events/events.go b/pkg/runtime/events/events.go index 1f9ffebfcf..48b3b4b8d2 100644 --- a/pkg/runtime/events/events.go +++ b/pkg/runtime/events/events.go @@ -154,6 +154,15 @@ type ArtifactInstallSuccess struct { func (ArtifactInstallSuccess) IsEvent() {} +// ArtifactInstallSkipped is fired in place of ArtifactInstallStarted/Success +// for an artifact that was skipped during unpack (for example an encrypted +// private artifact with no org key available) and so is never installed. +type ArtifactInstallSkipped struct { + ArtifactID strfmt.UUID +} + +func (ArtifactInstallSkipped) IsEvent() {} + type ArtifactUninstallStarted struct { ArtifactID strfmt.UUID } diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 1ce0c63fad..5f4b255e31 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -101,7 +101,9 @@ func (r *Runtime) Update(bp *buildplan.BuildPlan, hash string, setOpts ...SetOpt return errs.Wrap(err, "Failed to install runtime") } - if err := r.saveHash(hash); err != nil { + if setup.skippedAny() { + logging.Debug("Runtime has skipped artifacts; not saving hash so the next update retries them") + } else if err := r.saveHash(hash); err != nil { return errs.Wrap(err, "Failed to save hash") } diff --git a/pkg/runtime/setup.go b/pkg/runtime/setup.go index 9c9a41c829..c03ad81e1e 100644 --- a/pkg/runtime/setup.go +++ b/pkg/runtime/setup.go @@ -5,17 +5,20 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/ActiveState/cli/internal/constants" "github.com/ActiveState/cli/pkg/executors" "github.com/go-openapi/strfmt" "golang.org/x/net/context" + "github.com/ActiveState/cli/internal/artifactcrypto" "github.com/ActiveState/cli/internal/chanutils/workerpool" "github.com/ActiveState/cli/internal/errs" "github.com/ActiveState/cli/internal/fileutils" "github.com/ActiveState/cli/internal/httputil" "github.com/ActiveState/cli/internal/locale" + "github.com/ActiveState/cli/internal/logging" "github.com/ActiveState/cli/internal/osutils" "github.com/ActiveState/cli/internal/proxyreader" "github.com/ActiveState/cli/internal/sliceutils" @@ -102,6 +105,11 @@ type setup struct { // toUninstall encompasses all artifacts that will need to be uninstalled for this runtime. toUninstall map[strfmt.UUID]bool + + // skipped records encrypted artifacts that were skipped because no org key + // was available to decrypt them. + skipMutex sync.Mutex + skipped map[strfmt.UUID]struct{} } func newSetup(path string, bp *buildplan.BuildPlan, env *envdef.Collection, depot *depot, opts *Opts) (*setup, error) { @@ -202,9 +210,30 @@ func newSetup(path string, bp *buildplan.BuildPlan, env *envdef.Collection, depo toInstall: artifactsToInstall.ToIDMap(), toUninstall: artifactsToUninstall, ecosystems: ecosystems, + skipped: map[strfmt.UUID]struct{}{}, }, nil } +func (s *setup) markSkipped(id strfmt.UUID) { + s.skipMutex.Lock() + defer s.skipMutex.Unlock() + s.skipped[id] = struct{}{} +} + +func (s *setup) wasSkipped(id strfmt.UUID) bool { + s.skipMutex.Lock() + defer s.skipMutex.Unlock() + _, ok := s.skipped[id] + return ok +} + +// skippedAny reports whether any artifact was skipped this run. +func (s *setup) skippedAny() bool { + s.skipMutex.Lock() + defer s.skipMutex.Unlock() + return len(s.skipped) > 0 +} + func (s *setup) RunAndWait() (rerr error) { defer func() { // Handle success / failure event @@ -424,9 +453,32 @@ func (s *setup) unpack(artifact *buildplan.Artifact, b []byte) (rerr error) { return errs.Wrap(err, "unpack failed") } + // Decrypt and extract an encrypted private-ingredient payload, if present. + outcome, err := s.decryptPayload(artifact.Name(), unpackPath) + if err != nil { + if err2 := os.RemoveAll(unpackPath); err2 != nil { + return errs.Pack(err, errs.Wrap(err2, "unable to remove partially-unpacked directory")) + } + return errs.Wrap(err, "decrypt failed") + } + if outcome == decryptSkipped { + s.markSkipped(artifact.ArtifactID) + if err := os.RemoveAll(unpackPath); err != nil { + return errs.Wrap(err, "unable to remove skipped artifact directory") + } + logging.Warning("Skipping encrypted artifact %s (%s): no org key available", artifact.ArtifactID, artifact.Name()) + return nil + } + if err := s.depot.Put(artifact.ArtifactID); err != nil { return errs.Wrap(err, "Could not put artifact in depot") } + if outcome == decryptDone { + logging.Debug("Decrypted private artifact %s (%s)", artifact.ArtifactID, artifact.Name()) + if err := s.depot.MarkPrivate(artifact.ArtifactID); err != nil { + return errs.Wrap(err, "Could not mark decrypted artifact as private") + } + } // Camel artifacts do not have runtime.json, so in order to not have multiple paths of logic we generate one based // on the camel specific info in the artifact. @@ -445,6 +497,138 @@ func (s *setup) unpack(artifact *buildplan.Artifact, b []byte) (rerr error) { return nil } +type decryptOutcome int + +const ( + decryptNotEncrypted decryptOutcome = iota // no encrypted payload present + decryptDone // payload decrypted and extracted in place + decryptSkipped // encrypted, but no org key available +) + +// decryptPayload finds an encrypted private-ingredient payload among the +// artifact's top-level files (identified by envelope magic, not filename), +// decrypts it, and extracts the inner tar.gz archive in place of the ciphertext. +// +// A missing key returns decryptSkipped; a wrong key or corrupt payload returns +// an error. +func (s *setup) decryptPayload(artifactName, unpackPath string) (outcome decryptOutcome, rerr error) { + payloadPath, err := findEncryptedPayload(unpackPath) + if err != nil { + return decryptNotEncrypted, errs.Wrap(err, "could not scan for encrypted payload") + } + if payloadPath == "" { + return decryptNotEncrypted, nil + } + logging.Debug("Detected encrypted payload in artifact %s", artifactName) + + if len(s.opts.OrgKey) == 0 { + return decryptSkipped, nil + } + + // Confirm the key matches the payload header. + header, err := readPayloadHeader(payloadPath) + if err != nil { + return decryptNotEncrypted, errs.Wrap(err, "could not read encrypted payload header") + } + if err := header.CheckKey(s.opts.OrgKey); err != nil { + return decryptNotEncrypted, errs.Wrap(err, "org key does not match encrypted artifact %s", artifactName) + } + + // Decrypt to a private temp dir, then extract the archive in place. + tmpDir, err := os.MkdirTemp(filepath.Dir(unpackPath), ".decrypt-") + if err != nil { + return decryptNotEncrypted, errs.Wrap(err, "could not create decrypt temp dir") + } + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + rerr = errs.Pack(rerr, errs.Wrap(err, "could not remove decrypt temp dir")) + } + }() + + archivePath := filepath.Join(tmpDir, "payload") + src, err := os.Open(payloadPath) + if err != nil { + return decryptNotEncrypted, errs.Wrap(err, "could not open encrypted payload") + } + err = artifactcrypto.Decrypt(src, archivePath, s.opts.OrgKey) + if cerr := src.Close(); cerr != nil { + err = errs.Pack(err, errs.Wrap(cerr, "could not close encrypted payload")) + } + if err != nil { + return decryptNotEncrypted, errs.Wrap(err, "could not decrypt artifact %s", artifactName) + } + + // Remove the ciphertext from the artifact directory. + if err := os.Remove(payloadPath); err != nil { + return decryptNotEncrypted, errs.Wrap(err, "could not remove ciphertext") + } + + archive, err := os.Open(archivePath) + if err != nil { + return decryptNotEncrypted, errs.Wrap(err, "could not open decrypted payload") + } + defer func() { + if err := archive.Close(); err != nil { + rerr = errs.Pack(rerr, errs.Wrap(err, "could not close decrypted payload")) + } + }() + archiveUA := unarchiver.NewTarGz(unarchiver.WithUntrustedSource()) + if err := archiveUA.Unarchive(archive, unpackPath); err != nil { + return decryptNotEncrypted, errs.Wrap(err, "could not extract decrypted artifact %s", artifactName) + } + + // Restrict the decrypted artifact directory to owner-only (0700). + if err := os.Chmod(unpackPath, 0700); err != nil { + return decryptNotEncrypted, errs.Wrap(err, "could not restrict decrypted artifact directory") + } + + return decryptDone, nil +} + +// findEncryptedPayload returns the path of the single top-level file in dir that +// is an artifactcrypto envelope, or "" if none is. Subdirectories and plaintext +// files are ignored. +func findEncryptedPayload(dir string) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", errs.Wrap(err, "could not read artifact directory") + } + for _, e := range entries { + if e.IsDir() { + continue + } + path := filepath.Join(dir, e.Name()) + f, err := os.Open(path) + if err != nil { + return "", errs.Wrap(err, "could not open artifact file") + } + encrypted, err := artifactcrypto.IsEncrypted(f) + if cerr := f.Close(); cerr != nil { + err = errs.Pack(err, errs.Wrap(cerr, "could not close artifact file")) + } + if err != nil { + return "", errs.Wrap(err, "could not detect encrypted payload") + } + if encrypted { + return path, nil + } + } + return "", nil +} + +func readPayloadHeader(path string) (header artifactcrypto.Header, rerr error) { + f, err := os.Open(path) + if err != nil { + return artifactcrypto.Header{}, errs.Wrap(err, "could not open encrypted payload") + } + defer func() { + if cerr := f.Close(); cerr != nil { + rerr = errs.Pack(rerr, errs.Wrap(cerr, "could not close encrypted payload")) + } + }() + return artifactcrypto.ParseHeader(f) +} + func (s *setup) updateExecutors() error { execPath := ExecutorsPath(s.path) if err := fileutils.MkdirUnlessExists(execPath); err != nil { @@ -476,6 +660,16 @@ func (s *setup) updateExecutors() error { func (s *setup) install(artifact *buildplan.Artifact) (rerr error) { id := artifact.ArtifactID + + // Artifacts skipped during unpack are not in the depot. Report the skip so + // the install progress bar still accounts for them. + if s.wasSkipped(id) { + if err := s.fireEvent(events.ArtifactInstallSkipped{id}); err != nil { + return errs.Wrap(err, "Could not handle ArtifactInstallSkipped event") + } + return nil + } + defer func() { if rerr == nil { if err := s.fireEvent(events.ArtifactInstallSuccess{id}); err != nil { From d30489d14a88d5e12fd1fd66ac1ac1d7f6ff6c20 Mon Sep 17 00:00:00 2001 From: mitchell Date: Tue, 23 Jun 2026 13:53:43 -0400 Subject: [PATCH 2/4] Surface skipped private ingredients to the user When the organization key is unavailable, encrypted private ingredients are skipped during runtime setup. pkg/runtime cannot output to the user and the skip set lives in a transient setup object, so the skip is reported by way of the existing event stream: ArtifactInstallSkipped now carries the artifact name, a small collector handler in runbits accumulates them, and the caller prints a single warning naming the skipped ingredients after the update completes. This also covers the case where no key service is configured at all, which was previously skipped silently. ENG-1635 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../runtime/progress/progress_skip_test.go | 4 +-- internal/runbits/runtime/runtime.go | 27 ++++++++++++++++++- pkg/runtime/events/events.go | 1 + pkg/runtime/setup.go | 2 +- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/internal/runbits/runtime/progress/progress_skip_test.go b/internal/runbits/runtime/progress/progress_skip_test.go index b990b3b865..8a193da30d 100644 --- a/internal/runbits/runtime/progress/progress_skip_test.go +++ b/internal/runbits/runtime/progress/progress_skip_test.go @@ -25,7 +25,7 @@ func TestInstallSkippedCompletesBar(t *testing.T) { handle(t, p, events.Start{ArtifactsToInstall: expected}) handle(t, p, events.ArtifactInstallStarted{installed}) handle(t, p, events.ArtifactInstallSuccess{installed}) - handle(t, p, events.ArtifactInstallSkipped{skipped}) + handle(t, p, events.ArtifactInstallSkipped{skipped, "pkg"}) if got, want := p.installBar.Current(), int64(len(expected)); got != want { t.Errorf("install bar at %d/%d; want %d (a skipped artifact must still advance the bar)", got, p.installBar.total, want) @@ -39,7 +39,7 @@ func TestInstallSkippedCompletesBar(t *testing.T) { // No Started event fires, so the skip event itself must create the bar. expected := buildplan.ArtifactIDMap{skipped: nil} handle(t, p, events.Start{ArtifactsToInstall: expected}) - handle(t, p, events.ArtifactInstallSkipped{skipped}) + handle(t, p, events.ArtifactInstallSkipped{skipped, "pkg"}) if p.installBar == nil { t.Fatal("install bar was never created for an all-skipped install") diff --git a/internal/runbits/runtime/runtime.go b/internal/runbits/runtime/runtime.go index d6c55131b9..19e4b84559 100644 --- a/internal/runbits/runtime/runtime.go +++ b/internal/runbits/runtime/runtime.go @@ -6,6 +6,7 @@ import ( "net/url" "os" "strings" + "sync" anaConsts "github.com/ActiveState/cli/internal/analytics/constants" "github.com/ActiveState/cli/internal/analytics/dimensions" @@ -254,9 +255,11 @@ func Update( pg := progress.NewRuntimeProgressIndicator(prime.Output()) defer rtutils.Closer(pg.Close, &rerr) + skipped := &skipReporter{} + rtOpts := []runtime.SetOpt{ runtime.WithAnnotations(proj.Owner(), proj.Name(), commitID), - runtime.WithEventHandlers(pg.Handle, ah.handle), + runtime.WithEventHandlers(pg.Handle, ah.handle, skipped.handle), runtime.WithPreferredLibcVersion(prime.Config().GetString(constants.PreferredGlibcVersionConfig)), runtime.WithAuthToken(prime.Auth().BearerToken()), } @@ -308,6 +311,12 @@ func Update( return nil, locale.WrapError(err, "err_packages_update_runtime_install") } + if len(skipped.names) > 0 { + prime.Output().Notice(locale.Tl("warn_private_artifacts_skipped", + "[WARNING]Warning:[/RESET] These private ingredients were skipped because the organization key was unavailable: {{.V0}}. They will be installed on the next run once the key is available.", + strings.Join(skipped.names, ", "))) + } + return rt, nil } @@ -420,3 +429,19 @@ func (h *analyticsHandler) handle(event events.Event) error { return nil } + +// skipReporter collects the names of artifacts skipped during runtime setup so +// the caller can report them once the update completes. +type skipReporter struct { + mutex sync.Mutex + names []string +} + +func (r *skipReporter) handle(event events.Event) error { + if e, ok := event.(events.ArtifactInstallSkipped); ok { + r.mutex.Lock() + defer r.mutex.Unlock() + r.names = append(r.names, e.Name) + } + return nil +} diff --git a/pkg/runtime/events/events.go b/pkg/runtime/events/events.go index 48b3b4b8d2..9f17075402 100644 --- a/pkg/runtime/events/events.go +++ b/pkg/runtime/events/events.go @@ -159,6 +159,7 @@ func (ArtifactInstallSuccess) IsEvent() {} // private artifact with no org key available) and so is never installed. type ArtifactInstallSkipped struct { ArtifactID strfmt.UUID + Name string } func (ArtifactInstallSkipped) IsEvent() {} diff --git a/pkg/runtime/setup.go b/pkg/runtime/setup.go index c03ad81e1e..c3ad00646b 100644 --- a/pkg/runtime/setup.go +++ b/pkg/runtime/setup.go @@ -664,7 +664,7 @@ func (s *setup) install(artifact *buildplan.Artifact) (rerr error) { // Artifacts skipped during unpack are not in the depot. Report the skip so // the install progress bar still accounts for them. if s.wasSkipped(id) { - if err := s.fireEvent(events.ArtifactInstallSkipped{id}); err != nil { + if err := s.fireEvent(events.ArtifactInstallSkipped{id, artifact.Name()}); err != nil { return errs.Wrap(err, "Could not handle ArtifactInstallSkipped event") } return nil From 9e915219ae2565a637ebcad3fda08c7817d1feba Mon Sep 17 00:00:00 2001 From: mitchell Date: Tue, 23 Jun 2026 17:29:25 -0400 Subject: [PATCH 3/4] Improve messaging around private packages not being installed due to unavailable org key. --- internal/runbits/runtime/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/runbits/runtime/runtime.go b/internal/runbits/runtime/runtime.go index 19e4b84559..e56a67f668 100644 --- a/internal/runbits/runtime/runtime.go +++ b/internal/runbits/runtime/runtime.go @@ -313,7 +313,7 @@ func Update( if len(skipped.names) > 0 { prime.Output().Notice(locale.Tl("warn_private_artifacts_skipped", - "[WARNING]Warning:[/RESET] These private ingredients were skipped because the organization key was unavailable: {{.V0}}. They will be installed on the next run once the key is available.", + "[WARNING]Warning:[/RESET] These private packages were skipped because the organization key was unavailable: {{.V0}}. Ensure the key is available and run '[ACTIONABLE]state refresh[/RESET]' to try installing them again.", strings.Join(skipped.names, ", "))) } From 334c832e8a217d626512ac149bcbf0f720c66972 Mon Sep 17 00:00:00 2001 From: mitchell Date: Tue, 23 Jun 2026 17:34:49 -0400 Subject: [PATCH 4/4] Improve another org key message. --- internal/runbits/runtime/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/runbits/runtime/runtime.go b/internal/runbits/runtime/runtime.go index e56a67f668..dcdf29b446 100644 --- a/internal/runbits/runtime/runtime.go +++ b/internal/runbits/runtime/runtime.go @@ -296,7 +296,7 @@ func Update( key, keyID, err := orgKeyProvider.Key(context.Background()) if err != nil { prime.Output().Notice(locale.Tl("warn_orgkey_unavailable", - "[WARNING]Warning:[/RESET] Could not fetch the organization key: {{.V0}}. Encrypted private ingredients will be skipped and retried once the key service is reachable.", + "[WARNING]Warning:[/RESET] Could not fetch the organization key: {{.V0}}. Encrypted private artifacts will be skipped. Ensure the key is available and run '[ACTIONABLE]state refresh[/RESET]' to try installing them again.", errs.JoinMessage(err))) } else { rtOpts = append(rtOpts, runtime.WithDecryptionKey(key, keyID))