fix(hash): pin the service config-hash byte layout, decoupled from compose-go struct refactorings

pkg/compose/hash.go

@@ -17,15 +17,137 @@
 package compose
 
 import (
+	"bytes"
 	"encoding/json"
 
 	"github.com/compose-spec/compose-go/v2/types"
 	"github.com/opencontainers/go-digest"
 )
 
-// ServiceHash computes the configuration hash for a service.
-func ServiceHash(o types.ServiceConfig) (string, error) {
-	// remove the Build config when generating the service hash
+// serviceHashKeyOrder freezes the top-level JSON key order of the service
+// config-hash. Hashing json.Marshal of the struct directly would couple every
+// recorded hash to the DECLARATION ORDER of compose-go's fields —
+// encoding/json emits struct fields in that order, and flattens embedded
+// structs at their embedding position — so any compose-go refactoring moving
+// a field (or grouping fields into embedded specs) would change the bytes,
+// and with them the hash, of configurations that did not change at all:
+// every container recreated on the first `up` after an upgrade.
+//
+// This list pins the byte layout to the historical form instead, generated
+// by reflection over compose-go v2.15.1-0.20260908103050 (the last layout
+// every released hash was computed from), so existing container stamps stay
+// valid verbatim: no migration, no recreation. A root attribute missing from
+// the list (added to compose-go later) is emitted after the listed ones, in
+// sorted order — deterministic, and thanks to omitempty only configurations
+// using the new attribute see their hash move, exactly like a field addition
+// always did. Nested objects keep the struct marshal of their own types; a
+// reorder inside one of them would still move hashes — the golden tests
+// exist to turn that into a reviewed decision instead of a side effect.
+var serviceHashKeyOrder = []string{
+	"profiles",
+	"annotations",
+	"attach",
+	"build",
+	"develop",
+	"blkio_config",
+	"cap_add",
+	"cap_drop",
+	"cgroup_parent",
+	"cgroup",
+	"cpu_count",
+	"cpu_percent",
+	"cpu_period",
+	"cpu_quota",
+	"cpu_rt_period",
+	"cpu_rt_runtime",
+	"cpus",
+	"cpuset",
+	"cpu_shares",
+	"command",
+	"configs",
+	"container_name",
+	"credential_spec",
+	"depends_on",
+	"deploy",
+	"device_cgroup_rules",
+	"devices",
+	"dns",
+	"dns_opt",
+	"dns_search",
+	"dockerfile",
+	"domainname",
+	"entrypoint",
+	"provider",
+	"environment",
+	"env_file",
+	"expose",
+	"extends",
+	"external_links",
+	"extra_hosts",
+	"group_add",
+	"gpus",
+	"hostname",
+	"healthcheck",
+	"image",
+	"init",
+	"ipc",
+	"isolation",
+	"labels",
+	"label_file",
+	"links",
+	"logging",
+	"log_driver",
+	"log_opt",
+	"mem_limit",
+	"mem_reservation",
+	"memswap_limit",
+	"mem_swappiness",
+	"mac_address",
+	"models",
+	"net",
+	"network_mode",
+	"networks",
+	"oom_kill_disable",
+	"oom_score_adj",
+	"pid",
+	"pids_limit",
+	"platform",
+	"ports",
+	"privileged",
+	"pull_policy",
+	"read_only",
+	"restart",
+	"runtime",
+	"scale",
+	"secrets",
+	"security_opt",
+	"shm_size",
+	"stdin_open",
+	"stop_grace_period",
+	"stop_signal",
+	"storage_opt",
+	"sysctls",
+	"tmpfs",
+	"tty",
+	"ulimits",
+	"use_api_socket",
+	"user",
+	"userns_mode",
+	"uts",
+	"volume_driver",
+	"volumes",
+	"volumes_from",
+	"working_dir",
+	"pre_start",
+	"post_start",
+	"pre_stop",
+}
+
+// trimServiceHashFields removes the attributes deliberately excluded from the
+// service config-hash (build inputs, scaling knobs, dependency wiring). It is
+// the single definition of that exclusion set, shared with the continuity
+// test so the two cannot silently drift apart.
+func trimServiceHashFields(o types.ServiceConfig) types.ServiceConfig {
 	o.Build = nil
 	o.PullPolicy = ""
 	o.Scale = nil
@@ -36,31 +158,76 @@ func ServiceHash(o types.ServiceConfig) (string, error) {
 	}
 	o.DependsOn = nil
 	o.Profiles = nil
+	return o
+}
 
-	bytes, err := json.Marshal(o)
+// ServiceHash computes the configuration hash for a service.
+func ServiceHash(o types.ServiceConfig) (string, error) {
+	o = trimServiceHashFields(o)
+
+	raw, err := json.Marshal(o)
 	if err != nil {
 		return "", err
 	}
-	return digest.SHA256.FromBytes(bytes).Encoded(), nil
+	pinned, err := pinRootKeyOrder(raw, serviceHashKeyOrder)
+	if err != nil {
+		return "", err
+	}
+	return digest.SHA256.FromBytes(pinned).Encoded(), nil
+}
+
+// pinRootKeyOrder re-emits a JSON object with its top-level keys in the
+// given order (values kept byte-verbatim), keys absent from the list
+// appended in sorted order. For an object whose keys all follow the list,
+// the output is byte-identical to the input.
+func pinRootKeyOrder(raw []byte, order []string) ([]byte, error) {
+	var root map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &root); err != nil {
+		return nil, err
+	}
+	var buf bytes.Buffer
+	buf.WriteByte('{')
+	first := true
+	write := func(key string, val json.RawMessage) {
+		if !first {
+			buf.WriteByte(',')
+		}
+		first = false
+		name, _ := json.Marshal(key)
+		buf.Write(name)
+		buf.WriteByte(':')
+		buf.Write(val)
+	}
+	for _, key := range order {
+		if val, ok := root[key]; ok {
+			write(key, val)
+			delete(root, key)
+		}
+	}
+	for _, key := range sortedKeys(root) {
+		write(key, root[key])
+	}
+	buf.WriteByte('}')
+	return buf.Bytes(), nil
 }
 
 // NetworkHash computes the configuration hash for a network.
 func NetworkHash(o *types.NetworkConfig) (string, error) {
-	bytes, err := json.Marshal(o)
+	raw, err := json.Marshal(o)
 	if err != nil {
 		return "", err
 	}
-	return digest.SHA256.FromBytes(bytes).Encoded(), nil
+	return digest.SHA256.FromBytes(raw).Encoded(), nil
 }
 
 // VolumeHash computes the configuration hash for a volume.
 func VolumeHash(o types.VolumeConfig) (string, error) {
 	if o.Driver == "" { // (TODO: jhrotko) This probably should be fixed in compose-go
 		o.Driver = "local"
 	}
-	bytes, err := json.Marshal(o)
+	raw, err := json.Marshal(o)
 	if err != nil {
 		return "", err
 	}
-	return digest.SHA256.FromBytes(bytes).Encoded(), nil
+	return digest.SHA256.FromBytes(raw).Encoded(), nil
 }

pkg/compose/hash_test.go

@@ -17,9 +17,13 @@
 package compose
 
 import (
+	"encoding/json"
+	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/compose-spec/compose-go/v2/types"
+	"github.com/opencontainers/go-digest"
 	"gotest.tools/v3/assert"
 )
 
@@ -41,3 +45,154 @@ func serviceConfig(replicas int) types.ServiceConfig {
 		Image: "bar",
 	}
 }
+
+// TestServiceHashContinuity proves the pinned serializer reproduces the
+// historical bytes: while compose-go's struct order still matches the frozen
+// list — true on this branch's compose-go — the pinned hash and the plain
+// struct-marshal hash are byte-identical. The compose-go upgrade that first
+// reorders the struct (the container-spec layering) deletes this test in the
+// same commit: from that point the frozen list carries continuity alone,
+// locked by TestHashGoldenValues.
+func TestServiceHashContinuity(t *testing.T) {
+	svc := richServiceFixture()
+	pinned, err := ServiceHash(svc)
+	assert.NilError(t, err)
+
+	raw, err := json.Marshal(trimServiceHashFields(svc))
+	assert.NilError(t, err)
+	legacy := digest.SHA256.FromBytes(raw).Encoded()
+	assert.Equal(t, pinned, legacy)
+	t.Logf("GOLDEN service=%s", pinned)
+}
+
+func richServiceFixture() types.ServiceConfig {
+	replicas := 3
+	return types.ServiceConfig{
+		Name:        "web",
+		Image:       "nginx:latest",
+		Command:     types.ShellCommand{"nginx", "-g", "daemon off;"},
+		User:        "nobody",
+		Tty:         true,
+		StdinOpen:   true,
+		Restart:     types.RestartPolicyAlways,
+		Environment: types.MappingWithEquals{"A": strPtr("1"), "B": nil},
+		Labels:      types.Labels{"com.example": "v"},
+		Annotations: types.Mapping{"note": "x"},
+		CapAdd:      []string{"NET_ADMIN"},
+		ExtraHosts:  types.HostsList{"alpha": []string{"10.0.0.1"}},
+		Deploy:      &types.DeployConfig{Replicas: &replicas},
+		Ports: []types.ServicePortConfig{
+			{Target: 80, Published: "8080", Protocol: "tcp"},
+		},
+		HealthCheck: &types.HealthCheckConfig{
+			Test: types.HealthCheckTest{"CMD", "true"},
+		},
+		Volumes: []types.ServiceVolumeConfig{
+			{Type: types.VolumeTypeVolume, Source: "data", Target: "/data"},
+		},
+		Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
+	}
+}
+
+func strPtr(s string) *string { return &s }
+
+func TestHashGoldenValues(t *testing.T) {
+	svc, err := ServiceHash(richServiceFixture())
+	assert.NilError(t, err)
+	assert.Equal(t, svc, "75bc312132c71fac4971d123202631b6978ac9d796336621af744d46611b42a2")
+
+	nw, err := NetworkHash(&types.NetworkConfig{Name: "proj_default", Driver: "bridge"})
+	assert.NilError(t, err)
+	assert.Equal(t, nw, "eeef29d8955b1d4e9382986e213c80789065d989d7a6d035163e0180159aaac0")
+
+	vol, err := VolumeHash(types.VolumeConfig{Name: "proj_data"})
+	assert.NilError(t, err)
+	assert.Equal(t, vol, "dd3953f0ff20e0f9044086b0483690f2cc2b4653e57aaaa5fa8ea2735da61000")
+}
+
+// pinRootKeyOrder is the identity for an object whose keys already follow
+// the frozen order — the property that keeps every released hash valid.
+func TestPinRootKeyOrderIdentity(t *testing.T) {
+	in := []byte(`{"profiles":["p"],"command":["c"],"image":"img","user":"u"}`)
+	out, err := pinRootKeyOrder(in, serviceHashKeyOrder)
+	assert.NilError(t, err)
+	assert.Equal(t, string(out), string(in))
+}
+
+// The pinned form is a function of the configuration VALUES, not of the
+// struct layout it travels in: two types declaring the same JSON fields in a
+// different order digest identically.
+func TestPinRootKeyOrderIgnoresFieldOrder(t *testing.T) {
+	type a struct {
+		Image string   `json:"image"`
+		Ports []string `json:"ports,omitempty"`
+		User  string   `json:"user,omitempty"`
+	}
+	type b struct {
+		User  string   `json:"user,omitempty"`
+		Image string   `json:"image"`
+		Ports []string `json:"ports,omitempty"`
+	}
+	ra, _ := json.Marshal(a{Image: "nginx", Ports: []string{"80:80"}, User: "nobody"})
+	rb, _ := json.Marshal(b{Image: "nginx", Ports: []string{"80:80"}, User: "nobody"})
+	pa, err := pinRootKeyOrder(ra, serviceHashKeyOrder)
+	assert.NilError(t, err)
+	pb, err := pinRootKeyOrder(rb, serviceHashKeyOrder)
+	assert.NilError(t, err)
+	assert.Equal(t, string(pa), string(pb))
+}
+
+// Keys unknown to the frozen list are appended in sorted order, each emitted
+// exactly once: nothing an attribute addition brings can be silently dropped.
+func TestPinRootKeyOrderAppendsUnknownSorted(t *testing.T) {
+	in := []byte(`{"zz_new":"2","image":"img","aa_new":"1"}`)
+	out, err := pinRootKeyOrder(in, serviceHashKeyOrder)
+	assert.NilError(t, err)
+	assert.Equal(t, string(out), `{"image":"img","aa_new":"1","zz_new":"2"}`)
+}
+
+// Every root JSON key of compose-go's ServiceConfig must be in the frozen
+// list: a compose-go upgrade adding an attribute fails here, so extending the
+// hash surface is a reviewed decision — add the new key at the END of
+// serviceHashKeyOrder (its hash only moves for configurations using it).
+func TestServiceHashKeyOrderCoversStruct(t *testing.T) {
+	known := map[string]bool{}
+	for _, k := range serviceHashKeyOrder {
+		known[k] = true
+	}
+	var walk func(t reflect.Type)
+	walk = func(rt reflect.Type) {
+		for i := 0; i < rt.NumField(); i++ {
+			f := rt.Field(i)
+			tag := f.Tag.Get("json")
+			name, _, _ := strings.Cut(tag, ",")
+			if name == "-" {
+				continue
+			}
+			if f.Anonymous && name == "" {
+				// encoding/json dereferences embedded pointers and promotes
+				// the exported fields of even an unexported struct embed;
+				// match it, and leave non-struct embeds (named types,
+				// interfaces) to the regular field handling below
+				ft := f.Type
+				if ft.Kind() == reflect.Pointer {
+					ft = ft.Elem()
+				}
+				if ft.Kind() == reflect.Struct {
+					walk(ft)
+					continue
+				}
+			}
+			if !f.IsExported() {
+				// unexported fields never reach the JSON output
+				continue
+			}
+			if name == "" {
+				name = f.Name
+			}
+			assert.Assert(t, known[name],
+				"ServiceConfig root attribute %q is not in serviceHashKeyOrder: append it at the end of the list (a reviewed decision — the hash of configurations using it will move)", name)
+		}
+	}
+	walk(reflect.TypeOf(types.ServiceConfig{}))
+}