feat: use version patterns

docs/HOW-TO-CONTRIBUTE.md

@@ -1,16 +1,16 @@
 # How to contribute CRDs
 
-You need to create a pull request with changes to the sources list using the methods below:
+You need to create a pull request with changes to the sources list using the methods (prepared in order of preference) below:
 
-* [Helm charts](#helm-charts) - the preferred method
-* [Git](#git)
-* [OCI charts](#oci-charts)
-* [Uris](#uris) - when everything else fails
+- [Helm charts](#helm-charts)
+- [OCI charts](#oci-charts)
+- [Git](#git)
+- [Uris](#uris) - when everything else fails
 
 We prefer using the [Helm charts](#helm-charts) method to avoid issues like needing to specify each release version with CRD changes or version history being unavailable.
 
 > [!IMPORTANT]  
-> Please keep the `configuration.yaml` sorted by name.  
+> Please keep the `configuration.yaml` sorted by name.
 
 We encourage you to look through the [COMPARISON.md](COMPARISON.md), if you would like to help add relevant missing schemas.
 
@@ -24,14 +24,15 @@ To add CRDs for [ArgoCD](https://github.com/argoproj/argo-cd) you should apply t
   kind: helm
   name: argo
   repository: https://argoproj.github.io/argo-helm
+  # versionPattern: # regex pattern with capture group for semver major.minor.patch - default: `^v?([0-9]+\.[0-9]+\.[0-9]+)$`
 ```
 
 > [!NOTE]  
 > Each `repository` should only be listed once in the file, if possible.  
-> The `name` of a repository should be the name given by the developers, if possible.  
+> The `name` of a repository should be the name given by the developers, if possible.
 
 > [!IMPORTANT]  
-> Always add the repository that the developers publish to directly and avoid aggregated helm chart repositories like Truechart etc.  
+> Always add the repository that the developers publish to directly and avoid aggregated helm chart repositories like Truechart etc.
 
 If the above example already exists when you are trying to add [Argo Rollouts](https://github.com/argoproj/argo-rollouts). You should only add another chart entry to the existing repository entry, so the repository entry looks like the following.
 
@@ -54,15 +55,25 @@ To add CRDs for [eks-anywhere](https://github.com/aws/eks-anywhere) you should a
     - config/crd
   name: eks-anywhere
   repository: https://github.com/aws/eks-anywhere
-  versionPrefix: v # by default only major.minor.patch tags are used, but a prefix can be set
-  # versionSuffix: # regex matching end of versions, by default matching $
-  # includeHead: true # by default the head branch is ignored and only published tags are used
+  # versionPattern: # regex pattern with capture group for semver major.minor.patch - default: `^([0-9]+\.[0-9]+\.[0-9]+)$`
   # searchPaths: # paths to recursively find yaml files in (non-CRDs are discarded)
   #   - crds
   # genPaths: # paths to recursively find go files to generate CRDs from
   #   - api
 ```
 
+The `versionPattern` is a regex that must contain a capture group for the semver major.minor.patch version.
+
+Examples:
+
+- Match v-prefixed tags: `^v([0-9]+\.[0-9]+\.[0-9]+)$`
+- Match tags with custom prefix: `^prefix-([0-9]+\.[0-9]+\.[0-9]+)$`
+- Match tags with pre-release suffixes: `^v([0-9]+\.[0-9]+\.[0-9]+-.*)$`
+- Match versioned tags and branches: `^(([0-9]+\.[0-9]+\.[0-9]+)|(main|master))$`
+- Match branches only: `^((main|master))$`
+
+Git sources are sorted by commit date (most recent first), not by semver.
+
 ## OCI charts
 
 To add CRDs for [CrunchyData/postgres-operator](https://github.com/CrunchyData/postgres-operator) you should apply the following changes to `configuration.yaml`.
@@ -71,6 +82,7 @@ To add CRDs for [CrunchyData/postgres-operator](https://github.com/CrunchyData/p
 - kind: helm-oci
   name: crunchydata-pgo
   repository: oci://registry.developers.crunchydata.com/crunchydata/pgo
+  # versionPattern: # regex pattern with capture group for semver major.minor.patch - default: `^v?([0-9]+\.[0-9]+\.[0-9]+)$`
 ```
 
 ## Uris
@@ -90,14 +102,14 @@ To add CRDs for version 1.0.0 of the fictional Custom tool you should apply the
 ```
 
 > [!NOTE]  
-> Each `name` should only be listed once in the file.  
+> Each `name` should only be listed once in the file.
 
 ### `apiGroups`
 
 This entry should contain a complete list of unique value specified in the CRDs under the yaml path `.spec.group`.
 
 > [!IMPORTANT]  
-> You should only ever append to this list.  
+> You should only ever append to this list.
 
 ### `crds`
 

internal/configuration/configuration.go

@@ -26,16 +26,14 @@ type Configuration struct {
 	Downloads      []ConfigurationDownload `yaml:"crds"`
 	Entries        []string                `yaml:"entries"`
 	GenPaths       []string                `yaml:"genPaths"`
-	IncludeHead    bool                    `yaml:"includeHead"`
 	Kind           Kind                    `yaml:"kind"`
 	KustomizePaths []string                `yaml:"kustomizationPaths"`
 	Name           string                  `yaml:"name"`
 	Namespace      string                  `yaml:"namespace"`
 	Repository     string                  `yaml:"repository"`
 	SearchPaths    []string                `yaml:"searchPaths"`
 	Values         []ConfigurationValues   `yaml:"valuesFiles"`
-	VersionPrefix  string                  `yaml:"versionPrefix"`
-	VersionSuffix  string                  `yaml:"versionSuffix"`
+	VersionPattern string                  `yaml:"versionPattern"`
 }
 
 type ConfigurationDownload struct {

internal/configuration/configuration_test.go

@@ -3,6 +3,7 @@ package configuration
 import (
 	"fmt"
 	"os"
+	"regexp"
 	"slices"
 	"strings"
 	"testing"
@@ -33,6 +34,23 @@ func TestConfigurationIsSortedAndValid(t *testing.T) {
 	assert.Equal(t, sorted, unsorted)
 }
 
+func TestConfigurationAllPatternsAreValidRegex(t *testing.T) {
+	f, err := os.Open("../../configuration.yaml")
+	assert.Nil(t, err)
+	defer f.Close()
+
+	conf, err := UnmarshalConfigurations(f)
+	assert.Nil(t, err)
+	assert.NotNil(t, conf)
+
+	for _, c := range conf {
+		if c.VersionPattern != "" {
+			_, err := regexp.Compile(c.VersionPattern)
+			assert.Nil(t, err, "invalid regex for '%s': %s", c.Name, c.VersionPattern)
+		}
+	}
+}
+
 func TestUnmarshalCorrectConfigurations(t *testing.T) {
 	f, err := os.Open("testdata/correct.yaml")
 	assert.Nil(t, err)

internal/configuration/schema.json

@@ -72,10 +72,6 @@
           }
         }
       },
-      "includeHead": {
-        "description": "Specifies whether or not to include HEAD as a source of CRDs",
-        "type": "boolean"
-      },
       "genPaths": {
         "description": "Specifies paths to generate using go source from",
         "type": "array",
@@ -106,14 +102,8 @@
           "minLength": 1
         }
       },
-      "versionPrefix": {
-        "description": "Specifies a suffix to tags that is not part of the semver - becomes part of a regex",
-        "$comment": "Is also used to indicate that there are no versions by settings this property to 'unversioned'",
-        "type": "string",
-        "minLength": 1
-      },
-      "versionSuffix": {
-        "description": "Specifies a suffix to tags that is not part of the semver - becomes part of a regex",
+      "versionPattern": {
+        "description": "A regex pattern with a capture group for the semver major.minor.patch version",
         "type": "string",
         "minLength": 1
       },

internal/configuration/testdata/correct.yaml

@@ -16,15 +16,14 @@
       version: 2.0.0
   kind: http
   name: http
-- includeHead: false
-  kind: git
+- kind: git
   kustomizationPaths:
     - kustomizations
   name: giT
   repository: http:///repository/chart.git
   searchPaths:
     - crds
-  versionPrefix: v
+  versionPattern: ^([0-9]+\.[0-9]+\.[0-9]+)$
   genPaths:
     - ./source/...
 - entries:

internal/generator/factory.go

@@ -10,6 +10,7 @@ import (
 	"slices"
 	"sort"
 	"strconv"
+	"strings"
 
 	"github.com/CustomResourceDefinition/catalog/internal/configuration"
 	"github.com/CustomResourceDefinition/catalog/internal/crd"
@@ -32,23 +33,15 @@ func NewBuilder(config configuration.Configuration, reader crd.CrdReader, genera
 		return nil, err
 	}
 
-	if len(config.VersionPrefix) == 0 {
-		if config.Kind == configuration.Helm || config.Kind == configuration.HelmOci {
-			config.VersionPrefix = "v?"
-		} else {
-			config.VersionPrefix = ""
-		}
-	}
-
-	if len(config.VersionSuffix) == 0 {
-		config.VersionSuffix = "$"
-	}
-
 	if len(config.Namespace) == 0 {
 		config.Namespace = "namespace"
 	}
 
-	pattern := fmt.Sprintf(`^%s([0-9]{1,})\.([0-9]{1,})\.([0-9]{1,})%s`, config.VersionPrefix, config.VersionSuffix)
+	pattern := defaultVersionPattern(config.Kind)
+	if len(config.VersionPattern) > 0 {
+		pattern = config.VersionPattern
+	}
+
 	re, err := regexp.Compile(pattern)
 	if err != nil {
 		return nil, err
@@ -65,6 +58,15 @@ func NewBuilder(config configuration.Configuration, reader crd.CrdReader, genera
 	}, nil
 }
 
+func defaultVersionPattern(kind configuration.Kind) string {
+	switch kind {
+	case configuration.Helm, configuration.HelmOci:
+		return `^v?([0-9]+\.[0-9]+\.[0-9]+)$`
+	default:
+		return `^([0-9]+\.[0-9]+\.[0-9]+)$`
+	}
+}
+
 func (builder Builder) Build() error {
 	defer builder.generator.Close()
 
@@ -158,18 +160,19 @@ func (builder Builder) versions() ([]string, error) {
 
 	filtered := make([]string, 0)
 	for _, v := range versions {
-		if builder.versionFilter.MatchString(v) || v == referenceHead {
+		if builder.versionFilter.MatchString(v) {
 			filtered = append(filtered, v)
 		}
 	}
 
 	sort.Slice(filtered, func(i, j int) bool {
-		if filtered[i] == referenceHead {
-			return true
-		}
-		if filtered[j] == referenceHead {
-			return false
+		keyA, errA := builder.generator.VersionSortKey(filtered[i])
+		keyB, errB := builder.generator.VersionSortKey(filtered[j])
+
+		if errA == nil && errB == nil && keyA != 0 && keyB != 0 && keyA != keyB {
+			return keyA > keyB
 		}
+
 		a := normalizeVersion(builder.versionFilter.FindAllStringSubmatch(filtered[i], -1))
 		b := normalizeVersion(builder.versionFilter.FindAllStringSubmatch(filtered[j], -1))
 		return semver.Compare(a, b) > 0
@@ -178,10 +181,20 @@ func (builder Builder) versions() ([]string, error) {
 }
 
 func normalizeVersion(matches [][]string) string {
+	if len(matches) == 0 || len(matches[0]) < 2 {
+		return "v0.0.0"
+	}
+
+	version := matches[0][1]
+	parts := strings.Split(version, ".")
+	if len(parts) < 3 {
+		return "v0.0.0"
+	}
+
 	ints := make([]int, 3)
-	for i := 1; i <= 3; i++ {
-		n, _ := strconv.Atoi(matches[0][i])
-		ints[i-1] = n
+	for i := range 3 {
+		n, _ := strconv.Atoi(parts[i])
+		ints[i] = n
 	}
 
 	return fmt.Sprintf("v%d.%d.%d", ints[0], ints[1], ints[2])

internal/generator/factory_test.go

@@ -21,16 +21,16 @@ func TestBuilderVersionSorting(t *testing.T) {
 	for _, v := range seedVersions {
 		bundles = append(bundles, gitBundle{tag: v, paths: []gitPath{}})
 	}
-	expectedVersions := []string{"head", seedVersions[0], seedVersions[2], seedVersions[3], seedVersions[4], seedVersions[1]}
+	expectedVersions := []string{seedVersions[0], seedVersions[2], seedVersions[3], seedVersions[4], seedVersions[1]}
 
 	p, err := setupGit(t, bundles)
 	assert.Nil(t, err)
 	assert.NotNil(t, p)
 
 	config := configuration.Configuration{
-		Kind:        configuration.Git,
-		Repository:  fmt.Sprintf("file://%s", *p),
-		IncludeHead: true,
+		Kind:           configuration.Git,
+		Repository:     fmt.Sprintf("file://%s", *p),
+		VersionPattern: `^([0-9]+\.[0-9]+\.[0-9]+)$`,
 	}
 
 	b, err := NewBuilder(config, nil, "-", "-", "-", nil)
@@ -45,7 +45,7 @@ func TestBuilderVersionSorting(t *testing.T) {
 type testScenario struct {
 	versions         []string
 	expectedVersions []string
-	prefix, suffix   string
+	pattern          string
 }
 
 func TestBuilderVersionFiltering(t *testing.T) {
@@ -61,29 +61,37 @@ func TestBuilderVersionFiltering(t *testing.T) {
 		{
 			versions:         []string{"v2.0.0", "v1.3.0", "v1.0.0"},
 			expectedVersions: []string{"v2.0.0", "v1.3.0", "v1.0.0"},
-			prefix:           "v",
+			pattern:          `^v([0-9]+\.[0-9]+\.[0-9]+)$`,
 		},
 		{
 			versions:         []string{"2.0.0", "v1.3.0", "v1.0.0"},
 			expectedVersions: []string{"2.0.0", "v1.3.0", "v1.0.0"},
-			prefix:           "v?",
+			pattern:          `^v?([0-9]+\.[0-9]+\.[0-9]+)$`,
 		},
 		{
 			versions:         []string{"2.0.0", "v1.3.0v", "v1.0.0"},
 			expectedVersions: []string{"2.0.0", "v1.0.0"},
-			prefix:           "v?",
-			suffix:           "",
+			pattern:          `^v?([0-9]+\.[0-9]+\.[0-9]+)$`,
 		},
 		{
 			versions:         []string{"2.0.0-2", "1.3.0-1892", "1.0.0-01"},
 			expectedVersions: []string{"2.0.0-2", "1.3.0-1892", "1.0.0-01"},
-			suffix:           `\-\d*`,
+			pattern:          `^([0-9]+\.[0-9]+\.[0-9]+-\d+)$`,
 		},
 		{
 			versions:         []string{"v1.33.2+k0s.0"},
 			expectedVersions: []string{"v1.33.2+k0s.0"},
-			prefix:           "v",
-			suffix:           `\+k0s\.`,
+			pattern:          `^v([0-9]+\.[0-9]+\.[0-9]+\+k0s\.0)$`,
+		},
+		{
+			versions:         []string{"main", "v1.0", "master"},
+			expectedVersions: []string{"main", "master"},
+			pattern:          `^(main|master)$`,
+		},
+		{
+			versions:         []string{"main", "v1.0.0", "2.0.0", "master"},
+			expectedVersions: []string{"2.0.0", "main", "master"},
+			pattern:          `^([0-9]+\.[0-9]+\.[0-9]+)|(main|master)$`,
 		},
 	}
 
@@ -93,10 +101,9 @@ func TestBuilderVersionFiltering(t *testing.T) {
 			downloads = append(downloads, configuration.ConfigurationDownload{Version: v})
 		}
 		config := configuration.Configuration{
-			Kind:          configuration.Http,
-			Downloads:     downloads,
-			VersionPrefix: test.prefix,
-			VersionSuffix: test.suffix,
+			Kind:           configuration.Http,
+			Downloads:      downloads,
+			VersionPattern: test.pattern,
 		}
 
 		b, err := NewBuilder(config, nil, "-", "-", "-", nil)

internal/generator/generator.go

@@ -10,5 +10,6 @@ type Generator interface {
 	Versions() ([]string, error)
 	MetaData(version string) ([]crd.CrdMetaSchema, error)
 	Crds(version string) ([]crd.Crd, error)
+	VersionSortKey(version string) (int64, error)
 	io.Closer
 }