Add support for CNCF model-spec artifact (ModelPacks)

go.mod

@@ -5,6 +5,7 @@ go 1.24.0
 require (
 	github.com/google/licensecheck v0.3.1
 	github.com/moby/patternmatcher v0.6.0
+	github.com/modelpack/model-spec v0.0.8-0.20251029035601-816c546bfd6b
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.1
 	github.com/spf13/cobra v1.10.1

go.sum

@@ -14,6 +14,8 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
 github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
+github.com/modelpack/model-spec v0.0.8-0.20251029035601-816c546bfd6b h1:09eN9UkR8+HH3DzfxTAwkjNmqtWS8oc9MYBXRAUpXbA=
+github.com/modelpack/model-spec v0.0.8-0.20251029035601-816c546bfd6b/go.mod h1:5Go37og1RmvcTdVI5Remd+PpQRNLlKSNwSNbXmEqu50=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=

pkg/artifact/kitfile.go

@@ -20,7 +20,11 @@ import (
 	"bytes"
 	"encoding/json"
 	"io"
+	"slices"
+	"time"
 
+	modelspecv1 "github.com/modelpack/model-spec/specs-go/v1"
+	"github.com/opencontainers/go-digest"
 	"gopkg.in/yaml.v3"
 )
 
@@ -133,3 +137,63 @@ func (kf *KitFile) MarshalToYAML() ([]byte, error) {
 	}
 	return buf.Bytes(), nil
 }
+
+func (kf *KitFile) ToModelPackConfig(diffIDs []digest.Digest) modelspecv1.Model {
+	// Fill fields as best as we can, depending on what's available in the Kitfile
+	now := time.Now()
+
+	modelDescriptor := modelspecv1.ModelDescriptor{
+		CreatedAt:   &now,
+		Authors:     kf.Package.Authors,
+		Name:        kf.Package.Name,
+		Version:     kf.Package.Version,
+		Licenses:    kf.collectLicenses(),
+		Title:       kf.Package.Name,
+		Description: kf.Package.Description,
+	}
+
+	modelFS := modelspecv1.ModelFS{
+		Type:    "layers",
+		DiffIDs: diffIDs,
+	}
+
+	modelConfig := modelspecv1.ModelConfig{}
+	if kf.Model != nil {
+		modelConfig.Format = kf.Model.Format
+	}
+
+	model := modelspecv1.Model{
+		Descriptor: modelDescriptor,
+		ModelFS:    modelFS,
+		Config:     modelConfig,
+	}
+
+	return model
+}
+
+func (kf *KitFile) collectLicenses() []string {
+	var licenses []string
+	appendNotEmpty := func(l []string, s string) []string {
+		if s == "" {
+			return l
+		}
+		return append(l, s)
+	}
+	licenses = appendNotEmpty(licenses, kf.Package.License)
+	if kf.Model != nil {
+		licenses = appendNotEmpty(licenses, kf.Model.License)
+		for _, modelpart := range kf.Model.Parts {
+			licenses = appendNotEmpty(licenses, modelpart.License)
+		}
+	}
+	for _, ds := range kf.DataSets {
+		licenses = appendNotEmpty(licenses, ds.License)
+	}
+	for _, code := range kf.Code {
+		licenses = appendNotEmpty(licenses, code.License)
+	}
+	slices.Sort(licenses)
+	licenses = slices.Compact(licenses)
+
+	return licenses
+}

pkg/artifact/kitfile_test.go

@@ -91,3 +91,109 @@ func loadAllTestCasesOrPanic[T interface{ withName(string) T }](t *testing.T, te
 	}
 	return tests
 }
+
+func TestCollectLicenses(t *testing.T) {
+	testcases := []struct {
+		desc     string
+		kitfile  string
+		licenses []string
+	}{
+		{
+			desc: "Empty kitfile",
+			kitfile: `
+manifestVersion: 1.0.0
+`,
+			licenses: nil,
+		},
+		{
+			desc: "Kitfile with no licenses",
+			kitfile: `
+manifestVersion: 1.0.0
+package:
+  name: "test-package"
+model:
+  path: model-files
+`,
+			licenses: nil,
+		},
+		{
+			desc: "Kitfile with package license",
+			kitfile: `
+manifestVersion: 1.0.0
+package:
+  name: "test-package"
+  license: "Apache-2.0"
+model:
+  path: model-files
+`,
+			licenses: []string{"Apache-2.0"},
+		},
+		{
+			desc: "Kitfile with multiple licenses, to be sorted",
+			kitfile: `
+manifestVersion: 1.0.0
+package:
+  name: "test-package"
+  license: "license-g"
+model:
+  path: model-files
+  license: "license-h"
+  parts:
+  - path: part-files
+    license: "license-f"
+  - path: part-files
+    license: "license-e"
+datasets:
+- path: dataset
+  license: "license-c"
+- path: dataset-extra
+  license: "license-d"
+code:
+- path: code
+  license: "license-b"
+- path: code-extra
+  license: "license-a"
+`,
+			licenses: []string{"license-a", "license-b", "license-c", "license-d", "license-e", "license-f", "license-g", "license-h"},
+		},
+		{
+			desc: "Kitfile with multiple licenses, to be deduplicated",
+			kitfile: `
+manifestVersion: 1.0.0
+package:
+  name: "test-package"
+  license: Apache-2.0
+model:
+  path: model-files
+  license: MIT
+  parts:
+  - path: part-files
+    license: Apache-2.0
+  - path: part-files
+    license: MIT
+datasets:
+- path: dataset
+  license: Apache-2.0
+- path: dataset-extra
+  license: MIT
+code:
+- path: code
+  license: MIT
+- path: code-extra
+  license: MIT
+`,
+			licenses: []string{"Apache-2.0", "MIT"},
+		},
+	}
+	for _, tt := range testcases {
+		t.Run(tt.desc, func(t *testing.T) {
+			kf := &KitFile{}
+			err := kf.LoadModel(io.NopCloser(strings.NewReader(tt.kitfile)))
+			if !assert.NoError(t, err, "Unexpected error loading testcase Kitfile") {
+				return
+			}
+			actualLicenses := kf.collectLicenses()
+			assert.Equal(t, actualLicenses, tt.licenses, "Licenses should match, including order")
+		})
+	}
+}

pkg/cmd/dev/dev.go

@@ -29,10 +29,10 @@ import (
 	"github.com/kitops-ml/kitops/pkg/artifact"
 	"github.com/kitops-ml/kitops/pkg/lib/constants"
 	"github.com/kitops-ml/kitops/pkg/lib/filesystem"
+	"github.com/kitops-ml/kitops/pkg/lib/filesystem/unpack"
 	"github.com/kitops-ml/kitops/pkg/lib/harness"
 	kfutils "github.com/kitops-ml/kitops/pkg/lib/kitfile"
 	"github.com/kitops-ml/kitops/pkg/lib/repo/util"
-	"github.com/kitops-ml/kitops/pkg/lib/unpack"
 	"github.com/kitops-ml/kitops/pkg/output"
 )
 

pkg/cmd/diff/cmd.go

@@ -28,6 +28,8 @@ import (
 
 	"github.com/kitops-ml/kitops/pkg/cmd/options"
 	"github.com/kitops-ml/kitops/pkg/lib/constants"
+	"github.com/kitops-ml/kitops/pkg/lib/constants/mediatype"
+	"github.com/kitops-ml/kitops/pkg/lib/repo/util"
 	"github.com/kitops-ml/kitops/pkg/output"
 )
 
@@ -158,18 +160,18 @@ func (opts *diffOptions) complete(ctx context.Context, args []string) error {
 	opts.configHome = configHome
 
 	imageName := removePrefix(args[0])
-	refA, err := registry.ParseReference(imageName)
+	refA, _, err := util.ParseReference(imageName)
 	if err != nil {
 		return fmt.Errorf("failed to parse reference for ref1: %w", err)
 	}
-	opts.refA = &refA
+	opts.refA = refA
 
 	imageName = removePrefix(args[1])
-	refB, err := registry.ParseReference(imageName)
+	refB, _, err := util.ParseReference(imageName)
 	if err != nil {
 		return fmt.Errorf("failed to parse reference for ref2: %w", err)
 	}
-	opts.refB = &refB
+	opts.refB = refB
 
 	if err := opts.NetworkOptions.Complete(ctx, args); err != nil {
 		return err
@@ -190,7 +192,7 @@ func displayLayers(title string, layers []ocispec.Descriptor) {
 		output.Infof(layerTableHeadings)
 		for _, layer := range layers {
 			output.Infof(layerTableFormat,
-				constants.FormatMediaTypeForUser(layer.MediaType),
+				mediatype.FormatMediaTypeForUser(layer.MediaType),
 				layer.Digest[:17],
 				output.FormatBytes(layer.Size))
 		}

pkg/cmd/inspect/inspect.go

@@ -18,6 +18,7 @@ package inspect
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/kitops-ml/kitops/pkg/artifact"
@@ -66,7 +67,7 @@ func getRemoteInspect(ctx context.Context, opts *inspectOptions) (*inspectInfo,
 
 func getInspectInfo(ctx context.Context, repository oras.Target, ref string) (*inspectInfo, error) {
 	desc, manifest, kitfile, err := util.ResolveManifestAndConfig(ctx, repository, ref)
-	if err != nil {
+	if err != nil && !errors.Is(err, util.ErrNoKitfile) {
 		return nil, err
 	}
 	version := "unknown"

pkg/cmd/kitimport/hfimport.go

@@ -25,8 +25,8 @@ import (
 
 	"github.com/kitops-ml/kitops/pkg/artifact"
 	"github.com/kitops-ml/kitops/pkg/lib/constants"
-	"github.com/kitops-ml/kitops/pkg/lib/filesystem"
 	"github.com/kitops-ml/kitops/pkg/lib/filesystem/cache"
+	"github.com/kitops-ml/kitops/pkg/lib/filesystem/ignore"
 	"github.com/kitops-ml/kitops/pkg/lib/hf"
 	kfutils "github.com/kitops-ml/kitops/pkg/lib/kitfile"
 	kfgen "github.com/kitops-ml/kitops/pkg/lib/kitfile/generate"
@@ -127,7 +127,7 @@ func filterListingForKitfile(contents *kfgen.DirectoryListing, kitfile *artifact
 	// Repurpose the ignore implementation to find which files we need to download and which ones we can skip.
 	// This works because ignore is designed to _also_ ignore paths that are packed as part of another layer
 	// instead of the current one.
-	ignore, err := filesystem.NewIgnore(nil, kitfile)
+	ignore, err := ignore.New(nil, kitfile)
 	if err != nil {
 		return nil, fmt.Errorf("failed to process Kitfile to get file list: %w", err)
 	}

pkg/cmd/kitimport/util.go

@@ -29,7 +29,9 @@ import (
 
 	"github.com/kitops-ml/kitops/pkg/artifact"
 	"github.com/kitops-ml/kitops/pkg/lib/constants"
+	"github.com/kitops-ml/kitops/pkg/lib/constants/mediatype"
 	"github.com/kitops-ml/kitops/pkg/lib/filesystem"
+	"github.com/kitops-ml/kitops/pkg/lib/filesystem/ignore"
 	kfutils "github.com/kitops-ml/kitops/pkg/lib/kitfile"
 	kfgen "github.com/kitops-ml/kitops/pkg/lib/kitfile/generate"
 	"github.com/kitops-ml/kitops/pkg/lib/repo/local"
@@ -103,11 +105,15 @@ func packDirectory(ctx context.Context, configHome, contextDir string, kitfile *
 	if err != nil {
 		return err
 	}
-	ignore, err := filesystem.NewIgnoreFromContext(contextDir, kitfile)
+	ignore, err := ignore.NewFromContext(contextDir, kitfile)
 	if err != nil {
 		return err
 	}
-	manifestDesc, err := kfutils.SaveModel(ctx, localRepo, kitfile, ignore, constants.NoneCompression)
+	manifestDesc, err := filesystem.SaveModel(ctx, localRepo, kitfile, ignore, &filesystem.SaveModelOptions{
+		ModelFormat: mediatype.KitFormat,
+		Compression: mediatype.NoneCompression,
+		LayerFormat: mediatype.TarFormat,
+	})
 	if err != nil {
 		return err
 	}

pkg/cmd/list/list.go

@@ -49,9 +49,17 @@ func readInfoFromRepo(ctx context.Context, repo local.LocalRepo) ([]modelInfo, e
 	var infos []modelInfo
 	manifestDescs := repo.GetAllModels()
 	for _, manifestDesc := range manifestDescs {
-		manifest, config, err := util.GetManifestAndConfig(ctx, repo, manifestDesc)
-		if err != nil && !errors.Is(err, util.ErrNotAModelKit) {
-			return nil, err
+		manifest, config, err := util.GetManifestAndKitfile(ctx, repo, manifestDesc)
+		if err != nil {
+			if errors.Is(err, util.ErrNotAModelKit) {
+				// Shouldn't happen since this is a local repo, but either way it's not a supported artifact
+				continue
+			}
+			// Allow artifacts without Kitfiles as all that will be lacking is some metadata; we can still
+			// describe them
+			if !errors.Is(err, util.ErrNoKitfile) {
+				return nil, err
+			}
 		}
 		tags := repo.GetTags(manifestDesc)
 		// Strip localhost from repo if present, since we added it

pkg/cmd/list/remote.go

@@ -21,7 +21,7 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/kitops-ml/kitops/pkg/lib/constants"
+	"github.com/kitops-ml/kitops/pkg/lib/constants/mediatype"
 	"github.com/kitops-ml/kitops/pkg/lib/repo/remote"
 	"github.com/kitops-ml/kitops/pkg/lib/repo/util"
 
@@ -77,11 +77,11 @@ func listImageTag(ctx context.Context, repo registry.Repository, ref *registry.R
 	if err != nil {
 		return nil, fmt.Errorf("failed to resolve reference %s: %w", ref.Reference, err)
 	}
-	manifest, config, err := util.GetManifestAndConfig(ctx, repo, manifestDesc)
-	if err != nil {
+	manifest, config, err := util.GetManifestAndKitfile(ctx, repo, manifestDesc)
+	if err != nil && !errors.Is(err, util.ErrNoKitfile) {
 		return nil, fmt.Errorf("failed to read modelkit: %w", err)
 	}
-	if manifest.Config.MediaType != constants.ModelConfigMediaType.String() {
+	if _, err := mediatype.ModelFormatForManifest(manifest); err != nil {
 		return nil, nil
 	}
 	info := &modelInfo{

pkg/cmd/list/util.go

@@ -52,6 +52,8 @@ func (m *modelInfo) format() []string {
 	return lines
 }
 
+// fill adds information pulled from a manifest and kitfile into the modelInfo. Handles
+// cases where the Kitfile is nil
 func (m *modelInfo) fill(manifest *ocispec.Manifest, kitfile *artifact.KitFile) {
 	m.Size = getModelSize(manifest)
 	m.Author = getModelAuthor(kitfile)
@@ -67,15 +69,18 @@ func getModelSize(manifest *ocispec.Manifest) string {
 }
 
 func getModelAuthor(kitfile *artifact.KitFile) string {
-	if len(kitfile.Package.Authors) > 0 {
+	if kitfile != nil && len(kitfile.Package.Authors) > 0 {
 		return kitfile.Package.Authors[0]
 	} else {
 		return "<none>"
 	}
 }
 
 func getModelName(kitfile *artifact.KitFile) string {
-	name := kitfile.Package.Name
+	name := ""
+	if kitfile != nil {
+		name = kitfile.Package.Name
+	}
 	if name == "" {
 		name = "<none>"
 	}