feat: support build unknown type with annotation

cmd/modelfile/generate.go

@@ -62,7 +62,7 @@ func init() {
 	flags.StringVar(&generateConfig.Precision, "precision", "", "specify model precision, such as bf16, fp16, int8, etc")
 	flags.StringVar(&generateConfig.Quantization, "quantization", "", "specify model quantization, such as awq, gptq, etc")
 	flags.StringVarP(&generateConfig.Output, "output", "O", ".", "specify the output path of modelfilem, must be a directory")
-	flags.BoolVar(&generateConfig.IgnoreUnrecognizedFileTypes, "ignore-unrecognized-file-types", false, "ignore the unrecognized file types in the workspace")
+	flags.BoolVar(&generateConfig.IgnoreUnrecognizedFileTypes, "ignore-unrecognized-file-types", false, "[deprecated] ignore the unrecognized file types in the workspace")
 	flags.BoolVar(&generateConfig.Overwrite, "overwrite", false, "overwrite the existing modelfile")
 
 	if err := viper.BindPFlags(flags); err != nil {

pkg/backend/attach.go

@@ -107,6 +107,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 	pb.Start()
 	defer pb.Stop()
 
+	// TODO: Copy old flags to the new layer.
 	newLayers, err := proc.Process(ctx, builder, ".", processor.WithProgressTracker(pb))
 	if err != nil {
 		return fmt.Errorf("failed to process layers: %w", err)
@@ -266,11 +267,11 @@ func (b *backend) getProcessor(filepath string) processor.Processor {
 	}
 
 	if modelfile.IsFileType(filepath, modelfile.ModelFilePatterns) {
-		return processor.NewModelProcessor(b.store, modelspec.MediaTypeModelWeight, []string{filepath})
+		return processor.NewModelProcessor(b.store, modelspec.MediaTypeModelWeight, []string{filepath}, make(map[string]map[string]string))
 	}
 
 	if modelfile.IsFileType(filepath, modelfile.CodeFilePatterns) {
-		return processor.NewCodeProcessor(b.store, modelspec.MediaTypeModelCode, []string{filepath})
+		return processor.NewCodeProcessor(b.store, modelspec.MediaTypeModelCode, []string{filepath}, make(map[string]map[string]string))
 	}
 
 	if modelfile.IsFileType(filepath, modelfile.DocFilePatterns) {

pkg/backend/build.go

@@ -170,15 +170,15 @@ func (b *backend) getProcessors(modelfile modelfile.Modelfile, cfg *config.Build
 		if cfg.Raw {
 			mediaType = modelspec.MediaTypeModelWeightRaw
 		}
-		processors = append(processors, processor.NewModelProcessor(b.store, mediaType, models))
+		processors = append(processors, processor.NewModelProcessor(b.store, mediaType, models, modelfile.GetModelFlags()))
 	}
 
 	if codes := modelfile.GetCodes(); len(codes) > 0 {
 		mediaType := modelspec.MediaTypeModelCode
 		if cfg.Raw {
 			mediaType = modelspec.MediaTypeModelCodeRaw
 		}
-		processors = append(processors, processor.NewCodeProcessor(b.store, mediaType, codes))
+		processors = append(processors, processor.NewCodeProcessor(b.store, mediaType, codes, modelfile.GetCodeFlags()))
 	}
 
 	if docs := modelfile.GetDocs(); len(docs) > 0 {

pkg/backend/build/builder.go

@@ -54,8 +54,8 @@ const (
 
 // Builder is an interface for building artifacts.
 type Builder interface {
-	// BuildLayer builds the layer blob from the given file path.
-	BuildLayer(ctx context.Context, mediaType, workDir, path string, hooks hooks.Hooks) (ocispec.Descriptor, error)
+	// BuildLayer builds the layer blob from the given file path with optional extra annotations.
+	BuildLayer(ctx context.Context, mediaType, workDir, path string, annotations map[string]string, hooks hooks.Hooks) (ocispec.Descriptor, error)
 
 	// BuildConfig builds the config blob of the artifact.
 	BuildConfig(ctx context.Context, layers []ocispec.Descriptor, modelConfig *buildconfig.Model, hooks hooks.Hooks) (ocispec.Descriptor, error)
@@ -119,7 +119,7 @@ type abstractBuilder struct {
 	interceptor interceptor.Interceptor
 }
 
-func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, path string, hooks hooks.Hooks) (ocispec.Descriptor, error) {
+func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, path string, annotations map[string]string, hooks hooks.Hooks) (ocispec.Descriptor, error) {
 	info, err := os.Stat(path)
 	if err != nil {
 		return ocispec.Descriptor{}, fmt.Errorf("failed to get file info: %w", err)
@@ -223,6 +223,13 @@ func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, p
 	}
 	desc.Annotations[modelspec.AnnotationFileMetadata] = string(metadataStr)
 
+	// Add extra annotations if provided
+	if annotations != nil {
+		for key, value := range annotations {
+			desc.Annotations[key] = value
+		}
+	}
+
 	return desc, nil
 }
 

pkg/backend/build/builder_test.go

@@ -126,20 +126,20 @@ func (s *BuilderTestSuite) TestBuildLayer() {
 		s.mockOutputStrategy.On("OutputLayer", mock.Anything, "test/media-type.tar", "test-file.txt", mock.AnythingOfType("string"), mock.AnythingOfType("int64"), mock.AnythingOfType("*io.PipeReader"), mock.Anything).
 			Return(expectedDesc, nil)
 
-		desc, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempFile, hooks.NewHooks())
+		desc, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempFile, nil, hooks.NewHooks())
 		s.NoError(err)
 		s.Equal(expectedDesc.MediaType, desc.MediaType)
 		s.Equal(expectedDesc.Digest, desc.Digest)
 		s.Equal(expectedDesc.Size, desc.Size)
 	})
 
 	s.Run("file not found", func() {
-		_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, filepath.Join(s.tempDir, "non-existent.txt"), hooks.NewHooks())
+		_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, filepath.Join(s.tempDir, "non-existent.txt"), nil, hooks.NewHooks())
 		s.Error(err)
 	})
 
 	s.Run("directory not supported", func() {
-		_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempDir, hooks.NewHooks())
+		_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempDir, nil, hooks.NewHooks())
 		s.Error(err)
 		s.True(strings.Contains(err.Error(), "is a directory and not supported yet"))
 	})

pkg/backend/build_test.go

@@ -29,7 +29,9 @@ func TestGetProcessors(t *testing.T) {
 	modelfile := &modelfile.Modelfile{}
 	modelfile.On("GetConfigs").Return([]string{"config1", "config2"})
 	modelfile.On("GetModels").Return([]string{"model1", "model2"})
+	modelfile.On("GetModelFlags").Return(make(map[string]map[string]string))
 	modelfile.On("GetCodes").Return([]string{"1.py", "2.py"})
+	modelfile.On("GetCodeFlags").Return(make(map[string]map[string]string))
 	modelfile.On("GetDocs").Return([]string{"doc1", "doc2"})
 
 	b := &backend{}

pkg/backend/inspect.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
 	godigest "github.com/opencontainers/go-digest"
@@ -62,6 +63,32 @@ type InspectedModelArtifactLayer struct {
 	Size int64 `json:"Size"`
 	// Filepath is the filepath of the model artifact layer.
 	Filepath string `json:"Filepath"`
+	// Flags is the flags of the model artifact layer.
+	Flags string `json:"Flags,omitempty"`
+}
+
+// collectFlags collects all annotations from the layer (excluding known metadata annotations)
+// and formats them as key=value pairs
+func collectFlags(annotations map[string]string) string {
+	if annotations == nil {
+		return ""
+	}
+
+	var flags []string
+	// Skip the filepath annotation since it's already displayed separately
+	// Also skip the file metadata annotation since it's internal metadata
+	skipAnnotations := map[string]bool{
+		modelspec.AnnotationFilepath:     true,
+		modelspec.AnnotationFileMetadata: true,
+	}
+
+	for key, value := range annotations {
+		if !skipAnnotations[key] {
+			flags = append(flags, fmt.Sprintf("%s=%s", key, value))
+		}
+	}
+
+	return strings.Join(flags, ",")
 }
 
 // Inspect inspects the target from the storage.
@@ -107,6 +134,7 @@ func (b *backend) Inspect(ctx context.Context, target string, cfg *config.Inspec
 			Digest:   layer.Digest.String(),
 			Size:     layer.Size,
 			Filepath: layer.Annotations[modelspec.AnnotationFilepath],
+			Flags:    collectFlags(layer.Annotations),
 		})
 	}
 

pkg/backend/processor/base.go

@@ -46,8 +46,8 @@ type base struct {
 	patterns []string
 }
 
-// Process implements the Processor interface, which can be reused by other processors.
-func (b *base) Process(ctx context.Context, builder build.Builder, workDir string, opts ...ProcessOption) ([]ocispec.Descriptor, error) {
+// Process implements the Processor interface with file flags support.
+func (b *base) Process(ctx context.Context, builder build.Builder, workDir string, fileFlags map[string]map[string]string, opts ...ProcessOption) ([]ocispec.Descriptor, error) {
 	processOpts := &processOptions{}
 	for _, opt := range opts {
 		opt(processOpts)
@@ -103,7 +103,20 @@ func (b *base) Process(ctx context.Context, builder build.Builder, workDir strin
 
 		eg.Go(func() error {
 			return retry.Do(func() error {
-				desc, err := builder.BuildLayer(ctx, b.mediaType, workDir, path, hooks.NewHooks(
+				// Get relative path for looking up flags
+				relPath, relErr := filepath.Rel(absWorkDir, path)
+				if relErr != nil {
+					return relErr
+				}
+
+				anno := make(map[string]string)
+				if fileFlags != nil {
+					if flags, exists := fileFlags[relPath]; exists {
+						anno = flags
+					}
+				}
+
+				desc, buildErr := builder.BuildLayer(ctx, b.mediaType, workDir, path, anno, hooks.NewHooks(
 					hooks.WithOnStart(func(name string, size int64, reader io.Reader) io.Reader {
 						return tracker.Add(internalpb.NormalizePrompt("Building layer"), name, size, reader)
 					}),
@@ -114,9 +127,10 @@ func (b *base) Process(ctx context.Context, builder build.Builder, workDir strin
 						tracker.Complete(name, fmt.Sprintf("%s %s", internalpb.NormalizePrompt("Built layer"), desc.Digest))
 					}),
 				))
-				if err != nil {
+
+				if buildErr != nil {
 					cancel()
-					return err
+					return buildErr
 				}
 
 				mu.Lock()

pkg/backend/processor/code.go

@@ -30,26 +30,28 @@ const (
 )
 
 // NewCodeProcessor creates a new code processor.
-func NewCodeProcessor(store storage.Storage, mediaType string, patterns []string) Processor {
+func NewCodeProcessor(store storage.Storage, mediaType string, patterns []string, flags map[string]map[string]string) Processor {
 	return &codeProcessor{
 		base: &base{
 			name:      codeProcessorName,
 			store:     store,
 			mediaType: mediaType,
 			patterns:  patterns,
 		},
+		flags: flags,
 	}
 }
 
 // codeProcessor is the processor to process the code file.
 type codeProcessor struct {
-	base *base
+	base  *base
+	flags map[string]map[string]string
 }
 
 func (p *codeProcessor) Name() string {
 	return codeProcessorName
 }
 
 func (p *codeProcessor) Process(ctx context.Context, builder build.Builder, workDir string, opts ...ProcessOption) ([]ocispec.Descriptor, error) {
-	return p.base.Process(ctx, builder, workDir, opts...)
+	return p.base.Process(ctx, builder, workDir, p.flags, opts...)
 }

pkg/backend/processor/code_test.go

@@ -44,7 +44,7 @@ type codeProcessorSuite struct {
 func (s *codeProcessorSuite) SetupTest() {
 	s.mockStore = &storage.Storage{}
 	s.mockBuilder = &buildmock.Builder{}
-	s.processor = NewCodeProcessor(s.mockStore, modelspec.MediaTypeModelCode, []string{"*.py"})
+	s.processor = NewCodeProcessor(s.mockStore, modelspec.MediaTypeModelCode, []string{"*.py"}, make(map[string]map[string]string))
 	// generate test files for prorcess.
 	s.workDir = s.Suite.T().TempDir()
 	if err := os.WriteFile(filepath.Join(s.workDir, "test.py"), []byte(""), 0644); err != nil {
@@ -58,7 +58,7 @@ func (s *codeProcessorSuite) TestName() {
 
 func (s *codeProcessorSuite) TestProcess() {
 	ctx := context.Background()
-	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
+	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
 		Digest: godigest.Digest("sha256:1234567890abcdef"),
 		Size:   int64(1024),
 		Annotations: map[string]string{

pkg/backend/processor/doc.go

@@ -51,5 +51,5 @@ func (p *docProcessor) Name() string {
 }
 
 func (p *docProcessor) Process(ctx context.Context, builder build.Builder, workDir string, opts ...ProcessOption) ([]ocispec.Descriptor, error) {
-	return p.base.Process(ctx, builder, workDir, opts...)
+	return p.base.Process(ctx, builder, workDir, nil, opts...)
 }

pkg/backend/processor/doc_test.go

@@ -58,7 +58,7 @@ func (s *docProcessorSuite) TestName() {
 
 func (s *docProcessorSuite) TestProcess() {
 	ctx := context.Background()
-	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
+	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
 		Digest: godigest.Digest("sha256:1234567890abcdef"),
 		Size:   int64(1024),
 		Annotations: map[string]string{

pkg/backend/processor/model.go

@@ -30,26 +30,28 @@ const (
 )
 
 // NewModelProcessor creates a new model processor.
-func NewModelProcessor(store storage.Storage, mediaType string, patterns []string) Processor {
+func NewModelProcessor(store storage.Storage, mediaType string, patterns []string, flags map[string]map[string]string) Processor {
 	return &modelProcessor{
 		base: &base{
 			name:      modelProcessorName,
 			store:     store,
 			mediaType: mediaType,
 			patterns:  patterns,
 		},
+		flags: flags,
 	}
 }
 
 // modelProcessor is the processor to process the model file.
 type modelProcessor struct {
-	base *base
+	base  *base
+	flags map[string]map[string]string
 }
 
 func (p *modelProcessor) Name() string {
 	return modelProcessorName
 }
 
 func (p *modelProcessor) Process(ctx context.Context, builder build.Builder, workDir string, opts ...ProcessOption) ([]ocispec.Descriptor, error) {
-	return p.base.Process(ctx, builder, workDir, opts...)
+	return p.base.Process(ctx, builder, workDir, p.flags, opts...)
 }

pkg/backend/processor/model_config.go

@@ -51,5 +51,5 @@ func (p *modelConfigProcessor) Name() string {
 }
 
 func (p *modelConfigProcessor) Process(ctx context.Context, builder build.Builder, workDir string, opts ...ProcessOption) ([]ocispec.Descriptor, error) {
-	return p.base.Process(ctx, builder, workDir, opts...)
+	return p.base.Process(ctx, builder, workDir, nil, opts...)
 }

pkg/backend/processor/model_config_test.go

@@ -58,7 +58,7 @@ func (s *modelConfigProcessorSuite) TestName() {
 
 func (s *modelConfigProcessorSuite) TestProcess() {
 	ctx := context.Background()
-	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
+	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
 		Digest: godigest.Digest("sha256:1234567890abcdef"),
 		Size:   int64(1024),
 		Annotations: map[string]string{

pkg/backend/processor/model_test.go

@@ -44,7 +44,7 @@ type modelProcessorSuite struct {
 func (s *modelProcessorSuite) SetupTest() {
 	s.mockStore = &storage.Storage{}
 	s.mockBuilder = &buildmock.Builder{}
-	s.processor = NewModelProcessor(s.mockStore, modelspec.MediaTypeModelWeight, []string{"model"})
+	s.processor = NewModelProcessor(s.mockStore, modelspec.MediaTypeModelWeight, []string{"model"}, make(map[string]map[string]string))
 	// generate test files for prorcess.
 	s.workDir = s.Suite.T().TempDir()
 	if err := os.WriteFile(filepath.Join(s.workDir, "model"), []byte(""), 0644); err != nil {
@@ -58,7 +58,7 @@ func (s *modelProcessorSuite) TestName() {
 
 func (s *modelProcessorSuite) TestProcess() {
 	ctx := context.Background()
-	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
+	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
 		Digest: godigest.Digest("sha256:1234567890abcdef"),
 		Size:   int64(1024),
 		Annotations: map[string]string{

pkg/config/modelfile/modelfile.go

@@ -31,7 +31,7 @@ type GenerateConfig struct {
 	Name                        string
 	Version                     string
 	Output                      string
-	IgnoreUnrecognizedFileTypes bool
+	IgnoreUnrecognizedFileTypes bool // [deprecated] will be removed in the next release
 	Overwrite                   bool
 	Arch                        string
 	Family                      string
@@ -83,7 +83,7 @@ func (g *GenerateConfig) Validate() error {
 	// If the output path does not exist, we can create the modelfile.
 	if _, err := os.Stat(g.Output); err == nil {
 		if !g.Overwrite {
-			return fmt.Errorf("Modelfile already exists at %s - use --overwrite to overwrite", g.Output)
+			return fmt.Errorf("modelfile already exists at %s - use --overwrite to overwrite", g.Output)
 		}
 	}