Add a kw-based function to create a builder (#315)

A new function using kwargs has been added, called `from_config`. The
old function has been renamed `from_configpath` instead. This approach
has been favored over creating yet another data structure in python. The
validation stays in Rust, as checked in the pytests.

Closes #314

Author: javiermtorres

Makefile

@@ -11,7 +11,7 @@ setup:
 
 .PHONY: build-py
 build-py:
-	maturin develop -m encoderfile-py/Cargo.toml
+	uv run maturin develop -m encoderfile-py/Cargo.toml
 
 .PHONY: stubtest
 stubtest:

allowlist.txt

@@ -1,2 +1,3 @@
 # ignore missing __all__ in encoderfile
 encoderfile.encoderfile.__all__
+# encoderfile.encoderfile.EncoderfileBuilder.from_config

encoderfile-py/python/encoderfile/__init__.py

@@ -1,6 +1,6 @@
 # type: ignore
 from .encoderfile import *  # noqa # type: ignore
-
+from .enums import *  # noqa # type: ignore
 
 __doc__ = encoderfile.__doc__  # noqa # type: ignore
 if hasattr(encoderfile, "__all__"):  # noqa # type: ignore

encoderfile-py/python/encoderfile/encoderfile.pyi

@@ -1,19 +1,62 @@
 from typing import Optional, final
+from .enums import ModelType
+
+@final
+class TargetSpec:
+    arch: str
+    os: str
+    abi: str
+
+    @staticmethod
+    def parse(spec: str) -> "TargetSpec": ...
 
 @final
 class EncoderfileBuilder:
-    @classmethod
+    @staticmethod
     def from_config(
-        cls,
         config: str,
     ) -> "EncoderfileBuilder": ...
+    @staticmethod
+    def from_dict(
+        *,
+        name: str,
+        version: Optional[str] = "0.1.0",
+        model_type: ModelType,
+        path: str,
+        output_path: Optional[str] = None,
+        cache_dir: Optional[str] = None,
+        base_binary_path: Optional[str] = None,
+        transform: Optional[str] = None,
+        lua_libs: Optional[list[str]] = None,
+        tokenizer: Optional[TokenizerBuildConfig] = None,
+        validate_transform: bool = True,
+        target: Optional[str] = None,
+    ) -> "EncoderfileBuilder": ...
     def build(
         self,
-        working_dir: Optional[str] = None,
+        workdir: Optional[str] = None,
         version: Optional[str] = None,
         no_download: bool = False,
     ): ...
 
+@final
+class TokenizerBuildConfig:
+    pad_strategy: Optional[str]
+    truncation_side: Optional[str]
+    truncation_strategy: Optional[str]
+    max_length: Optional[int]
+    stride: Optional[int]
+
+    @staticmethod
+    def new(
+        *,
+        pad_strategy: Optional[str] = None,
+        truncation_side: Optional[str] = None,
+        truncation_strategy: Optional[str] = None,
+        max_length: Optional[int] = None,
+        stride: Optional[int] = None,
+    ) -> "TokenizerBuildConfig": ...
+
 @final
 class ModelConfig:
     model_type: str

encoderfile-py/python/encoderfile/enums.py

@@ -0,0 +1,24 @@
+from enum import StrEnum
+
+
+class ModelType(StrEnum):
+    Embedding = "embedding"
+    SequenceClassification = "sequence_classification"
+    TokenClassification = "token_classification"
+    SentenceEmbedding = "sentence_embedding"
+
+
+class TokenizerTruncationSide(StrEnum):
+    Left = "left"
+    Right = "right"
+
+
+class TokenizerTruncationStrategy(StrEnum):
+    LongestFirst = "longest_first"
+    OnlyFirst = "only_first"
+    OnlySecond = "only_second"
+
+
+class TokenizerPadStrategy(StrEnum):
+    BatchLongest = "batch_longest"
+    Fixed = "fixed"

encoderfile-py/python/tests/assets/test_config.yml

@@ -2,7 +2,7 @@ encoderfile:
   name: my-model-2
   path: models/token_classification
   model_type: token_classification
-  output_path: ./test-model.encoderfile
+  output_path: /tmp/test-model.encoderfile
   transform: |
     --- Applies a softmax across token classification logits.
     --- Each token classification is normalized independently.

encoderfile-py/python/tests/assets/test_config_lua.yml

@@ -2,7 +2,7 @@ encoderfile:
   name: my-model-2-lua
   path: models/token_classification
   model_type: token_classification
-  output_path: ./test-model.encoderfile
+  output_path: /tmp/test-model-lua.encoderfile
   lua_libs:
     - table
     - math

encoderfile-py/python/tests/conftest.py

@@ -1,6 +1,8 @@
 import os
+from pathlib import Path
 
 import yaml
+import json
 
 
 def asset_path(filename: str) -> str:
@@ -22,3 +24,13 @@ def load_yaml_asset(filename):
             return yaml.safe_load(f)
     else:
         raise ValueError("Only yaml files are supported for this fixture.")
+
+
+def load_json(path: Path):
+    """
+    Loads a json asset file from the assets directory.
+    """
+    if path.suffix != ".json":
+        raise ValueError("Only json files are supported for this fixture.")
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)