tok/api/openapi.yaml at c4cf9a4406ee58bd13d9e747f03a92cdac16efcb · GrayCodeAI/tok · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
openapi: "3.1.0"
info:
  title: tok — Token Optimizer API Reference
  description: |
    tok is a tokenizer, compressor, secrets scanner, and rate limiter for AI coding agents.
    It operates as a Go library and optional CLI — no HTTP server is exposed.
    This document describes the library's public API surface as a machine-readable reference.
  version: "0.1.0"
  license:
    name: MIT
    url: https://github.com/GrayCodeAI/tok/blob/main/LICENSE
  contact:
    url: https://github.com/GrayCodeAI/tok

# No servers — tok is a library, not a network service.

components:
  schemas:
    CompressRequest:
      type: object
      required: [text]
      properties:
        text:
          type: string
          description: Input text to compress
        tier:
          type: string
          enum: [surface, trim, extract, core, code, log, adaptive]
          default: code
          description: Compression tier profile
        mode:
          type: string
          enum: [minimal, aggressive]
          default: minimal
          description: Compression aggressiveness
        budget:
          type: integer
          description: Maximum output token count (0 = unlimited)
        query:
          type: string
          description: Goal context for relevance-based filtering

    CompressResponse:
      type: object
      properties:
        compressed:
          type: string
        original_tokens:
          type: integer
        final_tokens:
          type: integer
        savings_percent:
          type: number
          format: double

    EstimateRequest:
      type: object
      required: [text]
      properties:
        text:
          type: string
        precise:
          type: boolean
          default: false
          description: Use BPE-accurate estimation (slower)

    EstimateResponse:
      type: object
      properties:
        tokens:
          type: integer
        method:
          type: string
          enum: [approximate, precise]

    DetectSecretsRequest:
      type: object
      required: [text]
      properties:
        text:
          type: string
        entropy_threshold:
          type: number
          format: double
          default: 4.5

    DetectSecretsResponse:
      type: object
      properties:
        matches:
          type: array
          items:
            type: object
            properties:
              type:
                type: string
              value:
                type: string
              start:
                type: integer
              end:
                type: integer
              line:
                type: integer
        redacted:
          type: string

x-library-api:
  compress:
    description: Compress text using a tiered filter pipeline
    go_signature: "func Compress(text string, opts ...Option) (string, Stats)"
  new_compressor:
    description: Create a reusable compressor (caches tokenizer state)
    go_signature: "func NewCompressor(opts ...Option) *Compressor"
  estimate_tokens:
    description: Fast approximate token count (±5%)
    go_signature: "func EstimateTokens(text string) int"
  estimate_tokens_precise:
    description: BPE-accurate token count
    go_signature: "func EstimateTokensPrecise(text string) int"
  warmup_tokenizer:
    description: Pre-initialize BPE tokenizer in background
    go_signature: "func WarmupTokenizer()"
  detect_secrets:
    description: Detect secrets and credentials in text
    go_signature: "func (d *SecretDetector) DetectSecrets(text string) []SecretMatch"
  redact_secrets:
    description: Detect and redact secrets in text
    go_signature: "func (d *SecretDetector) RedactSecrets(text string) string"