feat: add token counting, markdown format, and YAML escaping fixes

- Add token counting module with tiktoken support and fallback approximation
- Add o200k_harmony encoding for newer models
- Add warning when --token-encoding used without --tokens
- Fix YAML escaping for \n, \r, \0, \x85, \u2028, \u2029 in filenames
- Add markdown output format with language-aware code fences
- Add comprehensive tests for tokens (23), markdown (56), YAML escaping (11)

Author: nikolay-e

CLAUDE.md

@@ -43,27 +43,44 @@ treemapper .                          # YAML to stdout
 treemapper . -o tree.yaml             # save to file
 treemapper . -o -                     # explicit stdout output
 treemapper . --format json            # JSON format
-treemapper . --format text            # tree-style text
+treemapper . --format text            # plain text with indentation
+treemapper . --format md              # Markdown with headings and fenced code blocks
 treemapper . --no-content             # structure only (no file contents)
 treemapper . --max-depth 3            # limit directory depth
 treemapper . --max-file-bytes 10000   # skip files larger than 10KB
 treemapper . -i custom.ignore         # custom ignore patterns
 treemapper . --no-default-ignores     # disable .gitignore/.treemapperignore (custom -i still works)
 treemapper . -v 2                     # verbose output (0=ERROR, 1=WARNING, 2=INFO, 3=DEBUG)
-treemapper . -c                       # copy output to clipboard (also outputs to stdout)
-treemapper . --copy-only              # copy to clipboard only (no stdout output)
+treemapper . -c                       # copy to clipboard (no stdout)
+treemapper . -c -o tree.yaml          # copy to clipboard + save to file
+treemapper . --tokens                 # show token count (for LLM context planning)
+treemapper . --tokens --token-encoding cl100k_base  # use GPT-4 tokenizer
 treemapper --version                  # show version
 ```
 
+## Token Counting
+
+Show token count for LLM context planning with `--tokens`:
+
+```bash
+treemapper . --tokens                 # 12,847 tokens (o200k_base)
+treemapper . --tokens --copy          # tokens + clipboard
+```
+
+**Encodings:**
+- `o200k_base` (default) — GPT-4o tokenizer
+- `o200k_harmony` — GPT-4.1/newer models tokenizer
+- `cl100k_base` — GPT-4/GPT-3.5 tokenizer
+
+Token count is displayed on stderr only when connected to a TTY (won't break pipes).
+
 ## Clipboard Support
 
 Copy output directly to clipboard with `-c` or `--copy`:
 
 ```bash
-treemapper . -c                       # copy to clipboard + stdout
+treemapper . -c                       # copy to clipboard (no stdout)
 treemapper . -c -o tree.yaml          # copy to clipboard + save to file
-treemapper . --copy-only              # copy to clipboard only
-treemapper . --copy-only -o tree.yaml # copy to clipboard + save to file (no stdout)
 ```
 
 **System Requirements:**
@@ -75,7 +92,7 @@ treemapper . --copy-only -o tree.yaml # copy to clipboard + save to file (no std
 ## Python API
 
 ```python
-from treemapper import map_directory, to_yaml, to_json, to_text
+from treemapper import map_directory, to_yaml, to_json, to_text, to_markdown
 
 # Full function signature
 tree = map_directory(
@@ -96,6 +113,7 @@ tree = map_directory(".", max_file_bytes=50000, ignore_file="custom.ignore")
 yaml_str = to_yaml(tree)
 json_str = to_json(tree)
 text_str = to_text(tree)
+md_str = to_markdown(tree)  # or to_md(tree)
 ```
 
 ## Ignore Patterns
@@ -134,8 +152,9 @@ Integration tests only - test against real filesystem. No mocking.
 src/treemapper/
 ├── cli.py        # argument parsing
 ├── ignore.py     # gitignore/treemapperignore handling
+├── tokens.py     # token counting (optional tiktoken)
 ├── tree.py       # directory traversal
-├── writer.py     # YAML/JSON/text output
+├── writer.py     # YAML/JSON/text/Markdown output
 └── treemapper.py # main entry point
 ```
 

pyproject.toml

@@ -70,6 +70,7 @@ classifiers = [
 dependencies = [
     "pathspec>=0.11,<2.0",
     "pyyaml>=6.0.2,<8.0",
+    "tiktoken>=0.7,<1.0",
 ]
 
 [project.urls]

src/treemapper/__init__.py

@@ -5,12 +5,14 @@
 from .ignore import get_ignore_specs
 from .tree import TreeBuildContext, build_tree
 from .version import __version__
-from .writer import write_tree_json, write_tree_text, write_tree_yaml
+from .writer import write_tree_json, write_tree_markdown, write_tree_text, write_tree_yaml
 
 __all__ = [
     "__version__",
     "map_directory",
     "to_json",
+    "to_markdown",
+    "to_md",
     "to_text",
     "to_yaml",
 ]
@@ -63,3 +65,12 @@ def to_text(tree: dict[str, Any]) -> str:
     buf = io.StringIO()
     write_tree_text(buf, tree)
     return buf.getvalue()
+
+
+def to_markdown(tree: dict[str, Any]) -> str:
+    buf = io.StringIO()
+    write_tree_markdown(buf, tree)
+    return buf.getvalue()
+
+
+to_md = to_markdown

src/treemapper/cli.py

@@ -19,7 +19,8 @@ class ParsedArgs:
     no_content: bool
     max_file_bytes: Optional[int]
     copy: bool
-    copy_only: bool
+    show_tokens: bool
+    token_encoding: str
 
 
 DEFAULT_IGNORES_HELP = """
@@ -49,31 +50,35 @@ def parse_args() -> ParsedArgs:
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
-    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    parser.add_argument("-v", "--version", action="version", version=f"%(prog)s {__version__}")
     parser.add_argument("directory", nargs="?", default=".", help="The directory to analyze")
     parser.add_argument("-i", "--ignore-file", default=None, help="Path to custom ignore file")
     parser.add_argument("-o", "--output-file", default=None, help="Output file (default: stdout, use '-' to force stdout)")
-    parser.add_argument("--format", choices=["yaml", "json", "text"], default="yaml", help="Output format")
+    parser.add_argument("-f", "--format", choices=["yaml", "json", "text", "md"], default="yaml", help="Output format")
     parser.add_argument("--no-default-ignores", action="store_true", help="Disable all default ignores")
     parser.add_argument("--max-depth", type=int, default=None, metavar="N", help="Maximum traversal depth")
     parser.add_argument("--no-content", action="store_true", help="Skip file contents (structure only)")
     parser.add_argument("--max-file-bytes", type=int, default=None, metavar="N", help="Skip files larger than N bytes")
-    parser.add_argument("-c", "--copy", action="store_true", help="Copy output to clipboard")
+    parser.add_argument("-c", "--copy", action="store_true", help="Copy to clipboard (suppresses stdout unless -o is used)")
     parser.add_argument(
-        "--copy-only", action="store_true", help="Copy to clipboard, suppress stdout (file output with -o still works)"
+        "--log-level",
+        choices=["error", "warning", "info", "debug"],
+        default="error",
+        help="Log level (default: error)",
     )
+    parser.add_argument("-t", "--tokens", action="store_true", help="Show token count for output")
     parser.add_argument(
-        "-v",
-        "--verbosity",
-        type=int,
-        choices=range(0, 4),
-        default=0,
-        metavar="[0-3]",
-        help="Verbosity: 0=ERROR, 1=WARNING, 2=INFO, 3=DEBUG",
+        "--token-encoding",
+        choices=["o200k_base", "o200k_harmony", "cl100k_base"],
+        default="o200k_base",
+        help="Tokenizer encoding (default: o200k_base)",
     )
 
     args = parser.parse_args()
 
+    if args.token_encoding != "o200k_base" and not args.tokens:
+        print("Warning: --token-encoding has no effect without --tokens", file=sys.stderr)
+
     if args.max_depth is not None and args.max_depth < 0:
         print(f"Error: --max-depth must be non-negative, got {args.max_depth}", file=sys.stderr)
         sys.exit(1)
@@ -97,6 +102,9 @@ def parse_args() -> ParsedArgs:
     output_file = None
     if args.output_file and args.output_file != "-":
         output_file = Path(args.output_file).resolve()
+        if output_file.is_dir():
+            print(f"Error: '{args.output_file}' is a directory, not a file.", file=sys.stderr)
+            sys.exit(1)
 
     ignore_file = None
     if args.ignore_file:
@@ -105,16 +113,20 @@ def parse_args() -> ParsedArgs:
             print(f"Error: Ignore file '{args.ignore_file}' does not exist.", file=sys.stderr)
             sys.exit(1)
 
+    log_level_map = {"error": 0, "warning": 1, "info": 2, "debug": 3}
+    verbosity = log_level_map[args.log_level]
+
     return ParsedArgs(
         root_dir=root_dir,
         ignore_file=ignore_file,
         output_file=output_file,
         no_default_ignores=args.no_default_ignores,
-        verbosity=args.verbosity,
+        verbosity=verbosity,
         output_format=args.format,
         max_depth=args.max_depth,
         no_content=args.no_content,
         max_file_bytes=args.max_file_bytes,
-        copy=args.copy or args.copy_only,
-        copy_only=args.copy_only,
+        copy=args.copy,
+        show_tokens=args.tokens,
+        token_encoding=args.token_encoding,
     )

src/treemapper/tokens.py

@@ -0,0 +1,36 @@
+import sys
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any, Optional
+
+
+@dataclass
+class TokenCountResult:
+    count: int
+    is_exact: bool
+    encoding: str
+
+
+@lru_cache(maxsize=4)
+def _get_encoder(encoding: str) -> Optional[Any]:
+    try:
+        import tiktoken
+
+        return tiktoken.get_encoding(encoding)
+    except (ImportError, Exception):
+        return None
+
+
+def count_tokens(text: str, encoding: str = "o200k_base") -> TokenCountResult:
+    encoder = _get_encoder(encoding)
+    if encoder:
+        return TokenCountResult(len(encoder.encode(text)), True, encoding)
+    return TokenCountResult(len(text) // 4, False, "approximation")
+
+
+def print_token_summary(text: str, encoding: str = "o200k_base") -> None:
+    if not sys.stderr.isatty():
+        return
+    result = count_tokens(text, encoding)
+    prefix = "" if result.is_exact else "~"
+    print(f"{prefix}{result.count:,} tokens ({result.encoding})", file=sys.stderr)

src/treemapper/treemapper.py

@@ -13,6 +13,7 @@ def main() -> None:
     from .clipboard import ClipboardError, copy_to_clipboard
     from .ignore import get_ignore_specs
     from .logger import setup_logging
+    from .tokens import print_token_summary
     from .tree import TreeBuildContext, build_tree
     from .writer import tree_to_string, write_string_to_file, write_tree_to_file
 
@@ -34,22 +35,24 @@ def main() -> None:
         "children": build_tree(args.root_dir, ctx),
     }
 
-    output_content = None
-    if args.copy:
+    output_content: str = ""
+    if args.copy or args.output_file or args.show_tokens:
         output_content = tree_to_string(directory_tree, args.output_format)
+
+    if args.show_tokens:
+        print_token_summary(output_content, args.token_encoding)
+
+    if args.copy:
         try:
             byte_size = copy_to_clipboard(output_content)
             print(f"Copied to clipboard ({_format_size(byte_size)})", file=sys.stderr)
         except ClipboardError as e:
             logging.warning(f"Clipboard: {e}")
 
-    if args.copy_only and args.output_file is None:
-        return
-
-    if output_content is not None:
+    if args.output_file:
         write_string_to_file(output_content, args.output_file, args.output_format)
-    else:
-        write_tree_to_file(directory_tree, args.output_file, args.output_format)
+    elif not args.copy:
+        write_tree_to_file(directory_tree, None, args.output_format)
 
 
 if __name__ == "__main__":