Merge pull request #4 from magicdrive/feature/improved_arklite

Fix improved arklite format.

Author: magicdrive

.gitignore

@@ -2,9 +2,9 @@
 ark-output.txt
 ark-output.md
 ark-output.xml
-ark-output.arklite
-*.arklite
-!/internal/**/*.arklite
+ark-output.arklite.txt
+*.arklite.txt
+!/internal/**/*.arklite.txt
 /build
 /tmp
 

internal/commandline/option.go

@@ -277,15 +277,16 @@ func (cr *Option) Normalize() error {
 		case model.XML:
 			cr.OutputFilename = "ark-output.xml"
 		case model.Arklite:
-			cr.OutputFilename = "ark-output.arklite"
+			cr.OutputFilename = "ark-output.arklite.txt"
 		default:
 			cr.OutputFilename = "ark-output.txt"
 		}
 	}
 
 	// compless
 	if cr.OutputFormat.CanCompless() && cr.ComplessFlag {
-		cr.OutputFilename = fmt.Sprintf("%s%s", cr.OutputFilename, ".arklite")
+		cr.DeleteCommentsFlag = true
+		cr.OutputFilename = fmt.Sprintf("%s%s", cr.OutputFilename, ".arklite.txt")
 	}
 
 	// gitignorerule

internal/commandline/option_test.go

@@ -36,8 +36,8 @@ func TestOptParse_ValidInputs(t *testing.T) {
 		t.Fatalf("Expected no error, got: %v", err)
 	}
 
-	if opt.OutputFilename != "out.txt.arklite" {
-		t.Errorf("Expected OutputFilename = out.txt.arklite, got %s", opt.OutputFilename)
+	if opt.OutputFilename != "out.txt.arklite.txt" {
+		t.Errorf("Expected OutputFilename = out.txt.arklite.txt, got %s", opt.OutputFilename)
 	}
 	if opt.ScanBuffer.String() != "20K" {
 		t.Errorf("Expected ScanBuffer = 20K, got %s", opt.ScanBuffer.String())

internal/textbank/description_template/arklite_compless_header.arklite

@@ -0,0 +1,72 @@
+# Arklite Format Overview
+
+**Project:** %s
+**Path:** %s
+
+This file is a **full project export** written in the Arklite format — a compact, LLM-optimized representation of a codebase.
+It is designed to make large-scale projects easy for LLMs and automation tools to consume and analyze.
+
+---
+
+## Quick Format Summary
+
+- The Arklite dump consists of **three sections**:
+  1. **Description** — This explanation and project info.
+  2. **Directory Structure (JSON)** — Hierarchical JSON tree of the project layout.
+  3. **File Dump (Arklite format)** — Compact, one-line-per-file contents.
+
+- **File Dump Rules:**
+  - Each file starts with a line like `@relative/path/to/file.ext`
+  - The **next line** contains the file’s entire contents, as a single line.
+  - All newlines (`\n`) are replaced with `␤` (Unicode U+2424).
+  - **All comments** (e.g. `//`, `#`, `/* */`, `<!-- -->`, etc.) are removed.
+  - File order and relative paths reflect the directory structure.
+
+---
+
+## How to Reconstruct Files (Step-by-Step)
+
+1. **Find file sections**: Look for lines starting with `@` (at-sign).
+   Each such line indicates the start of a new file and gives its relative path.
+2. **Get file content**: The line *immediately after* each `@` line is the entire content of that file, with all line breaks replaced by `␤`.
+3. **Restore line breaks**: In each file’s content, replace all `␤` with real newlines (`\n`) to reconstruct the original file.
+4. **Save files**: Use the given relative paths to rebuild the directory and file structure.
+
+**Sample:**
+
+```
+@main.go
+package main␤func main() {␤ println("hello, world")␤}
+@README.md
+Arklite project dump␤See main.go for code.␤
+```
+
+---
+
+## IMPORTANT — For LLMs and Automation Tools
+
+- **Always** reconstruct the full project files using the above steps *before* attempting any code analysis, summarization, or refactoring.
+- Do **not** process or interpret the file dump lines directly.
+  Always decode (expand) the contents first.
+- After decoding, you will obtain the original project in `%s` formats.
+
+---
+
+## Use Cases
+
+- Efficient LLM-based code analysis, summarization, and refactoring.
+- Fast, lossless sharing of large codebases for machine processing.
+- Compact project archival.
+
+---
+
+## Technical Notes
+
+- The Arklite format removes comments to reduce token count and focus on core code logic.
+- The directory tree in JSON format at the top of this file reflects the entire layout for context.
+- If you are an LLM or tool, **always decode before analysis** — never analyze the dump lines as-is.
+
+---
+
+*This format is designed for high-efficiency AI workflows and large-scale automation.*
+

internal/textbank/description_template/arklite_compless_header.arklite.txt

@@ -0,0 +1,61 @@
+# Arklite Format Overview
+
+**Project:** %s
+**Path:** %s
+
+This file is written in **Arklite format** — a compact, LLM-optimized snapshot of an entire project directory.
+
+---
+
+## Sections
+
+1. **Description**
+   Explanation of the project and the Arklite format.
+
+2. **Directory Structure (JSON)**
+   Hierarchical representation of the directory layout as a JSON tree.
+
+3. **File Dump (Arklite format)**
+   The full content of each file, stored compactly — one file per line.
+
+---
+
+## Arklite File Dump Format
+
+- Each file starts with a line like: `@<relative/path/to/file.ext>`
+- The **next line** is the entire file content, all on a single line.
+- **Newlines** inside files are replaced with `␤` (U+2424).
+- **All comments** (`//`, `#`, `/* */`, `<!-- -->`, etc.) are stripped out.
+- File order and relative paths match the original directory tree.
+
+---
+
+### How to Restore Original Files
+
+1. For each line that starts with `@`, treat it as a new file path.
+2. The line *immediately after* is the compressed file contents (with `␤` as newlines).
+3. Reconstruct each file by:
+    - Creating the path as given after `@`
+    - Replacing all `␤` with newline (`\n`) characters
+
+**Example:**
+
+```
+@main.go
+package main␤func main() {␤ println("hello, world")␤}
+@README.md
+Arklite project dump␤See main.go for code.␤
+```
+
+---
+
+## Note for LLMs & Automation Tools
+
+- **Always reconstruct** the files using the rules above *before* analysis or code search.
+- Do **not** analyze, search, or refactor on the compressed file dump lines directly — always decode (expand) first.
+- After decoding, you will have all original files as plain text, ready for any further processing.
+
+---
+
+This format enables *large projects* to be shared, analyzed, or archived in a highly efficient, LLM-friendly way.
+

internal/textbank/description_template/description.arklite

@@ -11,10 +11,10 @@ var DescriptionTemplateText string
 //go:embed description_template/description.xml
 var DescriptionTemplateXML string
 
-//go:embed description_template/description.arklite
+//go:embed description_template/description.arklite.txt
 var DescriptionTemplateArklite string
 
-//go:embed description_template/arklite_compless_header.arklite
+//go:embed description_template/arklite_compless_header.arklite.txt
 var ArkliteComplessHeaderTemplate string
 
 const (