Added a documentation site

.github/workflows/gh-pages.yml

@@ -0,0 +1,44 @@
+name: github pages
+
+on:
+  push:
+    branches:
+      - main  # Set a branch to deploy
+  pull_request:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          submodules: true  # Fetch Hugo themes (true OR recursive)
+          fetch-depth: 0    # Fetch all history for .GitInfo and .Lastmod
+
+      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe
+        with:
+          go-version: '^1.23'
+
+      - run: go version
+
+      - name: Setup Hugo
+        uses: peaceiris/actions-hugo@v2
+        with:
+          hugo-version: 'latest'
+          extended: true
+
+      - name: Prepare
+        run: |
+          make build compile
+          cp output/SQLiteHunter.zip ./docs/static/
+
+      - name: Build
+        run: cd docs/ && hugo --minify
+
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        if: github.ref == 'refs/heads/master'
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/public
+          force_orphan: true

.gitignore

@@ -1,3 +1,4 @@
 velociraptor*
 sqlitehunter_compiler*
-datastore
+datastore
+output/*.zip

.pyspelling.yml

@@ -0,0 +1,40 @@
+matrix:
+- name: Markdown
+  aspell:
+    lang: en
+    d: en_US
+  dictionary:
+    wordlists:
+      - .wordlist.txt
+    output: .wordlist.dict
+  sources:
+  - 'docs/content/**/*.md'
+  pipeline:
+    - pyspelling.filters.context:
+        context_visible_first: true
+        escapes: '\\[\\`]'
+        delimiters:
+          # Ignore multiline content between fences (fences can have 3 or more back ticks)
+          # ```
+          # content
+          # ```
+          - open: '(?s)^(?P<open> *`{3,})'
+            close: '^(?P=open)$'
+            # Insides of URL links
+          - open: '\]\('
+            close: '\)'
+            # Bolded words are usually terms
+          - open: '[*]+'
+            close: '[*]+'
+            # Inside HTML tags
+          - open: '(?s)(?P<open>[<])'
+            close: '[>]'
+            # Ignore text between inline back ticks
+          - open: '(?P<open>`+)'
+            close: '(?P=open)'
+
+            # Inside yaml headers
+          - open: '(?s)^(?P<open>---)$'
+            close: '^(?P=open)$'
+
+    - pyspelling.filters.url:

.wordlist.txt

@@ -0,0 +1,15 @@
+# docs/content/_index.md
+SQLiteHunter
+
+<url-free> docs/content/docs/sqlite_hunter/_index.md
+MacOS
+SQLECmd
+SQLiteHunter
+VQL
+Velociraptor
+YAML
+journaling
+SQLite
+Velociraptor
+de
+facto

Makefile

@@ -8,7 +8,7 @@ windows:
 	go build -o sqlitehunter_compiler.exe ./bin/*.go
 
 compile: FORCE
-	./sqlitehunter_compiler -config ./config.yaml -definition_directory ./definitions > output/SQLiteHunter.yaml
+	./sqlitehunter_compiler compile ./definitions/ ./output/SQLiteHunter.yaml --output_zip ./output/SQLiteHunter.zip --index ./docs/content/docs/rules/index.json
 
 golden: compile
 	./testing/velociraptor.bin --definitions ./output --config ./testing/test.config.yaml golden --env testFiles=`pwd`/test_files ./testing/testcases -v --filter=${GOLDEN}

api/api.go

@@ -5,26 +5,34 @@ type ConfigDefinitions struct {
 }
 
 type Definition struct {
-	Name                string      `yaml:"Name"`
-	Author              string      `yaml:"Author"`
-	Description         string      `yaml:"Description"`
-	Email               string      `yaml:"Email"`
-	Reference           string      `yaml:"Reference"`
-	Categories          []string    `yaml:"Categories"`
-	SQLiteIdentifyQuery string      `yaml:"SQLiteIdentifyQuery"`
-	SQLiteIdentifyValue interface{} `yaml:"SQLiteIdentifyValue"`
-	Globs               []string    `yaml:"Globs"`
-	FilenameRegex       string      `yaml:"FilenameRegex"`
-	Sources             []Source    `yaml:"Sources"`
+	Name                string      `yaml:"Name" json:"Name,omitempty"`
+	Author              string      `yaml:"Author" json:"Author,omitempty"`
+	Description         string      `yaml:"Description" json:"Description,omitempty"`
+	Email               string      `yaml:"Email" json:"Email,omitempty"`
+	Reference           string      `yaml:"Reference" json:"Reference,omitempty"`
+	Categories          []string    `yaml:"Categories" json:"Categories,omitempty"`
+	SQLiteIdentifyQuery string      `yaml:"SQLiteIdentifyQuery" json:"SQLiteIdentifyQuery,omitempty"`
+	SQLiteIdentifyValue interface{} `yaml:"SQLiteIdentifyValue" json:"SQLiteIdentifyValue,omitempty"`
+	Globs               []string    `yaml:"Globs" json:"Globs,omitempty"`
+	FilenameRegex       string      `yaml:"FilenameRegex" json:"FilenameRegex,omitempty"`
+	Sources             []Source    `yaml:"Sources" json:"Sources,omitempty"`
+
+	Filename_ string `yaml:"Filename" json:"Filename,omitempty"`
+	RawData_  string `yaml:"RawData" json:"RawData,omitempty"`
 }
 
 type Source struct {
 	Name string `yaml:"name"`
+
+	// VQL to include prior to the VQL query - for example contains
+	// custom VQL functions
+	Preamble string `yaml:"Preamble" json:"Preamble,omitempty"`
+
 	// Specialized VQL to post process the rows. Default is a
 	// passthrough `SELECT * FROM Rows`
-	VQL                 string      `yaml:"VQL"`
-	SQL                 string      `yaml:"SQL"`
-	SQLiteIdentifyQuery string      `json:"id_query"`
-	SQLiteIdentifyValue interface{} `json:"id_value"`
-	Filename            string      `json:"filename"`
+	VQL                 string      `yaml:"VQL" json:"VQL,omitempty"`
+	SQL                 string      `yaml:"SQL" json:"SQL,omitempty"`
+	SQLiteIdentifyQuery string      `json:"id_query,omitempty"`
+	SQLiteIdentifyValue interface{} `json:"id_value,omitempty"`
+	Filename            string      `json:"filename,omitempty"`
 }

api/spec.go

@@ -15,4 +15,5 @@ type GlobSpec struct {
 	Glob     string   `json:"glob"`
 	Tags     []string `json:"tags"`
 	Filename string   `json:"name"`
+	Rule     string   `json:"rule"`
 }

bin/compile.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"os"
+
+	"github.com/Velocidex/SQLiteHunter/api"
+	"github.com/Velocidex/SQLiteHunter/compile"
+	"github.com/Velocidex/SQLiteHunter/definitions"
+	"github.com/alecthomas/kingpin"
+	"gopkg.in/yaml.v3"
+
+	_ "embed"
+)
+
+//go:embed config.yaml
+var config_string string
+
+var (
+	compile_cmd = app.Command("compile",
+		"Build an artifact from a set of SQLiteHunter yaml files.")
+
+	compile_yaml = compile_cmd.Arg("input",
+		"Path to the SQLiteHunter yaml directory to compile").
+		Required().String()
+
+	output_artifact = compile_cmd.Arg("output",
+		"Where to write the final artifact").
+		Required().String()
+
+	output_zip = compile_cmd.Flag("output_zip",
+		"Produce a ZIP file we can use to hunt").
+		String()
+
+	output_index = compile_cmd.Flag("index", "Where to write the rules index").
+			String()
+)
+
+func loadConfig() (*api.ConfigDefinitions, error) {
+	config_obj := &api.ConfigDefinitions{}
+	err := yaml.Unmarshal([]byte(config_string), config_obj)
+	return config_obj, err
+}
+
+func doCompile() error {
+	config_obj, err := loadConfig()
+	if err != nil {
+		return err
+	}
+
+	defs, err := definitions.LoadDefinitions(*compile_yaml)
+	if err != nil {
+		return err
+	}
+
+	spec, err := compile.Compile(defs, config_obj)
+	if err != nil {
+		return err
+	}
+
+	// Serialize the artifact to YAML
+	res, err := spec.Yaml()
+	if err != nil {
+		return err
+	}
+
+	fd, err := os.OpenFile(*output_artifact,
+		os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	_, err = fd.Write([]byte(res))
+	if err != nil {
+		return err
+	}
+
+	if *output_zip != "" {
+		err := spec.WriteZip(*output_zip)
+		if err != nil {
+			return err
+		}
+	}
+
+	if *output_index != "" {
+		fd, err := os.OpenFile(*output_index,
+			os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+		if err != nil {
+			return err
+		}
+		defer fd.Close()
+
+		_, err = fd.Write(spec.BuildIndex())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func init() {
+	command_handlers = append(command_handlers, func(command string) bool {
+		switch command {
+		case compile_cmd.FullCommand():
+			err := doCompile()
+			kingpin.FatalIfError(err, "Compiling artifact")
+
+		default:
+			return false
+		}
+		return true
+	})
+}

bin/main.go

@@ -1,58 +1,28 @@
 package main
 
 import (
-	"flag"
-	"fmt"
-	"io/ioutil"
 	"os"
 
-	"github.com/Velocidex/SQLiteHunter/api"
-	"github.com/Velocidex/SQLiteHunter/compile"
-	"github.com/Velocidex/SQLiteHunter/definitions"
-	"gopkg.in/yaml.v3"
+	"github.com/alecthomas/kingpin"
 )
 
-func loadConfig(config_path string) (*api.ConfigDefinitions, error) {
-	fd, err := os.Open(config_path)
-	if err != nil {
-		return nil, err
-	}
+var (
+	app = kingpin.New("sqlitehunter_compiler",
+		"A tool for packaging the SQLiteHunt artifact.")
 
-	data, err := ioutil.ReadAll(fd)
-	if err != nil {
-		return nil, err
-	}
+	command_handlers []CommandHandler
+)
 
-	config_obj := &api.ConfigDefinitions{}
-	err = yaml.Unmarshal(data, config_obj)
-
-	return config_obj, err
-}
+type CommandHandler func(command string) bool
 
 func main() {
-	config_path := flag.String("config", "./config.yaml",
-		"The path to the config file")
-
-	definition_directory := flag.String("definition_directory", "./definitions",
-		"A directory containing all definitiions")
-
-	flag.Parse()
-
-	config_obj, err := loadConfig(*config_path)
-	if err != nil {
-		panic(err)
-	}
-
-	defs, err := definitions.LoadDefinitions(*definition_directory)
-	if err != nil {
-		panic(err)
+	app.HelpFlag.Short('h')
+	app.UsageTemplate(kingpin.CompactUsageTemplate)
+	command := kingpin.MustParse(app.Parse(os.Args[1:]))
+
+	for _, handler := range command_handlers {
+		if handler(command) {
+			break
+		}
 	}
-
-	spec, err := compile.Compile(defs, config_obj)
-	if err != nil {
-		panic(err)
-	}
-
-	// Serialize the artifact to YAML
-	fmt.Println(spec.Yaml())
 }