Added a documentation site

Refactored SQLiteHunter:

1. It is now possible to acquire only a single rules based on regex
   match
2. Notebooks are created lazily to avoid having to calculate many
   cells at creation time.
3. Notebook cell automatically adds new cells at the top for sources
   with results.

Author: scudette

.github/workflows/gh-pages.yml

@@ -0,0 +1,44 @@
+name: github pages
+
+on:
+  push:
+    branches:
+      - main  # Set a branch to deploy
+  pull_request:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          submodules: true  # Fetch Hugo themes (true OR recursive)
+          fetch-depth: 0    # Fetch all history for .GitInfo and .Lastmod
+
+      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe
+        with:
+          go-version: '^1.23'
+
+      - run: go version
+
+      - name: Setup Hugo
+        uses: peaceiris/actions-hugo@v2
+        with:
+          hugo-version: 'latest'
+          extended: true
+
+      - name: Prepare
+        run: |
+          make build compile
+          cd output/SQLiteHunter.zip ./docs/static/
+
+      - name: Build
+        run: cd docs/ && hugo --minify
+
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        if: github.ref == 'refs/heads/master'
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/public
+          force_orphan: true

.gitignore

@@ -1,3 +1,4 @@
 velociraptor*
 sqlitehunter_compiler*
-datastore
+datastore
+output/*.zip

.pyspelling.yml

@@ -0,0 +1,40 @@
+matrix:
+- name: Markdown
+  aspell:
+    lang: en
+    d: en_US
+  dictionary:
+    wordlists:
+      - .wordlist.txt
+    output: .wordlist.dict
+  sources:
+  - 'docs/content/**/*.md'
+  pipeline:
+    - pyspelling.filters.context:
+        context_visible_first: true
+        escapes: '\\[\\`]'
+        delimiters:
+          # Ignore multiline content between fences (fences can have 3 or more back ticks)
+          # ```
+          # content
+          # ```
+          - open: '(?s)^(?P<open> *`{3,})'
+            close: '^(?P=open)$'
+            # Insides of URL links
+          - open: '\]\('
+            close: '\)'
+            # Bolded words are usually terms
+          - open: '[*]+'
+            close: '[*]+'
+            # Inside HTML tags
+          - open: '(?s)(?P<open>[<])'
+            close: '[>]'
+            # Ignore text between inline back ticks
+          - open: '(?P<open>`+)'
+            close: '(?P=open)'
+
+            # Inside yaml headers
+          - open: '(?s)^(?P<open>---)$'
+            close: '^(?P=open)$'
+
+    - pyspelling.filters.url:

.wordlist.dict

@@ -0,0 +1,15 @@
+# docs/content/_index.md
+SQLiteHunter
+
+<url-free> docs/content/docs/sqlite_hunter/_index.md
+MacOS
+SQLECmd
+SQLiteHunter
+VQL
+Velociraptor
+YAML
+journaling
+SQLite
+Velociraptor
+de
+facto

.wordlist.txt

@@ -8,7 +8,7 @@ windows:
 	go build -o sqlitehunter_compiler.exe ./bin/*.go
 
 compile: FORCE
-	./sqlitehunter_compiler -config ./config.yaml -definition_directory ./definitions > output/SQLiteHunter.yaml
+	./sqlitehunter_compiler compile ./definitions/ ./output/SQLiteHunter.yaml --output_zip ./output/SQLiteHunter.zip --index ./docs/content/docs/rules/index.json
 
 golden: compile
 	./testing/velociraptor.bin --definitions ./output --config ./testing/test.config.yaml golden --env testFiles=`pwd`/test_files ./testing/testcases -v --filter=${GOLDEN}

Makefile

@@ -5,26 +5,34 @@ type ConfigDefinitions struct {
 }
 
 type Definition struct {
-	Name                string      `yaml:"Name"`
-	Author              string      `yaml:"Author"`
-	Description         string      `yaml:"Description"`
-	Email               string      `yaml:"Email"`
-	Reference           string      `yaml:"Reference"`
-	Categories          []string    `yaml:"Categories"`
-	SQLiteIdentifyQuery string      `yaml:"SQLiteIdentifyQuery"`
-	SQLiteIdentifyValue interface{} `yaml:"SQLiteIdentifyValue"`
-	Globs               []string    `yaml:"Globs"`
-	FilenameRegex       string      `yaml:"FilenameRegex"`
-	Sources             []Source    `yaml:"Sources"`
+	Name                string      `yaml:"Name" json:"Name,omitempty"`
+	Author              string      `yaml:"Author" json:"Author,omitempty"`
+	Description         string      `yaml:"Description" json:"Description,omitempty"`
+	Email               string      `yaml:"Email" json:"Email,omitempty"`
+	Reference           string      `yaml:"Reference" json:"Reference,omitempty"`
+	Categories          []string    `yaml:"Categories" json:"Categories,omitempty"`
+	SQLiteIdentifyQuery string      `yaml:"SQLiteIdentifyQuery" json:"SQLiteIdentifyQuery,omitempty"`
+	SQLiteIdentifyValue interface{} `yaml:"SQLiteIdentifyValue" json:"SQLiteIdentifyValue,omitempty"`
+	Globs               []string    `yaml:"Globs" json:"Globs,omitempty"`
+	FilenameRegex       string      `yaml:"FilenameRegex" json:"FilenameRegex,omitempty"`
+	Sources             []Source    `yaml:"Sources" json:"Sources,omitempty"`
+
+	Filename_ string `yaml:"Filename" json:"Filename,omitempty"`
+	RawData_  string `yaml:"RawData" json:"RawData,omitempty"`
 }
 
 type Source struct {
 	Name string `yaml:"name"`
+
+	// VQL to include prior to the VQL query - for example contains
+	// custom VQL functions
+	Preamble string `yaml:"Preamble" json:"Preamble,omitempty"`
+
 	// Specialized VQL to post process the rows. Default is a
 	// passthrough `SELECT * FROM Rows`
-	VQL                 string      `yaml:"VQL"`
-	SQL                 string      `yaml:"SQL"`
-	SQLiteIdentifyQuery string      `json:"id_query"`
-	SQLiteIdentifyValue interface{} `json:"id_value"`
-	Filename            string      `json:"filename"`
+	VQL                 string      `yaml:"VQL" json:"VQL,omitempty"`
+	SQL                 string      `yaml:"SQL" json:"SQL,omitempty"`
+	SQLiteIdentifyQuery string      `json:"id_query,omitempty"`
+	SQLiteIdentifyValue interface{} `json:"id_value,omitempty"`
+	Filename            string      `json:"filename,omitempty"`
 }

api/api.go

@@ -15,4 +15,5 @@ type GlobSpec struct {
 	Glob     string   `json:"glob"`
 	Tags     []string `json:"tags"`
 	Filename string   `json:"name"`
+	Rule     string   `json:"rule"`
 }

api/spec.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"os"
+
+	"github.com/Velocidex/SQLiteHunter/api"
+	"github.com/Velocidex/SQLiteHunter/compile"
+	"github.com/Velocidex/SQLiteHunter/definitions"
+	"github.com/alecthomas/kingpin"
+	"gopkg.in/yaml.v3"
+
+	_ "embed"
+)
+
+//go:embed config.yaml
+var config_string string
+
+var (
+	compile_cmd = app.Command("compile",
+		"Build an artifact from a set of SQLiteHunter yaml files.")
+
+	compile_yaml = compile_cmd.Arg("input",
+		"Path to the SQLiteHunter yaml directory to compile").
+		Required().String()
+
+	output_artifact = compile_cmd.Arg("output",
+		"Where to write the final artifact").
+		Required().String()
+
+	output_zip = compile_cmd.Flag("output_zip",
+		"Produce a ZIP file we can use to hunt").
+		String()
+
+	output_index = compile_cmd.Flag("index", "Where to write the rules index").
+			String()
+)
+
+func loadConfig() (*api.ConfigDefinitions, error) {
+	config_obj := &api.ConfigDefinitions{}
+	err := yaml.Unmarshal([]byte(config_string), config_obj)
+	return config_obj, err
+}
+
+func doCompile() error {
+	config_obj, err := loadConfig()
+	if err != nil {
+		return err
+	}
+
+	defs, err := definitions.LoadDefinitions(*compile_yaml)
+	if err != nil {
+		return err
+	}
+
+	spec, err := compile.Compile(defs, config_obj)
+	if err != nil {
+		return err
+	}
+
+	// Serialize the artifact to YAML
+	res, err := spec.Yaml()
+	if err != nil {
+		return err
+	}
+
+	fd, err := os.OpenFile(*output_artifact,
+		os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	_, err = fd.Write([]byte(res))
+	if err != nil {
+		return err
+	}
+
+	if *output_zip != "" {
+		err := spec.WriteZip(*output_zip)
+		if err != nil {
+			return err
+		}
+	}
+
+	if *output_index != "" {
+		fd, err := os.OpenFile(*output_index,
+			os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+		if err != nil {
+			return err
+		}
+		defer fd.Close()
+
+		_, err = fd.Write(spec.BuildIndex())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func init() {
+	command_handlers = append(command_handlers, func(command string) bool {
+		switch command {
+		case compile_cmd.FullCommand():
+			err := doCompile()
+			kingpin.FatalIfError(err, "Compiling artifact")
+
+		default:
+			return false
+		}
+		return true
+	})
+}