Configurable hf cache

.gitignore

@@ -2,5 +2,5 @@ __pycache__/
 .vscode/
 .venv/
 .env
-
-presets/
+presets/
+addons/

CHANGELOG.md

@@ -1,4 +1,73 @@
-# v1.1.1
+
+Api changes:
+Image interrogation via api receives two extra parameters; empty strings by
+default. `queue`: the name for a queue, which could be e.g. the person or
+subject name. You can leave it empty for the first interrogation, then the
+response will que in a new auto-generated unique name, listed in the response.
+# v1.2.0 (2023-09-16)
+
+Make sure you use this same name as queue, for all interrogations that you want
+to be grouped together. The second parameter is `name_in_queue`: the name for
+that particular image that is being queued, e.g. a file name.
+
+If both queue and name are empty, there is a single interrogation with response,
+which includes nested objects "ratings" and "tags", so:
+`{"ratings": {"sensitive": 0.5, ..}, "tags": {"tag1": 0.5, ..}}`
+
+If neither name nor queue are empty, the interrogation is queued under that name.
+If already in queue, that name is changed - clobbered - with #. An exception is if
+the given name is <sha256> in which case an image checksum will be used instead of
+a name. Duplicates are ignored.
+
+During queuing, the response is the number of all processed interrogations for all
+active queues.
+
+If name_in_queue is empty, but queue is not, that particular queue is finalized,
+A response is awaited for remaining interrogations in this queue (if any still).
+The response, only for this queue, is an object with the name_in_queue as key,
+and the tag with weights contained. Ratings have ther tag name prefixed with
+"rating:". Example:
+`{"name_in_queue": {"rating:sensitive": 0.5, "tag1": 0.5, ..}}`
+
+Fix in absence of tensrflow_io
+Fix deprecation warning
+Added three scripts in shell scripts under shell_scripts:
+ * A bash script to generate per safetensors file the fraction of images
+   that the model was trained on that was tagged with particular tokens.
+ * A python script to compare the interrogation results (read from db.json)
+   and find the top -c safetensors files that contain similar weights (or at
+   least, that was the intention, there may be better algorithms to compare,
+   but it seems to do the job).
+ * And finally a model_grep script which listts the tags and number of trained
+   images in a safetensors model.
+
+# v1.1.2 c9f8efd (2023-08-26)
+
+Explain recursive path usage better in ui
+Fix sending tags via buttons to txt2img and img2img
+type additions, inadvertently pushed, later retouched.
+allow setting gpu device via flag
+Fix inverted cumulative checkbox
+wrap_gradio_gpu_call fallback
+Fix for preload shared access
+preload update
+A few ui changes
+Fix not clearing the tags after writing them to files
+Fix: Tags were still added, beyond count threshold
+fix search/replace bug
+(here int based weights were reverted)
+circumvent when unable to load tensorflow
+fix for too many exclude_tags
+add db.json validation schema, add schema validation
+return fix for fastapi
+pick up huggingface cache dir from env, with default, configurable also via settings.
+leave tensorflow requirements to the user.
+Fix for Reappearance of gradio bug: duplicate image edit
+(index based weights, but later reverted)
+Instead of cache_dir use local_dir, leav
+
+
+# v1.1.1 eada050 (2023-07-20) 
 
 Internal cleanup, no separate interrogation for inverse
 Fix issues with search and sending selection to keep/exclude
@@ -11,7 +80,9 @@ fix some hf download issues
 fixes for fastapi
 added ML-Danbooru support, thanks to [CCRcmcpe](github.com/CCRcmcpe)
 
-# v1.1.0
+
+# v1.1.0 87706b7 (2023-07-16)
+
 fix: failed to install onnxruntime package on MacOS thanks to heady713
 fastapi: remote unload model, picked up from [here](https://github.com/toriato/stable-diffusion-webui-wd14-tagger/pull/109)
 attribute error fix from aria1th also reported by yjunej
@@ -39,7 +110,7 @@ changed internal error handling, It is a bit quirky, which I intend to fix, stil
 If you find it keeps complaining about an input field without reason, just try editing that one again (e.g. add a space there and remove it).
 
 
-# v1.0.0
+# v1.0.0 a1b59d6 (2023-07-10)
 
 You may have to remove the presets/default.json and save a new one.witth your desired defaults. Otherwise checkboxes may not have the right default values.
 

default/interrogators.json

@@ -0,0 +1,71 @@
+{
+  "MLDanbooruInterrogator": {
+    "mld-caformer.dec-5-97527" : {
+       "model_path" : "ml_caformer_m36_dec-5-97527.onnx",
+       "name" : "ML-Danbooru Caformer dec-5-97527",
+       "repo_id" : "deepghs/ml-danbooru-onnx"
+    },
+    "mld-tresnetd.6-30000" : {
+       "model_path" : "TResnet-D-FLq_ema_6-30000.onnx",
+       "name" : "ML-Danbooru TResNet-D 6-30000",
+       "repo_id" : "deepghs/ml-danbooru-onnx"
+    }
+  },
+  "WaifuDiffusionInterrogator": {
+    "wd-v1-4-moat-tagger.v2" : {
+      "name" : "WD14 moat tagger v2",
+      "repo_id" : "SmilingWolf/wd-v1-4-moat-tagger-v2"
+    },
+    "wd14-convnext.v1" : {
+      "name" : "WD14 ConvNeXT v1",
+      "repo_id" : "SmilingWolf/wd-v1-4-convnext-tagger"
+    },
+    "wd14-convnext.v2" : {
+      "name" : "WD14 ConvNeXT v2",
+      "repo_id" : "SmilingWolf/wd-v1-4-convnext-tagger-v2"
+    },
+    "wd14-convnextv2.v1" : {
+      "name" : "WD14 ConvNeXTV2 v1",
+      "repo_id" : "SmilingWolf/wd-v1-4-convnextv2-tagger-v2"
+    },
+    "wd14-swinv2-v1" : {
+      "name" : "WD14 SwinV2 v1",
+      "repo_id" : "SmilingWolf/wd-v1-4-swinv2-tagger-v2"
+    },
+    "wd14-vit.v1" : {
+      "name" : "WD14 ViT v1",
+      "repo_id" : "SmilingWolf/wd-v1-4-vit-tagger"
+    },
+    "wd14-vit.v2" : {
+      "name" : "WD14 ViT v2",
+      "repo_id" : "SmilingWolf/wd-v1-4-vit-tagger-v2"
+    }
+  },
+  "DeepDanbooruInterrogator": {
+    "deepdanbooru-v3-20211112-sgd-e28": {
+      "zip": "https://github.com/KichangKim/DeepDanbooru/releases/download/v3-20211112-sgd-e28/deepdanbooru-v3-20211112-sgd-e28.zip"
+    },
+    "deepdanbooru-v4-20200814-sgd-e30": {
+      "zip": "https://github.com/KichangKim/DeepDanbooru/releases/download/v4-20200814-sgd-e30/deepdanbooru-v4-20200814-sgd-e30.zip"
+    }
+  },
+  "FromFileInterrogator": {
+    "[name].[hash:sha1].[output_extension]": {
+      "format": "[name].[hash:sha1].[output_extension]",
+      "path": "",
+      "value": 1.0
+    }
+  },
+  "MLDanbooruInterrogator": {
+    "mld-caformer.dec-5-97527" : {
+       "model_path" : "ml_caformer_m36_dec-5-97527.onnx",
+       "name" : "ML-Danbooru Caformer dec-5-97527",
+       "repo_id" : "deepghs/ml-danbooru-onnx"
+    },
+    "mld-tresnetd.6-30000" : {
+       "model_path" : "TResnet-D-FLq_ema_6-30000.onnx",
+       "name" : "ML-Danbooru TResNet-D 6-30000",
+       "repo_id" : "deepghs/ml-danbooru-onnx"
+    }
+  }
+}

install.py

@@ -1,12 +1,13 @@
 """Install requirements for WD14-tagger."""
 import os
 import sys
-
+import json
 from launch import run  # pylint: disable=import-error
 
+local_dir = os.path.dirname(os.path.realpath(__file__))
+
 NAME = "WD14-tagger"
-req_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                        "requirements.txt")
+req_file = os.path.join(local_dir, "requirements.txt")
 print(f"loading {NAME} reqs from {req_file}")
 run(f'"{sys.executable}" -m pip install -q -r "{req_file}"',
     f"Checking {NAME} requirements.",

json_schema/db_json_v1_schema.json

@@ -10,28 +10,28 @@
                     "type": "array",
                     "prefixItems": [
                         {"type": "string" },
-                        {"type": "number", "minimum": 0}
-                    ],
-                    "minContains": 2,
-                    "maxContains": 2
-                }
-            }
-        },
-        "meta": {
-            "type": "object",
-            "properties": {
-              "index_shift": {
-                "type": "integer",
-                "minimum": 0,
-                "maximum": 16
+                      {"type": "number", "minimum": 0}
+                  ],
+                  "minContains": 2,
+                  "maxContains": 2
               }
+          }
+      },
+      "meta": {
+          "type": "object",
+          "properties": {
+            "index_shift": {
+              "type": "integer",
+              "minimum": 0,
+              "maximum": 16
             }
-        },
-        "add": { "type": "string" },
-        "exclude": { "type": "string" },
-        "keep": { "type": "string" },
-        "repl": { "type": "string" },
-        "search": { "type": "string" }
+          }
+      },
+      "add": { "type": "string" },
+      "exclude": { "type": "string" },
+      "keep": { "type": "string" },
+      "repl": { "type": "string" },
+      "search": { "type": "string" }
     },
     "required": ["rating", "tag", "query"],
     "additionalProperties": false,

json_schema/interrogators_v1_schema.json

@@ -0,0 +1,104 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Interrogator Configurations",
+  "type": "object",
+  "properties": {
+    "MLDanbooruInterrogator": {
+      "type": "object",
+      "patternProperties": {
+        ".*": { "$ref": "#/$defs/huggingfaceInterrogator" }
+      }
+    },
+    "WaifuDiffusionInterrogator": {
+      "type": "object",
+      "patternProperties": {
+        ".*": { "$ref": "#/$defs/huggingfaceInterrogator" }
+      }
+    },
+    "DeepDanbooruInterrogator": {
+      "type": "object",
+      "patternProperties": {
+        ".*": {
+          "type": "object",
+          "properties": {
+            "zip": { "type": "string" }
+          },
+          "additionalProperties": false
+        }
+      }
+    },
+    "MLDanbooruInterrogator": {
+      "type": "object",
+      "patternProperties": {
+        ".*": { "$ref": "#/$defs/huggingfaceInterrogator" }
+      }
+    },
+    "FromFileInterrogator": {
+      "type": "object",
+      "patternProperties": {
+        ".*": {
+          "type": "object",
+          "properties": {
+            "path": { "type": "string" },
+            "format": { "type": "string", "default": "[name].[output_extension]" },
+            "value": { "type": "number", "default": 1.0 }
+          },
+          "required": [
+            "path"
+          ],
+          "additionalProperties": false
+        }
+      }
+    },
+    "additionalProperties": false
+  },
+  "$defs": {
+    "huggingfaceInterrogator": {
+      "type": "object",
+      "properties": {
+        "name": { "type": "string" },
+        "local_model": { "type": "string" },
+        "local_tags": { "type": "string" },
+        "model_path": { "type": "string" },
+        "repo_id": { "type": "string" },
+        "filename": { "type": "string" },
+        "subfolder": { "type": "string" },
+        "repo_type": { "type": "string" },
+        "revision": { "type": "string" },
+        "endpoint": { "type": "string" },
+        "library_name": { "type": "string" },
+        "library_version": { "type": "string" },
+        "cache_dir": { "type": "string" },
+        "local_dir": { "type": "string" },
+        "local_dir_use_symlinks": { "type": "string" },
+        "user_agent": { "type": "string" },
+        "force_download": { "type": "boolean" },
+        "force_filename": { "type": "string" },
+        "proxies": { "type": "string" },
+        "etag_timeout": { "type": "number" },
+        "resume_download": { "type": "boolean" },
+        "token": { "type": "string" },
+        "local_files_only": { "type": "boolean" },
+        "legacy_cache_layout": { "type": "boolean" }
+      },
+      "$comment": "repo_id is required if you want to actually download the model",
+      "oneOf": [
+        {
+          "required": [
+            "name",
+            "repo_id"
+          ]
+        },
+        {
+          "required": [
+            "name",
+            "local_model",
+            "local_tags"
+          ]
+        }
+      ],
+      "additionalProperties": false
+    }
+  }
+}   
+

preload.py

@@ -1,6 +1,8 @@
 """ Preload module for DeepDanbooru or onnxtagger. """
-from pathlib import Path
 from argparse import ArgumentParser
+from pathlib import Path
+
+root_dir = Path(__file__).parent
 
 
 def preload(parser: ArgumentParser):
@@ -19,8 +21,9 @@ def preload(parser: ArgumentParser):
         type=str,
         help='Path to directory with Onnyx project(s).'
     )
+    # TODO allow using devices in parallel, specified as comma separed list
     parser.add_argument(
         '--additional-device-ids',
         type=str,
-        help='Extra device ID to use. cpu:0,gpu:1..',
+        help='Device ID to use. cpu:0, gpu:0 or gpu:1, etc.',
     )