Merge pull request #10 from shuangluoxss/main

2.0 artifacts support

Author: ProblemFactory

.gitignore

@@ -1,13 +1,10 @@
 ArtScanner/artifacts.genshinart.json
 ArtScanner/artifacts.mingyulab.json
-ArtScanner/artifacts.dat
-ArtScanner/artifacts.dat.index
-ArtScanner/artifacts.dat.lock
-ArtScanner/artifacts.dat.tmp
 **/__pycache__/**
 ArtScanner/build/**
 ArtScanner/dist/**
 ArtScanner/*.spec
 ArtScanner/artifacts/**
+ArtScanner/artifacts-all/**
 
 .vscode/**

ArtScanner/ArtsInfo.py

@@ -97,6 +97,8 @@
     ["游医的银莲", "游医的枭羽", "游医的怀钟", "游医的药壶", "游医的方巾"],
     ["勋绩之花", "昭武翎羽", "金铜时晷", "盟誓金爵", "将帅兜鏊"],
     ["无垢之花", "贤医之羽", "停摆之刻", "超越之盏", "嗤笑之面"],
+    ["祈望之心", "思忆之矢", "无常之面", "羁缠之花", "朝露之时"],
+    ["绯花之壶", "切落之羽", "华饰之兜", "明威之镡", "雷云之笼"]
 ]
 Users = [
     "空",
@@ -193,6 +195,8 @@
     "travelingDoctor",  # 游医
     "tenacityOfTheMillelith",  # 千岩牢固
     "paleFlame",  # 苍白之火
+    "shimenawaReminiscence", # 追忆之注连
+    "emblemOfSeveredFate", # 绝缘之旗印
 ]
 
 
@@ -254,4 +258,6 @@
     "traveling_doctor",  # 游医
     "tenacity_of_the_millelith",  # 千岩牢固
     "pale_flame",  # 苍白之火
+    "reminiscence_of_shime", # 追忆之注连
+    "seal_of_insulation", # 绝缘之旗印
 ]

ArtScanner/build.cmd

@@ -1 +1 @@
-pyinstaller --onefile --add-data "mn_model_weight.h5;." --add-data "Tools/ReliquaryLevelExcelConfigData.json;./Tools" --add-data "Tools/ReliquaryAffixExcelConfigData.json;./Tools" --hidden-import=h5py --hidden-import=h5py.defs --hidden-import=h5py.utils --hidden-import=h5py.h5ac --hidden-import=h5py._proxy --uac-admin -n ArtScanner main.py
+pyinstaller --onefile --add-data "mn_model_weight.h5;." --add-data "mn_model_weight_artnames.h5;." --add-data "Tools/ReliquaryLevelExcelConfigData.json;./Tools" --add-data "Tools/ReliquaryAffixExcelConfigData.json;./Tools" --hidden-import=h5py --hidden-import=h5py.defs --hidden-import=h5py.utils --hidden-import=h5py.h5ac --hidden-import=h5py._proxy --uac-admin -n ArtScanner main.py

ArtScanner/environment.yml

@@ -1,7 +1,7 @@
 name: ArtScanner
 channels:
-  - conda-forge
   - defaults
+  - conda-forge
   - msys2
 dependencies:
   - _tflow_select=2.3.0=eigen
@@ -117,4 +117,3 @@ dependencies:
     - keyboard==0.13.5
     - mouse==0.7.1
     - mss==6.1.0
-prefix: C:\Users\marti\miniforge3\envs\ArtScanner

ArtScanner/main.py

@@ -25,7 +25,7 @@ def is_admin():
 
 import win32gui
 import time
-import keyboard, mouse
+import mouse
 import os
 import json
 import ocr
@@ -90,7 +90,8 @@ def is_admin():
 # margin near level number, color=233,229,220
 
 # initialization
-ocr_model = ocr.OCR(scale_ratio=game_info.scale_ratio, model_weight=os.path.join(bundle_dir, 'mn_model_weight.h5'))
+ocr_model = ocr.OCR(scale_ratio=game_info.scale_ratio, model_weight=os.path.join(bundle_dir, 'mn_model_weight.h5'), 
+                    ocr_model_artnames=ocr.OCR_artnames(model_weight=os.path.join(bundle_dir, 'mn_model_weight_artnames.h5')))
 art_id = 0
 saved = 0
 skipped = 0
@@ -99,8 +100,6 @@ def is_admin():
 star_dist_saved = [0,0,0,0,0]
 
 
-os.makedirs('artifacts', exist_ok=True)
-
 input('请打开圣遗物背包界面，最好翻到圣遗物列表最上面。按回车继续')
 print('---------------------------------')
 if game_info.incomplete_lastrow:
@@ -116,6 +115,7 @@ def is_admin():
     level_threshold = input('请输入圣遗物等级阈值(0-20)(比如：16，则仅将保存16级及以上的圣遗物信息)。直接按回车则默认保存所有圣遗物信息。')
     rarity_threshold = input('请输入圣遗物星级阈值(1-5)(比如：5，则仅将保存5星的圣遗物信息)。直接按回车则默认保存所有圣遗物信息。')
     scroll_interval = input('请输入翻页时的检测延迟（秒），数值越大翻页速度越慢，可以解决一些翻页时的检测BUG，直接回车则为默认值0.05。')
+    save_all = input('请输入是否保存所有圣遗物截图，若是，请输入y，直接按回车则默认不保存。')
 print('---------------------------------')
 print('程序将于5秒后自动开始运行，若此条提示显示时未自动切换到原神窗口，请手动点击原神窗口切到前台')
 
@@ -144,8 +144,16 @@ def is_admin():
 except:
     exporter = art_data.exportGenshinArtJSON
     export_name = 'artifacts.genshinart.json'
+try:
+    save_all = save_all.lower().startswith('y')
+except:
+    save_all = False
 
 
+os.makedirs('artifacts', exist_ok=True)
+if save_all:
+    os.makedirs('artifacts-all', exist_ok=True)
+
 mouse.on_middle_click(art_scanner.interrupt)
 
 print('正在自动对齐')
@@ -160,7 +168,15 @@ def artscannerCallback(art_img):
     global skipped
     global failed
     global star_dist
+    global save_all
+    if save_all:
+        art_img.save(f'artifacts-all/{art_id}.png')
     info = ocr_model.detect_info(art_img)
+
+    # 修复部分错误识别
+    if info['main_attr_name'] == '生命值' and info['main_attr_value'] == '3,957':
+        info['main_attr_value'] = '3,967'
+
     star_dist[info['star']-1] += 1
     if decodeValue(info['level'])<level_threshold or decodeValue(info['star'])<rarity_threshold:
         skipped += 1

ArtScanner/mn_model_weight_artnames.h5

@@ -10,7 +10,7 @@
 from tensorflow import get_logger
 from tensorflow.keras.models import Model
 from tensorflow.keras.layers.experimental.preprocessing import StringLookup
-from tensorflow.keras.layers import Input, Reshape, Dense, Dropout, Bidirectional, LSTM
+from tensorflow.keras.layers import Input, Reshape, Dense, Dropout, Bidirectional, LSTM, Flatten
 from tensorflow.keras.backend import ctc_decode
 from mobilenetv3 import MobileNetV3_Small
 from tensorflow.strings import reduce_join
@@ -33,13 +33,13 @@ class Config:
     subattr_4_coords = [67, 636, 560, 676]
 
 class OCR:
-    def __init__(self, model_weight='mn_model_weight.h5', scale_ratio=1):
+    def __init__(self, model_weight='mn_model_weight.h5', scale_ratio=1, ocr_model_artnames=None):
         self.scale_ratio = scale_ratio
         self.characters = sorted(
                                 [
                                     *set(
                                         "".join(
-                                            sum(ArtsInfo.ArtNames, [])
+                                            sum(ArtsInfo.ArtNames[:-2], [])
                                             + ArtsInfo.TypeNames
                                             + list(ArtsInfo.MainAttrNames.values())
                                             + list(ArtsInfo.SubAttrNames.values())
@@ -63,12 +63,14 @@ def __init__(self, model_weight='mn_model_weight.h5', scale_ratio=1):
         self.max_length = 15
         self.build_model(input_shape=(self.width, self.height))
         self.model.load_weights(model_weight)
+        self.ocr_model_artnames = ocr_model_artnames
 
     def detect_info(self, art_img):
         info = self.extract_art_info(art_img)
         x = np.concatenate([self.preprocess(info[key]).T[None, :, :, None] for key in sorted(info.keys())], axis=0)
         y = self.model.predict(x)
         y = self.decode(y)
+        y[3] = self.ocr_model_artnames.reg(x[3][None])
         return {**{key:v for key, v in zip(sorted(info.keys()), y)}, **{'star':self.detect_star(art_img)}}
 
     def extract_art_info(self, art_img):
@@ -198,4 +200,51 @@ def build_model(self, input_shape):
         output = Dense(len(self.characters) + 2, activation="softmax", name="dense2")(x)
 
         # Define the model
-        self.model = Model(inputs=[input_img], outputs=output, name="ocr_model_v1")
+        self.model = Model(inputs=[input_img], outputs=output, name="ocr_model_v1")
+        
+class OCR_artnames:
+    def __init__(self, model_weight='mn_model_weight_artnames.h5'):
+        self.artnames = sorted(set(sum(ArtsInfo.ArtNames, [])))
+
+        self.model = self.build_model(input_shape=(240, 16))
+        self.model.load_weights(model_weight)
+        
+    def build_model(self, input_shape):
+        input_img = Input(
+            shape=(input_shape[0], input_shape[1], 1), name="image", dtype="float32"
+        )
+        mobilenet = MobileNetV3_Small(
+            (input_shape[0], input_shape[1], 1), 0, alpha=1.0, include_top=False
+        ).build()
+        x = mobilenet(input_img)
+        new_shape = ((input_shape[0] // 8), (input_shape[1] // 8) * 576)
+        x = Reshape(target_shape=new_shape, name="reshape")(x)
+        x = Dense(64, activation="relu", name="dense1")(x)
+        x = Dropout(0.2)(x)
+
+        # RNNs
+        x = Bidirectional(LSTM(128, return_sequences=True, dropout=0.25))(x)
+        x = Bidirectional(LSTM(64, return_sequences=True, dropout=0.25))(x)
+
+        # Output layer
+        x = Flatten(name="flatten")(x)
+        x = Dense(
+            len(self.artnames), activation="softmax", name="dense2"
+        )(x)
+
+        output = x
+
+        # Define the model
+        model = Model(inputs=[input_img], outputs=output, name="ocr_model_artnames")
+        
+        return model
+    
+    def decode_single(self, pred):
+        i = pred[0].argmax()
+        if pred[0][i] > 0.75:
+            return self.artnames[i]
+        else:
+            return 'Unknown'
+   
+    def reg(self, x):
+        return self.decode_single(self.model.predict(x))

ArtScanner/ocr.py

@@ -2,3 +2,10 @@
 Yet Another Genshin Impact Artifact Scanner
 
 check https://bbs.nga.cn/read.php?tid=26425531&_ff=650 for details
+
+Fork自[GenshinArtScanner](https://github.com/ProblemFactory/GenshinArtScanner)
+
+## 主要改动
+- 添加了对2.0新圣遗物`追忆之注连`与`绝缘之旗印`的支持
+- 将圣遗物名称识别部分单独建立为新模型，减小新增圣遗物后重新训练的工作量
+- 修正了1080P分辨率下16级五星生之花主属性3967被错误识别为3957的问题