Features/monsterfish picture load (#28)

* add cv2 as requirement

* feat(memory): terrain and height from image POC

* CI: sync PR build and test strategy

* add cv2 as dependency to setup.py

* add logic info

* unfinished logic values

* feat(get_terrain): working color display

* feat(get_height): working height and palette

* fix(set_terrain): make sure border edges are retained

* fix(set_terrain): change colors.Palette into Palette

* fix(main):  fixed relative import

* CI: add onedir and onefile

Author: gynt

.github/workflows/python-pr.yml

@@ -66,8 +66,9 @@ jobs:
       shell: bash
       run: |
         python -m pip install pyinstaller
-        pyinstaller --console --onefile ./sourcehold/__main__.py --name sourcehold
-        
+        pyinstaller --console --onefile ./sourcehold/__main__.py --name sourcehold --distpath dist/sourcehold-onefile
+        pyinstaller --console --onedir ./sourcehold/__main__.py --name sourcehold --distpath dist/sourcehold-onedir
+
     - name: Archive packages
       uses: actions/upload-artifact@v4
       with:
@@ -83,9 +84,9 @@ jobs:
       matrix:
         # os: [ubuntu-latest, windows-latest, macOS-latest]
         os: [windows-latest]
-        platform: [x86, x64]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
-        python-platform: [x86, x64]
+        platform: [x86]
+        python-version: ['3.8']
+        python-platform: [x86]
     steps:
     - name: Download package
       uses: actions/download-artifact@v4.1.7

requirements.txt

@@ -3,3 +3,4 @@ pymem
 dclimplode
 numpy
 build
+opencv-python

setup.py

@@ -21,7 +21,7 @@
         "Operating System :: OS Independent",
     ],
     python_requires='>=3.8',
-    install_requires=["pymem", "Pillow", "dclimplode", "numpy"],
+    install_requires=["pymem", "Pillow", "dclimplode", "numpy", "opencv-python"],
     test_suite="tests",
     entry_points={
         'console_scripts': ['sourcehold=sourcehold:entry_point']

sourcehold/__main__.py

@@ -10,6 +10,7 @@
 
 from sourcehold.tool.argparsers.common import main_parser
 from sourcehold.tool.argparsers.services import services_parser, convert_parser
+from sourcehold.tool.memory.map import memory_map
 from sourcehold.tool.modify.map import modify_map
 
 file_input_output = argparse.ArgumentParser(add_help=False)
@@ -75,13 +76,15 @@
 
 args = main_parser.parse_args()
 
-from .tool.convert.aiv import convert_aiv
+from sourcehold.tool.convert.aiv import convert_aiv
 
 def main():
   if convert_aiv(args):
       return
   if modify_map(args):
       return
+  if memory_map(args):
+      return
 
   if args.service == "aiv":
     if args.method == "file":

sourcehold/tool/argparsers/services.py

@@ -1,3 +1,4 @@
+import argparse
 from .common import main_parser, file_input_file_output
 
 services_parser = main_parser.add_subparsers(title="service", dest="service", required=True)
@@ -11,6 +12,25 @@
 convert_aiv_parser.add_argument('--to-format', required=False, default='')
 
 memory_parser = services_parser.add_parser('memory')
+memory_parser.add_argument('--game', choices=['SHC1.41-latin', "SHCE1.41-latin"], default="SHC1.41-latin")
+memory_subparsers = memory_parser.add_subparsers(dest='type', required=True, title='type')
+
+memory_map_parser = memory_subparsers.add_parser('map')
+memory_map_subparsers = memory_map_parser.add_subparsers(dest='action', required=True)
+
+memory_common = argparse.ArgumentParser(add_help=False)
+memory_common.add_argument('what', choices=['terrain', 'height'])
+memory_common.add_argument('--palette', default='', required=False)
+
+memory_map_get_parser = memory_map_subparsers.add_parser('get', parents=[memory_common])
+memory_map_get_parser.add_argument('--output', default='')
+memory_map_get_parser.add_argument('--output-format', default='png', choices=['png'])
+
+memory_map_set_parser = memory_map_subparsers.add_parser('set', parents=[memory_common])
+memory_map_set_parser.add_argument('--input', default='-')
+memory_map_set_parser.add_argument('--input-format', default='', choices=['png'])
+
+
 
 modify_parser = services_parser.add_parser('modify')
 modify_subparser = modify_parser.add_subparsers(dest='type', required=True, title='type')

sourcehold/tool/memory/__init__.py

@@ -0,0 +1,30 @@
+import pathlib, sys
+
+from sourcehold.tool.convert.aiv.exports import to_json
+from sourcehold.tool.convert.aiv.imports import from_json
+from sourcehold.tool.memory.map.height import get_height, set_height
+from sourcehold.tool.memory.map.terrain import get_terrain, set_terrain
+
+
+def memory_map(args):
+  #' returns None in case of non applicable
+  if args.service != "memory":
+    return None
+  
+  if args.type != "map":
+     return None
+  
+  if set_height(args):
+    return True
+  
+
+  if get_height(args):
+    return True
+
+  if set_terrain(args):
+    return True
+  
+  if get_terrain(args):
+    return True
+
+  return True

sourcehold/tool/memory/map/__init__.py

@@ -0,0 +1,19 @@
+import pathlib
+from sourcehold.debugtools.memory.access import SHC, SHCE
+
+def get_process_handle(version):
+  if version == "SHC1.41-latin":
+    return SHC()
+  # if version == "SHCE1.41-latin":
+  #   return SHCE()
+  raise NotImplementedError(f"process not implemented: {version}")
+
+
+def validate_input_path(img_path):
+  if not img_path:
+    raise Exception(f"no input file specified")
+  if img_path == "-":
+    raise NotImplementedError(f"stdin input not yet implemented for this action. Specify a file path using --input")
+
+  if not pathlib.Path(img_path).exists():
+    raise Exception(f"file does not exist: {img_path}")

sourcehold/tool/memory/map/common.py

@@ -0,0 +1,96 @@
+
+
+
+# python -m pip install Pillow
+import pathlib
+import struct
+import numpy
+from sourcehold.tool.memory.map.common import get_process_handle, validate_input_path
+from sourcehold.world import create_selection_matrix
+import cv2 as cv # type: ignore
+import sys
+
+def get_image_data_grayscale(img_path):
+  img = cv.imread(img_path)
+  img = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
+  return img
+
+selection = create_selection_matrix()
+
+# (Little endian) unsigned bytes
+def get_raw_height(process):
+  return struct.unpack("<80400B", process.read_section('1045'))
+
+def set_raw_height(process, data):
+  bytes_data = struct.pack("<80400B", *data)
+  # ChangedLayer
+  # process.write_bytes(0x01c5ad88, b'\x02' * 80400) # TODO: fix
+  # Logical terrain height layer: DefaultHeightLayer
+  process.write_section('1045', bytes_data)
+  # Visual height layer, I think also includes walls and towers: HeightLayer
+  process.write_section('1005', bytes_data)
+  # # LogicLayer
+  # process.write_section('1003', struct.pack("<80400I", *((v & 0xffffff7f) for v in struct.unpack("<80400I", process.read_section('1003')))))
+  # # Logic2Layer
+  # process.write_section('1037', b'\x04' * 80400)
+
+# def post_process_raw_height():
+#   # MiscDisplayLayer
+#   process.write_section('1007', struct.pack("<80400H", *(((v & 0xffdf) & 0xf83f) for v in struct.unpack("<80400H", process.read_section('1007')))))
+#   # LogicLayer, what a hot mess, probably not all required
+#   process.write_section('1003', struct.pack("<80400I", *(((((v & 0x5f81c436) & 0xffffff7f) & 0xbfffbfff) | 32768) for v in struct.unpack("<80400I", process.read_section('1003')))))
+#   # Logic2Layer
+#   process.write_section('1037', b'\x04' * 80400)
+#   # TODO: wall owner layer, and special logic2layer set to 4 or 8 depending on plateau
+#   # ChangedLayer
+#   process.write_bytes(0x01c5ad88, b'\x02' * 80400)
+
+
+# post_process_raw_height()
+
+
+def set_height(args):
+  #' returns None in case of non applicable
+  if args.what != 'height':
+    return None
+  
+  if args.action != "set":
+     return None
+  
+  img_path = args.input
+  validate_input_path(img_path)
+
+  img = get_image_data_grayscale(img_path)
+
+  process = get_process_handle(args.game)
+
+  set_raw_height(process, img[selection].flat)
+
+  return True
+
+
+
+def get_height(args):
+  #' returns None in case of non applicable
+  if args.what != 'height':
+    return None
+  
+  if args.action != "get":
+     return None
+  
+  img = numpy.zeros((400,400), dtype='uint8')
+
+  process = get_process_handle(args.game)
+
+  height = numpy.zeros((400, 400), dtype='uint8')
+  height[selection] = get_raw_height(process)
+
+  img[selection] = height[selection]
+
+  if not args.output:
+    print(args.output_format)
+    sys.stdout.buffer.write(cv.imencode(f".{args.output_format}", img)[1].tobytes())
+  else:
+    cv.imwrite(args.output, img=img)
+
+  return True

sourcehold/tool/memory/map/height/__init__.py

@@ -0,0 +1,137 @@
+MAP_SIZE = 400
+TILE_COUNT = MAP_SIZE * ((MAP_SIZE // 2) + 1 )
+# python -m pip install Pillow
+import struct, numpy
+from sourcehold.tool.memory.map.common import get_process_handle, validate_input_path
+from sourcehold.world import create_selection_matrix
+import cv2 as cv # type: ignore
+from .logics import logic1, logic1_vk, logic2, logic2_vk
+from .colors import DEFAULT_PALETTE, Palette
+import sys
+
+def get_image_data(img_path):
+  img = cv.imread(img_path)
+  return img
+
+selection = create_selection_matrix(size=MAP_SIZE)
+
+# (Little endian) unsigned bytes
+def get_raw_logic1(process):
+  return struct.unpack(f"<{TILE_COUNT}I", process.read_section('1003'))
+
+def set_raw_logic1(process, data):
+  serialized = struct.pack(f"<{TILE_COUNT}I", *data)
+  # Logical terrain height layer
+  process.write_section('1003', serialized)
+
+def get_raw_logic2(process):
+  return struct.unpack(f"<{TILE_COUNT}B", process.read_section('1037'))
+
+def set_raw_logic2(process, data):
+  serialized = struct.pack(f"<{TILE_COUNT}B", *data)
+  # Logical terrain height layer
+  process.write_section('1037', serialized)
+
+
+# 16 is used for the inaccessible parts of the map, including the outer border of the 800x800 space, and 32 is used for a border just within that
+
+def set_terrain(args):
+  #' returns None in case of non applicable
+  if args.what != 'terrain':
+    return None
+  
+  if args.action != "set":
+     return None  
+  
+  if args.palette:
+    palette = Palette(args.palette)
+  else:
+    palette = DEFAULT_PALETTE
+
+  img_path = args.input
+  validate_input_path(img_path)
+  img = get_image_data(img_path)
+
+  process = get_process_handle(args.game)
+
+  logic1matrix = numpy.zeros((400, 400), dtype='uint32')
+  borderlogic1matrix = numpy.zeros((400, 400), dtype='uint32')
+  borderlogic1matrix[selection] = get_raw_logic1(process) # We load it here to get the map borders only...
+  logic1matrix[borderlogic1matrix & logic1['border'] != 0] |= logic1['border']
+  logic1matrix[borderlogic1matrix & logic1['border_edge'] != 0] |= logic1['border_edge']
+  logic1matrix[borderlogic1matrix == 0] = 0 
+
+  logic2matrix = numpy.zeros((400, 400), dtype='uint8')
+  # logic2matrix[selection] = get_raw_logic2(process)
+
+  for color, name in palette.bgr_palette.items():
+    where = (img == color).all(2)
+    logic1matrix[where] |= logic1[name]
+    if name in logic2:
+      logic2matrix[where] = logic2[name]
+
+  set_raw_logic1(process, logic1matrix[selection].flat)
+  set_raw_logic2(process, logic2matrix[selection].flat)
+
+  return True
+
+def get_terrain(args):
+  #' returns None in case of non applicable
+  if args.what != 'terrain':
+    return None
+  
+  if args.action != "get":
+     return None  
+  
+  if args.palette:
+    palette = Palette(args.palette)
+  else:
+    palette = DEFAULT_PALETTE
+
+  process = get_process_handle(args.game)
+
+  logic1matrix = numpy.zeros((400, 400), dtype='uint32')
+  logic1matrix[selection] = get_raw_logic1(process)
+
+  logic2matrix = numpy.zeros((400, 400), dtype='uint8')
+  logic2matrix[selection] = get_raw_logic2(process)
+
+  img = numpy.zeros((400,400,3), dtype='uint8')
+
+  if args.debug:
+    print("logic1")
+  for flag, name in logic1_vk.items():
+    color = (0, 0, 0)
+    if name in palette.palette_bgr:
+      color = palette.palette_bgr[name]
+    else:
+      if args.debug:
+        print(f"skipping color for: {name}")
+    img[logic1matrix & flag != 0] = color
+    if args.debug:
+      print(f"set '{name}' {img[logic1matrix & flag != 0].sum()} times to color: {palette.bgr_palette[color]}")
+
+  if args.debug:
+    print("logic2")
+  for flag, name in logic2_vk.items():
+    if name == 'none':
+      continue
+    color = (0, 0, 0)
+    where = (logic1matrix & logic1['default_earth_or_texture']) != 0
+    if name in palette.palette_bgr:
+      color = palette.palette_bgr[name]
+    else:
+      if args.debug:
+        print(f"skipping color for: {name}")
+    img[where & (logic2matrix == flag)] = color
+    if args.debug:
+      print(f"set '{name}' {img[where & (logic2matrix == flag)].sum()} times to color: {palette.bgr_palette[color]}")
+
+  if not args.output:
+    if args.debug:
+      print(args.output_format)
+    sys.stdout.buffer.write(cv.imencode(f".{args.output_format}", img)[1].tobytes())
+  else:
+    cv.imwrite(args.output, img=img)
+
+  return True