Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions sample/cpp/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,10 @@ int main([[maybe_unused]] int argc, char** argv)
std::string user_path = "./";
MaaToolkitConfigInitOption(user_path.c_str(), "{}");

auto controller_handle = create_adb_controller();
// auto controller_handle = create_win32_controller();
// auto controller_handle = create_adb_controller();
auto controller_handle = create_win32_controller();
auto ctrl_id = MaaControllerPostConnection(controller_handle);
MaaControllerPostScreencap(controller_handle);

auto resource_handle = MaaResourceCreate();
std::string resource_dir = R"(E:\Code\MaaFramework\sample\resource)";
Expand Down Expand Up @@ -127,18 +128,15 @@ MaaController* create_win32_controller()
std::string class_name = MaaToolkitDesktopWindowGetClassName(window_handle);
std::string window_name = MaaToolkitDesktopWindowGetWindowName(window_handle);

if (window_name.find("二重螺旋") != std::string::npos) {
if (window_name.find("Chrome") != std::string::npos) {
hwnd = MaaToolkitDesktopWindowGetHandle(window_handle);
break;
}
}

// create controller by hwnd
auto controller_handle = MaaWin32ControllerCreate(
hwnd,
MaaWin32ScreencapMethod_DXGI_DesktopDup_Window,
MaaWin32InputMethod_SendMessage,
MaaWin32InputMethod_SendMessage);
auto controller_handle =
MaaWin32ControllerCreate(hwnd, MaaWin32ScreencapMethod_FramePool, MaaWin32InputMethod_SendMessage, MaaWin32InputMethod_SendMessage);

destroy();
return controller_handle;
Expand Down
56 changes: 53 additions & 3 deletions source/MaaWin32ControlUnit/Screencap/DesktopDupScreencap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
#include "MaaUtils/ImageIo.h"
#include "MaaUtils/Logger.h"

#include <array>

MAA_CTRL_UNIT_NS_BEGIN

DesktopDupScreencap::~DesktopDupScreencap()
Expand Down Expand Up @@ -231,7 +233,33 @@ bool DesktopDupScreencap::init_primary_output()

bool DesktopDupScreencap::init_output_duplication()
{
HRESULT ret = dxgi_output_->DuplicateOutput(d3d_device_, &dxgi_dup_);
// 尝试使用 DuplicateOutput1 以支持 HDR 格式
IDXGIOutput5* output5 = nullptr;
HRESULT ret = dxgi_output_->QueryInterface(__uuidof(IDXGIOutput5), reinterpret_cast<void**>(&output5));
if (SUCCEEDED(ret) && output5) {
// 指定支持的格式,包括 HDR 格式
// 优先使用原始格式以获得最佳质量
std::array<DXGI_FORMAT, 4> formats = {
DXGI_FORMAT_R16G16B16A16_FLOAT, // HDR scRGB
DXGI_FORMAT_R10G10B10A2_UNORM, // HDR10
DXGI_FORMAT_B8G8R8A8_UNORM, // SDR
DXGI_FORMAT_R8G8B8A8_UNORM, // SDR
};

ret = output5->DuplicateOutput1(d3d_device_, 0, static_cast<UINT>(formats.size()), formats.data(), &dxgi_dup_);
output5->Release();

if (SUCCEEDED(ret)) {
DXGI_OUTDUPL_DESC dup_desc;
dxgi_dup_->GetDesc(&dup_desc);
LogInfo << "DuplicateOutput1 succeeded with format" << VAR(static_cast<int>(dup_desc.ModeDesc.Format));
return true;
}
LogWarn << "DuplicateOutput1 failed, falling back to DuplicateOutput" << VAR(ret);
}

// 回退到旧的 API
ret = dxgi_output_->DuplicateOutput(d3d_device_, &dxgi_dup_);
if (FAILED(ret)) {
LogError << "DuplicateOutput failed" << VAR(ret);
return false;
Expand Down Expand Up @@ -300,6 +328,29 @@ void DesktopDupScreencap::uninit()
}
}

cv::Mat DesktopDupScreencap::process_texture_data(const D3D11_MAPPED_SUBRESOURCE& mapped)
{
int width = static_cast<int>(texture_desc_.Width);
int height = static_cast<int>(texture_desc_.Height);
int row_pitch = static_cast<int>(mapped.RowPitch);

switch (texture_desc_.Format) {
case DXGI_FORMAT_R16G16B16A16_FLOAT:
LogDebug << "Processing HDR R16G16B16A16_FLOAT format";
return hdr_float16_to_sdr_bgra(mapped.pData, width, height, row_pitch);

case DXGI_FORMAT_R10G10B10A2_UNORM:
LogDebug << "Processing HDR R10G10B10A2_UNORM format";
return hdr_r10g10b10a2_to_sdr_bgra(mapped.pData, width, height, row_pitch);

case DXGI_FORMAT_B8G8R8A8_UNORM:
case DXGI_FORMAT_R8G8B8A8_UNORM:
default:
// 标准 SDR 格式,直接返回
return cv::Mat(height, width, CV_8UC4, mapped.pData, row_pitch).clone();
}
}

std::optional<cv::Mat> DesktopDupScreencap::screencap_impl()
{
if (!d3d_context_ || !dxgi_dup_) {
Expand Down Expand Up @@ -353,8 +404,7 @@ std::optional<cv::Mat> DesktopDupScreencap::screencap_impl()
}
OnScopeLeave([&]() { d3d_context_->Unmap(readable_texture_, 0); });

cv::Mat mat(texture_desc_.Height, texture_desc_.Width, CV_8UC4, mapped.pData, mapped.RowPitch);
return mat;
return process_texture_data(mapped);
}

MAA_CTRL_UNIT_NS_END
1 change: 1 addition & 0 deletions source/MaaWin32ControlUnit/Screencap/DesktopDupScreencap.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ class DesktopDupScreencap : public ScreencapBase
bool init_texture(ID3D11Texture2D* raw_texture);
void uninit();
std::optional<cv::Mat> screencap_impl();
cv::Mat process_texture_data(const D3D11_MAPPED_SUBRESOURCE& mapped);

ID3D11Device* d3d_device_ = nullptr;
ID3D11DeviceContext* d3d_context_ = nullptr;
Expand Down
41 changes: 38 additions & 3 deletions source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,30 @@ std::optional<cv::Mat> FramePoolScreencap::screencap()
}
OnScopeLeave([&]() { d3d_context_->Unmap(readable_texture_.get(), 0); });

cv::Mat raw(texture_desc_.Height, texture_desc_.Width, CV_8UC4, mapped.pData, mapped.RowPitch);
// 根据纹理格式处理数据
cv::Mat raw;
int width = static_cast<int>(texture_desc_.Width);
int height = static_cast<int>(texture_desc_.Height);
int row_pitch = static_cast<int>(mapped.RowPitch);

switch (texture_desc_.Format) {
case DXGI_FORMAT_R16G16B16A16_FLOAT:
LogDebug << "Processing HDR R16G16B16A16_FLOAT format";
raw = hdr_float16_to_sdr_bgra(mapped.pData, width, height, row_pitch);
break;

case DXGI_FORMAT_R10G10B10A2_UNORM:
LogDebug << "Processing HDR R10G10B10A2_UNORM format";
raw = hdr_r10g10b10a2_to_sdr_bgra(mapped.pData, width, height, row_pitch);
break;

case DXGI_FORMAT_B8G8R8A8_UNORM:
case DXGI_FORMAT_R8G8B8A8_UNORM:
default:
// 标准 SDR 格式
raw = cv::Mat(height, width, CV_8UC4, mapped.pData, row_pitch).clone();
break;
}

// 先按 alpha 通道裁剪掉四周 alpha != 255 的边框
cv::Mat alpha_channel;
Expand Down Expand Up @@ -233,12 +256,24 @@ bool FramePoolScreencap::init()
return false;
}

auto d3d_device = inspectable.as<winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice>();

// 优先尝试 HDR 格式(R16G16B16A16Float),如果失败则回退到 SDR 格式
cap_frame_pool_ = winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool::Create(
inspectable.as<winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice>(),
winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized,
d3d_device,
winrt::Windows::Graphics::DirectX::DirectXPixelFormat::R16G16B16A16Float,
1,
cap_item_.Size());

if (!cap_frame_pool_) {
LogInfo << "Failed to create HDR frame pool, falling back to SDR format";
cap_frame_pool_ = winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool::Create(
d3d_device,
winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized,
1,
cap_item_.Size());
}

if (!cap_frame_pool_) {
LogError << "Direct3D11CaptureFramePool::Create failed";
return false;
Expand Down
188 changes: 188 additions & 0 deletions source/MaaWin32ControlUnit/Screencap/HwndUtils.hpp
Original file line number Diff line number Diff line change
@@ -1,10 +1,17 @@
#pragma once

#include <algorithm>
#include <cmath>
#include <utility>

#include "Common/Conf.h"
#include "MaaUtils/NoWarningCV.hpp"
#include "MaaUtils/SafeWindows.hpp"
#include "SafeDXGI.hpp"

// DirectXMath for accurate half float conversion
#include <DirectXMath.h>
#include <DirectXPackedVector.h>

MAA_CTRL_UNIT_NS_BEGIN

Expand Down Expand Up @@ -53,6 +60,187 @@ inline bool is_fullscreen(HWND hwnd)
return GetWindowLongPtr(hwnd, GWL_STYLE) & WS_POPUP;
}

inline bool is_hdr_format(DXGI_FORMAT format)
{
switch (format) {
case DXGI_FORMAT_R16G16B16A16_FLOAT:
case DXGI_FORMAT_R10G10B10A2_UNORM:
case DXGI_FORMAT_R32G32B32A32_FLOAT:
return true;
default:
return false;
}
}

// 使用 DirectXMath 将 half float (16-bit) 转换为 float
inline float half_to_float(uint16_t h)
{
return DirectX::PackedVector::XMConvertHalfToFloat(h);
}

// ACES Filmic Tone Mapping
// 这个曲线在 SDR 范围内接近线性,只压缩 HDR 高光
// 参考: https://knarkowicz.wordpress.com/2016/01/06/aces-filmic-tone-mapping-curve/
inline float aces_filmic_tonemap(float x)
{
constexpr float a = 2.51f;
constexpr float b = 0.03f;
constexpr float c = 2.43f;
constexpr float d = 0.59f;
constexpr float e = 0.14f;
return (x * (a * x + b)) / (x * (c * x + d) + e);
}

// scRGB/HDR 线性值转换为 sRGB gamma
inline float linear_to_srgb_gamma(float linear)
{
if (linear <= 0.0031308f) {
return linear * 12.92f;
}
return 1.055f * std::pow(linear, 1.0f / 2.4f) - 0.055f;
}

// 将 R16G16B16A16_FLOAT HDR (scRGB) 图像转换为 SDR BGRA 图像
// scRGB 色彩空间:1.0 = 80 nits(参考白点)
// sdr_white_level:Windows HDR 模式下 SDR 内容的白点亮度(nits),默认 200
inline cv::Mat hdr_float16_to_sdr_bgra(const void* data, int width, int height, int row_pitch, float sdr_white_level = 200.0f)
{
cv::Mat result(height, width, CV_8UC4);

const uint8_t* src_row = static_cast<const uint8_t*>(data);

// scRGB 中 1.0 = 80 nits
// Windows HDR 模式下,SDR 白点通常设置为 200 nits(可在系统设置中调整)
// 需要将 scRGB 值除以 (sdr_white_level / 80) 来归一化
constexpr float scrgb_white_nits = 80.0f;
const float normalization_factor = scrgb_white_nits / sdr_white_level;

for (int y = 0; y < height; ++y) {
const uint16_t* src = reinterpret_cast<const uint16_t*>(src_row);
cv::Vec4b* dst = result.ptr<cv::Vec4b>(y);

for (int x = 0; x < width; ++x) {
// R16G16B16A16_FLOAT: 每个通道 16 bit,顺序为 RGBA
float r = half_to_float(src[0]);
float g = half_to_float(src[1]);
float b = half_to_float(src[2]);
float a = half_to_float(src[3]);

// 处理负值(scRGB 允许负值表示超出 sRGB 色域的颜色)
r = std::max(0.0f, r);
g = std::max(0.0f, g);
b = std::max(0.0f, b);

// 根据 SDR 白点归一化:将 SDR 范围(0 到 sdr_white_level nits)映射到 0-1
r *= normalization_factor;
g *= normalization_factor;
b *= normalization_factor;

// 对超出 SDR 范围的 HDR 高光应用 ACES Filmic tone mapping
// ACES 曲线在 SDR 范围内接近线性,只压缩高光
r = aces_filmic_tonemap(r);
g = aces_filmic_tonemap(g);
b = aces_filmic_tonemap(b);

// 将线性值转换为 sRGB gamma
r = linear_to_srgb_gamma(r);
g = linear_to_srgb_gamma(g);
b = linear_to_srgb_gamma(b);

// 钳制到 [0, 1] 并转换为 8-bit
auto to_byte = [](float v) -> uint8_t {
return static_cast<uint8_t>(std::clamp(v, 0.0f, 1.0f) * 255.0f + 0.5f);
};

// OpenCV 使用 BGRA 顺序
uint8_t alpha = static_cast<uint8_t>(std::clamp(a, 0.0f, 1.0f) * 255.0f + 0.5f);
dst[x] = cv::Vec4b(to_byte(b), to_byte(g), to_byte(r), alpha);

src += 4;
}
src_row += row_pitch;
}

return result;
}

// 将 R10G10B10A2_UNORM HDR 图像转换为 SDR BGRA 图像
// 这种格式通常用于 HDR10,数据已经经过 PQ (ST.2084) 编码
// sdr_white_level:SDR 白点亮度(nits),默认 200
inline cv::Mat hdr_r10g10b10a2_to_sdr_bgra(const void* data, int width, int height, int row_pitch, float sdr_white_level = 200.0f)
{
cv::Mat result(height, width, CV_8UC4);

const uint8_t* src_row = static_cast<const uint8_t*>(data);

// PQ (ST.2084) EOTF 参数
constexpr float m1 = 0.1593017578125f;
constexpr float m2 = 78.84375f;
constexpr float c1 = 0.8359375f;
constexpr float c2 = 18.8515625f;
constexpr float c3 = 18.6875f;

// PQ EOTF: 将 PQ 编码值转换为线性光(输出范围 0-1 对应 0-10000 nits)
auto pq_eotf = [=](float e) -> float {
if (e <= 0.0f) return 0.0f;
float ep = std::pow(e, 1.0f / m2);
float num = std::max(ep - c1, 0.0f);
float den = c2 - c3 * ep;
if (den <= 0.0f) return 0.0f;
return std::pow(num / den, 1.0f / m1);
};

// PQ 峰值 10000 nits,需要归一化到 SDR 白点
constexpr float pq_peak_nits = 10000.0f;
const float normalization_factor = 1.0f / (sdr_white_level / pq_peak_nits);

for (int y = 0; y < height; ++y) {
const uint32_t* src = reinterpret_cast<const uint32_t*>(src_row);
cv::Vec4b* dst = result.ptr<cv::Vec4b>(y);

for (int x = 0; x < width; ++x) {
uint32_t pixel = src[x];

// R10G10B10A2_UNORM: R(10bit), G(10bit), B(10bit), A(2bit)
float r = static_cast<float>(pixel & 0x3FF) / 1023.0f;
float g = static_cast<float>((pixel >> 10) & 0x3FF) / 1023.0f;
float b = static_cast<float>((pixel >> 20) & 0x3FF) / 1023.0f;
float a = static_cast<float>((pixel >> 30) & 0x3) / 3.0f;

// 应用 PQ EOTF 获得线性光值(0-1 对应 0-10000 nits)
r = pq_eotf(r);
g = pq_eotf(g);
b = pq_eotf(b);

// 将 nits 归一化到 SDR 白点(sdr_white_level nits = 1.0)
r *= normalization_factor;
g *= normalization_factor;
b *= normalization_factor;

// 应用 ACES Filmic tone mapping
r = aces_filmic_tonemap(r);
g = aces_filmic_tonemap(g);
b = aces_filmic_tonemap(b);

// 将线性值转换为 sRGB gamma
r = linear_to_srgb_gamma(r);
g = linear_to_srgb_gamma(g);
b = linear_to_srgb_gamma(b);

// 钳制到 [0, 1] 并转换为 8-bit
auto to_byte = [](float v) -> uint8_t {
return static_cast<uint8_t>(std::clamp(v, 0.0f, 1.0f) * 255.0f + 0.5f);
};

uint8_t alpha = static_cast<uint8_t>(a * 255.0f + 0.5f);
dst[x] = cv::Vec4b(to_byte(b), to_byte(g), to_byte(r), alpha);
}
src_row += row_pitch;
}

return result;
}

inline cv::Mat bgra_to_bgr(const cv::Mat& src)
{
if (src.empty()) {
Expand Down
2 changes: 2 additions & 0 deletions source/MaaWin32ControlUnit/Screencap/SafeDXGI.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,5 @@

#include <d3d11.h>
#include <dxgi1_2.h>
#include <dxgi1_5.h>
#include <dxgi1_6.h>
Loading