diff --git a/lua/modules/Madokami.lua b/lua/modules/Madokami.lua index 535bc0613..41c5ba5b4 100644 --- a/lua/modules/Madokami.lua +++ b/lua/modules/Madokami.lua @@ -12,11 +12,13 @@ function Init() m.OnGetPageNumber = 'GetPageNumber' m.OnGetNameAndLink = 'GetNameAndLink' m.OnGetDirectoryPageNumber = 'GetDirectoryPageNumber' + m.OnDownloadImage = 'DownloadImage' m.MaxTaskLimit = 1 m.MaxConnectionLimit = 4 m.AccountSupport = true m.OnLogin = 'Login' m.OnAccountState = 'AccountState' + local fmd = require 'fmd.env' local slang = fmd.SelectedLanguage @@ -35,6 +37,7 @@ function Init() end } m.AddOptionSpinEdit('mdkm_delay', lang:get('delay'), 2) + m.AddOptionComboBox('jxl_convert_target', 'Save JXL Images as', table.concat(jxl_supported_converted_target(), "\r\n"), png_index-1) m.Storage['madokamiulist'] = '' end @@ -42,6 +45,8 @@ end -- Local Constants ---------------------------------------------------------------------------------------------------- local json = require("utils.json") +local jxlconverter= require("utils.jxlconverter") +local crypto = require('fmd.crypto') local madokamilist_chr = {} local madokamilist_custom = {'_', 'Oneshots'} -- Add A to Z @@ -52,6 +57,16 @@ end for _, value in ipairs(madokamilist_custom) do madokamilist_chr[value .. '/'] = true end + +local supported_extensions = {".jpg", ".png", ".gif", ".webp", ".jxl"} +png_index = nil + +for i, ext in ipairs(supported_extensions) do + if ext == ".png" then + png_index = i + break + end +end ---------------------------------------------------------------------------------------------------- -- Helper Functions ---------------------------------------------------------------------------------------------------- @@ -78,6 +93,18 @@ local function Delay() MODULE.Storage['lastDelay'] = os.time() end +function jxl_supported_converted_target() + local cleaned = {} + for _, ext in ipairs(supported_extensions) do + if ext ~= ".jxl" then + table.insert(cleaned, ext:sub(2)) + end + end + --local result = table.concat(cleaned, "\r\n") + --return result + return cleaned +end + ---------------------------------------------------------------------------------------------------- -- Event Functions ---------------------------------------------------------------------------------------------------- @@ -86,7 +113,7 @@ function Login() MODULE.ClearCookies() MODULE.Account.Status = asChecking local login_url=MODULE.RootURL - local crypto = require 'fmd.crypto' + --local crypto = require 'fmd.crypto' if not HTTP.GET(login_url) then MODULE.Account.Status = asUnknown return net_problem @@ -150,21 +177,66 @@ end -- Get the page count and/or page links for the current chapter. function GetPageNumber() - local crypto = require 'fmd.crypto' + --local crypto = require 'fmd.crypto' Delay() CheckAuth() HTTP.Headers.Values['charset'] = 'utf-8' - HTTP.GET(MODULE.RootURL .. URL) - if HTTP.ResultCode ~= 200 then - return net_problem - end - local x = CreateTXQuery(HTTP.Document) - local datapath = x.XPathString('//div[@id="reader"]/@data-path') - datapath = crypto.EncodeURLElement(datapath) - local datafiles = x.XPathString('//div[@id="reader"]/@data-files') - datafiles = json.decode(datafiles) - for i=1, #datafiles do - TASK.PageLinks.Add(MODULE.RootURL .. '/reader/image?path=' .. datapath .. '&file=' .. crypto.EncodeURLElement(datafiles[i])) + local u = MaybeFillHost(MODULE.RootURL, URL) + HTTP.GET(u) + if HTTP.ResultCode == 500 then + local x = CreateTXQuery(HTTP.Document) + local err_msg = x.XPathString('//div[@class="container"]/p') + print(err_msg) + if err_msg == "No valid image files found in archive" then + print("Trying alt method to get pages links") + local datapath = u:gsub(string.gsub("https://manga.madokami.al/reader/", "([^%w])", "%%%1"), "", 1) --extract datapath from url + datapath =crypto.DecodeURL(datapath) + local tmp_u = crypto.DecodeURL(datapath) --need to do decoding 2 times + tmp_u = MaybeFillHost(MODULE.RootURL, datapath) + HTTP.HEAD(tmp_u) + local file_size = tonumber(HTTP.Headers.Values["content-length"]) + HTTP.Reset() + HTTP.Headers.Values['Range'] = "bytes=" .. math.max(0, file_size - (1024*32)) .. "-" .. (file_size - 1) + HTTP.GET(tmp_u) + if HTTP.ResultCode == 206 then -- Partial Content success + local i = 0 + local body = HTTP.Document.ToString() + while i <= #body - 46 do + if body:sub(i, i+3) == "\x50\x4b\x01\x02" then + local b1 = body:byte(i + 28) + local b2 = body:byte(i + 29) + local name_len = b1 + (b2 * 256) + local filename = body:sub(i + 46, i + 46 + name_len - 1) + if #filename > 0 then + for _, ext in ipairs(supported_extensions) do + if string.sub(filename:lower(), -#ext) == ext then + TASK.PageLinks.Add(MODULE.RootURL .. '/reader/image?path=' .. datapath .. '&file=' .. crypto.EncodeURLElement(filename)) + end + end + end + i = i + 46 + name_len + else + i = i + 1 + end + end + if TASK.PageLinks.Count == 0 then + print('The Images files is in not supported type') + end + else + print("Error: Server does not support Range Requests.") + end + end + elseif HTTP.ResultCode ~= 200 then + return net_problem + else + local x = CreateTXQuery(HTTP.Document) + local datapath = x.XPathString('//div[@id="reader"]/@data-path') + datapath = crypto.EncodeURLElement(datapath) + local datafiles = x.XPathString('//div[@id="reader"]/@data-files') + datafiles = json.decode(datafiles) + for i=1, #datafiles do + TASK.PageLinks.Add(MODULE.RootURL .. '/reader/image?path=' .. datapath .. '&file=' .. crypto.EncodeURLElement(datafiles[i])) + end end end @@ -213,3 +285,16 @@ function GetNameAndLink() LINKS, NAMES) end end + +-- Download and decrypt image given the image URL. +function DownloadImage() + Delay() + CheckAuth() + if not HTTP.GET(URL) then return false end + if URL:sub(-4) == ".jxl" then + --URL is a JXL image + local jpg_data = jxlconverter.convert(HTTP.Document.ToString(), jxl_supported_converted_target()[MODULE.GetOption('jxl_convert_target')-1]) + HTTP.Document.WriteString(jpg_data) + end + return true +end diff --git a/lua/utils/jxl2jpg.lua b/lua/utils/jxl2jpg.lua new file mode 100644 index 000000000..ef37c479f --- /dev/null +++ b/lua/utils/jxl2jpg.lua @@ -0,0 +1,196 @@ +local json = require("utils.json") +local crypto = require('fmd.crypto') +local nodejs = require("utils.nodejs") + +local jxl2jpg_executor = {} +local debugging = false +local b64chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + +-- Centralized error handling function +local function handle_error(message) + return "Error: " .. (message or "An unknown error occurred.") +end + +local function stringify(value) + if type(value) == "table" then + return "" + elseif type(value) == "userdata" then + return "" + elseif type(value) == "function" then + return "" + else + return tostring(value) + end +end + +local function safe_concat(...) + local args = {...} + for i = 1, #args do + args[i] = stringify(args[i]) -- Convert each element to a string + end + return table.concat(args, " ") +end + +-- Centralized debugging function +local function debug_print(...) + if debugging then + local message = "Utils[JXL2JPG]: " .. safe_concat(...) + print(message) + end +end + +local function b64_encode(data) + local parts = {} + local i = 1 + while i <= #data do + local a = string.byte(data, i) + local b = string.byte(data, i + 1) + local c = string.byte(data, i + 2) + if a and b and c then + local n = a * 65536 + b * 256 + c + parts[#parts + 1] = b64chars:sub(math.floor(n / 262144) % 64 + 1, math.floor(n / 262144) % 64 + 1) + parts[#parts + 1] = b64chars:sub(math.floor(n / 4096) % 64 + 1, math.floor(n / 4096) % 64 + 1) + parts[#parts + 1] = b64chars:sub(math.floor(n / 64) % 64 + 1, math.floor(n / 64) % 64 + 1) + parts[#parts + 1] = b64chars:sub(n % 64 + 1, n % 64 + 1) + elseif a and b then + local n = a * 256 + b + parts[#parts + 1] = b64chars:sub(math.floor(n / 1024) % 64 + 1, math.floor(n / 1024) % 64 + 1) + parts[#parts + 1] = b64chars:sub(math.floor(n / 16) % 64 + 1, math.floor(n / 16) % 64 + 1) + parts[#parts + 1] = b64chars:sub((n % 16) * 4 + 1, (n % 16) * 4 + 1) + parts[#parts + 1] = "=" + else + parts[#parts + 1] = b64chars:sub(math.floor(a / 4) + 1, math.floor(a / 4) + 1) + parts[#parts + 1] = b64chars:sub((a % 4) * 16 + 1, (a % 4) * 16 + 1) + parts[#parts + 1] = "==" + end + i = i + 3 + end + return table.concat(parts) +end + +local function b64_decode(str) + str = str:gsub("%s", "") + local parts = {} + local i = 1 + while i < #str do + local c1, c2, c3, c4 = str:sub(i, i), str:sub(i + 1, i + 1), str:sub(i + 2, i + 2), str:sub(i + 3, i + 3) + if c1 == "" or c2 == "" then break end + local v1 = b64chars:find(c1, 1, true) - 1 + local v2 = b64chars:find(c2, 1, true) - 1 + local v3 = (c3 ~= "=") and (b64chars:find(c3, 1, true) - 1) or 0 + local v4 = (c4 ~= "=") and (b64chars:find(c4, 1, true) - 1) or 0 + local n = v1 * 262144 + v2 * 4096 + v3 * 64 + v4 + parts[#parts + 1] = string.char(math.floor(n / 65536) % 256) + if c3 ~= "=" then parts[#parts + 1] = string.char(math.floor(n / 256) % 256) end + if c4 ~= "=" then parts[#parts + 1] = string.char(n % 256) end + i = i + 4 + end + return table.concat(parts) +end + +local function __convert(jxl_data) + if not jxl_data then return handle_error("No JXL data was provided.") end + debug_print('JXL data size:', #jxl_data) + + --Convering JXL data to base64 + debug_print('Convering JXL data to base64') + local jxl_data64 = b64_encode(jxl_data) + debug_print('JXL base64 data size:', #jxl_data64) + + --Writing js_code to pass to nodejs + local js_code = [=[ + +var b64 = "]=] .. jxl_data64 .. [=["; + +const { fileURLToPath } = require('node:url'); +const { dirname, join } = require('node:path'); +const fs = require('fs'); +var initJXLDecode = require('@jsquash/jxl/decode.js').init; +var jxl_decode = require("@jsquash/jxl").decode; +var jpeg = require("jpeg-js"); + +function buf_to_arrybuf (buf) { + const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); + return arrayBuffer +} + +const wasmPath = join(__dirname, 'node_modules/@jsquash/jxl/codec/dec/jxl_dec.wasm'); + +var buf = Buffer.from(b64, "base64"); +var arrybuf = buf_to_arrybuf(buf); +b64 = null; + +(async () => { + try { + const wasmBuffer = fs.readFileSync(wasmPath); + const wasmModule = await WebAssembly.compile(wasmBuffer); + + // Initialize your @jsquash/jxl module here + await initJXLDecode(wasmModule); + + const imageData = await jxl_decode(arrybuf); + + var jpg = jpeg.encode(imageData, 90); + + var out = jpg.data.toString("base64"); + + console.log(JSON.stringify( + { + 'status': 'Success', + 'width': imageData.width, + 'height': imageData.height, + //'base64_test':out.slice(0, 100), + 'base64':out, + } + )); + + } catch (error) { + console.error("Initialization failed:", error); + process.exit(1); + } +})(); + +]=] + debug_print('js_code:', js_code) + --clean variable jxl_data64 + jxl_data64 = nil + debug_print('Start Running js_code in Nonejs') + local output = nodejs.run_js(js_code, nil, nil, false) + debug_print('Nodejs ouput size:',#output) + debug_print('Nodejs ouput:',output) + + --clean variable js_code + js_code = nil + + --Convering JXL data to base64 + debug_print('Convering JXL data to base64') + if output and output ~= "" then + --Parsing Nonejs output + debug_print('Parsing Nonejs output') + local jpg_data64 = json.decode(output)["base64"] + debug_print('jpg base64 data size:', #jpg_data64) + + --Convering JPG base64 to data + debug_print('Convering JPG base64 to data') + local jpg_data = crypto.DecodeBase64(jpg_data64) + debug_print('JPG data size:', #jpg_data) + + --clean variable output + output = nil + + if jpg_data and jpg_data ~= "" then + return jpg_data + end + end + return true +end + + + +-- Public functions +function jxl2jpg_executor.convert(jxl_data) + debug_print('Start Convering JXL to JPG...') + return __convert(jxl_data) +end + +return jxl2jpg_executor \ No newline at end of file diff --git a/lua/utils/jxlconverter.lua b/lua/utils/jxlconverter.lua new file mode 100644 index 000000000..7237ae87a --- /dev/null +++ b/lua/utils/jxlconverter.lua @@ -0,0 +1,224 @@ +local json = require("utils.json") +local crypto = require('fmd.crypto') +local nodejs = require("utils.nodejs") + +local jxlconverter_executor = {} +local debugging = false +local b64chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + +-- Centralized error handling function +local function handle_error(message) + return "Error: " .. (message or "An unknown error occurred.") +end + +local function stringify(value) + if type(value) == "table" then + return "
" + elseif type(value) == "userdata" then + return "" + elseif type(value) == "function" then + return "" + else + return tostring(value) + end +end + +local function safe_concat(...) + local args = {...} + for i = 1, #args do + args[i] = stringify(args[i]) -- Convert each element to a string + end + return table.concat(args, " ") +end + +-- Centralized debugging function +local function debug_print(...) + if debugging then + local message = "Utils[JXL_Converter]: " .. safe_concat(...) + print(message) + end +end + +local function b64_encode(data) + local parts = {} + local i = 1 + while i <= #data do + local a = string.byte(data, i) + local b = string.byte(data, i + 1) + local c = string.byte(data, i + 2) + if a and b and c then + local n = a * 65536 + b * 256 + c + parts[#parts + 1] = b64chars:sub(math.floor(n / 262144) % 64 + 1, math.floor(n / 262144) % 64 + 1) + parts[#parts + 1] = b64chars:sub(math.floor(n / 4096) % 64 + 1, math.floor(n / 4096) % 64 + 1) + parts[#parts + 1] = b64chars:sub(math.floor(n / 64) % 64 + 1, math.floor(n / 64) % 64 + 1) + parts[#parts + 1] = b64chars:sub(n % 64 + 1, n % 64 + 1) + elseif a and b then + local n = a * 256 + b + parts[#parts + 1] = b64chars:sub(math.floor(n / 1024) % 64 + 1, math.floor(n / 1024) % 64 + 1) + parts[#parts + 1] = b64chars:sub(math.floor(n / 16) % 64 + 1, math.floor(n / 16) % 64 + 1) + parts[#parts + 1] = b64chars:sub((n % 16) * 4 + 1, (n % 16) * 4 + 1) + parts[#parts + 1] = "=" + else + parts[#parts + 1] = b64chars:sub(math.floor(a / 4) + 1, math.floor(a / 4) + 1) + parts[#parts + 1] = b64chars:sub((a % 4) * 16 + 1, (a % 4) * 16 + 1) + parts[#parts + 1] = "==" + end + i = i + 3 + end + return table.concat(parts) +end + +local function b64_decode(str) + str = str:gsub("%s", "") + local parts = {} + local i = 1 + while i < #str do + local c1, c2, c3, c4 = str:sub(i, i), str:sub(i + 1, i + 1), str:sub(i + 2, i + 2), str:sub(i + 3, i + 3) + if c1 == "" or c2 == "" then break end + local v1 = b64chars:find(c1, 1, true) - 1 + local v2 = b64chars:find(c2, 1, true) - 1 + local v3 = (c3 ~= "=") and (b64chars:find(c3, 1, true) - 1) or 0 + local v4 = (c4 ~= "=") and (b64chars:find(c4, 1, true) - 1) or 0 + local n = v1 * 262144 + v2 * 4096 + v3 * 64 + v4 + parts[#parts + 1] = string.char(math.floor(n / 65536) % 256) + if c3 ~= "=" then parts[#parts + 1] = string.char(math.floor(n / 256) % 256) end + if c4 ~= "=" then parts[#parts + 1] = string.char(n % 256) end + i = i + 4 + end + return table.concat(parts) +end + +local function __convert(jxl_data, convert_target) + if not jxl_data then return handle_error("No JXL data was provided.") end + convert_target = convert_target or 'png' + + debug_print('JXL data size:', #jxl_data) + + --Convering JXL data to base64 + debug_print('Convering JXL data to base64') + local jxl_data64 = b64_encode(jxl_data) + debug_print('JXL base64 data size:', #jxl_data64) + + --Writing js_code to pass to nodejs + local js_code = [=[ + +var __input = {'convert_target': ']=] .. convert_target .. [=[', 'base64': ']=] .. jxl_data64 .. [=['}; +var b64 = __input.base64; +const { join } = require('node:path'); +const fs = require('fs'); +var initJXLDecode = require('@jsquash/jxl/decode.js').init; +var jxl_decode = require("@jsquash/jxl").decode; +var sharp = require("sharp"); + +const SUPPORTED_TARGETS = ['png', 'jpg', 'jpeg', 'webp', 'gif']; +const FORMAT_META = { + png: { mime: 'image/png', extension: 'png' }, + jpg: { mime: 'image/jpeg', extension: 'jpg' }, + jpeg: { mime: 'image/jpeg', extension: 'jpg' }, + webp: { mime: 'image/webp', extension: 'webp' }, + gif: { mime: 'image/gif', extension: 'gif' }, +}; + +function buf_to_arrybuf(buf) { + const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); + return arrayBuffer; +} + +function fail(error_msg) { + console.log(JSON.stringify({ status: 'Failed', error_msg })); + process.exit(1); +} + +const wasmPath = join(__dirname, 'node_modules/@jsquash/jxl/codec/dec/jxl_dec.wasm'); +var buf = Buffer.from(b64, "base64"); +var arrybuf = buf_to_arrybuf(buf); +b64 = null; + +(async () => { + try { + const rawTarget = __input.convert_target; + const target = SUPPORTED_TARGETS.includes(rawTarget) ? rawTarget : 'png'; + const meta = FORMAT_META[target]; + + const wasmBuffer = fs.readFileSync(wasmPath); + const wasmModule = await WebAssembly.compile(wasmBuffer); + await initJXLDecode(wasmModule); + + const imageData = await jxl_decode(arrybuf); + + const rawBuffer = Buffer.from(imageData.data); + const sharpImg = sharp(rawBuffer, { + raw: { width: imageData.width, height: imageData.height, channels: 4 } + }); + + var out; + if (target === 'png') { + out = (await sharpImg.png().toBuffer()).toString("base64"); + } else if (target === 'jpg' || target === 'jpeg') { + out = (await sharpImg.jpeg({ quality: 90 }).toBuffer()).toString("base64"); + } else if (target === 'webp') { + out = (await sharpImg.webp({ quality: 90 }).toBuffer()).toString("base64"); + } else if (target === 'gif') { + out = (await sharpImg.gif().toBuffer()).toString("base64"); + } + + console.log(JSON.stringify({ + status: 'Success', + convert_target: target, + mime: meta.mime, + extension: meta.extension, + width: imageData.width, + height: imageData.height, + base64: out, + })); + + } catch (error) { + fail(error.message || String(error)); + } +})(); + +]=] + debug_print('js_code:', js_code) + --clean variable jxl_data64 + jxl_data64 = nil + debug_print('Start Running js_code in Nonejs') + local output = nodejs.run_js(js_code, nil, nil, false) + debug_print('Nodejs ouput size:',#output) + debug_print('Nodejs ouput:',output) + + --clean variable js_code + js_code = nil + + --Convering JXL data to base64 + debug_print('Convering JXL data to base64') + if output and output ~= "" then + --Parsing Nonejs output + debug_print('Parsing Nonejs output') + local jpg_data64 = json.decode(output)["base64"] + debug_print('jpg base64 data size:', #jpg_data64) + + --Convering JPG base64 to data + debug_print('Convering JPG base64 to data') + local jpg_data = crypto.DecodeBase64(jpg_data64) + debug_print('JPG data size:', #jpg_data) + + --clean variable output + output = nil + + if jpg_data and jpg_data ~= "" then + return jpg_data + end + end + return true +end + + + +-- Public functions +function jxlconverter_executor.convert(jxl_data, convert_target) + convert_target = convert_target or 'png' + debug_print('Start Convering JXL to ' .. string.upper(convert_target) .. '...') + return __convert(jxl_data, convert_target) +end + +return jxlconverter_executor \ No newline at end of file diff --git a/lua/utils/nodejs.lua b/lua/utils/nodejs.lua index 63035ed0b..eb255818a 100644 --- a/lua/utils/nodejs.lua +++ b/lua/utils/nodejs.lua @@ -92,7 +92,7 @@ local function install_required_modules(js_code) local success, err = ensure_install_directory(install_dir) if not success then debug_print(err) return false, err end - modules = {"puppeteer"} + modules = {"puppeteer", "@jsquash/jxl", "sharp"} --for mod in js_code:gmatch("require%s*%(%s*['\"](.-)['\"]%s*%)") do -- auto install any npm modules required by the script for _, mod in pairs(modules) do if not is_module_installed(mod, install_dir) then @@ -123,17 +123,20 @@ local function execute_js_script(js_code) return output end -local function isolatevm_js(js_code, pass_page) +local function isolatevm_js(js_code, pass_page, timeout_ms, globals) if not js_code then return handle_error("No JavaScript code provided.") end if pass_page then return js_code end - local safe_js_code = string.format("%q", js_code) + timeout_ms = timeout_ms or 5000 + globals = globals or {} + --local safe_js_code = string.format("%q", js_code) return [[ const vm = require('vm'); (async () => { const sandbox = { + ]] .. table.concat(globals, ",") ..[[ console: { log: (...args) => { console.log(...args); }, error: (...args) => { console.error(...args); } @@ -141,14 +144,14 @@ local function isolatevm_js(js_code, pass_page) }; vm.createContext(sandbox); - const jsCode = ]] .. safe_js_code .. [[; + const jsCode = `]] .. js_code .. [[`; try { const p = vm.runInContext('(async () => { ' + jsCode + ' })()', sandbox); await Promise.race([ p, new Promise((_, reject) => - setTimeout(() => reject(new Error('Execution timed out')), 5000) + setTimeout(() => reject(new Error('Execution timed out')), ]] .. timeout_ms .. [[) ) ]); } catch (error) { @@ -208,8 +211,13 @@ local function run_html_with_js(url, js_code) end -- Public functions -function node_executor.run_js(js_code) - return execute_js_script(isolatevm_js(js_code)) +function node_executor.run_js(js_code, timeout_ms, globals, run_in_sandbox) + run_in_sandbox = run_in_sandbox and true + if run_in_sandbox then + return execute_js_script(isolatevm_js(js_code, nil, timeout_ms, globals)) + else + return execute_js_script(js_code) + end end function node_executor.run_html_load(url)