Trade: static stat data, better error handling, fix radius jewel (PathOfBuildingCommunity#2020)

* Update trader for 0.5: regenerate mods and switch to using local stat json

* Fix newlines in querymods.lua

* Defer loading of trade stat data and fix tests

* Check that trader result items parse base item before adding them

* Fix weight generation for radius jewels again

* regen data files

* Download trade site stats automatically and convert it to Lua

* Disable spell checker and sort to try and minimise diff problems

* Cspell fixes

* Try to fix querymods.lua rune category inconsistencies

* Regenerate all data fter having properly exported game data

* Use stat transform in trader, fixes lower is better stats not counting

* Fix modCache

* Stop download stats using async download

DownloadPage is async and could download the trade stats after query mods was generated so it could use stale data

* Fix jewel corruptions

---------

Co-authored-by: LocalIdentity <localidentity2@gmail.com>

Author: vaisest

spec/System/TestTradeQueryGenerator_spec.lua

@@ -5,10 +5,9 @@ describe("TradeQueryGenerator", function()
 		-- Pass: Mod line maps correctly to trade stat entry without error
 		-- Fail: Mapping fails (e.g., no match found), indicating incomplete stat parsing for curse mods, potentially missing curse-enabling items in queries
 		it("handles special curse case", function()
-			local mod = { tradeHashes = {[30642521] = {"You can apply an additional Curse"}} }
-			local tradeStatsParsed = { result = { [2] = { entries = { { text = "You can apply # additional Curses", id = "explicit.stat_30642521" } } } } }
-			mock_queryGen.modData = { Explicit = true }
-			mock_queryGen:ProcessMod(mod, tradeStatsParsed, 1)
+			local mod = { tradeHashes = {[30642521] = {"You can apply an additional Curse"}}, type = "Prefix", weightKey = {}, weightVal = {} }
+			mock_queryGen.modData = { Explicit = {} }
+			mock_queryGen:ProcessMod(mod)
 			-- Simplified assertion; in full impl, check modData
 			assert.is_true(true)
 		end)

src/Classes/TradeHelpers.lua

@@ -32,30 +32,23 @@ function M.modLineValue(line)
 	return tonumber(line:match("%-?[%d]+%.?[%d]*"))
 end
 
--- Helper: fetch and cache the trade API stats
-local _tradeStats = nil
-local _tradeStatsFetched = false
--- contains data for stats which have options, like allocates #
-local optionTradeStatMap = {}
---- @return table
-local function getTradeStatsLookup()
+local _tradeStats
+
+---@return table? tradeStats
+function M.getTradeStats()
 	if _tradeStats then return _tradeStats end
-	local tradeStats = ""
-	local easy = common.curl.easy()
-	if not easy then return nil end
-	easy:setopt_url("https://www.pathofexile.com/api/trade2/data/stats")
-	easy:setopt_useragent("Path of Building/" .. (launch.versionNumber or ""))
-	easy:setopt_writefunction(function(d)
-		tradeStats = tradeStats .. d
-		return true
-	end)
-	local ok = easy:perform()
-	easy:close()
-	if not ok or tradeStats == "" then return {} end
-	local parsed = dkjson.decode(tradeStats)
-	_tradeStats = parsed.result
-
-	for _, cat in ipairs(_tradeStats) do
+	_tradeStats = LoadModule("Data/TradeSiteStats")
+	return _tradeStats
+end
+
+local _optionTradeStatMap
+
+---@param tradeStats table table of data from https://www.pathofexile.com/api/trade2/data/stats
+---@return table optionTradeStatMap table containing helper data for matching trade option filters
+local function getOptionTradeStatMap(tradeStats)
+	if _optionTradeStatMap then return _optionTradeStatMap end
+	local optionTradeStatMap = {}
+	for _, cat in ipairs(tradeStats) do
 		if cat.id == "enchant" or cat.id == "explicit" or cat.id == "implicit" then
 			optionTradeStatMap[cat.id] = {}
 			for _, entry in ipairs(cat.entries) do
@@ -72,7 +65,8 @@ local function getTradeStatsLookup()
 			end
 		end
 	end
-	return _tradeStats
+	_optionTradeStatMap = optionTradeStatMap
+	return _optionTradeStatMap
 end
 
 -- Map source types used in OpenBuySimilarPopup to trade API category labels
@@ -120,7 +114,7 @@ function M.shouldBeInverted(tradeId, modLine, modType)
 	if not inverseKey then
 		return false
 	end
-	for _, category in ipairs(getTradeStatsLookup()) do
+	for _, category in ipairs(M.getTradeStats() or {}) do
 		if category.id == modType then
 			for _, stat in ipairs(category.entries) do
 				if tradeId == stat.id then
@@ -132,7 +126,9 @@ function M.shouldBeInverted(tradeId, modLine, modType)
 					end
 
 					-- test for inverted mod
-					if inverseKey and ((invertedLine == formattedTradeSiteText) or (invertedLine:gsub("^%+", "") == formattedTradeSiteText)) then
+					if inverseKey
+						and ((invertedLine == formattedTradeSiteText)
+							or (invertedLine:gsub("^%+", "") == formattedTradeSiteText)) then
 						return true
 					end
 
@@ -153,30 +149,27 @@ function M.formatDatabaseText(text)
 	-- (123-124) -> #
 	text = text:gsub("%(%d+%-%d+%)", "#")
 	text = text:gsub("%d+", "#")
-	-- remove radius jewel text. the same description is used for regular and
-	-- radius jewels in the exports
-	text = text:gsub("^Notable Passive Skills in Radius also grant ", "")
-	text = text:gsub("^Small Passive Skills in Radius also grant ", "")
 	return text
 end
 
+
 -- Helper: find the trade stat ID for a mod line
---- @param item table
---- @param modLine string
---- @param modType string
---- @param isDesecrated boolean
---- the
---- @return number? hash returned for most mods
---- @return string? optionTradeId returned if the mod is an option. e.g. Allocates X
---- @return number value returned if the mod is an option and uses values. e.g. timeless jewel
+---@param item         table
+---@param modLine      string
+---@param modType      string
+---@param isDesecrated boolean
+---@return number? hash          returned for most mods
+---@return string? optionTradeId returned if the mod is an option. e.g. Allocates X
+---@return number? value         returned if the mod is an option and uses values. e.g. timeless jewel
 function M.findTradeHash(item, modLine, modType, isDesecrated)
 	local formattedLine = M.formatDatabaseText(modLine)
 	-- the data export splits some mods into different parts, even though they
 	-- are technically just one stat. we handle that here
 	local isUnique = item.rarity == "UNIQUE" or item.rarity == "RELIC"
 	local function findStat(dbMod, ignoreWeights)
 		local excludeTags = (not isUnique) and { default = true } or nil
-		if not ignoreWeights and #(dbMod.weightKey or {}) > 0 and not (item:GetModSpawnWeight(dbMod, nil, excludeTags) > 0) then
+		if not ignoreWeights and #(dbMod.weightKey or {}) > 0
+			and not (item:GetModSpawnWeight(dbMod, nil, excludeTags) > 0) then
 			return nil
 		end
 		for tradeHash, description in pairs(dbMod.tradeHashes) do
@@ -196,10 +189,9 @@ function M.findTradeHash(item, modLine, modType, isDesecrated)
 		end
 	end
 
-	-- initialise optionTradeStatMap
-	if not _tradeStats then
-		getTradeStatsLookup()
-	end
+	local tradeStats = M.getTradeStats()
+	local optionTradeStatMap = getOptionTradeStatMap(tradeStats)
+	if not tradeStats or not optionTradeStatMap then return end
 
 	for _, v in ipairs(optionTradeStatMap[modType] or {}) do
 		if v.pattern then
@@ -211,7 +203,6 @@ function M.findTradeHash(item, modLine, modType, isDesecrated)
 			return nil, v.tradeId
 		end
 	end
-	
 
 	-- desecrate-only mods
 	if isDesecrated then
@@ -230,8 +221,8 @@ function M.findTradeHash(item, modLine, modType, isDesecrated)
 				return tradeHashMaybe
 			end
 		end
-	-- most implicit and explicit applicable to the type
-	elseif modType ~= "implicit" or modType ~= "explicit" then
+		-- most implicit and explicit applicable to the type
+	elseif modType == "implicit" or modType == "explicit" then
 		for _, dbMod in pairs(item.affixes) do
 			local tradeHashMaybe = findStat(dbMod)
 			if tradeHashMaybe then

src/Classes/TradeQuery.lua

@@ -777,7 +777,7 @@ function TradeQueryClass:GetResultEvaluation(row_idx, result_index, calcFunc, ba
 		self.onlyWeightedBaseOutput[row_idx][result_index] = onlyWeightedBaseOutput
 		self.lastComparedWeightList[row_idx][result_index] = self.statSortSelectionList
 	end
-	
+
 	local slotName = self.slotTables[row_idx].nodeId and "Jewel " .. tostring(self.slotTables[row_idx].nodeId) or self.slotTables[row_idx].slotName
 	if slotName == "Megalomaniac" then
 		local addedNodes = {}
@@ -787,7 +787,7 @@ function TradeQueryClass:GetResultEvaluation(row_idx, result_index, calcFunc, ba
 				addedNodes[node] = true
 			end
 		end
-		
+
 		local output = self:ReduceOutput(calcFunc({ addNodes = addedNodes }))
 		local weight = self.tradeQueryGenerator.WeightedRatioOutputs(baseOutput, output, self.statSortSelectionList)
 		result.evaluation = {{ output = output, weight = weight }}
@@ -968,31 +968,43 @@ function TradeQueryClass:PriceItemRowDisplay(row_idx, top_pane_alignment_ref, ro
 						self:SetNotice(context.controls.pbNotice, "")
 					end
 
+					-- ensure we only take in items that parse properly to avoid crash issues.
+					local itemsSafe = {}
+					for _, entry in ipairs(items) do
+						local item = new("Item", entry.item_string)
+						if item.base then
+							t_insert(itemsSafe, entry)
+						end
+					end
+
 					if self.tradeQueryGenerator.lastAugmentBehaviour == "Copy Current" or self.tradeQueryGenerator.lastAnointBehaviour == "Copy Current" then
-						for i, _ in ipairs(items) do
-							local item = new("Item", items[i].item_string)
-							self.itemsTab:CopyAnointsAndAugments(item, true, true, context.slotTbl.slotName)
-							items[i].item_string = item:BuildRaw()
+						for i, _ in ipairs(itemsSafe) do
+							local item = new("Item", itemsSafe[i].item_string)
+							-- avoid interacting with badly parsed stuff
+							if item.base and item.type then
+								self.itemsTab:CopyAnointsAndAugments(item, true, true, context.slotTbl.slotName)
+								itemsSafe[i].item_string = item:BuildRaw()
+							end
 						end
 					elseif self.tradeQueryGenerator.lastAugmentBehaviour == "Remove" then
-						for item_idx, _ in ipairs(items) do
-							local item = new("Item", items[item_idx].item_string)
+						for item_idx, _ in ipairs(itemsSafe) do
+							local item = new("Item", itemsSafe[item_idx].item_string)
 							-- sockets are kept as-is so the user can see e.g. exceptional or corrupted sockets
 							for rune_idx, _ in ipairs(item.runes or {}) do
 								item.runes[rune_idx] = "None"
 							end
 							item:UpdateRunes()
-							items[item_idx].item_string = item:BuildRaw()
+							itemsSafe[item_idx].item_string = item:BuildRaw()
 						end
 					elseif self.tradeQueryGenerator.lastAnointBehaviour == "Remove" then
-						for i, _ in ipairs(items) do
-							local item = new("Item", items[i].item_string)
+						for i, _ in ipairs(itemsSafe) do
+							local item = new("Item", itemsSafe[i].item_string)
 							item.enchantModLines = {}
-							items[i].item_string = item:BuildRaw()
+							itemsSafe[i].item_string = item:BuildRaw()
 						end
 					end
 
-					self.resultTbl[context.row_idx] = items
+					self.resultTbl[context.row_idx] = itemsSafe
 					self:UpdateControlsWithItems(context.row_idx)
 					context.controls["priceButton"..context.row_idx].label =  "Price Item"
 				end,