Fix cardinality estimations for semi/anti joins

Author: dyemanov

src/jrd/optimizer/Optimizer.h

@@ -52,6 +52,7 @@ inline constexpr double REDUCE_SELECTIVITY_FACTOR_LESS = 0.05;
 inline constexpr double REDUCE_SELECTIVITY_FACTOR_GREATER = 0.05;
 inline constexpr double REDUCE_SELECTIVITY_FACTOR_STARTING = 0.01;
 inline constexpr double REDUCE_SELECTIVITY_FACTOR_OTHER = 0.01;
+inline constexpr double REDUCE_SELECTIVITY_FACTOR_ANY = 0.5;
 
 // Cost of simple (CPU bound) operations is less than the page access cost
 inline constexpr double COST_FACTOR_MEMCOPY = 0.5;

src/jrd/recsrc/HashJoin.cpp

@@ -314,7 +314,8 @@ void HashJoin::init(thread_db* tdbb, CompilerScratch* csb, FB_SIZE_T count,
 		const auto subRsb = args[i];
 		fb_assert(subRsb);
 
-		m_cardinality *= subRsb->getCardinality();
+		if (m_joinType == JoinType::INNER || m_joinType == JoinType::OUTER)
+			m_cardinality *= subRsb->getCardinality();
 
 		SubStream sub;
 		sub.buffer = FB_NEW_POOL(csb->csb_pool) BufferedStream(csb, subRsb);
@@ -352,7 +353,10 @@ void HashJoin::init(thread_db* tdbb, CompilerScratch* csb, FB_SIZE_T count,
 	}
 
 	if (!selectivity)
-		selectivity = pow(REDUCE_SELECTIVITY_FACTOR_EQUALITY, keyCount);
+	{
+		selectivity = (m_joinType == JoinType::INNER || m_joinType == JoinType::OUTER) ?
+			pow(REDUCE_SELECTIVITY_FACTOR_EQUALITY, keyCount) : REDUCE_SELECTIVITY_FACTOR_ANY;
+	}
 
 	m_cardinality *= selectivity;
 }

src/jrd/recsrc/NestedLoopJoin.cpp

@@ -57,7 +57,10 @@ NestedLoopJoin::NestedLoopJoin(CompilerScratch* csb,
 	fb_assert(outer && inner);
 
 	m_impure = csb->allocImpure<Impure>();
-	m_cardinality = outer->getCardinality() * inner->getCardinality();
+
+	m_cardinality = outer->getCardinality();
+	m_cardinality *= (m_joinType == JoinType::INNER || m_joinType == JoinType::OUTER) ?
+		inner->getCardinality() : REDUCE_SELECTIVITY_FACTOR_ANY;
 
 	m_args.add(outer);
 	m_args.add(inner);