Handle outdated format error in a case of selectable procedure. Added ability to force-update version of an object in the cache.

Author: AlexPeshkoff

src/dsql/DsqlRequests.cpp

@@ -35,6 +35,7 @@
 #include "../dsql/errd_proto.h"
 #include "../dsql/movd_proto.h"
 #include "../jrd/exe_proto.h"
+#include "../common/utils_proto.h"
 
 using namespace Firebird;
 using namespace Jrd;
@@ -553,17 +554,47 @@ void DsqlDmlRequest::doExecute(thread_db* tdbb, jrd_tra** traHandle,
 	firstRowFetched = false;
 	const dsql_msg* message = dsqlStatement->getSendMsg();
 
-	if (!message)
-	{
-		JRD_start(tdbb, request, req_transaction);
-	}
-	else
+	for (int i = 0; i < EXEC_RESTARTS; ++i)
 	{
-		fb_assert(inMsg != nullptr);
+		try
+		{
+			if (!message)
+			{
+				JRD_start(tdbb, request, req_transaction);
+			}
+			else
+			{
+				fb_assert(inMsg != nullptr);
+
+				const ULONG inMsgLength = dsqlStatement->getStatement()->getMessage(message->msg_number)->
+					getFormat(request)->fmt_length;
+				JRD_start_and_send(tdbb, request, req_transaction, message->msg_number,
+					inMsgLength, inMsg);
+			}
 
-		const ULONG inMsgLength = dsqlStatement->getStatement()->getMessage(message->msg_number)->getFormat(request)->fmt_length;
-		JRD_start_and_send(tdbb, request, req_transaction, message->msg_number,
-			inMsgLength, inMsg);
+			break;
+		}
+		catch (const Exception& ex)
+		{
+			FbLocalStatus st;
+			ex.stuffException(&st);
+
+			if (fb_utils::containsErrorCode(st->getErrors(), isc_old_format))
+			{
+				// destroy existing request
+				auto* statement = request->getStatement();
+				EXE_release(tdbb, request);
+
+				// create fresh one (with changed metadata in cache)
+				request = statement->findRequest(tdbb);
+				tdbb->getAttachment()->att_requests.add(request);
+
+				tdbb->tdbb_status_vector->init();
+				continue;
+			}
+
+			throw;
+		}
 	}
 
 	// Selectable execute block should get the "proc fetch" flag assigned,

src/include/firebird/impl/msg/jrd.h

@@ -1021,3 +1021,4 @@ FB_IMPL_MSG(JRD, 1018, dsql_agg_param_not_accum, -204, "42", "000", "Aggregate f
 FB_IMPL_MSG(JRD, 1019, dsql_agg_exit_group, -204, "42", "000", "EXIT is not allowed in ON GROUP DO section of aggregate function")
 FB_IMPL_MSG(JRD, 1020, dsql_agg_return, -204, "42", "000", "RETURN is not allowed in ON START DO, ON ACCUMULATE DO or ON FINISH DO sections of aggregate function; use EXIT instead")
 FB_IMPL_MSG(JRD, 1021, hypfun_args_non_equal_sort_item, -833, "42", "000", "Number of arguments of hypothetical-set aggregate function @1 must match number of sort items in WITHIN GROUP clause")
+FB_IMPL_MSG(JRD, 1022, old_format, -804, "07", "000", "Statement format outdated, need to be reprepared")

src/include/gen/Firebird.pas

@@ -6063,6 +6063,7 @@ 	IPerformanceStatsImpl = class(IPerformanceStats)
 	 isc_dsql_agg_exit_group = 335545339;
 	 isc_dsql_agg_return = 335545340;
 	 isc_hypfun_args_non_equal_sort_item = 335545341;
+	 isc_old_format = 335545342;
 	 isc_gfix_db_name = 335740929;
 	 isc_gfix_invalid_sw = 335740930;
 	 isc_gfix_incmp_sw = 335740932;

src/jrd/CacheVector.h

@@ -115,9 +115,8 @@ namespace CacheFlag
 	static constexpr ObjectBase::Flag NOCOMMIT =	0x010;		// do not commit created version
 	static constexpr ObjectBase::Flag NOERASED =	0x020;		// never return erased version, skip till older object
 	static constexpr ObjectBase::Flag RETIRED = 	0x040;		// object is in a process of GC
-	static constexpr ObjectBase::Flag UPGRADE =		0x080;		// create new versions for already existing in a cache objects
-	static constexpr ObjectBase::Flag MINISCAN =	0x100;		// perform minimum scan and set cache entry to reload state
-	static constexpr ObjectBase::Flag DB_VERSION =	0x200;		// execute version upgrade in database
+	static constexpr ObjectBase::Flag MINISCAN =	0x080;		// perform minimum scan and set cache entry to reload state
+	static constexpr ObjectBase::Flag DB_VERSION =	0x100;		// execute version upgrade in database
 
 	// Useful combinations
 	static constexpr ObjectBase::Flag TAG_FOR_UPDATE = NOCOMMIT | MINISCAN | DB_VERSION;
@@ -163,11 +162,13 @@ class ListEntry : public HazardObject
 public:
 	enum State { INITIAL, RELOAD, MISSING, SCANNING, READY };
 
-	ListEntry(Versioned* object, TraNumber traNumber, ObjectBase::Flag fl)
+	ListEntry(Versioned* object, TraNumber traNumber, ObjectBase::Flag fl, ListEntry* link = nullptr)
 		: object(object), traNumber(traNumber), cacheFlags(fl), state(INITIAL)
 	{
 		if (fl & CacheFlag::ERASED)
 			fb_assert(!object);
+		if (link)
+			next.store(link);
 	}
 
 	~ListEntry()
@@ -548,6 +549,21 @@ class ListEntry : public HazardObject
 		return state == READY ? false : (thd == Thread::getCurrentThreadId()) && (state == SCANNING);
 	}
 
+	static bool upgradable(HazardPtr<ListEntry>& listEntry, const Versioned* from)
+	{
+		for (; listEntry; listEntry.set(listEntry->next))
+		{
+			if (listEntry->object == from)
+				return false;		// not found upgrade version
+
+			if (listEntry->getFlags() & CacheFlag::COMMITTED)
+				return true;		// already upgraded by someone else
+		}
+
+		fb_assert(false);
+		return false;				// miss from what to upgrade
+	}
+
 private:
 
 	// object (nill/not nill) & ERASED bit in cacheFlags together control state of cache element
@@ -951,6 +967,36 @@ class CacheElement : public ElementBase, public P
 		}
 	}
 
+	bool upgrade(thread_db* tdbb, const Versioned* from)
+	{
+		HazardPtr<ListEntry<Versioned>> l(list);
+
+		// list of versions should be present
+		fb_assert(l);
+		if (!l)
+			return false;
+
+		// if there is another version at the top nothing to be added
+		if (l->getVersioned() != from)
+			return ListEntry<Versioned>::upgradable(l, from);
+
+		// we have candidate for upgrade - make sure it's not half-done
+		fb_assert(l->getFlags() & CacheFlag::COMMITTED);
+		if (!(l->getFlags() & CacheFlag::COMMITTED))
+			return false;
+
+		// Try to upgrade
+		ListEntry<Versioned>* newEntry = FB_NEW ListEntry<Versioned>(nullptr, TransactionNumber::current(tdbb),
+			CacheFlag::COMMITTED | CacheFlag::MINISCAN | CacheFlag::DB_VERSION, l.getPointer());
+		if (l.replace(list, newEntry))
+			return true;
+		delete newEntry;
+
+		// Someone already added entry - see is it OK for us
+		l.set(list);
+		return ListEntry<Versioned>::upgradable(l, from);
+	}
+
 private:
 	void setNewResetAt(TraNumber oldVal, TraNumber newVal)
 	{
@@ -1063,7 +1109,6 @@ class CacheVector : public Firebird::PermanentStorage
 
 	Versioned* getVersioned(thread_db* tdbb, MetaId id, ObjectBase::Flag fl)
 	{
-
 //		In theory that should be endless cycle - object may arrive/disappear again and again.
 //		But in order to faster find devel problems we run it very limited number of times.
 #ifdef DEV_BUILD
@@ -1077,17 +1122,7 @@ class CacheVector : public Firebird::PermanentStorage
 			{
 				StoredElement* data = ptr->load(atomics::memory_order_acquire);
 				if (data)
-				{
-					if (fl & CacheFlag::UPGRADE)
-					{
-						auto val = makeObject(tdbb, id, fl);
-						if (val)
-							return val;
-						continue;
-					}
-
 					return data->getVersioned(tdbb, fl);
-				}
 			}
 
 			if (!(fl & CacheFlag::AUTOCREATE))
@@ -1170,6 +1205,28 @@ class CacheVector : public Firebird::PermanentStorage
 		return data;
 	}
 
+	bool upgrade(thread_db* tdbb, MetaId id, const Versioned* from)
+	{
+		fb_assert(id < getCount());
+
+		if (id < getCount())
+		{
+			auto ptr = getDataPointer(id);
+			fb_assert(ptr);
+
+			if (ptr)
+			{
+				StoredElement* data = ptr->load(atomics::memory_order_acquire);
+				fb_assert(data);
+
+				if (data)
+					return data->upgrade(tdbb, from);
+			}
+		}
+
+		return false;
+	}
+
 	template <typename F>
 	StoredElement* lookup(thread_db* tdbb, F&& cmp, ObjectBase::Flag fl) const
 	{
@@ -1188,6 +1245,7 @@ class CacheVector : public Firebird::PermanentStorage
 					auto listEntry = ptr->getEntry(tdbb, TransactionNumber::current(tdbb), fl | CacheFlag::MINISCAN);
 					if (listEntry && cmp(ptr))
 					{
+//						if (!(fl & (CacheFlag::ERASED | CacheFlag::MINISCAN)))
 						if (!(fl & CacheFlag::ERASED))
 							ptr->reload(tdbb, fl);		// found object to be reloaded w/o MINISCAN flag
 						return ptr;

src/jrd/Resources.cpp

@@ -55,7 +55,7 @@ void Resources::transfer(thread_db* tdbb, VersionedObjects* to, bool internal)
 
 [[noreturn]] void Resources::outdated()
 {
-	ERR_post(Arg::Gds(isc_random) << "Statement format outdated, need to be reprepared");
+	ERR_post(Arg::Gds(isc_old_format));
 }
 
 Resources::~Resources()

src/jrd/constants.h

@@ -520,4 +520,7 @@ inline constexpr USHORT MAX_ERROR_MSG_LENGTH = 1024 * METADATA_BYTES_PER_CHAR; /
 // Prefix of index that's getting dropped
 inline constexpr const char* TEMP_DEPEND = "RDB$TEMP_DEPEND";
 
+// How many times request can be restarted in attempts to sync formats in it
+inline constexpr int EXEC_RESTARTS = 8;
+
 #endif // JRD_CONSTANTS_H

src/jrd/met.h

@@ -391,6 +391,15 @@ class MetadataCache : public Firebird::PermanentStorage
 		return vector.newVersion(tdbb, id);
 	}
 
+	// Upgrade object in cache from requested version
+
+	template <typename C, typename V>
+	static bool upgrade(thread_db* tdbb, MetaId id, V* from)
+	{
+		auto& vector = Vector<C>::get(getCache(tdbb));
+		return vector.upgrade(tdbb, id, from);
+	}
+
 	// Mark object in cache as dropped
 
 	template <typename C>

src/jrd/recsrc/ProcedureScan.cpp

@@ -31,6 +31,7 @@
 #include "../jrd/trace/TraceManager.h"
 #include "../jrd/trace/TraceJrdHelpers.h"
 #include "../jrd/optimizer/Optimizer.h"
+#include "../common/utils_proto.h"
 
 #include "RecordSource.h"
 
@@ -116,7 +117,21 @@ void ProcedureScan::internalOpen(thread_db* tdbb) const
 		im = NULL;
 	}
 
-	Request* const proc_request = proc->getStatement()->findRequest(tdbb);
+	Request* proc_request = nullptr;
+	try
+	{
+		proc_request = proc->getStatement()->findRequest(tdbb);
+	}
+	catch (const Exception& ex)
+	{
+		FbLocalStatus st;
+		ex.stuffException(&st);
+
+		if (fb_utils::containsErrorCode(st->getErrors(), isc_old_format))
+			MetadataCache::upgrade<Cached::Procedure>(tdbb, m_procedure()->getId(), proc);
+
+		throw;
+	}
 	impure->irsb_req_handle = proc_request;
 
 	// req_proc_fetch flag used only when fetching rows, so