RE1-T117 PR#403 fixes

Author: ucswift

Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs

@@ -15,13 +15,17 @@ public interface IUtf8MaintenanceRepository
 	{
 		/// <summary>
 		/// Returns every base-table text column grouped by table, restricted to tables that have a
-		/// single-column primary key (required for stable keyset pagination).
+		/// single-column primary key of a pageable type — text/citext, an integer type, or uuid
+		/// (required for stable keyset pagination). Tables whose PK is some other type are skipped.
 		/// </summary>
 		Task<List<Utf8TextColumnTarget>> GetTextColumnTargetsAsync(CancellationToken cancellationToken);
 
 		/// <summary>
 		/// Reads the next batch of rows for <paramref name="target"/> ordered by primary key, starting
-		/// after <paramref name="lastKey"/> (null/empty for the first page).
+		/// after <paramref name="lastKey"/> (null/empty for the first page). The cursor is the previous
+		/// page's last key rendered as an invariant string; the implementation re-binds it to the PK's
+		/// native type (see <see cref="Utf8TextColumnTarget.PrimaryKeyType"/>) so the keyset comparison
+		/// works for text, integer and uuid keys alike.
 		/// </summary>
 		Task<Utf8RowBatch> GetRowBatchAsync(Utf8TextColumnTarget target, string lastKey, int batchSize, CancellationToken cancellationToken);
 
@@ -38,12 +42,33 @@ public interface IUtf8MaintenanceRepository
 		Task SaveProgressAsync(Utf8CleanupProgress progress, CancellationToken cancellationToken);
 	}
 
+	/// <summary>
+	/// The pageable category of a table's primary key. Keyset pagination renders the cursor as an
+	/// invariant string and re-binds it as this type for the <c>&gt;</c> / <c>=</c> predicates, so only
+	/// these PK types are swept; tables with any other PK type are skipped.
+	/// </summary>
+	public enum Utf8PrimaryKeyType
+	{
+		/// <summary>char/varchar/nchar/nvarchar/text/ntext and PostgreSQL citext — bound as a string.</summary>
+		Text = 0,
+
+		/// <summary>tinyint/smallint/int/bigint (and PostgreSQL integer types) — bound as Int64.</summary>
+		Integer = 1,
+
+		/// <summary>SQL Server uniqueidentifier / PostgreSQL uuid — bound as a Guid.</summary>
+		Guid = 2
+	}
+
 	/// <summary>A table plus its single-column primary key and the free-form text columns to scan.</summary>
 	public class Utf8TextColumnTarget
 	{
 		public string Schema { get; set; }
 		public string TableName { get; set; }
 		public string PrimaryKeyColumn { get; set; }
+
+		/// <summary>The PK's value category, so keyset cursors are bound as the PK's native type.</summary>
+		public Utf8PrimaryKeyType PrimaryKeyType { get; set; }
+
 		public List<string> TextColumns { get; set; } = new List<string>();
 
 		/// <summary>Stable identifier used as the watermark key (schema-qualified table name).</summary>
@@ -64,6 +89,7 @@ public class Utf8RowBatch
 	/// </summary>
 	public class Utf8TextRow
 	{
+		/// <summary>The row's primary key rendered as an invariant string (re-bound to its native type for queries).</summary>
 		public string Key { get; set; }
 		public Dictionary<string, string> Columns { get; set; } = new Dictionary<string, string>();
 	}

Core/Resgrid.Services/UsersService.cs

@@ -98,6 +98,12 @@ public IdentityUser GetUserByEmail(string emailAddress)
 		public async Task<bool> DoesUserHaveAnyActiveDepartments(string userName)
 		{
 			var user = await GetUserByNameAsync(userName);
+
+			// Defensive: the user has already authenticated by this point, but if the lookup returns
+			// null (e.g. a stale/edge data state) treat it as no active departments rather than NRE.
+			if (user == null)
+				return false;
+
 			var memberships = await _departmentMembersRepository.GetAllByUserIdAsync(user.UserId);
 
 			return memberships.Any(x => x.IsDeleted == false && (x.IsDisabled == null || x.IsDisabled == false));

Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs

@@ -63,7 +63,10 @@ public async Task<IdentityUser> GetUserByUserNameAsync(string userName)
 			{
 				using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString))
 				{
-					var result = await db.QueryAsync<IdentityUser>($"SELECT * FROM aspnetusers WHERE username = @userName", new { userName = userName });
+					// PostgreSQL text comparison is case-sensitive (unlike SQL Server's default collation),
+					// so match on normalizedusername the same way ASP.NET Identity sign-in does. Otherwise a
+					// user who authenticated with different casing than the stored username is not found here.
+					var result = await db.QueryAsync<IdentityUser>($"SELECT * FROM aspnetusers WHERE normalizedusername = @normalizedUserName", new { normalizedUserName = userName?.ToUpperInvariant() });
 
 					return result.FirstOrDefault();
 				}
@@ -87,7 +90,9 @@ public IdentityUser GetUserByEmail(string email)
 			{
 				using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString))
 				{
-					return db.Query<IdentityUser>($"SELECT * FROM aspnetusers WHERE email = @email", new { email = email }).FirstOrDefault();
+					// PostgreSQL text comparison is case-sensitive; LOWER() both sides so a different-cased email
+					// still matches (normalizedemail is not maintained on the PG update path, so it can't be relied on).
+					return db.Query<IdentityUser>($"SELECT * FROM aspnetusers WHERE LOWER(email) = LOWER(@email)", new { email = email }).FirstOrDefault();
 				}
 			}
 			else
@@ -107,7 +112,8 @@ public void UpdateUsername(string oldUsername, string newUsername)
 			{
 				using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString))
 				{
-					db.Execute($"UPDATE public.aspnetusers SET username = @newUsername, normalizedusername = @newUsernameUpper WHERE username = @oldUsername", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpper(), oldUsername = oldUsername });
+					// Case-insensitive match on the old username (PostgreSQL text comparison is case-sensitive).
+					db.Execute($"UPDATE public.aspnetusers SET username = @newUsername, normalizedusername = @newUsernameUpper WHERE LOWER(username) = LOWER(@oldUsername)", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpper(), oldUsername = oldUsername });
 				}
 			}
 			else
@@ -125,7 +131,9 @@ public void UpdateEmail(string userId, string newEmail)
 			{
 				using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString))
 				{
-					db.Execute($"UPDATE public.aspnetusers SET email = @newEmail WHERE id = @userId", new { userId = userId, newEmail = newEmail });
+					// Keep normalizedemail in sync (ASP.NET Identity's FindByEmailAsync looks up by it); the
+					// SQL Server branch already does this. Without it, email lookups go stale after a change.
+					db.Execute($"UPDATE public.aspnetusers SET email = @newEmail, normalizedemail = @newEmailUpper WHERE id = @userId", new { userId = userId, newEmail = newEmail, newEmailUpper = newEmail?.ToUpper() });
 				}
 			}
 			else

Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs

@@ -70,12 +70,17 @@ INNER JOIN information_schema.tables t
 							AND c.table_schema NOT IN ('pg_catalog', 'information_schema')";
 
 					pkSql = @"
-						SELECT tc.table_schema AS TableSchema, tc.table_name AS TableName, kcu.column_name AS ColumnName
+						SELECT tc.table_schema AS TableSchema, tc.table_name AS TableName, kcu.column_name AS ColumnName,
+							col.data_type AS DataType, col.udt_name AS UdtName
 						FROM information_schema.table_constraints tc
 						INNER JOIN information_schema.key_column_usage kcu
 							ON tc.constraint_name = kcu.constraint_name
 							AND tc.table_schema = kcu.table_schema
 							AND tc.table_name = kcu.table_name
+						INNER JOIN information_schema.columns col
+							ON col.table_schema = kcu.table_schema
+							AND col.table_name = kcu.table_name
+							AND col.column_name = kcu.column_name
 						WHERE tc.constraint_type = 'PRIMARY KEY'
 							AND tc.table_schema NOT IN ('pg_catalog', 'information_schema')";
 				}
@@ -89,12 +94,17 @@ INNER JOIN INFORMATION_SCHEMA.TABLES t
 						WHERE c.DATA_TYPE IN ('char', 'varchar', 'nchar', 'nvarchar', 'text', 'ntext')";
 
 					pkSql = @"
-						SELECT tc.TABLE_SCHEMA AS TableSchema, tc.TABLE_NAME AS TableName, kcu.COLUMN_NAME AS ColumnName
+						SELECT tc.TABLE_SCHEMA AS TableSchema, tc.TABLE_NAME AS TableName, kcu.COLUMN_NAME AS ColumnName,
+							col.DATA_TYPE AS DataType
 						FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc
 						INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
 							ON tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME
 							AND tc.TABLE_SCHEMA = kcu.TABLE_SCHEMA
 							AND tc.TABLE_NAME = kcu.TABLE_NAME
+						INNER JOIN INFORMATION_SCHEMA.COLUMNS col
+							ON col.TABLE_SCHEMA = kcu.TABLE_SCHEMA
+							AND col.TABLE_NAME = kcu.TABLE_NAME
+							AND col.COLUMN_NAME = kcu.COLUMN_NAME
 						WHERE tc.CONSTRAINT_TYPE = 'PRIMARY KEY'";
 				}
 
@@ -109,7 +119,7 @@ INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
 					var singlePk = pkRows
 						.GroupBy(p => p.TableSchema + "." + p.TableName)
 						.Where(g => g.Count() == 1)
-						.ToDictionary(g => g.Key, g => g.First().ColumnName, StringComparer.OrdinalIgnoreCase);
+						.ToDictionary(g => g.Key, g => g.First(), StringComparer.OrdinalIgnoreCase);
 
 					var targets = new List<Utf8TextColumnTarget>();
 
@@ -120,13 +130,20 @@ INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
 						if (ExcludedTables.Contains(first.TableName))
 							continue;
 
-						if (!singlePk.TryGetValue(group.Key, out var pkColumn))
+						if (!singlePk.TryGetValue(group.Key, out var pk))
 							continue; // no single-column PK -> cannot page safely
 
+						// The keyset cursor is a string re-bound to the PK's native type; only text,
+						// integer and uuid PKs are supported. Skip anything else (e.g. date/decimal PKs)
+						// so we never emit a query that compares an incompatible type against the cursor.
+						var keyType = ClassifyKeyType(pk);
+						if (keyType == null)
+							continue;
+
 						// Never clean the primary-key column itself.
 						var textColumns = group
 							.Select(c => c.ColumnName)
-							.Where(name => !string.Equals(name, pkColumn, StringComparison.OrdinalIgnoreCase))
+							.Where(name => !string.Equals(name, pk.ColumnName, StringComparison.OrdinalIgnoreCase))
 							.ToList();
 
 						if (textColumns.Count == 0)
@@ -136,7 +153,8 @@ INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
 						{
 							Schema = first.TableSchema,
 							TableName = first.TableName,
-							PrimaryKeyColumn = pkColumn,
+							PrimaryKeyColumn = pk.ColumnName,
+							PrimaryKeyType = keyType.Value,
 							TextColumns = textColumns
 						});
 					}
@@ -168,7 +186,7 @@ public async Task<Utf8RowBatch> GetRowBatchAsync(Utf8TextColumnTarget target, st
 				var dynamicParameters = new DynamicParameters();
 				dynamicParameters.Add("batchSize", batchSize);
 				if (hasCursor)
-					dynamicParameters.Add("lastKey", lastKey);
+					dynamicParameters.Add("lastKey", ConvertKey(lastKey, target.PrimaryKeyType));
 
 				using (DbConnection conn = _connectionProvider.Create())
 				{
@@ -238,7 +256,7 @@ public async Task<int> UpdateRowsAsync(Utf8TextColumnTarget target, IReadOnlyLis
 								dynamicParameters.Add(paramName, (object)column.Value ?? DBNull.Value);
 							}
 
-							dynamicParameters.Add("rg_key", row.Key);
+							dynamicParameters.Add("rg_key", ConvertKey(row.Key, target.PrimaryKeyType));
 
 							var sql = $"UPDATE {table} SET {string.Join(", ", setClauses)} WHERE {pk} = @rg_key";
 
@@ -289,24 +307,45 @@ public async Task SaveProgressAsync(Utf8CleanupProgress progress, CancellationTo
 		{
 			try
 			{
-				var updateSql = $@"
-					UPDATE {ProgressTable}
-					SET lastprocessedkey = @LastProcessedKey, lastcompletedutc = @LastCompletedUtc,
-						rowsscanned = @RowsScanned, rowsfixed = @RowsFixed, updatedonutc = @UpdatedOnUtc
-					WHERE tablename = @TableName";
+				// Atomic upsert: overlapping cleanup runs (long sweeps or multiple workers) can save the
+				// first watermark for the same table concurrently, which the old UPDATE-then-INSERT raced
+				// on (duplicate-PK on the second INSERT). The tablename PK makes both forms below race-safe.
+				string upsertSql;
 
-				var insertSql = $@"
-					INSERT INTO {ProgressTable} (tablename, lastprocessedkey, lastcompletedutc, rowsscanned, rowsfixed, updatedonutc)
-					VALUES (@TableName, @LastProcessedKey, @LastCompletedUtc, @RowsScanned, @RowsFixed, @UpdatedOnUtc)";
+				if (IsPostgres)
+				{
+					upsertSql = $@"
+						INSERT INTO {ProgressTable} (tablename, lastprocessedkey, lastcompletedutc, rowsscanned, rowsfixed, updatedonutc)
+						VALUES (@TableName, @LastProcessedKey, @LastCompletedUtc, @RowsScanned, @RowsFixed, @UpdatedOnUtc)
+						ON CONFLICT (tablename) DO UPDATE SET
+							lastprocessedkey = EXCLUDED.lastprocessedkey,
+							lastcompletedutc = EXCLUDED.lastcompletedutc,
+							rowsscanned = EXCLUDED.rowsscanned,
+							rowsfixed = EXCLUDED.rowsfixed,
+							updatedonutc = EXCLUDED.updatedonutc";
+				}
+				else
+				{
+					// UPDLOCK + SERIALIZABLE takes a key-range lock so the row-absent case serializes:
+					// one run inserts, the other blocks then updates. No MERGE (its upsert races are well known).
+					upsertSql = $@"
+						SET XACT_ABORT ON;
+						BEGIN TRANSACTION;
+						UPDATE {ProgressTable} WITH (UPDLOCK, SERIALIZABLE)
+						SET lastprocessedkey = @LastProcessedKey, lastcompletedutc = @LastCompletedUtc,
+							rowsscanned = @RowsScanned, rowsfixed = @RowsFixed, updatedonutc = @UpdatedOnUtc
+						WHERE tablename = @TableName;
+						IF @@ROWCOUNT = 0
+							INSERT INTO {ProgressTable} (tablename, lastprocessedkey, lastcompletedutc, rowsscanned, rowsfixed, updatedonutc)
+							VALUES (@TableName, @LastProcessedKey, @LastCompletedUtc, @RowsScanned, @RowsFixed, @UpdatedOnUtc);
+						COMMIT TRANSACTION;";
+				}
 
 				using (DbConnection conn = _connectionProvider.Create())
 				{
 					await conn.OpenAsync(cancellationToken);
 
-					var affected = await conn.ExecuteAsync(new CommandDefinition(updateSql, progress, cancellationToken: cancellationToken));
-
-					if (affected == 0)
-						await conn.ExecuteAsync(new CommandDefinition(insertSql, progress, cancellationToken: cancellationToken));
+					await conn.ExecuteAsync(new CommandDefinition(upsertSql, progress, cancellationToken: cancellationToken));
 				}
 			}
 			catch (Exception ex)
@@ -316,11 +355,72 @@ public async Task SaveProgressAsync(Utf8CleanupProgress progress, CancellationTo
 			}
 		}
 
+		/// <summary>
+		/// Maps a primary key column's declared type to the pageable category used for keyset cursors,
+		/// or null when the type cannot be paged with a string cursor (e.g. date/decimal PKs).
+		/// </summary>
+		private static Utf8PrimaryKeyType? ClassifyKeyType(MetaColumn pk)
+		{
+			var udtName = (pk.UdtName ?? string.Empty).ToLowerInvariant();
+			if (udtName == "citext")
+				return Utf8PrimaryKeyType.Text;
+
+			var dataType = (pk.DataType ?? string.Empty).ToLowerInvariant();
+			switch (dataType)
+			{
+				case "char":
+				case "varchar":
+				case "nchar":
+				case "nvarchar":
+				case "text":
+				case "ntext":
+				case "character":
+				case "character varying":
+					return Utf8PrimaryKeyType.Text;
+
+				case "tinyint":
+				case "smallint":
+				case "int":
+				case "integer":
+				case "bigint":
+					return Utf8PrimaryKeyType.Integer;
+
+				case "uniqueidentifier":
+				case "uuid":
+					return Utf8PrimaryKeyType.Guid;
+
+				default:
+					return null; // unsupported PK type -> table is skipped
+			}
+		}
+
+		/// <summary>
+		/// Re-binds a string cursor to the primary key's native CLR type so the keyset <c>&gt;</c> / <c>=</c>
+		/// predicates compare like types (PostgreSQL has no implicit text-to-integer/uuid coercion).
+		/// </summary>
+		private static object ConvertKey(string key, Utf8PrimaryKeyType keyType)
+		{
+			if (string.IsNullOrEmpty(key))
+				return key;
+
+			switch (keyType)
+			{
+				case Utf8PrimaryKeyType.Integer:
+					return long.Parse(key, CultureInfo.InvariantCulture);
+				case Utf8PrimaryKeyType.Guid:
+					return Guid.Parse(key);
+				default:
+					return key;
+			}
+		}
+
 		private class MetaColumn
 		{
 			public string TableSchema { get; set; }
 			public string TableName { get; set; }
 			public string ColumnName { get; set; }
+			public string DataType { get; set; }
+			public string UdtName { get; set; }
 		}
 	}
 }

Tools/Resgrid.Console/Commands/CleanUtf8Command.cs

@@ -38,8 +38,7 @@ public async Task<ExitCode> ExecuteMainAsync(string[] args, CancellationToken ca
 			}
 			catch (Exception ex)
 			{
-				logger.LogError("There was an error running the UTF-8 data cleanup, see the error output below:");
-				logger.LogError(ex.ToString());
+				Resgrid.Framework.Logging.LogException(ex, "There was an error running the UTF-8 data cleanup");
 				return ExitCode.Failed;
 			}
 

Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs

@@ -13,7 +13,7 @@ public class Utf8CleanupTask : IQuidjiboHandler<Utf8CleanupCommand>
 	{
 		public string Name => "UTF-8 Data Cleanup";
 		public int Priority => 1;
-		public ILogger _logger;
+		private readonly ILogger _logger;
 
 		public Utf8CleanupTask(ILogger logger)
 		{