[backport] Fix check in GPU histogram. (dmlc#11574) (dmlc#11582)

Author: trivialfis

src/tree/gpu_hist/histogram.cu

@@ -386,8 +386,6 @@ class DeviceHistogramBuilderImpl {
     if (!this->kernel_->shared) {  // Use global memory
       CHECK_EQ(this->kernel_->smem_size, 0);
       if (matrix.IsDenseCompressed()) {
-        // Dense must use shared memory except for testing.
-        CHECK(this->kernel_->force_global);
         launcher(this->kernel_->global_dense_kernel);
       } else {
         launcher(this->kernel_->global_kernel);

tests/cpp/tree/gpu_hist/test_histogram.cu

@@ -274,7 +274,7 @@ void ValidateCategoricalHistogram(size_t n_categories, common::Span<GradientPair
 // Test 1 vs rest categorical histogram is equivalent to one hot encoded data.
 void TestGPUHistogramCategorical(size_t num_categories) {
   auto ctx = MakeCUDACtx(0);
-  size_t constexpr kRows = 340;
+  size_t kRows = std::max(static_cast<decltype(num_categories)>(340), num_categories);
   size_t constexpr kBins = 256;
   auto x = GenerateRandomCategoricalSingleColumn(kRows, num_categories);
   auto cat_m = GetDMatrixFromData(x, kRows, 1);
@@ -332,6 +332,11 @@ TEST(Histogram, GPUHistCategorical) {
   for (size_t num_categories = 2; num_categories < 8; ++num_categories) {
     TestGPUHistogramCategorical(num_categories);
   }
+  // Larger than the shared memory size, must use global memory since there's no feature
+  // group with a single feature.
+  auto max_shmem = dh::MaxSharedMemoryOptin(0);
+  auto n_categories = common::DivRoundUp(max_shmem, sizeof(GradientPairInt64)) * 2;
+  TestGPUHistogramCategorical(n_categories);
 }
 
 namespace {