From 4203d6601d78750ec2dd1c67f3c8db03036993d1 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Wed, 18 Mar 2026 06:14:13 -0700 Subject: [PATCH 01/13] Add CPU V6 SHAP algorithm --- doc/parameter.rst | 8 + src/gbm/gbtree.h | 5 + src/predictor/cpu_predictor.cc | 16 ++ src/predictor/interpretability/shap.cc | 233 +++++++++++++++++++++++++ src/predictor/interpretability/shap.h | 4 + tests/cpp/predictor/test_shap.cc | 80 +++++++++ 6 files changed, 346 insertions(+) diff --git a/doc/parameter.rst b/doc/parameter.rst index 46891cbb9736..72b0b8c4423f 100644 --- a/doc/parameter.rst +++ b/doc/parameter.rst @@ -189,6 +189,14 @@ Parameters for Tree Booster - ``approx``: Approximate greedy algorithm using quantile sketch and gradient histogram. - ``hist``: Faster histogram optimized approximate greedy algorithm. +* ``shap_algorithm`` string [default= ``treeshap``] + + - CPU algorithm used for ``pred_contribs`` with tree boosters. + - Choices: ``treeshap``, ``v6``. + + - ``treeshap``: Existing exact TreeSHAP implementation. + - ``v6``: Quadrature plus telescoping SHAP implementation for CPU prediction. + * ``scale_pos_weight`` [default=1] - Control the balance of positive and negative weights, useful for unbalanced classes. A typical value to consider: ``sum(negative instances) / sum(positive instances)``. See :doc:`Parameters Tuning ` for more discussion. Also, see Higgs Kaggle competition demo for examples: `R `_, `py1 `_, `py2 `_, `py3 `_. diff --git a/src/gbm/gbtree.h b/src/gbm/gbtree.h index 2d1e63133f52..4b97bb5bd06f 100644 --- a/src/gbm/gbtree.h +++ b/src/gbm/gbtree.h @@ -62,6 +62,8 @@ struct GBTreeTrainParam : public XGBoostParameter { TreeProcessType process_type; // tree construction method TreeMethod tree_method; + // CPU SHAP implementation used for pred_contribs. + std::string shap_algorithm; // declare parameters DMLC_DECLARE_PARAMETER(GBTreeTrainParam) { DMLC_DECLARE_FIELD(updater_seq).describe("Tree updater sequence.").set_default(""); @@ -80,6 +82,9 @@ struct GBTreeTrainParam : public XGBoostParameter { .add_enum("exact", TreeMethod::kExact) .add_enum("hist", TreeMethod::kHist) .describe("Choice of tree construction method."); + DMLC_DECLARE_FIELD(shap_algorithm) + .set_default("treeshap") + .describe("CPU algorithm used for SHAP feature contributions."); } }; diff --git a/src/predictor/cpu_predictor.cc b/src/predictor/cpu_predictor.cc index 85947b907760..40bfc9aab968 100644 --- a/src/predictor/cpu_predictor.cc +++ b/src/predictor/cpu_predictor.cc @@ -745,6 +745,16 @@ class CPUPredictor : public Predictor { public: explicit CPUPredictor(Context const *ctx) : Predictor::Predictor{ctx} {} + void Configure(Args const &cfg) override { + for (auto const &kv : cfg) { + if (kv.first == "shap_algorithm") { + CHECK(kv.second == "treeshap" || kv.second == "v6") + << "Unknown SHAP algorithm: " << kv.second; + shap_algorithm_ = kv.second; + } + } + } + void PredictBatch(DMatrix *dmat, PredictionCacheEntry *predts, gbm::GBTreeModel const &model, bst_tree_t tree_begin, bst_tree_t tree_end = 0, std::vector const *tree_weights = nullptr) const override { @@ -868,6 +878,9 @@ class CPUPredictor : public Predictor { if (approximate) { interpretability::ApproxFeatureImportance(this->ctx_, p_fmat, out_contribs, model, ntree_limit, tree_weights); + } else if (shap_algorithm_ == "v6" && condition == 0 && condition_feature == 0) { + interpretability::cpu_impl::V6ShapValues(this->ctx_, p_fmat, out_contribs, model, ntree_limit, + tree_weights); } else { interpretability::ShapValues(this->ctx_, p_fmat, out_contribs, model, ntree_limit, tree_weights, condition, condition_feature); @@ -881,6 +894,9 @@ class CPUPredictor : public Predictor { interpretability::ShapInteractionValues(this->ctx_, p_fmat, out_contribs, model, ntree_limit, tree_weights, approximate); } + + private: + std::string shap_algorithm_{"treeshap"}; }; XGBOOST_REGISTER_PREDICTOR(CPUPredictor, "cpu_predictor") diff --git a/src/predictor/interpretability/shap.cc b/src/predictor/interpretability/shap.cc index a6608a4133fb..c7c7473831cc 100644 --- a/src/predictor/interpretability/shap.cc +++ b/src/predictor/interpretability/shap.cc @@ -4,6 +4,9 @@ #include "shap.h" #include // for fill +#include // for array +#include // for abs +#include // for numeric_limits #include // for remove_const_t #include // for vector @@ -60,6 +63,165 @@ void CalculateApproxContributions(tree::ScalarTreeView const &tree, RegTree::FVe CalculateContributionsApprox(tree, feats, mean_values, out_contribs->data()); } +constexpr std::size_t kV6QuadraturePoints = 30; +constexpr double kV6Qeps = 1e-15; +constexpr double kV6Unseen = -999.0; +using V6Quad = std::array; + +V6Quad const &V6Nodes() { + static constexpr V6Quad kNodes = { + 1.55325796267524740557e-03, 8.16593836012641238753e-03, 1.99890675158462260974e-02, + 3.68999762853628454629e-02, 5.87197321039736319648e-02, 8.52171188086158215569e-02, + 1.16111283947586907406e-01, 1.51074752603342077339e-01, 1.89736908505378554235e-01, + 2.31687925928990068325e-01, 2.76483115230955422970e-01, 3.23647637234560914266e-01, + 3.72681536916055100583e-01, 4.23065043195708256896e-01, 4.74264078722341164696e-01, + 5.25735921277658890816e-01, 5.76934956804291743104e-01, 6.27318463083944899417e-01, + 6.76352362765439085734e-01, 7.23516884769044521519e-01, 7.68312074071009876164e-01, + 8.10263091494621390254e-01, 8.48925247396657978172e-01, 8.83888716052413148105e-01, + 9.14782881191384178443e-01, 9.41280267896026368035e-01, 9.63100023714637210048e-01, + 9.80010932484153718391e-01, 9.91834061639873532101e-01, 9.98446742037324752594e-01}; + return kNodes; +} + +V6Quad const &V6Weights() { + static constexpr V6Quad kWeights = { + 3.98409624808451681005e-03, 9.23323415554584518705e-03, 1.43923539416612698838e-02, + 1.93995962848134140266e-02, 2.42013364152968944720e-02, 2.87465781088096158924e-02, + 3.29871149410902175791e-02, 3.68779873688524495456e-02, 4.03779476147099816719e-02, + 4.34498936005414254646e-02, 4.60612611188929710337e-02, 4.81843685873219601534e-02, + 4.97967102933974670176e-02, 5.08811948742026384784e-02, 5.14263264467792954870e-02, + 5.14263264467792954870e-02, 5.08811948742026384784e-02, 4.97967102933974670176e-02, + 4.81843685873219601534e-02, 4.60612611188929710337e-02, 4.34498936005414254646e-02, + 4.03779476147099816719e-02, 3.68779873688524495456e-02, 3.29871149410902175791e-02, + 2.87465781088096158924e-02, 2.42013364152968944720e-02, 1.93995962848134140266e-02, + 1.43923539416612698838e-02, 9.23323415554584518705e-03, 3.98409624808451681005e-03}; + return kWeights; +} + +V6Quad Scale(V6Quad h_vals, double scale) { + for (auto &v : h_vals) { + v *= scale; + } + return h_vals; +} + +V6Quad Add(V6Quad lhs, V6Quad const &rhs) { + for (std::size_t i = 0; i < lhs.size(); ++i) { + lhs[i] += rhs[i]; + } + return lhs; +} + +double ExtractTermV6(V6Quad const &h_vals, double p_value) { + if (p_value == kV6Unseen) { + return 0.0; + } + auto alpha = p_value - 1.0; + if (std::abs(alpha) < kV6Qeps) { + return 0.0; + } + auto const &nodes = V6Nodes(); + auto const &weights = V6Weights(); + double acc = 0.0; + for (std::size_t i = 0; i < h_vals.size(); ++i) { + acc += alpha * h_vals[i] / (1.0 + alpha * nodes[i]) * weights[i]; + } + return acc; +} + +bool GoesLeftV6(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, bst_node_t nidx) { + auto split_index = tree.SplitIndex(nidx); + auto fvalue = feat.GetFvalue(split_index); + auto missing = feat.IsMissing(split_index); + auto const &cats = tree.GetCategoriesMatrix(); + bst_node_t next = RegTree::kInvalidNodeId; + if (tree.HasCategoricalSplit()) { + next = missing ? predictor::GetNextNode(tree, nidx, fvalue, true, cats) + : predictor::GetNextNode(tree, nidx, fvalue, false, cats); + } else { + next = missing ? predictor::GetNextNode(tree, nidx, fvalue, true, cats) + : predictor::GetNextNode(tree, nidx, fvalue, false, cats); + } + return next == tree.LeftChild(nidx); +} + +double ChildWeightV6(tree::ScalarTreeView const &tree, bst_node_t parent, bst_node_t child) { + auto parent_cover = tree.Stat(parent).sum_hess; + CHECK_GT(parent_cover, 0.0f); + return tree.Stat(child).sum_hess / parent_cover; +} + +V6Quad TreeShapV6(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, bst_node_t nidx, + V6Quad const &c_vals, double w_prod, std::vector *p_vals, double *phi) { + if (tree.IsLeaf(nidx)) { + return Scale(c_vals, w_prod * tree.LeafValue(nidx)); + } + + auto split_index = tree.SplitIndex(nidx); + auto left = tree.LeftChild(nidx); + auto right = tree.RightChild(nidx); + auto left_weight = ChildWeightV6(tree, nidx, left); + auto right_weight = ChildWeightV6(tree, nidx, right); + auto goes_left = GoesLeftV6(tree, feat, nidx); + auto p_old = (*p_vals)[split_index]; + + auto visit_child = [&](bst_node_t child, double child_weight, bool satisfies) { + double p_e = 0.0; + double p_up = 0.0; + if (p_old == kV6Unseen) { + p_e = satisfies ? 1.0 / child_weight : 0.0; + p_up = 1.0; + } else if (std::abs(p_old) < kV6Qeps) { + p_e = 0.0; + p_up = 0.0; + } else { + p_e = satisfies ? p_old / child_weight : 0.0; + p_up = p_old; + } + + auto c_child = c_vals; + auto const &nodes = V6Nodes(); + auto alpha_e = p_e - 1.0; + for (std::size_t i = 0; i < c_child.size(); ++i) { + c_child[i] *= 1.0 + alpha_e * nodes[i]; + } + + if (p_old != kV6Unseen) { + auto alpha_old = p_old - 1.0; + if (std::abs(alpha_old) >= kV6Qeps) { + for (std::size_t i = 0; i < c_child.size(); ++i) { + c_child[i] /= 1.0 + alpha_old * nodes[i]; + } + } + } + + (*p_vals)[split_index] = p_e; + auto h_child = TreeShapV6(tree, feat, child, c_child, w_prod * child_weight, p_vals, phi); + (*p_vals)[split_index] = p_old; + phi[split_index] += ExtractTermV6(h_child, p_e); + phi[split_index] -= ExtractTermV6(h_child, p_up); + return h_child; + }; + + auto left_h = visit_child(left, left_weight, goes_left); + auto right_h = visit_child(right, right_weight, !goes_left); + return Add(std::move(left_h), right_h); +} + +void CalculateContributionsV6(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, + std::vector *mean_values, double *out_contribs) { + out_contribs[feat.Size()] += (*mean_values)[0]; + + if (tree.IsLeaf(RegTree::kRoot)) { + return; + } + + V6Quad c_init; + c_init.fill(1.0); + std::vector p_vals(feat.Size(), kV6Unseen); + TreeShapV6(tree, feat, RegTree::kRoot, c_init, 1.0, &p_vals, out_contribs); +} + template void DispatchByBatchView(Context const *ctx, DMatrix *p_fmat, EncAccessor acc, Fn &&fn) { using AccT = std::decay_t; @@ -165,6 +327,77 @@ void ShapValues(Context const *ctx, DMatrix *p_fmat, HostDeviceVector *ou LaunchShap(ctx, p_fmat, model, process_view); } +void V6ShapValues(Context const *ctx, DMatrix *p_fmat, HostDeviceVector *out_contribs, + gbm::GBTreeModel const &model, bst_tree_t tree_end, + std::vector const *tree_weights) { + CHECK(!model.learner_model_param->IsVectorLeaf()) << "Predict contribution" << MTNotImplemented(); + CHECK(!p_fmat->Info().IsColumnSplit()) + << "Predict contribution support for column-wise data split is not yet implemented."; + MetaInfo const &info = p_fmat->Info(); + tree_end = predictor::GetTreeLimit(model.trees, tree_end); + CHECK_GE(tree_end, 0); + ValidateTreeWeights(tree_weights, tree_end); + auto const n_trees = static_cast(tree_end); + auto const n_threads = ctx->Threads(); + size_t const ncolumns = model.learner_model_param->num_feature + 1; + std::vector &contribs = out_contribs->HostVector(); + contribs.resize(info.num_row_ * ncolumns * model.learner_model_param->num_output_group); + std::fill(contribs.begin(), contribs.end(), 0.0f); + + std::vector> mean_values(n_trees); + common::ParallelFor(n_trees, n_threads, [&](auto i) { + FillNodeMeanValues(model.trees[i]->HostScView(), &(mean_values[i])); + }); + + auto const n_groups = model.learner_model_param->num_output_group; + CHECK_NE(n_groups, 0); + auto const base_score = model.learner_model_param->BaseScore(DeviceOrd::CPU()); + auto const h_tree_groups = model.TreeGroups(DeviceOrd::CPU()); + std::vector feats_tloc(n_threads); + std::vector> contribs_tloc(n_threads, std::vector(ncolumns)); + + auto device = ctx->Device().IsSycl() ? DeviceOrd::CPU() : ctx->Device(); + auto base_margin = info.base_margin_.View(device); + + auto process_view = [&](auto &&view) { + common::ParallelFor(view.Size(), n_threads, [&](auto i) { + auto tid = omp_get_thread_num(); + auto &feats = feats_tloc[tid]; + if (feats.Size() == 0) { + feats.Init(model.learner_model_param->num_feature); + } + auto &this_tree_contribs = contribs_tloc[tid]; + auto row_idx = view.base_rowid + i; + auto n_valid = view.DoFill(i, feats.Data().data()); + feats.HasMissing(n_valid != feats.Size()); + for (bst_target_t gid = 0; gid < n_groups; ++gid) { + float *p_contribs = &contribs[(row_idx * n_groups + gid) * ncolumns]; + for (bst_tree_t j = 0; j < tree_end; ++j) { + if (h_tree_groups[j] != gid) { + continue; + } + std::fill(this_tree_contribs.begin(), this_tree_contribs.end(), 0.0); + auto const sc_tree = model.trees[j]->HostScView(); + CalculateContributionsV6(sc_tree, feats, &mean_values[j], this_tree_contribs.data()); + auto const weight = tree_weights == nullptr ? 1.0f : (*tree_weights)[j]; + for (size_t ci = 0; ci < ncolumns; ++ci) { + p_contribs[ci] += static_cast(this_tree_contribs[ci] * weight); + } + } + if (base_margin.Size() != 0) { + CHECK_EQ(base_margin.Shape(1), n_groups); + p_contribs[ncolumns - 1] += base_margin(row_idx, gid); + } else { + p_contribs[ncolumns - 1] += base_score(gid); + } + } + feats.Drop(); + }); + }; + + LaunchShap(ctx, p_fmat, model, process_view); +} + void ApproxFeatureImportance(Context const *ctx, DMatrix *p_fmat, HostDeviceVector *out_contribs, gbm::GBTreeModel const &model, bst_tree_t tree_end, std::vector const *tree_weights) { diff --git a/src/predictor/interpretability/shap.h b/src/predictor/interpretability/shap.h index 2c32d1d84554..d9853cfe9229 100644 --- a/src/predictor/interpretability/shap.h +++ b/src/predictor/interpretability/shap.h @@ -19,6 +19,10 @@ void ShapValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* ou gbm::GBTreeModel const& model, bst_tree_t tree_end, std::vector const* tree_weights, int condition, unsigned condition_feature); +void V6ShapValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, + gbm::GBTreeModel const& model, bst_tree_t tree_end, + std::vector const* tree_weights); + void ApproxFeatureImportance(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, bst_tree_t tree_end, std::vector const* tree_weights); diff --git a/tests/cpp/predictor/test_shap.cc b/tests/cpp/predictor/test_shap.cc index b86cb498733f..d7d9f970358d 100644 --- a/tests/cpp/predictor/test_shap.cc +++ b/tests/cpp/predictor/test_shap.cc @@ -291,6 +291,86 @@ TEST(Predictor, DartShapOutputCPU) { CheckDartShapOutput(&ctx); } +TEST(Predictor, V6PrototypeMatchesTreeShapCPU) { + Context ctx; + size_t constexpr kRows = 256; + size_t constexpr kCols = 1; + + auto dmat = RandomDataGenerator(kRows, kCols, 0.0).Device(ctx.Device()).GenerateDMatrix(); + SetLabels(dmat.get(), 1); + + auto args = BaseParams(&ctx, "binary:logistic", "6"); + args.emplace_back("tree_method", "exact"); + + std::shared_ptr p_dmat{dmat.get(), [](DMatrix*) {}}; + std::unique_ptr learner{Learner::Create({p_dmat})}; + learner->SetParams(args); + learner->Configure(); + for (size_t i = 0; i < 3; ++i) { + learner->UpdateOneIter(i, p_dmat); + } + + HostDeviceVector margin_predt; + learner->Predict(p_dmat, true, &margin_predt, 0, 0, false, false, false, false, false); + + LearnerModelParam mparam; + auto gbtree = LoadGBTreeModel(learner.get(), dmat->Ctx(), args, &mparam); + + HostDeviceVector treeshap; + interpretability::ShapValues(dmat->Ctx(), p_dmat.get(), &treeshap, *gbtree, 0, nullptr, 0, 0); + + HostDeviceVector v6_shap; + interpretability::cpu_impl::V6ShapValues(dmat->Ctx(), p_dmat.get(), &v6_shap, *gbtree, 0, + nullptr); + + auto const& h_treeshap = treeshap.ConstHostVector(); + auto const& h_v6 = v6_shap.ConstHostVector(); + ASSERT_EQ(h_treeshap.size(), h_v6.size()); + for (size_t i = 0; i < h_treeshap.size(); ++i) { + EXPECT_NEAR(h_treeshap[i], h_v6[i], 1e-4f); + } + + CheckShapAdditivity(kRows, kCols, v6_shap, margin_predt); +} + +TEST(Predictor, V6SelectorMatchesTreeShapCPU) { + Context ctx; + size_t constexpr kRows = 256; + size_t constexpr kCols = 1; + + auto dmat = RandomDataGenerator(kRows, kCols, 0.0).Device(ctx.Device()).GenerateDMatrix(); + SetLabels(dmat.get(), 1); + + std::unique_ptr learner{Learner::Create({dmat})}; + learner->SetParams(BaseParams(&ctx, "binary:logistic", "6")); + learner->SetParam("tree_method", "exact"); + learner->Configure(); + for (size_t i = 0; i < 3; ++i) { + learner->UpdateOneIter(i, dmat); + } + + HostDeviceVector margin_predt; + learner->Predict(dmat, true, &margin_predt, 0, 0, false, false, false, false, false); + + HostDeviceVector treeshap; + learner->Predict(dmat, false, &treeshap, 0, 0, false, false, true, false, false); + + learner->SetParam("shap_algorithm", "v6"); + learner->Configure(); + + HostDeviceVector v6_shap; + learner->Predict(dmat, false, &v6_shap, 0, 0, false, false, true, false, false); + + auto const& h_treeshap = treeshap.ConstHostVector(); + auto const& h_v6 = v6_shap.ConstHostVector(); + ASSERT_EQ(h_treeshap.size(), h_v6.size()); + for (size_t i = 0; i < h_treeshap.size(); ++i) { + EXPECT_NEAR(h_treeshap[i], h_v6[i], 1e-4f); + } + + CheckShapAdditivity(kRows, kCols, v6_shap, margin_predt); +} + TEST(Predictor, ApproxContribsBasic) { Context ctx; size_t constexpr kRows = 64; From 08881298fc0aeeeb72fd527a3b2bf9dc4b0d8692 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Thu, 19 Mar 2026 02:02:05 -0700 Subject: [PATCH 02/13] Refactor and optimize CPU QuadratureSHAP --- doc/parameter.rst | 4 +- src/predictor/cpu_predictor.cc | 8 +- src/predictor/interpretability/shap.cc | 350 ++++++++++++++++++------- src/predictor/interpretability/shap.h | 6 +- src/predictor/treeshap.cc | 232 ---------------- src/predictor/treeshap.h | 33 --- tests/cpp/predictor/test_shap.cc | 119 +++++++-- 7 files changed, 374 insertions(+), 378 deletions(-) delete mode 100644 src/predictor/treeshap.cc delete mode 100644 src/predictor/treeshap.h diff --git a/doc/parameter.rst b/doc/parameter.rst index 72b0b8c4423f..404f8403fd66 100644 --- a/doc/parameter.rst +++ b/doc/parameter.rst @@ -192,10 +192,10 @@ Parameters for Tree Booster * ``shap_algorithm`` string [default= ``treeshap``] - CPU algorithm used for ``pred_contribs`` with tree boosters. - - Choices: ``treeshap``, ``v6``. + - Choices: ``treeshap``, ``quadratureshap``. - ``treeshap``: Existing exact TreeSHAP implementation. - - ``v6``: Quadrature plus telescoping SHAP implementation for CPU prediction. + - ``quadratureshap``: Quadrature plus telescoping SHAP implementation for CPU prediction. * ``scale_pos_weight`` [default=1] diff --git a/src/predictor/cpu_predictor.cc b/src/predictor/cpu_predictor.cc index 40bfc9aab968..961e10e3342c 100644 --- a/src/predictor/cpu_predictor.cc +++ b/src/predictor/cpu_predictor.cc @@ -748,7 +748,7 @@ class CPUPredictor : public Predictor { void Configure(Args const &cfg) override { for (auto const &kv : cfg) { if (kv.first == "shap_algorithm") { - CHECK(kv.second == "treeshap" || kv.second == "v6") + CHECK(kv.second == "treeshap" || kv.second == "quadratureshap") << "Unknown SHAP algorithm: " << kv.second; shap_algorithm_ = kv.second; } @@ -878,9 +878,9 @@ class CPUPredictor : public Predictor { if (approximate) { interpretability::ApproxFeatureImportance(this->ctx_, p_fmat, out_contribs, model, ntree_limit, tree_weights); - } else if (shap_algorithm_ == "v6" && condition == 0 && condition_feature == 0) { - interpretability::cpu_impl::V6ShapValues(this->ctx_, p_fmat, out_contribs, model, ntree_limit, - tree_weights); + } else if (shap_algorithm_ == "quadratureshap" && condition == 0 && condition_feature == 0) { + interpretability::cpu_impl::QuadratureShapValues(this->ctx_, p_fmat, out_contribs, model, + ntree_limit, tree_weights); } else { interpretability::ShapValues(this->ctx_, p_fmat, out_contribs, model, ntree_limit, tree_weights, condition, condition_feature); diff --git a/src/predictor/interpretability/shap.cc b/src/predictor/interpretability/shap.cc index c7c7473831cc..d739fc4618c9 100644 --- a/src/predictor/interpretability/shap.cc +++ b/src/predictor/interpretability/shap.cc @@ -3,9 +3,10 @@ */ #include "shap.h" -#include // for fill +#include // for copy, fill #include // for array #include // for abs +#include // for uint32_t #include // for numeric_limits #include // for remove_const_t #include // for vector @@ -15,7 +16,6 @@ #include "../../tree/tree_view.h" // for ScalarTreeView #include "../data_accessor.h" // for GHistIndexMatrixView #include "../predict_fn.h" // for GetTreeLimit -#include "../treeshap.h" // for CalculateContributions #include "dmlc/omp.h" // for omp_get_thread_num #include "xgboost/base.h" // for bst_omp_uint #include "xgboost/logging.h" // for CHECK @@ -60,118 +60,289 @@ void CalculateApproxContributions(tree::ScalarTreeView const &tree, RegTree::FVe std::vector *mean_values, std::vector *out_contribs) { CHECK_EQ(out_contribs->size(), feats.Size() + 1); - CalculateContributionsApprox(tree, feats, mean_values, out_contribs->data()); + CHECK_GT(mean_values->size(), 0U); + bst_feature_t split_index = 0; + float node_value = (*mean_values)[0]; + out_contribs->back() += node_value; + if (tree.IsLeaf(RegTree::kRoot)) { + return; + } + + bst_node_t nidx = RegTree::kRoot; + auto const &cats = tree.GetCategoriesMatrix(); + while (!tree.IsLeaf(nidx)) { + split_index = tree.SplitIndex(nidx); + nidx = predictor::GetNextNode(tree, nidx, feats.GetFvalue(split_index), + feats.IsMissing(split_index), cats); + auto new_value = (*mean_values)[nidx]; + (*out_contribs)[split_index] += new_value - node_value; + node_value = new_value; + } + (*out_contribs)[split_index] += tree.LeafValue(nidx) - node_value; +} + +struct PathElement { + int feature_index; + float zero_fraction; + float one_fraction; + float pweight; + PathElement() = default; + PathElement(int i, float z, float o, float w) + : feature_index(i), zero_fraction(z), one_fraction(o), pweight(w) {} +}; + +void ExtendPath(PathElement *unique_path, std::uint32_t unique_depth, float zero_fraction, + float one_fraction, int feature_index) { + unique_path[unique_depth].feature_index = feature_index; + unique_path[unique_depth].zero_fraction = zero_fraction; + unique_path[unique_depth].one_fraction = one_fraction; + unique_path[unique_depth].pweight = (unique_depth == 0 ? 1.0f : 0.0f); + for (int i = static_cast(unique_depth) - 1; i >= 0; --i) { + unique_path[i + 1].pweight += + one_fraction * unique_path[i].pweight * (i + 1) / static_cast(unique_depth + 1); + unique_path[i].pweight = zero_fraction * unique_path[i].pweight * (unique_depth - i) / + static_cast(unique_depth + 1); + } +} + +void UnwindPath(PathElement *unique_path, std::uint32_t unique_depth, std::uint32_t path_index) { + auto const one_fraction = unique_path[path_index].one_fraction; + auto const zero_fraction = unique_path[path_index].zero_fraction; + float next_one_portion = unique_path[unique_depth].pweight; + + for (int i = static_cast(unique_depth) - 1; i >= 0; --i) { + if (one_fraction != 0.0f) { + auto const tmp = unique_path[i].pweight; + unique_path[i].pweight = + next_one_portion * (unique_depth + 1) / static_cast((i + 1) * one_fraction); + next_one_portion = tmp - unique_path[i].pweight * zero_fraction * (unique_depth - i) / + static_cast(unique_depth + 1); + } else { + unique_path[i].pweight = unique_path[i].pweight * (unique_depth + 1) / + static_cast(zero_fraction * (unique_depth - i)); + } + } + + for (auto i = path_index; i < unique_depth; ++i) { + unique_path[i].feature_index = unique_path[i + 1].feature_index; + unique_path[i].zero_fraction = unique_path[i + 1].zero_fraction; + unique_path[i].one_fraction = unique_path[i + 1].one_fraction; + } } -constexpr std::size_t kV6QuadraturePoints = 30; -constexpr double kV6Qeps = 1e-15; -constexpr double kV6Unseen = -999.0; -using V6Quad = std::array; - -V6Quad const &V6Nodes() { - static constexpr V6Quad kNodes = { - 1.55325796267524740557e-03, 8.16593836012641238753e-03, 1.99890675158462260974e-02, - 3.68999762853628454629e-02, 5.87197321039736319648e-02, 8.52171188086158215569e-02, - 1.16111283947586907406e-01, 1.51074752603342077339e-01, 1.89736908505378554235e-01, - 2.31687925928990068325e-01, 2.76483115230955422970e-01, 3.23647637234560914266e-01, - 3.72681536916055100583e-01, 4.23065043195708256896e-01, 4.74264078722341164696e-01, - 5.25735921277658890816e-01, 5.76934956804291743104e-01, 6.27318463083944899417e-01, - 6.76352362765439085734e-01, 7.23516884769044521519e-01, 7.68312074071009876164e-01, - 8.10263091494621390254e-01, 8.48925247396657978172e-01, 8.83888716052413148105e-01, - 9.14782881191384178443e-01, 9.41280267896026368035e-01, 9.63100023714637210048e-01, - 9.80010932484153718391e-01, 9.91834061639873532101e-01, 9.98446742037324752594e-01}; +float UnwoundPathSum(PathElement const *unique_path, std::uint32_t unique_depth, + std::uint32_t path_index) { + auto const one_fraction = unique_path[path_index].one_fraction; + auto const zero_fraction = unique_path[path_index].zero_fraction; + float next_one_portion = unique_path[unique_depth].pweight; + float total = 0.0f; + for (int i = static_cast(unique_depth) - 1; i >= 0; --i) { + if (one_fraction != 0.0f) { + auto const tmp = + next_one_portion * (unique_depth + 1) / static_cast((i + 1) * one_fraction); + total += tmp; + next_one_portion = + unique_path[i].pweight - + tmp * zero_fraction * ((unique_depth - i) / static_cast(unique_depth + 1)); + } else if (zero_fraction != 0.0f) { + total += (unique_path[i].pweight / zero_fraction) / + ((unique_depth - i) / static_cast(unique_depth + 1)); + } else { + CHECK_EQ(unique_path[i].pweight, 0.0f) << "Unique path " << i << " must have zero weight"; + } + } + return total; +} + +void TreeShap(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, float *phi, + bst_node_t nidx, std::uint32_t unique_depth, PathElement *parent_unique_path, + float parent_zero_fraction, float parent_one_fraction, int parent_feature_index, + int condition, std::uint32_t condition_feature, float condition_fraction) { + if (condition_fraction == 0.0f) { + return; + } + + PathElement *unique_path = parent_unique_path + unique_depth + 1; + std::copy(parent_unique_path, parent_unique_path + unique_depth + 1, unique_path); + if (condition == 0 || condition_feature != static_cast(parent_feature_index)) { + ExtendPath(unique_path, unique_depth, parent_zero_fraction, parent_one_fraction, + parent_feature_index); + } + + auto const split_index = tree.SplitIndex(nidx); + if (tree.IsLeaf(nidx)) { + for (std::uint32_t i = 1; i <= unique_depth; ++i) { + auto const w = UnwoundPathSum(unique_path, unique_depth, i); + auto const &el = unique_path[i]; + phi[el.feature_index] += + w * (el.one_fraction - el.zero_fraction) * tree.LeafValue(nidx) * condition_fraction; + } + return; + } + + auto const &cats = tree.GetCategoriesMatrix(); + auto hot_index = predictor::GetNextNode(tree, nidx, feat.GetFvalue(split_index), + feat.IsMissing(split_index), cats); + auto const cold_index = + (hot_index == tree.LeftChild(nidx) ? tree.RightChild(nidx) : tree.LeftChild(nidx)); + auto const w = tree.Stat(nidx).sum_hess; + auto const hot_zero_fraction = tree.Stat(hot_index).sum_hess / w; + auto const cold_zero_fraction = tree.Stat(cold_index).sum_hess / w; + float incoming_zero_fraction = 1.0f; + float incoming_one_fraction = 1.0f; + + std::uint32_t path_index = 0; + for (; path_index <= unique_depth; ++path_index) { + if (static_cast(unique_path[path_index].feature_index) == split_index) { + break; + } + } + if (path_index != unique_depth + 1) { + incoming_zero_fraction = unique_path[path_index].zero_fraction; + incoming_one_fraction = unique_path[path_index].one_fraction; + UnwindPath(unique_path, unique_depth, path_index); + unique_depth -= 1; + } + + float hot_condition_fraction = condition_fraction; + float cold_condition_fraction = condition_fraction; + if (condition > 0 && split_index == condition_feature) { + cold_condition_fraction = 0.0f; + unique_depth -= 1; + } else if (condition < 0 && split_index == condition_feature) { + hot_condition_fraction *= hot_zero_fraction; + cold_condition_fraction *= cold_zero_fraction; + unique_depth -= 1; + } + + TreeShap(tree, feat, phi, hot_index, unique_depth + 1, unique_path, + hot_zero_fraction * incoming_zero_fraction, incoming_one_fraction, split_index, + condition, condition_feature, hot_condition_fraction); + TreeShap(tree, feat, phi, cold_index, unique_depth + 1, unique_path, + cold_zero_fraction * incoming_zero_fraction, 0.0f, split_index, condition, + condition_feature, cold_condition_fraction); +} + +void CalculateContributions(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, + std::vector *mean_values, float *out_contribs, int condition, + std::uint32_t condition_feature) { + if (condition == 0) { + out_contribs[feat.Size()] += (*mean_values)[RegTree::kRoot]; + } + + auto const maxd = tree.MaxDepth() + 2; + std::vector unique_path_data((maxd * (maxd + 1)) / 2); + TreeShap(tree, feat, out_contribs, RegTree::kRoot, 0, unique_path_data.data(), 1.0f, 1.0f, -1, + condition, condition_feature, 1.0f); +} + +constexpr std::size_t kQuadratureShapPoints = 16; +constexpr double kQuadratureShapQeps = 1e-15; +constexpr double kQuadratureShapUnseen = -999.0; +using QuadratureRule = std::array; + +QuadratureRule const &QuadratureNodes() { + static constexpr QuadratureRule kNodes = { + 2.80850447628076738593e-05, 7.67982016833174672456e-04, 4.51374344293495755737e-03, + 1.49567508630415318960e-02, 3.65046411479570120928e-02, 7.34364533252638285177e-02, + 1.29023364563242204373e-01, 2.04750589337593075223e-01, 2.99763099175230585125e-01, + 4.10626915342501119799e-01, 5.31453230982491087175e-01, 6.54380885550600810419e-01, + 7.70361159218044599939e-01, 8.70144945830766847195e-01, 9.45343005090065746643e-01, + 9.89429020036412754102e-01}; return kNodes; } -V6Quad const &V6Weights() { - static constexpr V6Quad kWeights = { - 3.98409624808451681005e-03, 9.23323415554584518705e-03, 1.43923539416612698838e-02, - 1.93995962848134140266e-02, 2.42013364152968944720e-02, 2.87465781088096158924e-02, - 3.29871149410902175791e-02, 3.68779873688524495456e-02, 4.03779476147099816719e-02, - 4.34498936005414254646e-02, 4.60612611188929710337e-02, 4.81843685873219601534e-02, - 4.97967102933974670176e-02, 5.08811948742026384784e-02, 5.14263264467792954870e-02, - 5.14263264467792954870e-02, 5.08811948742026384784e-02, 4.97967102933974670176e-02, - 4.81843685873219601534e-02, 4.60612611188929710337e-02, 4.34498936005414254646e-02, - 4.03779476147099816719e-02, 3.68779873688524495456e-02, 3.29871149410902175791e-02, - 2.87465781088096158924e-02, 2.42013364152968944720e-02, 1.93995962848134140266e-02, - 1.43923539416612698838e-02, 9.23323415554584518705e-03, 3.98409624808451681005e-03}; +QuadratureRule const &QuadratureWeights() { + static constexpr QuadratureRule kWeights = { + 1.43895341220884505091e-04, 1.72520006395474869917e-03, 6.39316739866999800279e-03, + 1.52418484801773411463e-02, 2.85820905344451973995e-02, 4.58399977309956255245e-02, + 6.55908224919272003772e-02, 8.57252162327300087918e-02, 1.03725394222338646033e-01, + 1.17012592552996438910e-01, 1.23316521664007014425e-01, 1.21013898282131507345e-01, + 1.09387122775356740445e-01, 8.87653442838226142131e-02, 6.05283238746927090834e-02, + 2.70085640705332932776e-02}; return kWeights; } -V6Quad Scale(V6Quad h_vals, double scale) { - for (auto &v : h_vals) { +void ScaleInPlace(QuadratureRule *h_vals, double scale) { + for (auto &v : *h_vals) { v *= scale; } - return h_vals; } -V6Quad Add(V6Quad lhs, V6Quad const &rhs) { - for (std::size_t i = 0; i < lhs.size(); ++i) { - lhs[i] += rhs[i]; +void AddInPlace(QuadratureRule *lhs, QuadratureRule const &rhs) { + for (std::size_t i = 0; i < lhs->size(); ++i) { + (*lhs)[i] += rhs[i]; } - return lhs; } -double ExtractTermV6(V6Quad const &h_vals, double p_value) { - if (p_value == kV6Unseen) { - return 0.0; - } - auto alpha = p_value - 1.0; - if (std::abs(alpha) < kV6Qeps) { +double ExtractQuadratureDelta(QuadratureRule const &h_vals, double p_enter, double p_exit) { + auto const alpha_enter = p_enter - 1.0; + auto const alpha_exit = p_exit - 1.0; + auto const has_enter = + (p_enter != kQuadratureShapUnseen) && (std::abs(alpha_enter) >= kQuadratureShapQeps); + auto const has_exit = + (p_exit != kQuadratureShapUnseen) && (std::abs(alpha_exit) >= kQuadratureShapQeps); + if (!has_enter && !has_exit) { return 0.0; } - auto const &nodes = V6Nodes(); - auto const &weights = V6Weights(); + auto const &nodes = QuadratureNodes(); + auto const &weights = QuadratureWeights(); double acc = 0.0; for (std::size_t i = 0; i < h_vals.size(); ++i) { - acc += alpha * h_vals[i] / (1.0 + alpha * nodes[i]) * weights[i]; + auto const weighted_h = h_vals[i] * weights[i]; + if (has_enter) { + acc += alpha_enter * weighted_h / (1.0 + alpha_enter * nodes[i]); + } + if (has_exit) { + acc -= alpha_exit * weighted_h / (1.0 + alpha_exit * nodes[i]); + } } return acc; } -bool GoesLeftV6(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, bst_node_t nidx) { +bool GoesLeftQuadrature(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, + bst_node_t nidx) { auto split_index = tree.SplitIndex(nidx); - auto fvalue = feat.GetFvalue(split_index); - auto missing = feat.IsMissing(split_index); auto const &cats = tree.GetCategoriesMatrix(); - bst_node_t next = RegTree::kInvalidNodeId; - if (tree.HasCategoricalSplit()) { - next = missing ? predictor::GetNextNode(tree, nidx, fvalue, true, cats) - : predictor::GetNextNode(tree, nidx, fvalue, false, cats); - } else { - next = missing ? predictor::GetNextNode(tree, nidx, fvalue, true, cats) - : predictor::GetNextNode(tree, nidx, fvalue, false, cats); - } + auto next = predictor::GetNextNode(tree, nidx, feat.GetFvalue(split_index), + feat.IsMissing(split_index), cats); return next == tree.LeftChild(nidx); } -double ChildWeightV6(tree::ScalarTreeView const &tree, bst_node_t parent, bst_node_t child) { +double ChildWeightQuadrature(tree::ScalarTreeView const &tree, bst_node_t parent, + bst_node_t child) { auto parent_cover = tree.Stat(parent).sum_hess; CHECK_GT(parent_cover, 0.0f); return tree.Stat(child).sum_hess / parent_cover; } -V6Quad TreeShapV6(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, bst_node_t nidx, - V6Quad const &c_vals, double w_prod, std::vector *p_vals, double *phi) { +void TreeShapQuadrature(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, + bst_node_t nidx, QuadratureRule const &c_vals, double w_prod, + std::vector *p_vals, double *phi, QuadratureRule *out_h) { if (tree.IsLeaf(nidx)) { - return Scale(c_vals, w_prod * tree.LeafValue(nidx)); + *out_h = c_vals; + ScaleInPlace(out_h, w_prod * tree.LeafValue(nidx)); + return; } auto split_index = tree.SplitIndex(nidx); auto left = tree.LeftChild(nidx); auto right = tree.RightChild(nidx); - auto left_weight = ChildWeightV6(tree, nidx, left); - auto right_weight = ChildWeightV6(tree, nidx, right); - auto goes_left = GoesLeftV6(tree, feat, nidx); + auto left_weight = ChildWeightQuadrature(tree, nidx, left); + auto right_weight = ChildWeightQuadrature(tree, nidx, right); + auto goes_left = GoesLeftQuadrature(tree, feat, nidx); auto p_old = (*p_vals)[split_index]; + auto const &nodes = QuadratureNodes(); + QuadratureRule child_h; - auto visit_child = [&](bst_node_t child, double child_weight, bool satisfies) { + auto visit_child = [&](bst_node_t child, double child_weight, bool satisfies, + QuadratureRule *out_child) { double p_e = 0.0; double p_up = 0.0; - if (p_old == kV6Unseen) { + if (p_old == kQuadratureShapUnseen) { p_e = satisfies ? 1.0 / child_weight : 0.0; p_up = 1.0; - } else if (std::abs(p_old) < kV6Qeps) { + } else if (std::abs(p_old) < kQuadratureShapQeps) { p_e = 0.0; p_up = 0.0; } else { @@ -180,15 +351,14 @@ V6Quad TreeShapV6(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, b } auto c_child = c_vals; - auto const &nodes = V6Nodes(); auto alpha_e = p_e - 1.0; for (std::size_t i = 0; i < c_child.size(); ++i) { c_child[i] *= 1.0 + alpha_e * nodes[i]; } - if (p_old != kV6Unseen) { + if (p_old != kQuadratureShapUnseen) { auto alpha_old = p_old - 1.0; - if (std::abs(alpha_old) >= kV6Qeps) { + if (std::abs(alpha_old) >= kQuadratureShapQeps) { for (std::size_t i = 0; i < c_child.size(); ++i) { c_child[i] /= 1.0 + alpha_old * nodes[i]; } @@ -196,30 +366,29 @@ V6Quad TreeShapV6(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, b } (*p_vals)[split_index] = p_e; - auto h_child = TreeShapV6(tree, feat, child, c_child, w_prod * child_weight, p_vals, phi); + TreeShapQuadrature(tree, feat, child, c_child, w_prod * child_weight, p_vals, phi, out_child); (*p_vals)[split_index] = p_old; - phi[split_index] += ExtractTermV6(h_child, p_e); - phi[split_index] -= ExtractTermV6(h_child, p_up); - return h_child; + phi[split_index] += ExtractQuadratureDelta(*out_child, p_e, p_up); }; - auto left_h = visit_child(left, left_weight, goes_left); - auto right_h = visit_child(right, right_weight, !goes_left); - return Add(std::move(left_h), right_h); + visit_child(left, left_weight, goes_left, out_h); + visit_child(right, right_weight, !goes_left, &child_h); + AddInPlace(out_h, child_h); } -void CalculateContributionsV6(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, - std::vector *mean_values, double *out_contribs) { +void CalculateContributionsQuadrature(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, + std::vector *mean_values, double *out_contribs, + std::vector *p_vals) { out_contribs[feat.Size()] += (*mean_values)[0]; if (tree.IsLeaf(RegTree::kRoot)) { return; } - V6Quad c_init; + QuadratureRule c_init; c_init.fill(1.0); - std::vector p_vals(feat.Size(), kV6Unseen); - TreeShapV6(tree, feat, RegTree::kRoot, c_init, 1.0, &p_vals, out_contribs); + QuadratureRule h_vals; + TreeShapQuadrature(tree, feat, RegTree::kRoot, c_init, 1.0, p_vals, out_contribs, &h_vals); } template @@ -327,9 +496,9 @@ void ShapValues(Context const *ctx, DMatrix *p_fmat, HostDeviceVector *ou LaunchShap(ctx, p_fmat, model, process_view); } -void V6ShapValues(Context const *ctx, DMatrix *p_fmat, HostDeviceVector *out_contribs, - gbm::GBTreeModel const &model, bst_tree_t tree_end, - std::vector const *tree_weights) { +void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, + HostDeviceVector *out_contribs, gbm::GBTreeModel const &model, + bst_tree_t tree_end, std::vector const *tree_weights) { CHECK(!model.learner_model_param->IsVectorLeaf()) << "Predict contribution" << MTNotImplemented(); CHECK(!p_fmat->Info().IsColumnSplit()) << "Predict contribution support for column-wise data split is not yet implemented."; @@ -355,6 +524,9 @@ void V6ShapValues(Context const *ctx, DMatrix *p_fmat, HostDeviceVector * auto const h_tree_groups = model.TreeGroups(DeviceOrd::CPU()); std::vector feats_tloc(n_threads); std::vector> contribs_tloc(n_threads, std::vector(ncolumns)); + std::vector> p_vals_tloc( + n_threads, + std::vector(model.learner_model_param->num_feature, kQuadratureShapUnseen)); auto device = ctx->Device().IsSycl() ? DeviceOrd::CPU() : ctx->Device(); auto base_margin = info.base_margin_.View(device); @@ -367,6 +539,7 @@ void V6ShapValues(Context const *ctx, DMatrix *p_fmat, HostDeviceVector * feats.Init(model.learner_model_param->num_feature); } auto &this_tree_contribs = contribs_tloc[tid]; + auto &p_vals = p_vals_tloc[tid]; auto row_idx = view.base_rowid + i; auto n_valid = view.DoFill(i, feats.Data().data()); feats.HasMissing(n_valid != feats.Size()); @@ -378,7 +551,8 @@ void V6ShapValues(Context const *ctx, DMatrix *p_fmat, HostDeviceVector * } std::fill(this_tree_contribs.begin(), this_tree_contribs.end(), 0.0); auto const sc_tree = model.trees[j]->HostScView(); - CalculateContributionsV6(sc_tree, feats, &mean_values[j], this_tree_contribs.data()); + CalculateContributionsQuadrature(sc_tree, feats, &mean_values[j], + this_tree_contribs.data(), &p_vals); auto const weight = tree_weights == nullptr ? 1.0f : (*tree_weights)[j]; for (size_t ci = 0; ci < ncolumns; ++ci) { p_contribs[ci] += static_cast(this_tree_contribs[ci] * weight); diff --git a/src/predictor/interpretability/shap.h b/src/predictor/interpretability/shap.h index d9853cfe9229..18d17718fa3e 100644 --- a/src/predictor/interpretability/shap.h +++ b/src/predictor/interpretability/shap.h @@ -19,9 +19,9 @@ void ShapValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* ou gbm::GBTreeModel const& model, bst_tree_t tree_end, std::vector const* tree_weights, int condition, unsigned condition_feature); -void V6ShapValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, - gbm::GBTreeModel const& model, bst_tree_t tree_end, - std::vector const* tree_weights); +void QuadratureShapValues(Context const* ctx, DMatrix* p_fmat, + HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, + bst_tree_t tree_end, std::vector const* tree_weights); void ApproxFeatureImportance(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, diff --git a/src/predictor/treeshap.cc b/src/predictor/treeshap.cc deleted file mode 100644 index bae297c973a8..000000000000 --- a/src/predictor/treeshap.cc +++ /dev/null @@ -1,232 +0,0 @@ -/** - * Copyright 2017-2025, XGBoost Contributors - */ -#include "treeshap.h" - -#include // copy -#include // std::uint32_t - -#include "../tree/tree_view.h" // for ScalarTreeView -#include "predict_fn.h" // GetNextNode -#include "xgboost/base.h" // bst_node_t -#include "xgboost/logging.h" -#include "xgboost/tree_model.h" // RegTree - -namespace xgboost { -void CalculateContributionsApprox(tree::ScalarTreeView const& tree, const RegTree::FVec& feat, - std::vector* mean_values, float* out_contribs) { - CHECK_GT(mean_values->size(), 0U); - bst_feature_t split_index = 0; - // update bias value - float node_value = (*mean_values)[0]; - out_contribs[feat.Size()] += node_value; - if (tree.IsLeaf(RegTree::kRoot)) { - // nothing to do anymore - return; - } - - bst_node_t nidx = 0; - auto const& cats = tree.GetCategoriesMatrix(); - - while (!tree.IsLeaf(nidx)) { - split_index = tree.SplitIndex(nidx); - nidx = predictor::GetNextNode(tree, nidx, feat.GetFvalue(split_index), - feat.IsMissing(split_index), cats); - bst_float new_value = (*mean_values)[nidx]; - // update feature weight - out_contribs[split_index] += new_value - node_value; - node_value = new_value; - } - float leaf_value = tree.LeafValue(nidx); - // update leaf feature weight - out_contribs[split_index] += leaf_value - node_value; -} - -// Used by TreeShap -// data we keep about our decision path -// note that pweight is included for convenience and is not tied with the other attributes -// the pweight of the i'th path element is the permutation weight of paths with i-1 ones in them -struct PathElement { - int feature_index; - float zero_fraction; - float one_fraction; - float pweight; - PathElement() = default; - PathElement(int i, float z, float o, float w) - : feature_index(i), zero_fraction(z), one_fraction(o), pweight(w) {} -}; - -// extend our decision path with a fraction of one and zero extensions -void ExtendPath(PathElement* unique_path, std::uint32_t unique_depth, float zero_fraction, - float one_fraction, int feature_index) { - unique_path[unique_depth].feature_index = feature_index; - unique_path[unique_depth].zero_fraction = zero_fraction; - unique_path[unique_depth].one_fraction = one_fraction; - unique_path[unique_depth].pweight = (unique_depth == 0 ? 1.0f : 0.0f); - for (int i = unique_depth - 1; i >= 0; i--) { - unique_path[i + 1].pweight += - one_fraction * unique_path[i].pweight * (i + 1) / static_cast(unique_depth + 1); - unique_path[i].pweight = zero_fraction * unique_path[i].pweight * (unique_depth - i) / - static_cast(unique_depth + 1); - } -} - -// undo a previous extension of the decision path -void UnwindPath(PathElement* unique_path, std::uint32_t unique_depth, std::uint32_t path_index) { - const float one_fraction = unique_path[path_index].one_fraction; - const float zero_fraction = unique_path[path_index].zero_fraction; - float next_one_portion = unique_path[unique_depth].pweight; - - for (int i = unique_depth - 1; i >= 0; --i) { - if (one_fraction != 0) { - const float tmp = unique_path[i].pweight; - unique_path[i].pweight = - next_one_portion * (unique_depth + 1) / static_cast((i + 1) * one_fraction); - next_one_portion = tmp - unique_path[i].pweight * zero_fraction * (unique_depth - i) / - static_cast(unique_depth + 1); - } else { - unique_path[i].pweight = (unique_path[i].pweight * (unique_depth + 1)) / - static_cast(zero_fraction * (unique_depth - i)); - } - } - - for (auto i = path_index; i < unique_depth; ++i) { - unique_path[i].feature_index = unique_path[i + 1].feature_index; - unique_path[i].zero_fraction = unique_path[i + 1].zero_fraction; - unique_path[i].one_fraction = unique_path[i + 1].one_fraction; - } -} - -// determine what the total permutation weight would be if -// we unwound a previous extension in the decision path -float UnwoundPathSum(const PathElement* unique_path, std::uint32_t unique_depth, - std::uint32_t path_index) { - const float one_fraction = unique_path[path_index].one_fraction; - const float zero_fraction = unique_path[path_index].zero_fraction; - float next_one_portion = unique_path[unique_depth].pweight; - float total = 0; - for (int i = unique_depth - 1; i >= 0; --i) { - if (one_fraction != 0) { - const float tmp = - next_one_portion * (unique_depth + 1) / static_cast((i + 1) * one_fraction); - total += tmp; - next_one_portion = - unique_path[i].pweight - - tmp * zero_fraction * ((unique_depth - i) / static_cast(unique_depth + 1)); - } else if (zero_fraction != 0) { - total += (unique_path[i].pweight / zero_fraction) / - ((unique_depth - i) / static_cast(unique_depth + 1)); - } else { - CHECK_EQ(unique_path[i].pweight, 0) << "Unique path " << i << " must have zero weight"; - } - } - return total; -} - -/** - * \brief Recursive function that computes the feature attributions for a single tree. - * \param feat dense feature vector, if the feature is missing the field is set to NaN - * \param phi dense output vector of feature attributions - * \param node_index the index of the current node in the tree - * \param unique_depth how many unique features are above the current node in the tree - * \param parent_unique_path a vector of statistics about our current path through the tree - * \param parent_zero_fraction what fraction of the parent path weight is coming as 0 (integrated) - * \param parent_one_fraction what fraction of the parent path weight is coming as 1 (fixed) - * \param parent_feature_index what feature the parent node used to split - * \param condition fix one feature to either off (-1) on (1) or not fixed (0 default) - * \param condition_feature the index of the feature to fix - * \param condition_fraction what fraction of the current weight matches our conditioning feature - */ -void TreeShap(tree::ScalarTreeView const& tree, const RegTree::FVec& feat, float* phi, - bst_node_t nidx, std::uint32_t unique_depth, PathElement* parent_unique_path, - float parent_zero_fraction, float parent_one_fraction, int parent_feature_index, - int condition, std::uint32_t condition_feature, float condition_fraction) { - // stop if we have no weight coming down to us - if (condition_fraction == 0) return; - - // extend the unique path - PathElement* unique_path = parent_unique_path + unique_depth + 1; - std::copy(parent_unique_path, parent_unique_path + unique_depth + 1, unique_path); - - if (condition == 0 || condition_feature != static_cast(parent_feature_index)) { - ExtendPath(unique_path, unique_depth, parent_zero_fraction, parent_one_fraction, - parent_feature_index); - } - const std::uint32_t split_index = tree.SplitIndex(nidx); - - // leaf node - if (tree.IsLeaf(nidx)) { - for (std::uint32_t i = 1; i <= unique_depth; ++i) { - const float w = UnwoundPathSum(unique_path, unique_depth, i); - const PathElement& el = unique_path[i]; - phi[el.feature_index] += - w * (el.one_fraction - el.zero_fraction) * tree.LeafValue(nidx) * condition_fraction; - } - - // internal node - } else { - // find which branch is "hot" (meaning x would follow it) - auto const& cats = tree.GetCategoriesMatrix(); - bst_node_t hot_index = predictor::GetNextNode( - tree, nidx, feat.GetFvalue(split_index), feat.IsMissing(split_index), cats); - - const auto cold_index = - (hot_index == tree.LeftChild(nidx) ? tree.RightChild(nidx) : tree.LeftChild(nidx)); - const float w = tree.Stat(nidx).sum_hess; - const float hot_zero_fraction = tree.Stat(hot_index).sum_hess / w; - const float cold_zero_fraction = tree.Stat(cold_index).sum_hess / w; - float incoming_zero_fraction = 1; - float incoming_one_fraction = 1; - - // see if we have already split on this feature, - // if so we undo that split so we can redo it for this node - std::uint32_t path_index = 0; - for (; path_index <= unique_depth; ++path_index) { - if (static_cast(unique_path[path_index].feature_index) == split_index) break; - } - if (path_index != unique_depth + 1) { - incoming_zero_fraction = unique_path[path_index].zero_fraction; - incoming_one_fraction = unique_path[path_index].one_fraction; - UnwindPath(unique_path, unique_depth, path_index); - unique_depth -= 1; - } - - // divide up the condition_fraction among the recursive calls - float hot_condition_fraction = condition_fraction; - float cold_condition_fraction = condition_fraction; - if (condition > 0 && split_index == condition_feature) { - cold_condition_fraction = 0; - unique_depth -= 1; - } else if (condition < 0 && split_index == condition_feature) { - hot_condition_fraction *= hot_zero_fraction; - cold_condition_fraction *= cold_zero_fraction; - unique_depth -= 1; - } - - TreeShap(tree, feat, phi, hot_index, unique_depth + 1, unique_path, - hot_zero_fraction * incoming_zero_fraction, incoming_one_fraction, split_index, - condition, condition_feature, hot_condition_fraction); - - TreeShap(tree, feat, phi, cold_index, unique_depth + 1, unique_path, - cold_zero_fraction * incoming_zero_fraction, 0, split_index, condition, - condition_feature, cold_condition_fraction); - } -} - -void CalculateContributions(tree::ScalarTreeView const& tree, const RegTree::FVec& feat, - std::vector* mean_values, float* out_contribs, int condition, - std::uint32_t condition_feature) { - // find the expected value of the tree's predictions - if (condition == 0) { - float node_value = (*mean_values)[0]; - out_contribs[feat.Size()] += node_value; - } - - // Preallocate space for the unique path data - bst_node_t const maxd = tree.MaxDepth() + 2; - std::vector unique_path_data((maxd * (maxd + 1)) / 2); - - TreeShap(tree, feat, out_contribs, 0, 0, unique_path_data.data(), 1, 1, -1, condition, - condition_feature, 1); -} -} // namespace xgboost diff --git a/src/predictor/treeshap.h b/src/predictor/treeshap.h deleted file mode 100644 index 69423dd9d4bd..000000000000 --- a/src/predictor/treeshap.h +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2017-2025, XGBoost Contributors - */ -#pragma once - -#include // for vector - -#include "xgboost/tree_model.h" // for RegTree - -namespace xgboost { -/** - * @brief calculate the approximate feature contributions for the given root - * - * This follows the idea of http://blog.datadive.net/interpreting-random-forests/ - * - * @param feat dense feature vector, if the feature is missing the field is set to NaN - * @param out_contribs output vector to hold the contributions - */ -void CalculateContributionsApprox(tree::ScalarTreeView const& tree, const RegTree::FVec& feat, - std::vector* mean_values, float* out_contribs); - -/** - * @brief calculate the feature contributions (https://arxiv.org/abs/1706.06060) for the tree - * - * @param feat dense feature vector, if the feature is missing the field is set to NaN - * @param out_contribs output vector to hold the contributions - * @param condition fix one feature to either off (-1) on (1) or not fixed (0 default) - * @param condition_feature the index of the feature to fix - */ -void CalculateContributions(tree::ScalarTreeView const& tree, const RegTree::FVec& feat, - std::vector* mean_values, float* out_contribs, int condition, - unsigned condition_feature); -} // namespace xgboost diff --git a/tests/cpp/predictor/test_shap.cc b/tests/cpp/predictor/test_shap.cc index d7d9f970358d..bc64d7e48027 100644 --- a/tests/cpp/predictor/test_shap.cc +++ b/tests/cpp/predictor/test_shap.cc @@ -20,10 +20,58 @@ #include "../../../src/common/param_array.h" #include "../../../src/gbm/gbtree_model.h" #include "../../../src/predictor/interpretability/shap.h" +#include "../../../src/tree/tree_view.h" #include "../helpers.h" namespace xgboost { namespace { +struct CoverStats { + double min_child_weight{1.0}; + double min_path_weight{1.0}; + std::size_t internal_nodes{0}; + std::size_t leaves{0}; + std::size_t count_lt_1e2{0}; + std::size_t count_lt_1e3{0}; + std::size_t count_lt_1e4{0}; + std::size_t count_lt_1e5{0}; +}; + +void AccumulateCoverStats(tree::ScalarTreeView const& tree, bst_node_t nidx, double path_min_weight, + CoverStats* stats) { + if (tree.IsLeaf(nidx)) { + ++stats->leaves; + stats->min_path_weight = std::min(stats->min_path_weight, path_min_weight); + return; + } + + ++stats->internal_nodes; + auto left = tree.LeftChild(nidx); + auto right = tree.RightChild(nidx); + auto parent_cover = static_cast(tree.Stat(nidx).sum_hess); + CHECK_GT(parent_cover, 0.0); + + auto visit_child = [&](bst_node_t child) { + auto child_weight = static_cast(tree.Stat(child).sum_hess) / parent_cover; + stats->min_child_weight = std::min(stats->min_child_weight, child_weight); + if (child_weight < 1e-2) { + ++stats->count_lt_1e2; + } + if (child_weight < 1e-3) { + ++stats->count_lt_1e3; + } + if (child_weight < 1e-4) { + ++stats->count_lt_1e4; + } + if (child_weight < 1e-5) { + ++stats->count_lt_1e5; + } + AccumulateCoverStats(tree, child, std::min(path_min_weight, child_weight), stats); + }; + + visit_child(left); + visit_child(right); +} + void SetLabels(DMatrix* dmat, bst_target_t n_classes) { size_t const rows = dmat->Info().num_row_; dmat->Info().labels.Reshape(rows, 1); @@ -291,7 +339,7 @@ TEST(Predictor, DartShapOutputCPU) { CheckDartShapOutput(&ctx); } -TEST(Predictor, V6PrototypeMatchesTreeShapCPU) { +TEST(Predictor, QuadratureShapPrototypeMatchesTreeShapCPU) { Context ctx; size_t constexpr kRows = 256; size_t constexpr kCols = 1; @@ -319,21 +367,21 @@ TEST(Predictor, V6PrototypeMatchesTreeShapCPU) { HostDeviceVector treeshap; interpretability::ShapValues(dmat->Ctx(), p_dmat.get(), &treeshap, *gbtree, 0, nullptr, 0, 0); - HostDeviceVector v6_shap; - interpretability::cpu_impl::V6ShapValues(dmat->Ctx(), p_dmat.get(), &v6_shap, *gbtree, 0, - nullptr); + HostDeviceVector quadrature_shap; + interpretability::cpu_impl::QuadratureShapValues(dmat->Ctx(), p_dmat.get(), &quadrature_shap, + *gbtree, 0, nullptr); auto const& h_treeshap = treeshap.ConstHostVector(); - auto const& h_v6 = v6_shap.ConstHostVector(); - ASSERT_EQ(h_treeshap.size(), h_v6.size()); + auto const& h_quadrature = quadrature_shap.ConstHostVector(); + ASSERT_EQ(h_treeshap.size(), h_quadrature.size()); for (size_t i = 0; i < h_treeshap.size(); ++i) { - EXPECT_NEAR(h_treeshap[i], h_v6[i], 1e-4f); + EXPECT_NEAR(h_treeshap[i], h_quadrature[i], 1e-4f); } - CheckShapAdditivity(kRows, kCols, v6_shap, margin_predt); + CheckShapAdditivity(kRows, kCols, quadrature_shap, margin_predt); } -TEST(Predictor, V6SelectorMatchesTreeShapCPU) { +TEST(Predictor, QuadratureShapSelectorMatchesTreeShapCPU) { Context ctx; size_t constexpr kRows = 256; size_t constexpr kCols = 1; @@ -355,20 +403,59 @@ TEST(Predictor, V6SelectorMatchesTreeShapCPU) { HostDeviceVector treeshap; learner->Predict(dmat, false, &treeshap, 0, 0, false, false, true, false, false); - learner->SetParam("shap_algorithm", "v6"); + learner->SetParam("shap_algorithm", "quadratureshap"); learner->Configure(); - HostDeviceVector v6_shap; - learner->Predict(dmat, false, &v6_shap, 0, 0, false, false, true, false, false); + HostDeviceVector quadrature_shap; + learner->Predict(dmat, false, &quadrature_shap, 0, 0, false, false, true, false, false); auto const& h_treeshap = treeshap.ConstHostVector(); - auto const& h_v6 = v6_shap.ConstHostVector(); - ASSERT_EQ(h_treeshap.size(), h_v6.size()); + auto const& h_quadrature = quadrature_shap.ConstHostVector(); + ASSERT_EQ(h_treeshap.size(), h_quadrature.size()); for (size_t i = 0; i < h_treeshap.size(); ++i) { - EXPECT_NEAR(h_treeshap[i], h_v6[i], 1e-4f); + EXPECT_NEAR(h_treeshap[i], h_quadrature[i], 1e-4f); + } + + CheckShapAdditivity(kRows, kCols, quadrature_shap, margin_predt); +} + +TEST(Predictor, QuadratureShapExactCoverStatsCPU) { + Context ctx; + size_t constexpr kRows = 256; + size_t constexpr kCols = 1; + + auto dmat = RandomDataGenerator(kRows, kCols, 0.0).Device(ctx.Device()).GenerateDMatrix(); + SetLabels(dmat.get(), 1); + + auto args = BaseParams(&ctx, "binary:logistic", "6"); + args.emplace_back("tree_method", "exact"); + + std::shared_ptr p_dmat{dmat.get(), [](DMatrix*) {}}; + std::unique_ptr learner{Learner::Create({p_dmat})}; + learner->SetParams(args); + learner->Configure(); + for (size_t i = 0; i < 3; ++i) { + learner->UpdateOneIter(i, p_dmat); } - CheckShapAdditivity(kRows, kCols, v6_shap, margin_predt); + LearnerModelParam mparam; + auto gbtree = LoadGBTreeModel(learner.get(), dmat->Ctx(), args, &mparam); + + CoverStats stats; + for (auto const& tree : gbtree->trees) { + AccumulateCoverStats(tree->HostScView(), RegTree::kRoot, 1.0, &stats); + } + + std::cout << "QuadratureShap exact cover stats: internal_nodes=" << stats.internal_nodes + << " leaves=" << stats.leaves << " min_child_weight=" << stats.min_child_weight + << " max_first_occurrence_p=" << (1.0 / stats.min_child_weight) + << " min_path_weight=" << stats.min_path_weight + << " max_path_first_occurrence_p=" << (1.0 / stats.min_path_weight) + << " count_lt_1e-2=" << stats.count_lt_1e2 << " count_lt_1e-3=" << stats.count_lt_1e3 + << " count_lt_1e-4=" << stats.count_lt_1e4 << " count_lt_1e-5=" << stats.count_lt_1e5 + << std::endl; + + EXPECT_GT(stats.internal_nodes, 0); } TEST(Predictor, ApproxContribsBasic) { From e257d391f618033de39379cf77b7085c2a1c60e4 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Thu, 19 Mar 2026 02:56:49 -0700 Subject: [PATCH 03/13] Add configurable quadrature point count --- doc/parameter.rst | 5 + src/gbm/gbtree.h | 6 + src/predictor/cpu_predictor.cc | 17 ++- src/predictor/interpretability/shap.cc | 179 +++++++++++++++++-------- src/predictor/interpretability/shap.h | 3 +- tests/cpp/predictor/test_shap.cc | 3 +- 6 files changed, 152 insertions(+), 61 deletions(-) diff --git a/doc/parameter.rst b/doc/parameter.rst index 404f8403fd66..0500d232fda4 100644 --- a/doc/parameter.rst +++ b/doc/parameter.rst @@ -197,6 +197,11 @@ Parameters for Tree Booster - ``treeshap``: Existing exact TreeSHAP implementation. - ``quadratureshap``: Quadrature plus telescoping SHAP implementation for CPU prediction. +* ``quadratureshap_points`` [default= ``16``] + + - Experimental number of quadrature points used by CPU ``quadratureshap``. + - Valid range: ``[2, 64]``. + * ``scale_pos_weight`` [default=1] - Control the balance of positive and negative weights, useful for unbalanced classes. A typical value to consider: ``sum(negative instances) / sum(positive instances)``. See :doc:`Parameters Tuning ` for more discussion. Also, see Higgs Kaggle competition demo for examples: `R `_, `py1 `_, `py2 `_, `py3 `_. diff --git a/src/gbm/gbtree.h b/src/gbm/gbtree.h index 4b97bb5bd06f..c93ae99d48ad 100644 --- a/src/gbm/gbtree.h +++ b/src/gbm/gbtree.h @@ -64,6 +64,8 @@ struct GBTreeTrainParam : public XGBoostParameter { TreeMethod tree_method; // CPU SHAP implementation used for pred_contribs. std::string shap_algorithm; + // Number of quadrature points for CPU QuadratureSHAP. + std::size_t quadratureshap_points; // declare parameters DMLC_DECLARE_PARAMETER(GBTreeTrainParam) { DMLC_DECLARE_FIELD(updater_seq).describe("Tree updater sequence.").set_default(""); @@ -85,6 +87,10 @@ struct GBTreeTrainParam : public XGBoostParameter { DMLC_DECLARE_FIELD(shap_algorithm) .set_default("treeshap") .describe("CPU algorithm used for SHAP feature contributions."); + DMLC_DECLARE_FIELD(quadratureshap_points) + .set_default(16) + .set_range(2, 64) + .describe("Experimental number of quadrature points used by CPU QuadratureSHAP."); } }; diff --git a/src/predictor/cpu_predictor.cc b/src/predictor/cpu_predictor.cc index 961e10e3342c..2591453c7b24 100644 --- a/src/predictor/cpu_predictor.cc +++ b/src/predictor/cpu_predictor.cc @@ -6,6 +6,7 @@ #include // for size_t #include // for uint32_t, int32_t, uint64_t #include // for unique_ptr, shared_ptr +#include // for invalid_argument, out_of_range #include // for vector #include "../collective/allreduce.h" // for Allreduce @@ -751,6 +752,18 @@ class CPUPredictor : public Predictor { CHECK(kv.second == "treeshap" || kv.second == "quadratureshap") << "Unknown SHAP algorithm: " << kv.second; shap_algorithm_ = kv.second; + } else if (kv.first == "quadratureshap_points") { + std::size_t points{0}; + try { + points = std::stoul(kv.second); + } catch (std::invalid_argument const &) { + LOG(FATAL) << "Invalid quadratureshap_points: " << kv.second; + } catch (std::out_of_range const &) { + LOG(FATAL) << "quadratureshap_points out of range: " << kv.second; + } + CHECK_GE(points, 2) << "quadratureshap_points must be >= 2"; + CHECK_LE(points, 64) << "quadratureshap_points must be <= 64"; + quadrature_shap_points_ = points; } } } @@ -880,7 +893,8 @@ class CPUPredictor : public Predictor { ntree_limit, tree_weights); } else if (shap_algorithm_ == "quadratureshap" && condition == 0 && condition_feature == 0) { interpretability::cpu_impl::QuadratureShapValues(this->ctx_, p_fmat, out_contribs, model, - ntree_limit, tree_weights); + ntree_limit, tree_weights, + quadrature_shap_points_); } else { interpretability::ShapValues(this->ctx_, p_fmat, out_contribs, model, ntree_limit, tree_weights, condition, condition_feature); @@ -897,6 +911,7 @@ class CPUPredictor : public Predictor { private: std::string shap_algorithm_{"treeshap"}; + std::size_t quadrature_shap_points_{16}; }; XGBOOST_REGISTER_PREDICTOR(CPUPredictor, "cpu_predictor") diff --git a/src/predictor/interpretability/shap.cc b/src/predictor/interpretability/shap.cc index d739fc4618c9..4e04d5e5fc61 100644 --- a/src/predictor/interpretability/shap.cc +++ b/src/predictor/interpretability/shap.cc @@ -3,13 +3,15 @@ */ #include "shap.h" -#include // for copy, fill -#include // for array -#include // for abs -#include // for uint32_t -#include // for numeric_limits -#include // for remove_const_t -#include // for vector +#include // for copy, fill +#include // for array +#include // for abs +#include // for uint32_t +#include // for numeric_limits +#include // for mutex +#include // for remove_const_t +#include // for unordered_map +#include // for vector #include "../../common/threading_utils.h" // for ParallelFor #include "../../gbm/gbtree_model.h" // for GBTreeModel @@ -236,46 +238,106 @@ void CalculateContributions(tree::ScalarTreeView const &tree, RegTree::FVec cons condition, condition_feature, 1.0f); } -constexpr std::size_t kQuadratureShapPoints = 16; +constexpr std::size_t kDefaultQuadratureShapPoints = 16; +constexpr std::size_t kMaxQuadratureShapPoints = 64; constexpr double kQuadratureShapQeps = 1e-15; constexpr double kQuadratureShapUnseen = -999.0; -using QuadratureRule = std::array; - -QuadratureRule const &QuadratureNodes() { - static constexpr QuadratureRule kNodes = { - 2.80850447628076738593e-05, 7.67982016833174672456e-04, 4.51374344293495755737e-03, - 1.49567508630415318960e-02, 3.65046411479570120928e-02, 7.34364533252638285177e-02, - 1.29023364563242204373e-01, 2.04750589337593075223e-01, 2.99763099175230585125e-01, - 4.10626915342501119799e-01, 5.31453230982491087175e-01, 6.54380885550600810419e-01, - 7.70361159218044599939e-01, 8.70144945830766847195e-01, 9.45343005090065746643e-01, - 9.89429020036412754102e-01}; - return kNodes; +struct QuadratureRule { + std::size_t points{0}; + std::array nodes{}; + std::array weights{}; +}; + +using QuadratureBuffer = std::array; + +double LegendrePolynomial(std::size_t n, double x) { + double p0 = 1.0; + if (n == 0) { + return p0; + } + double p1 = x; + if (n == 1) { + return p1; + } + for (std::size_t k = 2; k <= n; ++k) { + double pk = + ((2.0 * static_cast(k) - 1.0) * x * p1 - (static_cast(k) - 1.0) * p0) / + static_cast(k); + p0 = p1; + p1 = pk; + } + return p1; +} + +double LegendreDerivative(std::size_t n, double x, double pn) { + auto n_d = static_cast(n); + return n_d * (x * pn - LegendrePolynomial(n - 1, x)) / (x * x - 1.0); } -QuadratureRule const &QuadratureWeights() { - static constexpr QuadratureRule kWeights = { - 1.43895341220884505091e-04, 1.72520006395474869917e-03, 6.39316739866999800279e-03, - 1.52418484801773411463e-02, 2.85820905344451973995e-02, 4.58399977309956255245e-02, - 6.55908224919272003772e-02, 8.57252162327300087918e-02, 1.03725394222338646033e-01, - 1.17012592552996438910e-01, 1.23316521664007014425e-01, 1.21013898282131507345e-01, - 1.09387122775356740445e-01, 8.87653442838226142131e-02, 6.05283238746927090834e-02, - 2.70085640705332932776e-02}; - return kWeights; +QuadratureRule MakeEndpointQuadrature(std::size_t n) { + CHECK_GE(n, 2); + CHECK_LE(n, kMaxQuadratureShapPoints); + + QuadratureRule rule; + rule.points = n; + std::vector> nodes_weights; + nodes_weights.reserve(n); + + for (std::size_t i = 0; i < n; ++i) { + double theta = M_PI * (static_cast(i) + 0.75) / (static_cast(n) + 0.5); + double x = std::cos(theta); + for (std::size_t iter = 0; iter < 64; ++iter) { + auto pn = LegendrePolynomial(n, x); + auto dpn = LegendreDerivative(n, x, pn); + auto dx = pn / dpn; + x -= dx; + if (std::abs(dx) < kQuadratureShapQeps) { + break; + } + } + + auto pn = LegendrePolynomial(n, x); + auto dpn = LegendreDerivative(n, x, pn); + auto w = 2.0 / ((1.0 - x * x) * dpn * dpn); + double s = 0.5 * (x + 1.0); + double ws = 0.5 * w; + nodes_weights.emplace_back(s * s, 2.0 * s * ws); + } + + std::sort(nodes_weights.begin(), nodes_weights.end(), + [](auto const &l, auto const &r) { return l.first < r.first; }); + for (std::size_t i = 0; i < n; ++i) { + rule.nodes[i] = nodes_weights[i].first; + rule.weights[i] = nodes_weights[i].second; + } + return rule; +} + +QuadratureRule const &GetQuadratureRule(std::size_t n) { + static std::mutex cache_mutex; + static std::unordered_map cache; + std::lock_guard guard{cache_mutex}; + auto it = cache.find(n); + if (it == cache.cend()) { + it = cache.emplace(n, MakeEndpointQuadrature(n)).first; + } + return it->second; } -void ScaleInPlace(QuadratureRule *h_vals, double scale) { +void ScaleInPlace(QuadratureBuffer *h_vals, double scale) { for (auto &v : *h_vals) { v *= scale; } } -void AddInPlace(QuadratureRule *lhs, QuadratureRule const &rhs) { - for (std::size_t i = 0; i < lhs->size(); ++i) { +void AddInPlace(QuadratureBuffer *lhs, QuadratureBuffer const &rhs, std::size_t points) { + for (std::size_t i = 0; i < points; ++i) { (*lhs)[i] += rhs[i]; } } -double ExtractQuadratureDelta(QuadratureRule const &h_vals, double p_enter, double p_exit) { +double ExtractQuadratureDelta(QuadratureRule const &rule, QuadratureBuffer const &h_vals, + double p_enter, double p_exit) { auto const alpha_enter = p_enter - 1.0; auto const alpha_exit = p_exit - 1.0; auto const has_enter = @@ -285,16 +347,14 @@ double ExtractQuadratureDelta(QuadratureRule const &h_vals, double p_enter, doub if (!has_enter && !has_exit) { return 0.0; } - auto const &nodes = QuadratureNodes(); - auto const &weights = QuadratureWeights(); double acc = 0.0; - for (std::size_t i = 0; i < h_vals.size(); ++i) { - auto const weighted_h = h_vals[i] * weights[i]; + for (std::size_t i = 0; i < rule.points; ++i) { + auto const weighted_h = h_vals[i] * rule.weights[i]; if (has_enter) { - acc += alpha_enter * weighted_h / (1.0 + alpha_enter * nodes[i]); + acc += alpha_enter * weighted_h / (1.0 + alpha_enter * rule.nodes[i]); } if (has_exit) { - acc -= alpha_exit * weighted_h / (1.0 + alpha_exit * nodes[i]); + acc -= alpha_exit * weighted_h / (1.0 + alpha_exit * rule.nodes[i]); } } return acc; @@ -317,8 +377,9 @@ double ChildWeightQuadrature(tree::ScalarTreeView const &tree, bst_node_t parent } void TreeShapQuadrature(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, - bst_node_t nidx, QuadratureRule const &c_vals, double w_prod, - std::vector *p_vals, double *phi, QuadratureRule *out_h) { + bst_node_t nidx, QuadratureRule const &rule, QuadratureBuffer const &c_vals, + double w_prod, std::vector *p_vals, double *phi, + QuadratureBuffer *out_h) { if (tree.IsLeaf(nidx)) { *out_h = c_vals; ScaleInPlace(out_h, w_prod * tree.LeafValue(nidx)); @@ -332,11 +393,10 @@ void TreeShapQuadrature(tree::ScalarTreeView const &tree, RegTree::FVec const &f auto right_weight = ChildWeightQuadrature(tree, nidx, right); auto goes_left = GoesLeftQuadrature(tree, feat, nidx); auto p_old = (*p_vals)[split_index]; - auto const &nodes = QuadratureNodes(); - QuadratureRule child_h; + QuadratureBuffer child_h{}; auto visit_child = [&](bst_node_t child, double child_weight, bool satisfies, - QuadratureRule *out_child) { + QuadratureBuffer *out_child) { double p_e = 0.0; double p_up = 0.0; if (p_old == kQuadratureShapUnseen) { @@ -352,43 +412,44 @@ void TreeShapQuadrature(tree::ScalarTreeView const &tree, RegTree::FVec const &f auto c_child = c_vals; auto alpha_e = p_e - 1.0; - for (std::size_t i = 0; i < c_child.size(); ++i) { - c_child[i] *= 1.0 + alpha_e * nodes[i]; + for (std::size_t i = 0; i < rule.points; ++i) { + c_child[i] *= 1.0 + alpha_e * rule.nodes[i]; } if (p_old != kQuadratureShapUnseen) { auto alpha_old = p_old - 1.0; if (std::abs(alpha_old) >= kQuadratureShapQeps) { - for (std::size_t i = 0; i < c_child.size(); ++i) { - c_child[i] /= 1.0 + alpha_old * nodes[i]; + for (std::size_t i = 0; i < rule.points; ++i) { + c_child[i] /= 1.0 + alpha_old * rule.nodes[i]; } } } (*p_vals)[split_index] = p_e; - TreeShapQuadrature(tree, feat, child, c_child, w_prod * child_weight, p_vals, phi, out_child); + TreeShapQuadrature(tree, feat, child, rule, c_child, w_prod * child_weight, p_vals, phi, + out_child); (*p_vals)[split_index] = p_old; - phi[split_index] += ExtractQuadratureDelta(*out_child, p_e, p_up); + phi[split_index] += ExtractQuadratureDelta(rule, *out_child, p_e, p_up); }; visit_child(left, left_weight, goes_left, out_h); visit_child(right, right_weight, !goes_left, &child_h); - AddInPlace(out_h, child_h); + AddInPlace(out_h, child_h, rule.points); } void CalculateContributionsQuadrature(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, - std::vector *mean_values, double *out_contribs, - std::vector *p_vals) { + QuadratureRule const &rule, std::vector *mean_values, + double *out_contribs, std::vector *p_vals) { out_contribs[feat.Size()] += (*mean_values)[0]; if (tree.IsLeaf(RegTree::kRoot)) { return; } - QuadratureRule c_init; - c_init.fill(1.0); - QuadratureRule h_vals; - TreeShapQuadrature(tree, feat, RegTree::kRoot, c_init, 1.0, p_vals, out_contribs, &h_vals); + QuadratureBuffer c_init{}; + std::fill_n(c_init.begin(), rule.points, 1.0); + QuadratureBuffer h_vals{}; + TreeShapQuadrature(tree, feat, RegTree::kRoot, rule, c_init, 1.0, p_vals, out_contribs, &h_vals); } template @@ -498,7 +559,8 @@ void ShapValues(Context const *ctx, DMatrix *p_fmat, HostDeviceVector *ou void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, HostDeviceVector *out_contribs, gbm::GBTreeModel const &model, - bst_tree_t tree_end, std::vector const *tree_weights) { + bst_tree_t tree_end, std::vector const *tree_weights, + std::size_t quadrature_points) { CHECK(!model.learner_model_param->IsVectorLeaf()) << "Predict contribution" << MTNotImplemented(); CHECK(!p_fmat->Info().IsColumnSplit()) << "Predict contribution support for column-wise data split is not yet implemented."; @@ -520,6 +582,7 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, auto const n_groups = model.learner_model_param->num_output_group; CHECK_NE(n_groups, 0); + auto const &rule = GetQuadratureRule(quadrature_points); auto const base_score = model.learner_model_param->BaseScore(DeviceOrd::CPU()); auto const h_tree_groups = model.TreeGroups(DeviceOrd::CPU()); std::vector feats_tloc(n_threads); @@ -551,7 +614,7 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, } std::fill(this_tree_contribs.begin(), this_tree_contribs.end(), 0.0); auto const sc_tree = model.trees[j]->HostScView(); - CalculateContributionsQuadrature(sc_tree, feats, &mean_values[j], + CalculateContributionsQuadrature(sc_tree, feats, rule, &mean_values[j], this_tree_contribs.data(), &p_vals); auto const weight = tree_weights == nullptr ? 1.0f : (*tree_weights)[j]; for (size_t ci = 0; ci < ncolumns; ++ci) { diff --git a/src/predictor/interpretability/shap.h b/src/predictor/interpretability/shap.h index 18d17718fa3e..bcbc0be02e5c 100644 --- a/src/predictor/interpretability/shap.h +++ b/src/predictor/interpretability/shap.h @@ -21,7 +21,8 @@ void ShapValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* ou void QuadratureShapValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, - bst_tree_t tree_end, std::vector const* tree_weights); + bst_tree_t tree_end, std::vector const* tree_weights, + std::size_t quadrature_points); void ApproxFeatureImportance(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, diff --git a/tests/cpp/predictor/test_shap.cc b/tests/cpp/predictor/test_shap.cc index bc64d7e48027..5292224b4d79 100644 --- a/tests/cpp/predictor/test_shap.cc +++ b/tests/cpp/predictor/test_shap.cc @@ -369,7 +369,7 @@ TEST(Predictor, QuadratureShapPrototypeMatchesTreeShapCPU) { HostDeviceVector quadrature_shap; interpretability::cpu_impl::QuadratureShapValues(dmat->Ctx(), p_dmat.get(), &quadrature_shap, - *gbtree, 0, nullptr); + *gbtree, 0, nullptr, 16); auto const& h_treeshap = treeshap.ConstHostVector(); auto const& h_quadrature = quadrature_shap.ConstHostVector(); @@ -404,6 +404,7 @@ TEST(Predictor, QuadratureShapSelectorMatchesTreeShapCPU) { learner->Predict(dmat, false, &treeshap, 0, 0, false, false, true, false, false); learner->SetParam("shap_algorithm", "quadratureshap"); + learner->SetParam("quadratureshap_points", "8"); learner->Configure(); HostDeviceVector quadrature_shap; From 60611db6714b11dadfda19b7e6830ccd6ba0f48f Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Thu, 19 Mar 2026 04:53:22 -0700 Subject: [PATCH 04/13] Add QuadratureSHAP benchmark harness --- demo/guide-python/quadratureshap_benchmark.py | 484 ++++++++++++++++++ quadratureshap_n8_accuracy_vs_nodes.png | Bin 0 -> 78004 bytes quadratureshap_n8_speedup_vs_nodes.png | Bin 0 -> 71642 bytes quadratureshap_n8_summary.json | 92 ++++ 4 files changed, 576 insertions(+) create mode 100644 demo/guide-python/quadratureshap_benchmark.py create mode 100644 quadratureshap_n8_accuracy_vs_nodes.png create mode 100644 quadratureshap_n8_speedup_vs_nodes.png create mode 100644 quadratureshap_n8_summary.json diff --git a/demo/guide-python/quadratureshap_benchmark.py b/demo/guide-python/quadratureshap_benchmark.py new file mode 100644 index 000000000000..b164461c8184 --- /dev/null +++ b/demo/guide-python/quadratureshap_benchmark.py @@ -0,0 +1,484 @@ +""" +QuadratureSHAP benchmark harness +================================ + +This script benchmarks CPU ``quadratureshap`` against ``treeshap`` on: + +- real datasets from scikit-learn +- an easy synthetic binary task +- harder synthetic noisy tasks + +It emits JSON results and two charts: + +- accuracy convergence vs quadrature point count +- runtime speedup vs TreeSHAP + +Example +------- + +Run from the repository root with the local package and library on the path: + + LD_LIBRARY_PATH=$PWD/lib \ + PYTHONPATH=$PWD/python-package \ + python demo/guide-python/quadratureshap_benchmark.py +""" + +from __future__ import annotations + +import argparse +import json +import statistics +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +import matplotlib.pyplot as plt +import numpy as np +import xgboost as xgb +from sklearn.datasets import load_breast_cancer, load_diabetes, load_digits +from sklearn.model_selection import train_test_split + + +@dataclass(frozen=True) +class Workload: + """A benchmark workload definition.""" + + name: str + family: str + objective: str + rounds: int + num_class: int | None + build: Callable[ + [np.random.Generator], tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray] + ] + + +DEFAULT_POINTS = [4, 6, 8, 10, 12, 16, 20] +DEFAULT_DEPTHS = [4, 8, 16, 30] +DEFAULT_SEED = 20260320 +DEFAULT_THREADS = 35 +DEFAULT_TEST_ROWS = 512 +DEFAULT_RUNS = 3 + + +def _split_dataset( + X: np.ndarray, + y: np.ndarray, + seed: int, + cap_rows: int, + *, + stratify: bool, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + X_train, X_test, y_train, y_test = train_test_split( + X, + y, + test_size=0.25, + random_state=seed, + stratify=y if stratify else None, + ) + if X_test.shape[0] > cap_rows: + X_test = X_test[:cap_rows] + y_test = y_test[:cap_rows] + return X_train, X_test, y_train, y_test + + +def make_real_workloads(seed: int, cap_rows: int) -> list[Workload]: + """Build benchmark specs for real scikit-learn datasets.""" + + def breast(_: np.random.Generator): + data = load_breast_cancer() + return _split_dataset( + data.data.astype(np.float32), + data.target.astype(np.float32), + seed, + cap_rows, + stratify=True, + ) + + def diabetes(_: np.random.Generator): + data = load_diabetes() + return _split_dataset( + data.data.astype(np.float32), + data.target.astype(np.float32), + seed, + cap_rows, + stratify=False, + ) + + def digits(_: np.random.Generator): + data = load_digits() + return _split_dataset( + data.data.astype(np.float32), + data.target.astype(np.int32), + seed, + cap_rows, + stratify=True, + ) + + return [ + Workload("breast_cancer", "real", "binary:logistic", 200, None, breast), + Workload("diabetes", "real", "reg:squarederror", 300, None, diabetes), + Workload("digits", "real", "multi:softprob", 250, 10, digits), + ] + + +def make_synthetic_workloads(cap_rows: int) -> list[Workload]: + """Build benchmark specs for synthetic workloads.""" + + def easy_linear(rng: np.random.Generator): + X = rng.standard_normal((40000, 50), dtype=np.float32) + y = (X[:, 0] + 0.5 * X[:, 1] - 0.25 * X[:, 2] > 0).astype(np.float32) + X_test = rng.standard_normal((cap_rows, 50), dtype=np.float32) + y_test = (X_test[:, 0] + 0.5 * X_test[:, 1] - 0.25 * X_test[:, 2] > 0).astype( + np.float32 + ) + return X, X_test, y, y_test + + def random_labels(rng: np.random.Generator): + X = rng.standard_normal((40000, 50), dtype=np.float32) + y = rng.integers(0, 2, size=40000, dtype=np.int32).astype(np.float32) + X_test = rng.standard_normal((cap_rows, 50), dtype=np.float32) + y_test = rng.integers(0, 2, size=cap_rows, dtype=np.int32).astype(np.float32) + return X, X_test, y, y_test + + def random_regression(rng: np.random.Generator): + X = rng.standard_normal((40000, 50), dtype=np.float32) + y = rng.standard_normal(40000).astype(np.float32) + X_test = rng.standard_normal((cap_rows, 50), dtype=np.float32) + y_test = rng.standard_normal(cap_rows).astype(np.float32) + return X, X_test, y, y_test + + return [ + Workload("easy_linear", "synthetic", "binary:logistic", 200, None, easy_linear), + Workload( + "random_labels", "synthetic", "binary:logistic", 200, None, random_labels + ), + Workload( + "random_regression", + "synthetic", + "reg:squarederror", + 200, + None, + random_regression, + ), + ] + + +def margin_shape(predt: np.ndarray) -> np.ndarray: + """Normalize output-margin predictions into a comparable array shape.""" + + predt = np.asarray(predt) + if predt.ndim == 1: + return predt + return predt + + +def contrib_sum(predt: np.ndarray) -> np.ndarray: + """Sum SHAP contributions across the feature axis.""" + + predt = np.asarray(predt) + return predt.sum(axis=-1) + + +def tree_stats(bst: xgb.Booster) -> dict[str, float]: + """Collect simple structural statistics from a booster dump.""" + + dump = bst.get_dump(dump_format="json", with_stats=True) + + def walk(node: dict, depth: int = 0) -> tuple[int, int, int]: + children = node.get("children", []) + if not children: + return depth, 1, 1 + max_depth = depth + node_count = 1 + leaf_count = 0 + for child in children: + child_depth, child_nodes, child_leaves = walk(child, depth + 1) + max_depth = max(max_depth, child_depth) + node_count += child_nodes + leaf_count += child_leaves + return max_depth, node_count, leaf_count + + max_depths = [] + node_counts = [] + leaf_counts = [] + for tree_json in dump: + tree = json.loads(tree_json) + max_depth, nodes, leaves = walk(tree) + max_depths.append(max_depth) + node_counts.append(nodes) + leaf_counts.append(leaves) + + return { + "mean_max_depth": statistics.mean(max_depths), + "max_max_depth": max(max_depths), + "mean_nodes": statistics.mean(node_counts), + "mean_leaves": statistics.mean(leaf_counts), + } + + +def evaluate_model( + bst: xgb.Booster, dtest: xgb.DMatrix, n_points: int, runs: int +) -> dict[str, float]: + """Measure accuracy and runtime for one quadrature point count.""" + + bst.set_param({"shap_algorithm": "treeshap"}) + margin = margin_shape(bst.predict(dtest, output_margin=True)) + treeshap = np.asarray(bst.predict(dtest, pred_contribs=True)) + + bst.set_param( + {"shap_algorithm": "quadratureshap", "quadratureshap_points": str(n_points)} + ) + quadrature = np.asarray(bst.predict(dtest, pred_contribs=True)) + + diff = np.abs(treeshap - quadrature) + add = np.abs(contrib_sum(quadrature) - margin) + + quadrature_times = [] + for _ in range(runs): + t0 = time.perf_counter() + bst.predict(dtest, pred_contribs=True) + quadrature_times.append(time.perf_counter() - t0) + + bst.set_param({"shap_algorithm": "treeshap"}) + treeshap_times = [] + for _ in range(runs): + t0 = time.perf_counter() + bst.predict(dtest, pred_contribs=True) + treeshap_times.append(time.perf_counter() - t0) + + quad_mean = statistics.mean(quadrature_times) + tree_mean = statistics.mean(treeshap_times) + return { + "points": n_points, + "max_abs_diff": float(diff.max()), + "mean_abs_diff": float(diff.mean()), + "max_additivity_err": float(add.max()), + "mean_additivity_err": float(add.mean()), + "quadrature_mean_s": quad_mean, + "treeshap_mean_s": tree_mean, + "speedup_vs_treeshap": tree_mean / quad_mean, + } + + +# pylint: disable=too-many-arguments,too-many-positional-arguments +def train_model( + workload: Workload, + X_train: np.ndarray, + y_train: np.ndarray, + depth: int, + threads: int, + seed: int, +) -> xgb.Booster: + """Train one benchmark model for a workload/depth pair.""" + params: dict[str, int | float | str] = { + "objective": workload.objective, + "tree_method": "hist", + "max_depth": depth, + "eta": 0.1, + "subsample": 1.0, + "colsample_bytree": 1.0, + "min_child_weight": 0.0, + "seed": seed, + "nthread": threads, + } + if workload.num_class is not None: + params["num_class"] = workload.num_class + + dtrain = xgb.DMatrix(X_train, label=y_train) + return xgb.train(params, dtrain, num_boost_round=workload.rounds) + + +# pylint: disable=too-many-arguments,too-many-locals +def run_benchmarks( + workloads: list[Workload], + *, + points: list[int], + depths: list[int], + seed: int, + threads: int, + runs: int, +) -> list[dict]: + """Run the full benchmark sweep and emit row-wise JSON records.""" + + rng = np.random.default_rng(seed) + results: list[dict] = [] + + for workload in workloads: + X_train, X_test, y_train, y_test = workload.build(rng) + dtest = xgb.DMatrix(X_test, label=y_test) + + for depth in depths: + bst = train_model(workload, X_train, y_train, depth, threads, seed) + stats = tree_stats(bst) + for n_points in points: + row = { + "dataset": workload.name, + "family": workload.family, + "depth": depth, + **stats, + **evaluate_model(bst, dtest, n_points, runs), + } + results.append(row) + print(json.dumps(row), flush=True) + + return results + + +def plot_results(results: list[dict], out_dir: Path) -> tuple[Path, Path]: + """Create summary plots for accuracy and speed trends.""" + + families = ["real", "synthetic"] + acc_path = out_dir / "quadratureshap_benchmark_accuracy.png" + speed_path = out_dir / "quadratureshap_benchmark_speedup.png" + + fig, axes = plt.subplots( + 1, len(families), figsize=(12, 4.5), constrained_layout=True + ) + if len(families) == 1: + axes = [axes] + for ax, family in zip(axes, families): + rows = [r for r in results if r["family"] == family] + for name in sorted({r["dataset"] for r in rows}): + group = [r for r in rows if r["dataset"] == name] + best = {} + for points in sorted({r["points"] for r in group}): + pts = [r["max_abs_diff"] for r in group if r["points"] == points] + best[points] = max(pts) + ax.plot( + list(best.keys()), + list(best.values()), + marker="o", + linewidth=2, + label=name, + ) + ax.axhline(1e-5, color="black", linestyle="--", linewidth=1, alpha=0.6) + ax.set_title(f"{family.title()} workloads") + ax.set_xlabel("Quadrature points") + ax.set_ylabel("Worst-case max abs diff") + ax.set_yscale("log") + ax.grid(True, alpha=0.3) + axes[-1].legend(loc="upper right") + fig.suptitle("QuadratureSHAP accuracy convergence") + fig.savefig(acc_path, dpi=180) + + fig, axes = plt.subplots( + 1, len(families), figsize=(12, 4.5), constrained_layout=True + ) + if len(families) == 1: + axes = [axes] + for ax, family in zip(axes, families): + rows = [r for r in results if r["family"] == family] + for name in sorted({r["dataset"] for r in rows}): + group = [r for r in rows if r["dataset"] == name] + best = {} + for points in sorted({r["points"] for r in group}): + pts = [r["speedup_vs_treeshap"] for r in group if r["points"] == points] + best[points] = statistics.mean(pts) + ax.plot( + list(best.keys()), + list(best.values()), + marker="o", + linewidth=2, + label=name, + ) + ax.axhline(1.0, color="black", linestyle="--", linewidth=1, alpha=0.6) + ax.set_title(f"{family.title()} workloads") + ax.set_xlabel("Quadrature points") + ax.set_ylabel("Mean speedup vs TreeSHAP") + ax.grid(True, alpha=0.3) + axes[-1].legend(loc="upper right") + fig.suptitle("QuadratureSHAP runtime trend") + fig.savefig(speed_path, dpi=180) + + return acc_path, speed_path + + +def print_summary(results: list[dict], target_error: float) -> None: + """Print a compact per-dataset summary table.""" + + print("\nSUMMARY") + print( + "dataset family depth points max_diff add_err " + "speedup mean_nodes" + ) + for name in sorted({r["dataset"] for r in results}): + rows = [r for r in results if r["dataset"] == name] + safe_rows = [ + r + for r in rows + if r["max_abs_diff"] <= target_error + and r["max_additivity_err"] <= target_error + ] + if safe_rows: + best = min( + safe_rows, key=lambda r: (r["points"], -r["speedup_vs_treeshap"]) + ) + else: + best = min(rows, key=lambda r: (r["max_abs_diff"], r["points"])) + print( + f"{best['dataset']:<20} {best['family']:<9} {best['depth']:<5} " + f"{best['points']:<6} {best['max_abs_diff']:<12.3e} " + f"{best['max_additivity_err']:<12.3e} {best['speedup_vs_treeshap']:<7.2f} " + f"{best['mean_nodes']:<10.1f}" + ) + + +def parse_args() -> argparse.Namespace: + """Parse command line arguments for the benchmark harness.""" + + parser = argparse.ArgumentParser( + description="Benchmark QuadratureSHAP against TreeSHAP." + ) + parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + parser.add_argument("--threads", type=int, default=DEFAULT_THREADS) + parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) + parser.add_argument("--test-rows", type=int, default=DEFAULT_TEST_ROWS) + parser.add_argument("--points", type=int, nargs="+", default=DEFAULT_POINTS) + parser.add_argument("--depths", type=int, nargs="+", default=DEFAULT_DEPTHS) + parser.add_argument( + "--workloads", + nargs="+", + choices=["real", "synthetic", "all"], + default=["all"], + help="Which workload families to run.", + ) + parser.add_argument("--out-dir", type=Path, default=Path.cwd()) + parser.add_argument("--target-error", type=float, default=1e-5) + return parser.parse_args() + + +def main() -> None: + """Entry point for the benchmark harness.""" + + args = parse_args() + args.out_dir.mkdir(parents=True, exist_ok=True) + + workloads: list[Workload] = [] + selected = set(args.workloads) + if "all" in selected or "real" in selected: + workloads.extend(make_real_workloads(args.seed, args.test_rows)) + if "all" in selected or "synthetic" in selected: + workloads.extend(make_synthetic_workloads(args.test_rows)) + + results = run_benchmarks( + workloads, + points=args.points, + depths=args.depths, + seed=args.seed, + threads=args.threads, + runs=args.runs, + ) + + out_json = args.out_dir / "quadratureshap_benchmark_results.json" + out_json.write_text(json.dumps(results, indent=2)) + acc_path, speed_path = plot_results(results, args.out_dir) + print_summary(results, args.target_error) + print(f"\nSAVED_JSON={out_json}") + print(f"SAVED_ACC={acc_path}") + print(f"SAVED_SPEED={speed_path}") + + +if __name__ == "__main__": + main() diff --git a/quadratureshap_n8_accuracy_vs_nodes.png b/quadratureshap_n8_accuracy_vs_nodes.png new file mode 100644 index 0000000000000000000000000000000000000000..4abfb87a3233bb2a08e82073371f4a4a8a1365ac GIT binary patch literal 78004 zcmd?RcQ~8v|34hn(%z-I?pvisTPCQMGql6@pMBcF^iB(Nfgjd$qP8p+gk0 zB?!_sc8nSkMEITQ{rP;4-|zR&^ZfDr_gu%}D0=0(uJgRk^F3a}t2>4|ynjLefoJg+TUYKp>n?_j7^YSQy0jfqztdZ(91Ac*1=B?|VBz4DS0ra`*Ighd(&w=j81J z_wFq4lVuS1gS2^(LmX!|#air!Rlwk(`T!M-EM-Qbtk2rNvi! zx`MrA_w`=9c>U3->kkvI*(g-`)^3KSQFp)Q=#Eqw1jVc|SRvQGB*#t=f(;F0rq!kc zzql(T8-PdnbNv4F$U*bee|~`M%>WPa=Lo&dCHJ3io2B+^asKD~?I)1`pZ@xZp{3;q zukUSbA2FAH_eJn&xHxzwCMF_&68*4M-FphYKW}VvC{=u8Ro#{Q0v2 zNi)|z$@_p@gJ_MmZ5o*7_ArA^T&aaOsclZg%8!dHDk}bXa@J?MD^6dk$rPMqkW2yT)KD$axjtEJ=>qJIJ0V)sxla0&pHT!T+_)J6oNpW z=85W{+e)k|=XYy|-COyFUkC@a^@V17^+2kqHB;ryOSm1nRQ=8tEmt>I>?qIr;@BmA z`+!Vx#x@qkd^p>e+v?Q*{=!O|Xmp#M|Cp1mCWmFmXxO0>SF~EAMJV^Wiyu!fkZbi{ z91?oim!r$O-Uxv-+u()jmCpyI+8UOymyFbwpRO%YD0QV(Ri=wrrbpoNH-3plc}PGW<-*#p z4p}=0Bz)<}pBp55V&U6SuV2S>NH%oVm01}$gl?V_MZY~PBI4OS)X~uqtHl@(Ui#i1 zcPWatFj5^$31u4Cqv&S!vw6uAG5V6y6{!iAtsYb*3J3h)5PihUUeCyz5Gdq_k4=sH zJ5TIpYn_Jg%$@p$8nW%4eyxwmN?ut_qn(b?>SB)Cs*7(qOv7`}3l{e*Jpu+O;Qw{-a*4PkomfSJB7$ z`N;{SqdYJixR7Ic8ujZjBe{4|=-leyGuV`?)tlaIos%>x*{JGrI2_(;IkWy_$jILj zMYM8O9+W|*DD^ZBka!hG@A3V@uC!m$9-Gyw31)aOk*OgcRq(>WB`gfr5KWK4nPi5d zpcvnqA9JSGG=OFN{Tsi-Q2sifhv7q_k(L&nI_6$c1rDvxWK{nTpf;u%iHwE%} zjMnuIkzDTKz?fS#ZZU3djh|Et__&OZ9@OLYs|>h>t=F^V z@lqPIRKqbl*E-K28P;fl#O1L1Ncl_-RFiy@_?k3fS|MSOl*K%c!qKS6`61)7`YEVY zt%qGHHrHIDn=JCt+U?WFY zxem4u)gVu5PN(=xnn)1|CH;D-nzC_jk8QW2@$@lr^~)nsYh| zhBHVfpTzKfVDz$L#1H;+`AUu5So9oLhr|GD6j6m@F4W!?2@0#^!z_*t@i(*ludH?7v>VqVT(?4R(rm#OQB&9t}H*+&gvIF%>Y&a zim!99T zu}mEs$7N(R`_1Mi5Cg)R+{rbBSuCI0?{HR4Id-dPB$QZHQj=Iqs{rxreqe} z6e(4d=@xF0wIWJW5WQkyw7#_mPd>?4*&DdKd>sklL{rR)0Q>v zi7YQk+_txk-ONzHDkfTE%Y}?V(u#y=Ns>tEd(99g%g5V05((Q08?E#5Sjz55?3iJd zX3@UR;}1$T-dU)~Pw{^J$@a#Lrw9rbg$y-Bc?MNVK0hSnstRINRCGS`-O$hwRK%#3 zQ6xxeDKaZ|2f@#$-eC~AvC<*cfGSt9H&pt$mrN|RQD5JzCfQfs)X^agO~)Fle4oTV zd-km6h^Tge{Bt8sRuzf%jo*#PUX%B0inc ztxAEF!pPmRYB!7M=)uhE+_mJ=TBZ@c8xs=~17(&aw6)zLgFw4fvos>&nMPXnshD%~ zGqtvMbpwge%f{L|sWdC=vw7;uinpV(o&8ciquiCTI9BtRlbbk%FD}Q9EU=m~9R)y2 zQ0w~r^UIjzDp=Uc>J&2zyGg%#umyl?MggCdp9U1J>B!9ysK$-xe;!9CNu+s$Z^D@?Gfo=Oftn?eshPp zv_Q^l(699vi(JZ)Z^A13Obs19cFb+wBv#D8b^aJc?&+=F=d~P=s3=$p{GSBO@6QLL zZp$DyClflF7~^gaW8eaZ2sMRWyE_a`(MzOG$$>}=m<38(dvB)*L2nz}A*y1}VuiT; zb@R%5gibz!cK>`;zmeux+o4|fQh7Jx2>}J%?oM!o0eWN1pCT{`J#*$vbtFydS?ktxPnaf?WBFhR(cqZ-R0=869|4g-*@leG`=L` z+~H#DykDiY<1%yS-Ly?k^(}nue9r_C`JAYB^ak zuT-J4iihok*XN~)+2}-%{v_i9XMLCT3h~NJ^`7pZ7)eBb1^YTMLR4j z>1$1Q#+ZR%7qwbGbaw6l>m$mm=dwCVQyCWOYTf|DC$6bycg(Hlp-8L`Sbq7AJC&b6 zdJq?N8+=z|7sjHuh8j`2F?{_CD(BAKY;SKrD=v<=)TTHt&S>~7j@Gv#3Hynx0vP$6 zn3&$xtB>7Bs&wNe?X1f?n=z458u_pb$%#P`T)Z;l(MkJCnf6Vnsm{dK1(oL*-huZV zh0eA!TO`C?*qx*e5J|Xe!`S#}DL?Yq8w)rh;^Vtw2!TxaD`d63!Emk2)l3Pjn5OrSpT&W2&&YqvbrhLYAHX}FDV4ng{{M)TS}OvM>*+(gDjkD1NI-*Xt^E<*TtK$ z7W@*XM)RRDj*TtSkuwkbDq#efF57;VFVdMWuwl306^w)CT!_`7%U-AKZlQ8&)|gQq zOqe-lu>ncRBELl!*U&=s7!utY%#!86NsnZcpk(X|-;;f)r)gsu)S_~OZu9=75{wb4 z({l_*6YyA^S6PkHp@ldvzPkept|H%6Jk>WXw9L)niRB1N)&OR7D$#`_rA zY0EhR!*BkPC>peS0rC~^(J3y7926~p*N?yKg+cUWTwN!RI+Z8YxDP+jzjfR2&hLrxYU_r@%tx3iZ)38PRQVb0Dctvb;f+rO_feP(DSk{xZGUzOK~91+NB z7y;_Y3eFPHGg7C&Cr2*KRdk_pUAcS*rJ=I0+o>`c(Sphkm>bk+ly@(Wod2474ojAq zMgD>$**OP;7lQ!ZN!r49@$@P>sO4v?XM#$j+12!9lNtT;K0=vnx~!~h|AIHWpjqgU zb0e0FFmhloK^`fWTtF{fMHy``h3&3xoJbOIg=x?#wzlP+u=bBy_DOaq!G2g=S`NaVR91xKHa-}i%`XD{YY6OyO>>Z497j&wioom`(E zHbHPz*>|bXF4vMreY(}Bt9E^BBdMX5QWyz~1hew=OH&m|feY~A;H4%Kb`(U;g;nGl z7{`#vZdJeGY8{0n`=WDF5;JqAw!@TgB3(VWrb8x}v50K9b-;!MR!2_~tWK&N3TlnX z?}pPmq;Mx_h9av4+31xIh&coE;#!t1#c$<&QxO8m8jvO^oVw8)!lbG!d~RC zF-dMdVRr{Ze2%0rYOU9m+1C{8Ppt`MZ3U{m(I?#mOJ<&IX0~P5+*qClpITIXrIAOi z`E?<Rn-O=)06 z_8|)esA)5nLSfs>t&V5UofBZ{`U~Q?9YB2~8C~f3h>WOJjw);*O>d~ryb@3QJtT!v zV+nGFYiY5YBw6sNRosX*9sNO(Z1m*cix`?@bTl( z*W~O;58Wyx@UQ9MHmP^Q2I_pel&1$UYp+CEUITdsC#gHo+bhTA0_cMl3*Fh8k=qNM zvfWh)zdou!N`n}S3(^@>1Bf+e8k&3yC7Gu8_yakTYhaDLBA>~aAMRJGuEpGok(%X0 zu7<~^VeI2Pe4uIti>BV5O4!!#vEmjDCF6?(tX@R?6r33Z^&ec>fBiX+v`7CbCj`FV ztD(353#(AUQP!#bII}|5xvRv_O1;imk~!oUZFVx8C|x8OUVPQU$I9PHU6Anwqi$xA zR8UaR79SJy;BxU*QS}UqRM{@2FP7#VCuUKodbFY1@(t=vkY1?4q(|qeLaxnt+GIuH!APbRR2^TP`D81c(Nia?9AX{K>)>6vIf{?wOVyE{)HX_qh3i2;p zROhXqyO0pHRi8FevNjOxAL|cKPFr_Rdu==RRXJ^jMbK?qt3s(usN9*Eoea8&lbV57 zs0g6@OcaXag?jow(AV^H^ULRM*jMM~@3`H%ylH!HkY~<+% zv+&i9c!||vl|GXRi*kE}EvXLz-kmfvGd_W}>|+71DNG`hJ)ql)J7un3VQB z&#QjWRQjv{xxMyc`GrWR%eLjfaF_hs+yygu13H<=gH$}E(q)}rAz8Ip#APWzllge6 zNYfmdnpj;!f8jW&Qisf>OHsmhQB*JaNBQO5V;l!7|H&dIR>cs%42sNJ=bLd}59&?& zG(I=<$uVN!G)aVRt$5e%n(Gw{zTY!fd3bYXmE1~=N{!3N=yeCuW8zu~A8)e@>Nf9= zOS3`59_YLmBdFT107vPzZm2ZLHx%|H{fOYQ9sux)DdLVPTx+yM(O!NgLta_lvS`U_ zEPb}+04h45vS}{ZDnc{pkuHS~vow(LO~5(j=cn-o9A3Njr0l&Sjv02D(25_hZquXX zSmft1*lqfQy*a*`&WoOEz0yT!B2j*}#?o%V%SXhqkCeG2N_AAF|Z zrl;DsFgQWrP`7JU`;p018k?e3>^{<|)nsiCgn#Pe9Jda0;L94&K{#YD7FXLhY(mG* z{WaqpL5w8Qv$#E~mS935o;99$mbn45A<2WZqrqD1Z>6txsdjH%CM8YG&dv^w^Fc~( zuye$*EvM1#&tY?g`R&qpL8_l}s{6O?Px&0QA`Tb4)Iv3@cd7&bn;$Q-Bj3cuYdZKI zF;U)0=rU_TW{QjY7HF73sK(8a17TMrxaeW>3bL0{Mi+nLqa&fGd^X? zY`Y!{A-;Mo*OM#Y5R?~Ay?THE=@Yi&7PwbbgOvJzc6gfOTesuF*e{<62ucs!s}{$kA4eY ztjE(LDbGhdX$sj(^1P4cre7-^rxUmLwTMJ3kuXMN)s}e)cuGR zYS+(U3URsh{s#micWb>;H>@C8^RC5KQPRKomc`AR@V)2Fm=WEtt;+k11J8PFc-;j(qzfe(6K{e+20V)*mC{598m z*EsKYtF1=jGjd_IQ6ie-b!CauXj+4U+^vEZMcIuKZW)ipWo>FbU8gb)?OtZY+jh}^ zn1vlh16**PgSN{2j?DY|Q``rUJn za&{H<{Q|cZf0ffsh49^evNI*zVT4ZqRll*FPSo4_lr;DQE^Ki;1G$#!jBQvThoG!Aa`P%Q$JcBa7q*0DH8tZx z$LFl}2mLnbI1I-XtGWQT>lKd+M5}3*Z$Dmy1ijt+Mv>zH>!*s>uN%S1jszTL=Y+J# z^)QG__yFaqi~rTiEn=Sn%{wXocktHfEb8_mbZVBT}^O@}a6 zwXv#MrL6p;yd{G$l88S~-^|s&@SM@~$yrlB9W*UlgIiQ^shFByy-CY1qqTh!Njj$q zaJvKAps~U&TTm#N-BnYQp(5+*AoHlu?w)%eLoK{YOx#2Ldl`k2)9m4-^heynNFnya z?pZOUpq4}#O<4Hw<=*?~dYBGPKr*Ny&B@q~R$^uv2o6D- zWl|1QHrceBmd~R5=2;kYJ7Otx)k=>y^7}H~g49^p+{dscmqPxr=^3mJUnu;PHI9?U z&_$3|M9|J<9N~Yn8JdA)qIVa8P}+|w=DH$CBMq9$4jgk(drrO3F7I?@Gw7(A?Ed1a zRtKcj6apjC0+#; zLwhqJ-hsKoF>IY0bG~vgZHq?uYKT5}x^Z*aer#(1aown!aXiGMZDHsnIyOGYMCGej zV$vQabeqI@2aBU!ScRUyBAYGm<~33U7hWpn?vwlA3+Mp>rtKIkE#TJy=2a0w%~-8x z`bI4E!aDYlSQ6bry|>egx8i(iNw>)+?CT4b=a zIM)ED>r}9LL>kP_UczXi(4(e4ZB3ngC--`1b46bSG$^B;M~gPBfOh8ja22-dw_1+X z>8!ZJTOD02fl}lBAi}0ccIEfmCA{Kc(V=u^x)rA_-(RK{u#&t5^0F6l&*a=p=x6X0 z{#Cw_Y0s7c=z5Y|opzO!t;5IkD|Y?~2_Iy$r*j19mnDg4m7hii{FT~OTy~wqLxZ5r za^ZVB8(lDG$XJ8zx#%hO-CM_Pss7-2J(XsSo9bMUA3QMd<9XWF0^#WkYg|1xaU{vf zk?TnHV&zVoH0`!)=~cW)$WF-ERz#2Vm*PB=o|6x}HFc5*PB4Yp8het-N7Q%enG#KH zw_h9At=)`?&Zwm7%j&%h9P#7YG_ziz3YM36qj77 zuh-??pC_7$7{b(LIy9uLICbS5zB6yG;-Rm5 zQE_pRz}KnyrKC(4>=J2O=hsYcb`;Gz+DeugBENYLVPEUvD8Fx?kP)KXjmy&xu3c;> z=gj3TL`yUG7vZhj`1Pv{R%v+KiR+ArnFE}`C5uofO~Xgzxg z%2Tq4DwY3jFrDge-;}0bd7toV3)Yn~%b0gGqTQm{z*y$$j?KwhR@bE&ng6~Rw|W6j zy_x`KV|6aKE`i5c!Y-GoSNSxq#*M2=JjrUV4(D3XeH_0cu(flg(ziH%u;*TOAn|pY zyc4Voa|w6fBN~<{X0WHgBOT*CsmJs`rG6!!=@{0t#E%J6WYAUWol#GERF0%82LfQ! zXW?AIXz%9!&QF7!4D>NZU;~Qg{1}Zw3TleTwJpvvlRc8<)y+?ZDnPWN+#!&GF96&w zh3mCCZEq~wQ>3J%y8jUnfRB~~K%@qsxl9BgY(D{%6FqG&?0`Wt|~|MZ6Fl+_I0q?y?U+e*~6V=0-zS3G4WP zjh`o&_*a5TAd=U$j$)9oa=@9CGg>6Mrys--HVqm zuWcw{773`7P;(5Er+0aFxH7*VLqsm2sln~Z3-t=+n;2SIrPMA!7Rdllqm`F0V*wF? z?RVBg(i%%jI@g!+B4qo3pZjc3?-LTpfPtE4o?i8_WvH7)2`cx&m779Xbds=$4esEF zj@;bbVPd(%Qbi=1P3I#V7B{EU{h5~4tdQcxrlRy>_QnOIq~!FgO>7@-QEmC_v330H zNOkeDP<|0mDtseWIO#LJeWp6StfoFVqCZh~z;)_J%b3}xuRm8pP|{j|%~y8UNcj-t zNhfG^bJJOi&VnKxW3=Fz%omYMj)N#h(8_o5g=nuqGmAjmLlEd+KxU%V^g;73y&_=w zm-U$c#_yl&OxN(jxoA;5q*x<*rzZ>`MSsxQWvX%5;Z{7pfpxlAvj+fd2pn$5rGHZH zQE@lD^`z&J2ae@u1Ed6MT%JM^j?lw6Dq7Wj3Ac+yv_=YOL$uOuB23YmA5cfai6ZyV zS3B73VL;Q?D6~yFA_MwT8F1G{yjqv=A|pY{ph1ihfvKROYbZedBECH5m3_i3Pd5q^ zRNgr?*uk!4R`>CGS0Tv6ojM&MH`f9lQnO zBtWk7I#_NEhwMW)o${xd9S%DW<~7$g0YGc)$dvQ!Uoht@3jFJ>@7%!m{cUbNCQF(= zQ@0(-#0bYJ$C<}U_>A~^N{{rwrv|oIgcZMts}fNWM$=Y7VZaC)8}N7c9TO85Cohoz z3@UoKl+}N)3BAi=P&`XS)n8_I%%~JeLQsqK)6g(_#<*y-+5>lS%ppNeRcJGqSfjD^ z!Ws@2dB1tfd_0krQC~jDLYGPLS|>5pShO|^+AVy$myIelQZ|y2wwR6#?q#KG5D$o_ z+zZ2QmEn`(Q$ZKf|CnP82GDY6O4uT~wtH6VO?^q3=UDlS(u@iTFU>clg09VnRE4$z zE;{rln|~vus*-jZhlhuep#;ELe3B9o5<=j1fUblX;Gq8Y0UEcdoKvntiYoI&l|p|v zZ2Wq&K--=xF0UC(pfYmi4iy+C%6x+>vpjw+o+``Mm#qH^547{dq#+yn=~ElRB0!OW zzY#vNsOcJ1KIdG#Io8Q@ts!_ZX-^Mybb*4y^>YZb#`DXKG#P76h{g1^ET)Tp8fx8& zDYLcd{q1jy6HA^fjiU5b%x*CT6%0M7t~IArwl!wXP~OrYAt>8C#E9QhR7@}mP2r?B z<)lBG@>gMn45D6AgZsv=T0vY~*upntjqBjWe8_D?eqVCCE9pE0sir;G{lX z8?-=4m~&+i7$uGh3Hfhez3{iySo@IWn+dm3_%~bvYuxqJI&&SqYsy?`H9NVwJ7#vo z$6N_T!6I|HfSwSh1VaPXVY}HJTy~0Rx!~Sp=G}kjySk&MG z`8sx01kWNp=RFxItD&a;yv%8IrZfN1e)1t+@$dHhFG`sZ5a~-jtYY+zV zBhIE$>*|#uZoD*+6AL|2TGkzO_zvHb!QUE~cq!L=gT<~4Kvw<`~ zwi-Ie4EXj&4ACb)e6yxsepNKQf$o^S8j<9ZR|ES=TvJ&KHbs&>5Dby=Mhp5qTZ5}J z;PnGc3-%%yGt(lj-b|fUEzwwVO0Y9uaGc@L6B6}o{RL!X#p?}-f7$oJCT!BU=hV&g zY0xN^!w?XyFQ*~AkOEnJVbtu%zb}-!Bw7tjqUL)=|;26&mt&dBe|HTl7kxe6iD3a6oGI$n8gt!1D!AgZJLY zOZCL;vA%d?Qxt+ob1ii;sQUU0>#%Vn4_Vy8qx+7|VGj8Q z4Xq( zayPq0AC*3L^(^lU&4D0D>Mtr*C5+o3g--Hzx@cD{z5tak^Ol{6ieb>ybJ$+Pbd9m| zuc+t-772*k5EKf{(k+k&l?r=zrBG_u`aEba>NJs1>+lorLa3iaPwMl>ZYmvi6q z#T7D*ghbTq@)Nf)o~wfzrutu$)-5XQ#(EK-m&5ZziNX4{2m+C~q_CB$rFn@iYl7ns z@PQ?3?&*OYwx(Q9d(tz$H^absL>qT-rebTHNGfDrW_iq%eN9qLS}u9)uIp0`IlX*Ds;P{5hfq>xZGQ+- zay)N$!1e+Z0{OxB2UGfl7nUsp>>l15qz@T6h${lr&wr?aEg<(u6(j6{K(uD|*J6N{ z_f)|9`1p2^I`7+hl50vMLCG^G`A7P@=L9G$AU~=T973(Ufi{VzI3AFFLt9%W2*?~E zkm!v;pm$Uo0o7}X<)>?UdU_AR0$l;A>Er^?EWWkzY>IpLM=WgmhOHF-?Lwa$lyM)r zl3(-j;}y0R3&<4{|9i$!K?tPzl7vmo`>PRL-z^+a=RlFBkKpxvph6CK}-y__CtpWY_*x1LO#XraYn_u6|2^0P&^OUIQmpRG*d~E-N z|MMXN-X|u`uI++J)CQ7Y_7P6nua(`xevd!DyvQvmxOMT;C4s=^sNcUIy?XN|c4Fe; zD>NDw`R81%JE9O?T;;HPOmgxiK;=HH;*dIi{CHcMdTX4 z@oMPYys7)};n}P5&4iXDAi275>5}Y@C)TsCukXaAKc^mkzhCRil9OWRsigR0E?^k} zP3hqfuA&3i65hXW>q=E!SW5p#!uk&SU-9_f+(*T#;XQfs=?2gcbX3&24Ib0$;>-cY zE3Ct9ARdks_;c0$dV4c|TB!WYu=2Yuo6ZrSLy_9T8Uv*o&&mJo|ACOQ)xlMRv%32! zkr72@f<+0AQ$vTtwIlZRLw;lbeW6hY|DWH(w)B_3e`srKb5WlJozx6%?ED~oy*28j zcz?c;^NoKm13e;>(R+S^KmI!g!9DSU-VlY`+P$~>BZ3bat8CuH*3u>`ARe~8J-q^Y z4Sg%C-%UW3$UdHm$z=RAhQadscjbo!ZUg20b_G1WvB4#~(=O>;u2bAPIV=0K(+%7q z?^)&4$($~UCyUk99%Cb+>YADk>Y)4FNi$*b@$zL@@B5;Pj;B8j*XYK%tYgcU2VD6T`!q<%!2Y!AfFiM=aec(C0<`goSMEP)q!Vr8>3r;491fA&NWrbU6*q94`g-RH=V^fu zVP;!`a74G+tT(1{T*|WAlVZ=7SPRIxyj|X9*XnmcSBR`-e;NYmX*&|GeXFIQ&!;t1W*Fnz4ZM1 z^6h~-1OjP!#^dtzcQ_@OQQ`6Rl2L;b+*TS7MPyzH3)L#k4mmWQBl#=5&h#s_N%ZceBPXAtS z|2YuNuHK?4G-OvN0m;eVucD%=YvIF@EavWq_g)}T9Ti^OtTO?+xn+NV3iQ=_I^>h% zFI%1I&ohX1>WJ6%^Q#dAHO%DB$f5tt3xM~ostsrC5D0`gK*`<)Sx-1o*(cAqe6ve+ zsY;3s$S~<(hynBs>*?WMP*O=4X>8`05Pbkla#lrUM7WL^Ue%ECA43aNO79Wm>TU6p zxApa7Y2`S34IqQ3?m{msxLqxN{rWE^lGog=2b1>(rk4!D+Cx4KHqNvH&V(L2>}p=E zk!D}@yvp$&$Z723&O@y)-q@6ar8546c09$sc$I}u8P-i}oHae5Eg&H9r*E<9sU7ux zbL2enwo~abL|+X^)TBADKdkuD+tb%4p7Gm1YKY{@{O>w%16!(oEyZyTx!b`-TG}yh z*xg}9Wvl{uv=h5kPmZ7H`Mysm;(Ikv9UY>2MhmNtj+a?=qXkI+K2`WDh}^Yy-Bg>l zW_tw9*P)5>N=Gr_o*xkSb=>5)duSUr@lGeFq~!J)QPIhw;xd^Lx$`X0=$SK5Og(*; zCuWdzA}nlklE1bCNPs&koI3syY5Eh9K&U$kg3kqO&cJpF*2L3nSN>bXNqT6|?mY%u zxgH<{3^1`E*9ahXH}IN~KYRlN>v@D!Oxb0)lEJNjnV!>DAMZVmJgI2dr9|b)$M;VT zvQ>{6HOjp|069Jh5Ss*0UYJ<}<}#!F$iL%hc?^*&D!nitaNr*h*<@BlIGehj5+{8F zn;g#f#GK65iB>4n1VXpgr;iEAii%>#)Mj&HBOxnbNRnXJ^#Q0%^h-T>EOw2O9qJ2* z7xq)5gj9*b?}Spzc+py1tv0vE7k!^<=Onv+tT{= zNw0BNa?owlrzwgVv0e?5{1;oCW4WH2ftA|`{1B~lw$*{{PdT*;JRs*JBnbPp-I}dD zPs+j~+3KJQ-`TtQcwp3sd3KP$os&=P6x%eh99GI41=wLAJA8mpDLZHQA1{{KvthZI zy#Xzkjf|5GU|`eSwX{6Hyu928q}3g*%D9iWD;USy-eq~~zI^k3&HPo&aS`jQ&QH^{ z0(UcZKn94y0Z@)d9MovaShr@28iVAsIyEeJ+w${Dr=F25RL6w>5fZOix%;mDul)57 z?+uTR6*YOOXHviSEzcE}96$B-K?wbbRe3Py)oT$1I~BP~%)beG?`3?mSJ$Ys6SL}- zVXfkls&-a#o?h4du2Qu{e8Q~+Ccj>Qsc!`RZ>`@Md;cvuASuTeET?78EH7YmX-8^b zKUIAZ+rGXO!_wEhx0j=iZPWR47{2x^K?=O9^Rqjrkee;HB-=V!lQL|iI5fKrdGbH0 zeh7Z5z6SxpBrhu>2hJvt(eCya_Tf<7=l49i`rj-0IdOYO>FyVLb`!7XPRov@>EhER z59lzw1wN%H-`3dQ{P%wghoQfn8i-+Nx8TO7H*cIE0SNRgV&Y+f)4J5|Ej>R^=d&D2 z>@M@qp;|ks*w>)K!fnF&^-L?}rk`EBbk2X=C-N*r{J++^dlv5}q2cJ=-L3BR@Vll} zIfojCo$ALN8YTa}xfv-2QQgz?@5Me|@=o9w8&`&u-sN)e9C+?$Y8-HDJw!gDckex& z|BizznIQJe!&S?I5N>Y#T_Sfd{F3d6&$NbUO}n1<@o=y^{=FCv7;kRYkMMpMmw2Ds zdg8UB?^PaT>Dzd9i7D5)N{ar~%$E86Inp{jtJLj3pE)y%z7xXq=$pv#0hgS*?R10V z#D9%|%|8&EygG&1VhvS7J9K^J3WR+2U*Z3~+0LE!lVY4IO>Wg+`qw5-A0@9VC9K<4 zWN^IY_;;ehUG*PTm1}USTvi)JNX#@*8bzJB{blR4M0F1QC%nHTz+$f{a?B*@Jt{C4 za6EVBQ=&<7X`|`NKiJN;JyC-HeLc5^AKMlxdH2B2e$H|{Czfjxm3{0;@~P79`mS7c z&z3iLt;|2q?1^G`HSxi}Te#0%zIL?hpi9g%~-jT(K}6q^^ApADc@ZG&$Gf@@NeRSL4cYJ0s(O0l5J7b_}T5*mgA_ zh&4?g>~a>1)4 z)YMt;%`ls5)0GtHer6LE|HTa#FE+->D6xR-T!%nU-;C(qblc_Wc^(5&2()DyWb;Hh zc#JJjz-v$W1O(Ipyu1ULSRS@K*|!23XzQ`Tw*w$RZGw)%UTVe6Og)J7daG!MrNtSSRZHHz^~BwtpucpZ16!8px&%8bMlKKta8- zu{o{T@p^kwDy+~eu#~qHVD_HrN}fP2KR7=9`cm1W#Qk^lT|O{KG*AS}0BytUKw<3H z)>b_lv2^m+8!5kJfzeNnI?A*X;Gt9$!8e*(oO!tX>zmAjulwPHMRCh&b{Jpj_J@E< zv0y?E38|coR$u?Hwm`N3G>e$Dv~*vYooYRB8$KMCna*C)b126C6Kn53pG~z4kUel8 z4(HIGMT1Uy z@Vw~HE+7-10D$|IG6(-Lq(_1k@Vwa~2hA#gEiYTRqL0R|*`0Q_H&=j_hiyB2=wKg# zR}sk4sObwY$p*>34wVL=b3L{>lPLoo7=+Xk-x5imTzb!B^XQs&BuFVfS%sdpz(lT_ z;AsHGR@^FVnZ>adJl6qMK5lDW<9-3uh4MS)Bq0$SPewdf0s!_UfCl^%&Y@}-7&zVm zOM)l_8@3yos3QM+?hQM)g_F0s5e>IKBvTyZ54?H#e&BxXEuu}>c8%lBWsXQJvEGHlWQtrGplH_vFbQ(Gw*CSf~pYbUvx^q)Cmf8Br!+{AmG!z|dlwMg?`T zbo-l`n-OGaBT;t`U`0Zy>%b(Y2kZRs(_53!MJ#dT)o=iU^$GwVQ|T_ass)kVjTc8O zvA~f!8Ed5RiDJaeXZg-oI*JG}9fVM6fb%)g;!J2H-ovfiNm(Cq0wjqHP$RPgi$(OG zl*mg!ZAOg@hWWks?y5d2ysEPjQ&>@_wH~smohoun>k)YS@iu<;_ngdWXP;x|`U^#d zLspcmyM86PmAwPrCWBBG-9mY1gE|S!hYEVCYjcAgZSIT2(q!^wwtUa%?%lhQ;bn$* z3S#-qbEAtv!|VJ$>XDA@2l*fF@r)i!Kd?*vsav$CmSWE7N4itj*49*ThP(=zK#Z$h zQ+WZ{PS4{qTkLrERY?)f5z*XPXeBrrA*={6|vw$8~zT!Mj(4oIxwt*BG^R!=@lLy9`i6FNbc zdEMns17_tUY^quy#7_=XcmNSmC`*nuv4A!=0gh{Q97e6YBmhYSVf-(T|SLAIbPgvcE} zD@V1I4fJ0ez2MnIk3TX$kkkSDZ6TWixG4DScvC~i(+H^LAwIqYk+ALFy1I@~qIEA` z$Ug-?O?t&rqwCYulUsgl^;};3`t?R_Z8f~Vb;0Md9mW)(I#D!E?n82WmKPNH#j&Zn#YfsEvi8+flE4;=cNw5|jn@u|}f?h>48<$xaB)1&pa_tgU1 zhA8?7Yyr&7j-oU_rj@Wa*s zXBnhb2d)ZEPEHywfC0f_K1hC(`}Z?9UK3v{7*pAxjn*WV& z`i&HPmqcvR;%~J~nbAYtA?!;1@Wca0$0+O0bkusH@fRz&#MHin|J0>M-l1vH&OSkt zkSkY9{}zh8c=2K+&RY0J^hmG|Fx>yvcNiV~@#hU_S$vqx+wmbwY83XTg#0>Ho2%yi z{RtP8SM2oD$q;$tbZcfbkK@snty^;nS{U`13oW&jd79cq>0K5Vg`+UIrmb3BE5g9QsOvvGvuPf|m zn)ups2bMGbMrzf&*oPhZB$etV<|3|d=e77*%%;fXKO9dbWilQg{$;6I@2X|xrsV)k z@9Q#xXnF?0&!K->4nFlslkMwY?{wrXQS=p&FGx>I^RNX56Ti}>KR|=|C2x~mWyz*c zdb&`x+N#W}+a9w6uuD?0jr*iF=anIQCBa)gJcL0Ov+aTMlFl&AVgTh~M$3-zyGl5v z<~#xheR~59OOfy2zpvA{`BX^9R^#5kKMt&ZI}dEx5?xop{G!+sTUpT-cF4G82B7&8 z*q#l;=s&F{huM=a2E&n#Z_Woo_KLA@ZV7UU?`+?<^V!m% zHq0J~&=ue5&zJ^4)rH;7?y$%jfH2(J-C>Qz7hj2}2CU(->q^U(64>tN*qwI@-Qu6M z_ZLheRfsBuS=dF;TYdSE(wlQkHWU;C+Nm@)NjQrIMNAw3HJ~I)$f`kt?x7|xuw~kS zx>)qwxpOiuJ*Tp(Uafz#)r1}o1Db_N%I=PTlfxf$awBHP&?kXop%_#R#Ot0hAKT97 z*Q|UAc3M(g+E@dl5N40!CFr)=+w~F^J=3In@8uh&Mjky0^&P||EvNzb$I+^32XvjP zaq*871u|C{6A_>U`_s3RmVo~y`^GKsw-DHVQ0*R~1zKh9yj7LUyX88zFLBOB(T#XJ z3HbLx57}2_c1any5~I9&y-MbupWIP8x(g~h0n|VRbjy<8A>@-qcX@YatcKshh}j`Q zrF#Rv6w)2Ubq07=xrGCoKL{RWuf9FH!;b|E&u_~<)_JCb*S%eg) z0QlIFusWc1@7~dCr}py*$lOry7zMN@5SiFm0Y&SP&ViUC&a6M-fJYW{bh+WY!HWCYy1IA zta?gqY89R=ZZ*QL2>%&a{kFC{nR@Q>na|VRaVp+f^Q zu78O7a{ITRqoc?`7R>=_UZEJ2vi_la+N%8!dPN)CiRrH})MN2WMv) z))DnZ8|ynA;lz=?LQ%20s-^EKrzba=EbwPJN;{A$5DJv6RbB;)s{7AtLVFvkK-L5^D=w=qj(vsju*L zS)ZsspDVgJ`V)lTApQdf8J^e&4?`-dpaF3cT}a7>cdARux3I4Yy1{XUjxXgPz+#H@QR$uk^Hum<4;$^pw8j#-r`u;OHkf_M6 zzaKgsI21QPzx`^-w?bdlsz=M^G$e|v@1J zPwa|{`#VN;X_IRKeEM_|2kw8-_1^JR_J92NF%q|wXdxmiyNqNXvX4zNqmaEt_Ng=! z*_2&KvU2P_Dj8*EWD_DXB71#b*Ew~6?%((K`}N;_KMt;QUGMAte!ZUS_4vsXp}p9c z7}y6W(x`5$;OPw5M?&xylEMu;efbLxW{+b`$60e9j!bsd_;zONX#1ttZHx~Q-Jbar-j71`>FLC1J@#U z2zjn>+4ocCBxA{BDWBKzN8pCwCcBoZ%QGD%H4h5($;Bb&gMmiOpM9uGbIee)VJMIA zx#7xE%bPdFixG3S+s}fJEs$En4uyUsL>fGb2{Cm1(v=RHL9p-(0DYwR>LLoEm3sYp zaSZxrB#{M89gm<)j~gy)IF5I^(nNK2;-uL?$2fphW^Nj1757bi6yM9X&Jf_A*H?TzT>XM$FbFHoe^A{{1j z-q?=TR3o(Rvl(ab!bT=1FO*J`;xD?Mvtw2!L72qYUiND7^jvBB&kiQc7HR6TiZVyuqI>Y`7IEf_^TeM&N)Lum;_ySkzHN?(gSarJqe7v8gYAS}nOE}rHGBW{6 z?O+W@&^+ax1`65@z+5C6VG0h5XsBJgH;W1Pt5NqZYmMnu3ewy*dI5=%-Adh1J7dG( zpVxlBDXJcm#%zRoXlRK3AP(6CA=8C4Ypmj{B5^7mX{dv*-0VNvAh(@nT3p3vWh=>L8tB8f*AC|;}) zK-9XMg$Ep{lCj~!t88dsr_>|jzu%K9h#6;0&wWlu6XX&SH4K0ZJDz%F>56_5;5H3_P$Cs zQJnFBD5_WqH6RONAl*G7bV4Du)=9Tw2i3lVzrg@eGW0_|7{O>Nf?}klW!>FgKWWA# zARx*gJ&wQks8=r?L|0S3RGwOmBT-OOJ!l&+yNTQ1Hq@9eFHDJr*ZAhAo&(IcN!v!) zn>Yn_;5UZM!J*?`8M1I1>Kr>0voe<#?a30L$93H1$*;OWnhK!vT;VkSXh7BTg||9i zQe_Zp&X{kCJ;NTjTN?ZOAbL20Xr5%QFJDyE;WhAHz9wx|>h`MkSz(~x>)S2|Aj=7% z!4knCi~!eM=gx%#;AI535FgSVsU^;c`|{$yfKB2*g?Cv|Nw-a8RAz>ZpE_*S0t7VB3jy`A1NK=b@S0LUhgoYVG8z5rj2jsTwKrIp{ zu~2pyw$`~_0%61KfpK|XdZY?fl&q3`(GKU}v!#={w#y?O@vSPCX0Rg4%AZ6_0{#ZG zjLc|&l(h87gxk|nv5`z^#V{sE6TkXFde!)PD%29w>IM-CEAz{gWCYjm;AXi^AGdb* ztG*5mwszG~X*KQkEzD3?JSPQw45DfTO(SM5rj+O;%c^)lhZuy>|UkG%t*1yaamkM zr1*{xDeA@jU4Zt-D#>y@&QG6=E7<_dJXf88LXWSl)-=HA{1tA@lb?8bfn1jtM=kh`4_^sBjEFOkGIvh zhly;?%yeZu33|AHA=xU znsQXs!?Gek6YW$2;jfFN`ddPH4i4V4O@IO{HM|`ywuj0^s>ac4_Bc9}7Sc~coIW)r!r8qOdSH)(~ zgU{2Fixv$~cnCBJ&jXks5;mncxOco*0aF5EBe z&-)E^7OlXATyfG5)S_bEZ{KHlx!Sg~P|}m%zJ4+ESwkN55I6|>qF0h9X#?GXmmO@o z35mCQ4I9l0g&&foK6&yabblSG+trXg@0^k(FYl*~Rt^$Waa2NreG0Oh)%9)U5jmgU z`Qg4fAMrxN5!j4K+GHa-VUlO-n>@JH7>2}whO=oIA9)7ZE9gdx@1R3}!OJ-@1`3_* zDcG&!;QeIa5rShMF(V)uE>_S4$THx;4}g!a?28$+I6pJGX~F^+P11nFR7T|bW^3#! z1^P3P)O}zy!d9^G0B^lPJ1R03`dPq))%S#^yPbXD2imTipG;*%u!hc~JeAU0nYzZ7 zZahCW?P7zJJ{t83Fh2_dERS&0_N~!gR28xsjGl$SxQG|WoEc&DjjOT-eS=;Yq8LK{ zv8c^eov_NM+@d+nU7=uX8!%vI3cQD!RU0=mTI`G<@o7s-OG0~rA}#Oo@7K}3{WXhE zlH=M9qg&zT(HEi1Fka5nH6CB#yF9p0 z6ZV5O_+!%du4D<{H&9t=N&V4v3=&V_FNpc;oXkNpAVr`MXlRXK1G(AO_x{F;l zuk_i<+GjW6j+H?Hi~l6`ODMWA3gg!^OO~G94z}%)y=!VIr)ztwH-;Zjws-p4PVh4S z=i`rYLu%+}BUc(4F0--$S#(#~;F+%Rj}hMf+8x@~Ez&we%b0x-Z%r4NX%PQpZp2uO%IG z*r|`)L%ArD2}`zo`sA?{Xb_`I*Z|Pfv18-~C=a*wLbjTB6gp+QS2NcV#dN2hH7_E! zHjtTwz+Ewu_B&i$XTLP?P#?TR-*uEKZptq1?*-~qIA^l^6EAW8?GadBdzh8z9h;IU zzW?z(My+*?#?C0&HA|kO)XlE&N;GC+=OphOEmr2QqRWOCA@)=aZqMAm*N|qBvD(34 zI^ZSXr@*cH58n^?V9*HDA1c#{llf5bGQ(4&Bjwugx+M*?P_DI(zZ0XzzO;C!Ek*I3zKt!nen zi@7Hkq|2Z!l>YaG3{Iut2^uu@xuJrR&Y02j_Z8%oXCG1Ec0*M(=6!{%RV!PWqTtww zU8S-Kk=uVucm>d`x|8qB68%y1B?XYq_wTFB0z5ju9p`_Jy|UD}i!-(0FNsV&%YKR87G+d?X+EE( zN4TfF0H@pU!{3}z$Qfnh$|HR2lu<|*SJz71xo)t6sRR<}2xKw=Q7*nLw zb0Jvy9R@(vh6xG@na*dr)-_VQF@3N~qj-Uidu<1%d-#YYl=HtPo z%*N_tm;O-T7kC#3QQo2+egD0c+`qp{KY$kcB(1D<>>dMmP zq!ageZ34`b8^URg?yZSVlmME^y5t8oq9zc}Sb=UG2iZ^vm6Z;ajAGp%jz93+C(k78 z`8OO9Ejff^Uxk8E?u3?zDog;G|uLl z#YCZG%F;cy&AFP0NhcECxs%j(yp(7e2q{%u5(Aj(m1YXR|JHB377juhr=ZJ=0gBFc zpeV}^xO+jDG4%E>u<)#q=YueZ-XL?I%*ss0&9Vnog@hc+uEeh`RFzf*C+5EoyRrRV z`h*5>(OatZjX-3G6=u?4$>d&;e!P2tGUVrXz+gMc&vSBcw4}=33rSsuipUB`;pL9Y zN5l@K#Kevw?X|1rBD7j>-)s~9$Q{#wG}EiBYsBq3O6s1r5W${HrAFX*Nh;pH)dI5`n_!n1S>Ar1XH0zj>{<=LNs5gg z+S=OpeiP(L%;AKRx!8iZAwEb+QDyFWDL{mq%T!zfl0OSqNrZ349!QOfiVCIgpnv4) z0jUPt?}L&B6W13<1wf12XhHan!k+gRh&$#f(7tJ_VtuN&)v9!j6|`%98ty!3?>*A- z%VNx0O8yx738xpntVm(AxYP9 zEm+NtKUc7^913|~eTU=6YYY_7X*Uh2W&kZ}0%0K!())vWPdWcuwI$$I%aB0EP;c#+ zB7W-)2*Bi!H^#S_ib_=m0*f03AuaIV2SfD2Lo-(I#2HsR$}sh4D6e?ubl<8lUXWdu z4s6cKWpmCxy)iBX?eD49tph=K&nGZ(6r#Z$ng|$1qh-+3s6ksVj~fk0Yv1HDQtqDF zIRb!6f2CKxYvncIO)cTALWE;H6{F{#&o#ePCqv!UcXDznc1Iw+B~y}BSMwgp{-tN` zk=Gh7_)i9L#tOYHUK&h$hwOq$1CbTUT*FMe@vid96!v4MQmoi70WCHKYbzYdm15HC zZ3V!G6I$aukeM`RrUclLCKvld)TNH0p{YmN&BZP4^6J5Am^&p_JvohvS=!Q zTk)yGUO}g`1zIb1Oba2qkbW8>sUWXV2_Y$1#~hH?H-ZJ2ac4xy!6qqbg5(vSG;z7} z{-eanhlEPTCIt3WP(ESyEf)B0=QpcZV@xXZ_$L5;Nk&HlBOCjDt5o3U9%TbBQ**CJ+{CpjEw3n&Q4Z^p)q z5q<)_`<0s~y4B}ih^iP^M(Y}H zj#uNnnkx@+QF9a7BsshVygerK>$;`DKQB)A z;`7by=`@m1O*C0*Or@x>vm5^EvGIbb&#`Z7JOmrwJjV4ljrTn1-@$58{BaSa`Y1^B zmh;u$*;0|Lh-16;WAP#-J$hd23o#QM<>$rmzk~Fgu~o(~t4gv|uKIBXXd*CsTh2)X zhx6ij6e@gpd5(mzHi)5>?l7RC}HGXyP$@_E<58L*n zl@RDs?jEP>SULysdq4I+rQ%y(G&XKjd9h{D>kKF>!r^>rS^H5cQM|ZmS5Z{?TRT|I z;d^!_u^jzh^j`LL(>!TVci>uv{=K}|rLXdtB`uM1{ktFLr2UzgqyOzgbPV(U zz`TB}W^7XMPiKWA7n)CZ@WR6MhX--VUe+D1+em0X9r#3I!-jIyl@fyO(J=+D;aY_ zSc~=VXJlWU^G(7SoA`1t|2|3ev$FN>$oeiXSAW8M>GT1)f15LA-LiN7WWqa{Q*?34 zdQr!w8(aQqPS}@xcUbSZX*du2pvLc!LN*|PA@xfll=W~VoP4@)ZQl>Hm8k+tX zaG;A9??O^>3@FEpdpQZ~32mO8lTP|jLFd5`vxJPF33S?PC6Dit;MM`IT@7c*nLR<> z(Ew(G&$AlLm!8qmy=_LJ=O@2^rw`QC)un?KHRkTuXXE$S%Q-5=c?M`6xILLw{*P41 z8S1`I=SNDJ)%@r_0lGY~AJ1>moX{vBBCUB#_Vm}UJi?tKUtVS~+V4NaU0;g;@U}c7 z<+6wOcEm0pDf=s-WPWt4JOTSwYBhyHRW37%t$f>%f4|EN=aEI$mE&B`|AdX87zd zSpM{L$*yr7{iU8b;E?eF=gR_Oc(f(jdAOpucfG&bw^VX#CQZ<~?eyJt{8guR{?ZFb ztFeBbws!U#4g1tjd$#F9F2Ge+QsRnghgin7I}h=gTcIkzM1M`d7-YJ-uhcH^!#S(C zvD89AGR69;pMz0$6Ab`ev+lgUrsM(Mb&5q(c)rL{mAjM3q;PtZ@_ zdEZ5IS|kf2K&0R z1ze07-lHpcVdvCOf94L74#e*rF(rAufwURo^kHGPfZd*jLDVT4iC=GcM^Z9Jct%vu zh}2O7sz7&^7cur%=`06sw?W%^hhywz8Rbd-Xb{viBXB&V*T+IWLit=?r6pwMxq-c3 zRZT6V$CU6zZCpU3%GGiW|B~CYGX=)8o%VkW(BjW50FKFpV|%c9c5w2$wM(rxfuhYS z@g?=qqbyKeWN7=OfighfD_{FRpx)Si%%3MYar9^kZzo?D9V_Jz(r^Tjd6m^BlCqw^ zVE}K0P>TmD^!XHB2O{(U8%90?7j6Qt9Xhu1?1DuIeLJYTK)X%&vYJTaPfw&}knLSd z$@a!Oh7>C7=7jA;z-oveSkZNWEet6Wn?ZHPVr6BeIEW=82@!9HGPdB2u}e6(wuG7d z2&b2M(o06@4qoq{KGl%+lKs}qXWYc_b|L@?0j6C70a~z=^Ug7J*a)RcAVJNS?3Xsv zqckFsX!R*ka%n=VS;p-VMFdZg=kjr8od+fY*E9|% zRiD1(f_AF-@d^?-N3L>m|GDeY0FT@%Oc!uD9@8ynM0JX%@?`aFcd`&d-u>o)Q^EXe z$2miBiWj}U`i6#vw<=`VS@5?4Z2`Md^vj>C{PU!#B$(NzV1A;R$f3_}<~=61Bi2QL z!P8rzwje&^?QGVc(^|DTd0xj*X9hMPRKMj6A2V!NmJ8;MxpuhSqQd3xgu{ z&kq#rDEeG&JA4HFhfL4kRQ(?+i&Ye*AY2QoO81$IQ zZ6_~tt^DOa8y%@O1VJ@mL5JsKt4C|vmtJW9+wWVhrig5d0iGMVS0{0S?@I=V$14zg zoBLuy=ymzDe>Yt0@tfKJ!DfUePIl>+&7Tyj;`jpZoj&tlh|iLiy0J8K&;7iz1MJ6i;BZbN-v_Y-z~DPj!m4zW|It}4}u9u*)e3+P+_@R8lfv-A0b`dnS zAfdYqb3b-RM0e^FhnUe9=}_eK|1bZfoU!Y|iOtPdfRUa37Bv-JdRUO+I(7eQvu>ow z3P0eKUrB##8kNX%2p8v=*(5g5Uurygp04ATcPFVFez~A6cx$hdg;Sr_X{%`@za;Sn za8ID3uN$UdAJAwK%?w=f+juLlq+~q&2U@P>8Y*bZ;HzPT`;gG|{$|%~&MyS2NjYwO zz6p5Vo0 zmk>Yfq0^V%4n#75yE3}izK=dLKGA=zpQ=L_zlgAP-1T?HX#2h!*!r|6iAbbgVDpsz07n7~9k#8h$z^N*@)0Sv;NW04KnZ-SG(oZrr9TGRhtO>$d3j;9CgEGr zUu#}))G;(ds)H0>?ay2)^f?Ie?K)qH(e9;>$*}eG* zy2IJLIuiE>!>E2s6tfM!>R5G$tHdtSFZJFzbF=k9;8RaKjAUMl$Nu#Cp&QhLyB~!6 zm%oWuJIS0!Ske1*iZkTl`0ry;_l4SO{$05f|!^%L?l&Y z`y~WzhnU-5WiJq0<1N(+VrOItJ9>SF<+^?rFQ1#;lSj^HwbgV^3U(=~c^G?6bqG9x zHh7(T)~dh|mfQuxYJL6i z&r|dPh7GWhjR`-lmc7rgTd^==_Yi8i4o8;Cc!jUf6*OfGM6+PdY6T#yr>^>t3BW;N z5K!F=at^CoR|)spjDz`SW7&geH#0*bpYpO| zwWE>JL^UV4h|Wq&UxQ6AX+St_9m+5O3cejP=ou^IuZ7(9l(p-4X~o8p_^8=-R73;| zFnmIL`TpHC`6lIEWpG^*5ae^iwm0W_tN*QXB*7+oy7QwHLM@oV7wX4h1$8R19#|?gxix#zKSX`P}ck^N4%o#jT4J zVIem*j&>7spmOP|>9Hekuiw+g$=P1~1w%#70W|a0Z>`^&hJnF8y5QOQ@i1N5qEGh) zl3W|J2_t{QluhStOt8#~yvrHJoMe)%KY1aPyrW z<{iX$hge~cqDG5sqdIHAD&ebTJP6_+wxdUnUXvpDy2(F5e{}ZP3$9xmxslnoQW(F> z@!-jZZ7rj<+OJ(TL@cyeCNa14s_li!Wss%zF=4?GoG^9&Dy*v?YvDizv3_~=JoXJ@3W`{$p3 zLblp{vStyB=H9~Bf_JCz?hzFA#$#MxdC`YbYvKm26A3=i!gKCUkx7G%~FTJfx5(Ja6~`iQCq0J4-z?c^^_3lU)5_c9&CJCw3F@ zCc}Ip#GJS_>8%U)fn}yEh>7ZsFV*c7$Xe@qsX&<`c?SQKmui7M6-@n-wSRgRN-sB1 za5{>8h zYvf$PiwdvB1y`|^i@996M5~BuWX`*oZ0*2FlQGarb*c)z5t$(z?{{lS56T(q^a-)m z=pNx3+KSTkZNqF;`hOlY(iu@}AxF9JKZgo= zOu(;?oUf$iAADnSje1b^@un2h^0)S?E_PJzt<=K4+~}t7(N`5*{(&Hk7l;;wi>IIV zDDzFDvFYKIX z4%iRKEMdir#!Te-$w9+u$WAV9iQY!*=tO=A3QE2-^8Gg9DWEj}6oeLCK1-`KCyq~a z>$^^-{%lUPg(Gi^8MGIU14RX9qkwUKoI*q-c-{rTKG@3%j^%8N#9;FHGzP9ZS%dXQ(@{I^LK1P(t=Jymki;(^@9llIKNFdB>L+Mr0#rdO|438PY(#KeaUb0XR_oK$iQ1eae0h7oZk< z%_o?ejf}7RIb7KnkhQZvATE9<)xsA}ZwzWHh*smgU=m!epJCNT7-P_g*}ZnK+vNV* zzy9t)bE<5f(p&uXqbKys-l^|S|IVXf){Gh#w#{ZuTWdD*+KSYbNo*xDPnnLmW!E0CtK@_zx_<85u zk-rS^bMnufr4-yml5f(0PFuPD(~4hL$Z{iElJ@U9_|BhzAePsser(Pj4z}xR!3D6C z-4}5QX--_MA_LLzE9x1&yF_hCIQ4q_YuGE*QqRkg+?l_e4N{ zdCCXBoaoUn61LjG9(~j#h{X|1&8VVS`)0FJ`N+zo=5s8mhM*MzU7oaTiUiOF75-xQ zpTcVC7g;1>7^XzcRVCN-R4%;=YsF4}dfvj!GG-V{WC~^%uz*AIf;8C(1Qbp3B(bTfy)Pyib$1662L=tKYxxL20<1<%R!3sJ*K=-|VldNtkd&HsP4pH?t1O zBo;|ei}!EVKFq)sHI3+NM+^AY`02u-V_8HU4qPcjfHXnzGh1~?B$&`OzK$JbUbyq- zY*aj^wXm^yq20{Wu03fx1f19k(50UeCOzFs+yVI{g@17jVnyu}wA2m0wji`8O$Gg0 z%bl1)z|Y+2RPmohQUkAOQVAe%zCQj+Hw?Ac5U;JLDVKL;Qt~Lb z2y{^3rj{4MSm}*NXlOl0@bc{%=h3JxlJbz##Hw1PsSv2fjU-%qK=cRVnSw&`1<1k$ z83Px_t>bWf zYK>t0iS%N(>bP=^O_>}?*r8Yc#8nsOsh%DS!hqL1~NeAf71qI^=i)>jTxGfUGR4^tciAcwUa~YTG zuko*xgyL7`P!Covs%kzon;sokDp&VUytV~Srk<@ zLix-Et1t>Cc%WgVT<+|N|A^W)R9r_+6kN&bVTkyu_A)+0@cWX{jcHxB9~)oT_e&QO z5yyTE2j}4%Y@N4-QVVgokXmFMio2ZTtm!6Iq;N;WT7^st$l1-kPRNyjV59BOea9p(T> zazq9uuBGRynPc}I*bJXYLJqqk^MD7O&zNmw9yHX3l=zxtGXdiZ@dMr3mU6Tzbhk4A zv96j1F>G7e%U{sbuFemF60)tiIRYRgmD+&a_csgsofknI5pwE+9GHf$TVe=XD{0!d zD!v;$Jg#<6lpQ#<)s2DwA*}_F$%959v!tuq5Er=l5s{j_Mip^EbBSb7ZUSdJ1@w3n z(se_0As=MZBF$|%bOIs!Ve}#`vw45sgq)oRZ7~PBuQ(W4+j>5>ui~{{(<3_`OVNZX zN`sh2eI#5RaV0?M&MD|NsT}<1(fPZb;@&LyFN;1$n#xs6WTwmrg6>@Nck4QDT@R2P z!jSr*2GV7xLCMex4q*;Jh!JL9qUYfsLE?$~z;zaVD4w%ZF{3t$th9!*iWa1q-@^}3 z*^>fqpba1rGLH)fhD>~v8ggd~X-=d4ZRt*W8Fp;frm(a77Sfpumo+^IR3&}#KV1j9 z3Bs6_JKIK02Ds{p%xt|I*>Mx|zF=T*dN`kFvJ9%5d4YXSlF-JZ*uC!w&fthf5@0p| zNb^!zj!iqB0J{N*Pj9#f+YQ7YN2QR5J&lQG5p8MPmzwNOE?qiq>pL#Pc9tVKLUHC` zsz_x?jZo$6E({p<@8!JI3c5r(XZ-!ql2Krqm077|sOk2Z3B}E&1wPKL>W1^j1-*8& zhCTP8NxONPYd5x}-(j@|{}U)uU*rv`>wB3{0&;tkSx-SZV=H!&rycFsJy0$WmDp=- zfKlG>VAALzBES;NnJji-|W$Rmje~5sC}2P@=`DS_eKW7Z^m4TBRad3n0j4`^KT%`XdgE$K9S=Jq|NE?O#9Mi? zrlsTE;$I$EIGmsqov*mCx0L;;j7^2kBNX1Zo#_l*PITu(v%l!x+cMIFW6rrsL)3=ge|}?{gU&OuPv&;y3;0LfcI@YMRgWJW`M!I z3_V%GRn>auQH)O97eh+&Ts)>bTYX6AJ^ndj%>`;@{Ya?Tku7?B=9-)33O8QHN`Yqm z{kPs2&-nC7Epvb64!)DZI&(6o?~u_)HhMxq6RX~Cvd~=rd?f2Nm}daA65%)DH1ymw|aW7B3KUPdUz$=JA`P)w9!K_f?4l$21fQE^Ff z8|ge8{@GOnl0E>CQeucwATSXBi7{*Zor7IZg8h>D{pk8WjDbfE$biOK`d4}m>63$* z@nI>F=mvox$V&}KS_Cu>s<fE}X#s4s0L3@~G<0;saHOyok#6W1V9!BaKzFiLN#+ zDC8sIApVNLlBhH^G#e_Alcuh$tV}_k%Edfn0*s2|M6?Yy`zKC%m_}}+MT64 zOqK$NJ|B?oqIcVYNB@Czk#tf%MeMtTjqG)910zVEx&t5F`Znkl1!dL*UVp&8D?5YZ zMIpAHiABIeX4mj-1*{o+lQS?6*$+aup_Hg?nrw~=W&RBld65JTn~p#1UAF?rN<1Wq zYz%k_ddAuSE~S#a8!Ca+@gRjmV$d&rww&pB;p7|$d}+ua67YsRVX!o5B9nZdgAna# zcrWvB$ZY#M_V2|4u{abvhRW&wo44bcdw`16A0-Z=A29J{*kA2Sg2({-P@2rEn*Qo1TuqVBf#`v(&uD?xIg%L;UxFYh5q@H^#>)-<#45 z^w8vdAVTBe=AHz&)XDkFR@)mu6%?r}xIZU40Sr^6vET3$=lEAZ;v-QXux;& zqO4JhdlJTX2HrR2YbGNzcp_{d>#&5)dYq;qqhb(J#29qD%8~u%U?Qez;j^*u@!i7`li!6Sv?3 zA{of;UhdY_Y=)?1Bv8f;iRYbR=l<_J9&YstT2?vgo9Jxt@#%pe)z(rn;p8nuJqlqii)+$GFD>A$ zZd`?gQBj*f0^?$rcFC*oYYPTgyC@?>`$9%T#0iaITnUMc-%W>)=ngA%HW-eZkoo{P zc3hu-Z}35wS3UMi8YGH}V@CT-x4N~dz795W5~N9O_*2XND^qM&@f zYYf^h80^P~j6)rAYvpqTh)H5tUtgaCJgkw4iKkn)2>hs43KT7=kJTJFz|6SVm1lJc zihIzW)$=ifZxyLXk!S^^x@z3Nr6ZRcEbKTSoQe7kIXi3!Zvx18k~l$`#LV2!-I8H5 zO%}Iy8$$G4z|gG>t2=1`;Dji+t9ISNWOoHgDyphX9vnI2M4B=&F=6%o(NVy^=WvN5 zuViOIl;E36kAGK~J0KY81Q6MX#|9H@7k>kMAkOcLr8GcNGmdOx+$4|208WlYAm7lC z?^FyI@y`n4egqs4TN6XzbbHEZT{WRWeXX<^`aAE?nY(fW5E}J2(gWa z?KA-fuGou?1A3JX5IKkl3VUiB6|vh!LSPorheBVT=83;7EdYn1R(Y~_}Z&Jy*6HG(!cgyP?m-I7|{$; z^22ftMSG|t2?@M$v9Z0({)D>dHI=bK`LX$-F+_Jp8nyK75)l3HP^#L^Vp}55Vzz?D zXn{)sN)K*PTaxhv&WO_}31Vmh7XSDf90BO0LGFFR%}7NN_3P)4>fw*IvuTL~Ei3EI zBhDiQu<`K|0X+a&vtU{%hXhABRZ)>K9uzlkWdDTPj)lsBTVz;mhK#fV zANTP42~tn1iQMo;s}sL^w2Y*sg8ucqV5+9r)Uk5-4H;!+U&|apDBZ|*e`_5YdC>v> znvENX(u!{Za*L78A^Rm`9+*(RR0`^mtAcj41w1Nl&n;Iv1k9AbM74t-g#0jEr9L7H zI(mB#(ArQn$TLic@z%2f(pwM~bL$oXSkPjmYP)j0FBzheGvBBAsvr=2;4lryw#0Oy zxVa684g|L%N+&DuA`ANbwz2JedH<+KF9x`%LuaH-q&FvNH=$``zj(2Cc7ag!mVp^1 zC>}ss5G(9mYM-}EK-adnP=G)n9;JT##?@*2R~hQERW*wV_%aQIIIWWII)$%{8YyRo0;2VF5PUCT4@ zR|Om-W0Ym6-w00yCwQ{3ZI?XKs3EoGmwDvHN3uno%ErmK`1p!*{+w?k8t7A_PbtWc z?wp;7%IK-f28MH5iZkX4Zto8E5V{u9sStL};$=M?XWOJfPm#nim_-wVWIZ8iq=sgM z-Fb*>0FgN4nO-HMWMR4WKKJbP>l%(3+c>LzTo~+c#Kyq9^yQlgAgFb8OiXhS1udel z?g9X?v=sp8UUD@Ud!9)(qDfdMvylD!g7@O8jcxC?0AAq-z82CNcm?;TW%e(q+95Mz z1aHmiA)K(hynLCdCq#-gjzT+q(dEyXEBQr`^h4#vvLaphU2F(`Spt!8DtiyC5Y&}V zhxk48J6sUJ;0D!GUzr1(8f?3P+e<_Fj*C@2u$d(X0~tE-Nl-sE}!;TGL~vdmy} zNEr5G$Z=fr$H;_7R!ac5Kzy|h96FHY3|esQK||4Rvo>R!0P;dE!WXhRGYdijcYH75 z)Co21d|J%!2)o)jyUzWy%|3JdToTXLtu3tXPqz796A$rkXvt%#BUK~wK#nR3a#5!t zg9#jX)T1Cw_QyaBohANq&|~L5xq-X2cTw;i2!3Y-F2exDXDnSE6D5#2be#zzvTreFO1F?n3E`x~HRDk}*^oPRe$EzB!bsz?EytW<|XX_T>YmU9fRjEDp za&)0Prs~41_b2h;dskl6(E&UiM63x53Ra>c=`$FtGzpUtWA$^5==$5u`5>3Kn4X;K;|wGSXAr_Hm|6!Ev<=)tb*k+@*=h$$XP5io<5uD(yyBH65IGf z44fBo-8LeCPXK-DoDL_mj*$^Jl5Jx66lQ>9AeX?k8bak71@Dnj;VS+ZKs@hDn6x^- z2Qie8oUz^6-j-PeO`x#olCl9)`z8Xq4A={gspJlfc}on1*H`soag2nU+rj;7C_MfPUvJ)RHM=g9#Q?VorGUDG9Qd52ApT7mgnCZf`B=bG~Iq`Q&j# za{fZ(P@=G{)sC^`lLb`j!DNJ0Cs5L`S>l-2&0y{$0J;7%;?=K7j2g3`T%Hb67$K z)>QDottPr`FYTMKX%aKyy!w6MKpvR#7iS^73)TdE9ApaGB$;OZ4OlSjoKOq2b~4IydogbN{Nsw*l)={f%T9Z5v)7VxdoX7JayDr+WnsWNVqv z&%Hwp(n%j7a#`S;*f=;8ovK&$kpmtLFWar%GTy4x`o;BEBg&R=6VTN!9 zM4SlP_nu*x{QcugC>Cd!i9qbj(704q=a=JiQmI>zrQC&U@sWBkBIB!h$KyxGCu;;< z8nHsdmK>`6&2^XUg}bvD?#IDDr?#jL^kzarLhJz2?(HK{phQ;4i{Z3CPGeXlXuMZ> zyHJy9QTb@bXCNIjTb2d6+BSy~w$>&@xF+uZJ3!$wt3W+SHsRrf_Ke(9z}ZY`7`~On z#O4q49ghi2JpP@M!j)(x<_HQKt`}MP_h8Ot<1WvDgE^d@Mv&ox z!@#3zSw`QJ>cm(dASPIa<+YP7FVEnt>X#Xp-)%|hZ0=_?`EhY@%XJyEZ|%4e0k!x~ z;kjC(OaFw0M8YCydg`@s%Wib#!(3!Q=RF#_luuZZvv@Tsy;j=@>+Tg~On17C?h6tuv8QlPqP}7#NIm_uBrS$)CPx1V~5*ZAL3WHV%z9`lC=c#0g zB40M+)VJw9sstGP)b2^2fg7OBLS$2?pr=D=`}~5wr-0uFB+No6QvCN#M1$K+Pk|xn zP*6}1grQ;Z*fg%e*Q2&6nf}f|WL=%pkE`un*L03yzPwcwRqy|0YD%-W`KpX~cl$~g z1ba`tPfO_A3P7P!J@M+eC(eB_pd^h$;5bwToAZ;?;7tdh9+P$^rSa^in=9+VDGM-6 zkD>5J@57PWszk$KhR5YKyaR~9Z|aY&Aa~)P9fy^+ca3a!={@!wx9g2YYyIcs={)Tu z=mtuwJ?6SU?e!aL>2X+`+0+ThRbbu@N{es(6nH55ec5ZHbrKY{$wuDwTv#ca<6We2 zD&)TEqrl~3ALRtzetmjt(>=35c_Ep_^vvGlU$$%)q+2mA)SsZBuUD=8=-kW$Ap&l5hWV*4cDR&z=fA0N#kZBJrUewWc-)CUTT>Ek;XMiKtrC=?E%4rE6yUSleX*&-%IMARyDe$_v>W>Pffy(zCnmQ3%egSDP-j$sE$cR~U% zkRfaHjtB+z<4aZs2?~9So14dKYD-EncBKJc^7T6}ji{c*8ZF#%;9>pZt(@YaV*k7j zOM>c9){Bi(`IQGJP!A1_-gOYxL15~fC?Ve7_F zbEC(&+xlmYsp)E~>-^cLh_84&De2lNs%@!ZUlfP_<2iRO*;8wpdFq=$_=7eY3EIj| zmYzfmhi)*&0k2Oc*0=S(&ubiIO`23|@SN}9GRFSa{-dSyKL+_eJX2mu)E!W5edQW3 z2S*$EVdODu4B!>`4fEMkOY!;X%@ibbZPZqVsQ51GnyY%hKXScub=_emL!o1$^_xm- zzqd@c7Fzy#ljOFcY?w>NA$V3zV=mdwQv8TGyJqj2l^(2Y)TZ>{wMN+ z;{ZdA7x3d!F(?o6G*YEUh!qxL$JV+SV9rvsu~Yf|FVOvlYPi~#`>nV?jrg%M6(%vI z9I;|sAQJreP~0BpUSLQ0)e2#ikTv`*bK&uT0_rp^HE|Evf($R}!)y7!u#EFQERRq{ z_Zk}HHX&lks^`s>Wk7=I0}&2$km>dq5n^<)KivWGzO1Kqpx(rXm7P5+_f%@l0O5nD zdX1f9W^6|H9#KIqlPpXYLONtbHw@^F_k3}mB_N!&Aook)QF)vA1(kftcP;5=n?ZK7HE0pXPa8qU&C7#|3H7 zLv@U~v?oi=$3g!vDDx>hO?DKtsC4zyAWK6O#)#7Wzp7C3^Jbp?R)WqWl5yMi-`#)& ztunGjyzV0Qy|=4O3-gfJC#0UYgn6OTl;D#zFMg+_3KMGrNt5Do+9IGc~dub+O+!9|)vB#@ry9%{M2C<`XQ8_r+mAoYa9anDV{ z;?#|eo~f}Ap|)@c!jhoskXj#_SWSP7AWp8P5K~fb$rs^@j9Ixe1H;wyzPG11Tguw^ zA5x+(>J71SW@BUPE>lDLwpy^wug+c|LXqnoq*R@J5(8IQYnH@gb-b8ToU>7(xj(Z= zq{aTX%mwuuFIjhbL9g6V*6OUaPq*~+rdX-?9GHdns~}%^T$w=(0zt?PAJFvPjQKQP z5r4tr!LO^B+hwhmz4RSWaHeRVG%8HXCQG%raU&go^=~}T2Mg|j{&T<@c4GA9ey=iv znhlmga&ZCU1kS1B#S$VpDsn}M!^Zd8<}=LJdlf^i;Se$7{sgHL-AdrF1VzSyw7o)= zIk`o%yr~+bNfwTX4u3$uV$| zAiBQobm-7n2Dn$1Ax31Xt_8>&@Cedi4_}Zyj+F+L-BCSt4UJed4o2>zy?-zZLCGJd zJ`EPVDvI5*0fBpU$~J5QaX|RcMIC;XnaNO`F7nPuDD~F2_rU-6Z3D@_6gRYcA2#ax zWnRP0L8gzB2lI;@Xn5{zr!!26_;{&AG?ByFheqNU-V%gaY*`{H4i~pW3>2W0NH?E2 z5_wzAxlvFani3xnLN7zc^r`-;c&?u|K$##?Lkk)iWa3{l00Hn@CeP3(OrQSGxDIa0kp6ub+t7zB&Iv0Hp+e?zKZ z{#U@8c4@ixXl?NPYpmPnY*+6z2=-UaJ#rX`5h&lruodiP(H!5OR{OPS*qUDDlJap{ z`O%YbA&ohnxD-(>jhSxwvkmV;`hL$G{MowsDK}t{^256Bl`4*2+QDlReloVWUYHd0 zW7(ATnVD{Ce&){{N5PfsQTaEgC7rj{PE&+!%B(GrRDGV^+1}vuUW~c^k=0j1f{iWw zeLveB!FL>5HmaCAa(7~58Ev)&vhQ@sjUR}Nj+Sas!?Lw8-~0K#zcqON=adwCb)VSN zAgku5R>rZs{xW^H_y@y&OG^8&dGC14Nk1{3a$Nqf9XvR>yB?LH?ksk^NBxA;YbSR5 z<2!?c>~7Ee^FCPQZ!U1wQl*ai%<-8PYy+Cnf1m?VLjV-j{Q2|o)^gmro`oB*)x*$) zspCBzU;`*=pWD7~wt|b_7XHBKHS|IWGXP);rtd1BtjTcZi%})UN{-baiO~SqXbZND zb_*IEzXg0N__)%XB*O~Ye#hOt-PP%9D$i98r*QX7IA~`K-{O7sPi6j<4HLuL@12Yd zl(B5=L8=df&IdmcZb~PL5PWN&qvZT$^XV}CV&)3`dM90x1x_Yk$&?McaXy*W!otG# z?NTzC)`Z0X?v_EGhyTo z**zdps$!{kCL~f3xjr| z!MqX16ko7dK(;fT9S-ge^Xfi~x_S(Rjt}-h*!T3XQdYU<<(ja_*w~w;W+~GCY~0*$ zM*Q`RBjdMU=L2h~y(m4zLHG*s%R&uXFD>R%G;8+U7wg7}xhRyQlODFU6nx2e<7xPw znnkwwl|4Cm$+q~CiHL`T<=fvsPw`ZD-C*Zp8CpmZ-{Ezx*=|00#`mAn;O*d#OzoV3 z+jO1bo5e6pDvXqw0A^xQAdzU|1eGtjB zikQI`(ulgGJ5q*Kf3Hk2ivK@!eRWinakupVA_58uk`gKFkrlD*1=Ewv5KsLAhwxoV}A5{Ilf``g}JAat#vPR`4KNuXvOZ@9^Q{8P} zlYs<}@a^Rg>Xdy!1AD`ajk${h6I;q)iFVU_Pp4kzZ{>FKU(c10(c-PgOJN;1k047b_$jCMbbvxUvo#tENZB&w=*gfXCW|Dd|97~?#2Q~Yt8{A! zaW8shgz5I%F3?-KRP{t#lML7O?zx%p`0$#G?GD)Q1Rs;j_bXv{Gd?J`Hm5Ah?90|{ zU`8e+aSEGFdR{mvWXNf9kMzMH8WIKztTiSJ_R8aHH@)pg6JWsMrughAgSRN7!%0f- zsi#`GAm!y#sM^N~>syOA^Pf06Vxm3!HXYBoeQjv^JK3M$upL5O(gsXrJPRJ?n<}n( zeBR4HWGse#=j`p9bt|yaHPPHy){bPEl+g7rC?R%s*^+WZmyKQ;8T|E3l16YX)r(g> zL(RA3b?Gy^@8G{F6UJ-61G#Zs#Oy-*W}|NIsyd*%(e(UMc6Xkg^nYbe3imVJ^YXK z$kAeE6hf&*xkW0#qKR`R*O~dHx7g??Te%Z{$k3Oe5&v@^O~d@ivr%TUIK^o=z(X(! zRgPS#J&4=Nu7pu55jDll{x$?grq;xX z+Aq>X`_&T?5@3{QK_yU$igdf%d3Xf_Bn`n*B{AcRiiirV|^8k zi!j+!sFa0A0$F%p12NYEhfO~R^R;U-ds|pe{t??(i1dw&j&5oY-X8Id`|4b;RJNIshn04WZGh~so9t5&GvLtf^c z!=by)yg9ZO$Ec)xQpqI)*xRhSpRW>jC)h+mTWRj;ZMoyuzP;6rMLnBeZ@ck+>1KEA`Gk?1eXy{e{4KdBC*7#o{~8 zt#F@j*DQpONCfv9#<9hW`+F3!dxLmJXU|TVl-$rK7l#ahb}|Zz6snbq=`fvo0n4b%KB)DxCLv;mXz5`ZVGLh0v1gQ5=rBFECEW#B=i`dSh`Aua)}m=hu>~&DV&pbWE#(bG8XnmZ>#S^6m;ua+QBgaFXqAO!+ z@JmOgI4#QO*J4w%k=BlJj=qfT;S%}OJaLz`v83r*S(mH%axvKxW;7g|w!S?hSbz~K z`?mQw{FY*Kip~2NnmpC8;o?ZSwdwmay0X!0mpZ&TCyH=3AV>&}frM&CdC2zSMlZqG za|EJ9?H963rNX+i8&&u1_ zZRniY^HRCPPOMY1Pt-MOg zE!;)l2m0ofd?L)ZTO9E(!p@KJ=^5=QeLmnGAGQ`-o3_HmsMkVuD*6`R11x44l)=9+ zMG1qfXCBrfd!JI3PdOXE0cBQ>O>Tmyv)XL(HK-4_=!LeIs9QVrp*rzC z$;s;K4=Z|;+J#4J&i?-7fTpt2Vs04JVNQJPy>x~b!8&m4CS}F|i8(cw4v&=zrt;5( z7QFzku7k`LBM*9a(NXDr`qG?-Va8E3)d75NrTpOJ=+&peqs8&R?-Tle*!~@j@+KF? zkP%|LlJ<5r>$`LHgK}zIdyUs6IEmuHFP%jR_>L}2F9 z3*82vkFhljM$r|AGj@i3ASb{U<2SI$o2&5qE=Zwiuf;4yV1K#d9<`{db6tA+^$tf-b-;$)c?sl2|2Ks$#BlT~Qds17ULizB9 zS|+MvMo*y8Em8_mDjS`l_yzUYj$SCtMKRc0ovrCJ*|FYT7!#@rp(Kx#2fA7i& zz5Y9JHu0f-__QEv*iJ%2(7WP}#-})T$GL+vsvzGD^GqGfhYVp?Wv#DTsma1^K>2)? zjdm-c+daUqs9p>;3usQKSe75oyD3n0@=+7p2sA1W70O|I=1Jn`}YUz9rs~VYo(P%ec*cAg`Hxl)sK<-zf0~@gzlX0FI_d ziDo9YFlQEgn(Nx#cPz92`mHYI5YPWK%~(l{?v1+&tRxCyck&ERa0c8=w)f#3&~(GaN399_NeZ;FW~>zH$lB%C2GSBYGiT?Kn>S25={4q<-=_aPB?4mj_)J-v@lH zIf;ZXEKG}~k~7(!R4Fw3IJ(7T(8lVsC2^`N?ZjD;;s#FqZT8HxAk(N%O?8ZFm}s_! z6gBv)i{>GI`*FlEzR5BVO}?EJOP?^~oUG3uf_;-0f8(%=>5IrY6gyDCDL-Yp-8x2< zTQcnQ6nmV!i18v+Se(jp{VtD-DJ>O{m6Hp?4twu?7lk<|Au%zK6shh;awrs|ii$dd zNy6|m9he>d7_zfFbE8Q}ZnQ+apx=;q`RBCr=QLa7c<}l4)98+T!(4s;4=0dPaN-uqZVD1rv`0VWL!nd-o5Ur-*+jFW5 zXxVEV*&!^K*3jnZ*-dFg$xU2x&^Aof(RUy^L0Mg6!+L2+sy={T`XX$3ZJAe+Idb z;NdiI9+DInHJYY2`xDmJxTTX5dy0s{-gSvLQ)fxc-l7q?EL$fz_Yex%gu+A92F1+q zY(|BYH?gr-QKZkr?7sf>6p(M5W+}FwpRIfU;JIv%>9*at3t_-PYD`4LHE@Z^H|u0T z64The0wzuApbTx@UdWJ~4b}N1m(pC+rKNT!fH8ez5 zWxE}irSR_vJbP3Lu4KI*Z{nj|UOmP)T{|4o1*9?N^f2RLPL(mM&{Y}MvkKkV+frNd zA)hZFodKw2PB=E^D|oHA>UH4)*A=YrLO`m9C$p)qv%8fjN;g~ptM7}_KJ*tX*m8JZ zvm&RMPIX=LZrAozkY*zuu#iRhamWC_^V!gRQuAN!0Hs^8LgPf)DyYKO$KA0)ZKD=~t?;%mQ5w3G1+nW4(+5Hh(3h<~EvfCb8Ekn{QFul&*8j zoN>lyg<7YSPMgU*=jgXAu0$O+rv{sMmVKnSPPTL>pu-I24I$!tuk?@*HG%&r zmZv*m{UwY*Mmxx#;TVTA|9IV$3}5e?D3i z!48!BGkDQR1w8UKH3T!aXrbVuU5}mY3~#tGgnbS$c$BqFqL$5H20{NaOZMo zfWhf5e0+SPTeQ|yaiK_#jfBKYwy&>*z+B$e236XSVEiC|rIAkSn~-$zqA-9JS;EQk z7T2{ERhqu4kJKOgpEJU*_E{Hg3pM*BpvtRLlOqtbpynB2S=@5QCYa-A?d5>>bK(mY z?13(Gp7s;{21C|joAvHNPOan20;0{v4zGN1G0nNYP4V&*3;ctn{VxI4sb8;aT~bA?W4e>ZL(7xy8WOD)vi>U$0I=Fe2Y^rPZ42^n7MeoSn!XNRPj-dK&XQSw8Y*srfq=gF=t~qKp$tz{Cqj^BBV?0Zg zd4G*aSJXmJ=zi-(y}4)(kzOMmYm6>A4)}N3rjr%~zd7Pq4{$eNoN-_TKs;A}ww6;h zT9oUOcfYKL=s@3e&M_O7bz~AcT6cKbDe2(xMl7JTAVCZ&L>>8KNcke)B(`+lq8aVf z!tWfE)-%AbGSl|gzpwnJWAw36>2tWroe6MlOATy22xLn!LV{%*^f1G!-hqT7R&Wa@?QS%89fbWD0< zgN1C>{RkOEwWiki`leoR*v8)FQWIj`>#Qp%oPeGD?Y-;5r`j4x-=+W9+%j%wy?$R} zSa{2@_-#8*V79Z!Ih1XGF*prRexh(1Hu8NKtRLXf+%Jc0z6a3K8@fz0AS zc7q|6bY2dFq}!1{#zhGj-iR_i%$e`n9lkASzxXh{b=E?8`;Oa%@3K_MGIAC3&Xw>$ zyCnS@@)aCEe+8!-%bYGr(AiFks+57_L-1oim$W0rY{5Zlr?KI_=PvQBru0* zKBo^PLz}rQo#}E(V!YczgLh4Nu(|p&^gRXHHo1Hjni(*#_p3?51@S!Ze7(n(xnnKz z1ZSAQo0>VcHJ!JM-%wb9-&*XWRn%-gp!IS%`2*kera7;JY#h`U7i+N9o&x`2PUx2* z1N(6F(X`m%5t`;80e2`8nesv2PiiU-$}2s;g)6g!KKKm>T0e@@j7V=m$@~_zi6$e= z_?mwZt0e2`Mx20=u5Y4kVrsk#xtmQ}aC3ism{BA z)=E*US?JQKD>b5~r@ga%PQx?q`bJjWN-%NQl1n>|?F|cp(Jnp-2|Oe>UHf)1^7^d@ zQ;8BIcYq3===6+mKu2T^o4DvO&g@)loS$|uAZTED|J{6J1lol=FD~%)*c@OHbc5|^ z>CQb)aKC>E_uT`X57HeQm9Fa>jS&&lh@>4`eEX{-0V>`mWm+xYYW)IIa>J~ef~WF1 z!WA`?;~zGERKSZ-;SN|?>vt4Ia&=yf(oQpHVcz*R*)buB%CrJ`8%Nj6g|c__2Qmw{Qxcg->#OMhrXJGr$SQ^&)Wf zV3`o*#k|_HPlt<1)XPk^-KKp%&o^SczQ)ESaVFQ(@l@^ZG?iLY;%1r9f2KQSTTcD& zzKPTD3utKw1<-zGHoUS?Yfq37ghFMPkrwD}<`M&9> zvw%%*F`KSO75aFz!pn93?2$6~LYPp5a7Z!Dy#0<-ldVaVbk^%t&S-_+)Y zLoya-0iOyeY+?>aqHSeYpAL*E^hd)vAeZ(mcruIBK_Y|tm1Zz5s%zz&8n;mCGGoc0 zwOG-bq-@k`^y@;|N!4;1sP%=o(8c^6F5*CoGFukYQw0webqU_Np^4zN&<>`(qW=zM z^whc|g>YvMRlETFacN=`FrE~Q80>9@U-v^KczDEw^`u)Q2{|;d)lSv&us?&T8 z@(38^-Y^7bA97>mcZD^7d73?sWh=0a)$V#hsbar4sN$^^?vjajez~1F_-PYlvHZh% zsM_97(Oh9~#AaIRZVTpnZ4=rR(LWykt^9A;1NbJi7KPk-ZO&YPRM%x}B|f|^T%P*Y z6t^dIjYsyZ)DZ~|cnLWLd$YJQ9zWnxqbRoint0RRe$E@2khr34symqc&eMMU$(k#a z^)>qwV5c13$Qr{7615`s0aVR(4xt@UThaEe$6Kq0H+FR~cT3GK5l+O3g)w5yHcOk1MFqH+342#X^>p@(V7IJpi79^PR@V#-v zpq#JOQWJVU>OC}lC|pQ5@M4{%zSl&&a)H< z#}PT~99AoH6bo24{9Okgvyf-jMHB8kWHL2)s_W?6nMpAFY<^stlg$bg?{bH{i(bJ92E14za@K4VFmx9?-mm{$bUMZ}O zzHcyxgKYzr#S}8L%AM6fZ8BfA1QMSVkXzh%+C~jQe|CQD-Q(SOv;tzAOpdb&1qhF) zWTv*cm|?@-eZ$n{OAdM|mP`_}h=*xCgUT)-EjV(y;Vas<;)a|f=gjz`OnXN~k`FMV z6<8qiLU?MtMtK!e65hCryDOk#fT9+Q6>@Ch9+2*ZnOa9WV-ie(;w*CmnS4J{lzC*h zIr{gjbwHhvEBQ;USQIcB2lHqyq=`8DvCnT$O@n8^JTyU?jqr@$#_y3XH>4Vs?l`Px zddqA@Z4X)B1S)-Ez=8Q%`B9yw!IKr=2WMg+6>lIZEluVs)50;6yk2U2ytssfgtnCz zN;!DYDzJTD@#NWeW%JV$y*H;fl_@z3JRv*#2lBHhai+*tdB@KOk1hRaan|z=^?Q5= zX*ry8E(pMFPdhb+J%th{Zt+?dWgn~2|3Up^n}==lCHmL0udMBG;W`6TVz2ldy3vZi zv(MQ@;bhx%5Nzg_jt_tv9B|ijqz`UR-p26>I~&8lD4Omr#H$<3DXkO^lpxM1nN?`V zvq$$}#x@_9x2T{X^tR%NCbNcg9Vh*+yXxJyqHTZ;Ip&vR>gYM6^`Cl%U-Y=;a|B_` zJU|BKBA(UdWQaX4Y-928@Pr}0(NEjIq-37z6GcrZvKDCl!U`m0M%U;?aW-~_lzaGJ z5gEoHFP*bF`DBm4`% z=Ur#f&$K~~SI8AWiJlj~^ic|NQgGZq*KY(-=J#ReZ>H;_@b&A;1fAad!{ZyGs4hB= zjEDI`3lT5?GYAJn#;Lnc^YnVt8_(f6x!LysJBNQ?U!q3&jPr$uH$`q52wR65a)>VE zxpw=)47%2#lGR83<*QfIAQlSoWpI-b-NP{Ht$G~n*m-KfrYQQs2Ev_!>TqR4^-w}e z>TPN&s#MK2Mb_)LJscE%zh>p`*`)wBJ56nJ-_5~l5yGhR>z9+7dm_I z_WBBnmi?0O*0VAoXmmc0*8o% z(<6=n&dvVO@%>SK$u}K&fmcq3Z)r)S1-H$My{t0#uCEItzvPmuDeBxZ7*$(r7ZuSH zbq$4ts5SQ}_?}E1@+ZHAglLo%dfR?JL0AdvrB|BCC^{r0;aRg{M8GL!gDe45-#2IC zaa>~Rg5;V7^_d$|Ldh;rGN)s&vT)pON-<7rnQ8mzYx;ejj$SleEkbul?3RG|&$Ek{ zVT!{6Ln;}Q2+MI$#s9u}>a?uP@Fypn0-IpFrFD3vh@iBiJkG-bpBqm}J6gFNaIpmM8M`!B2^g`$_||Z>wturWAv(IkR#^qGx5aNr`L@yz!dJoqM*c{sP*_y z#5&>VV=}%^WBn|R5^q@cz7juXk0ZHywRw5^i$71k$sa?6`BjbN;_>Xnyt|Bm=4~{? zAi`dy{e+nT{IMl~PM5*DB7r0B60N8XR=`C)Hn!q>isw|~OVcU*bRjeYOjl14AV&c5 zr}PqBq<}{vaSP8Xy6MP_BusjMs?m&dBBOqS9{!Kty$=)ep-GfU*e-Z*KV)-?;q~Sq7K0| zw*?l+*l=)g5b0Iqb;Fmbsi)!z?LSe;K1eBRo?R)c;0cM-8+ng#H}I#H*i`{_*vuB! z1jY#05Z5GFMQmTnA3t1jUh9$P9jsQ5%d`g5_h^ErMDcnK?X5w?G1F*Mk6Gks)!z}Nb=EsJaMRp^IOUF&qw|<@h{XlfYUie zKoiyXmWlbAxXO^!odsxMS2S)P(+HPMc@FMRZ90WZZ_=`R#%ayJh=h_EfBWW#Z~?oi z*QZXOmV*tw(XHn-hw5ifzdD#C_3oN1SW6Wo;|3hCawWs>K7I@ZkHW!XHWn6H$dIKI zwEuy36TeH>Ci-`k?Ed(zbc=m>j%ML=oZ)kE{8_1jW|KR|W1(DIK57-Rg#5+FfTsHHvdvWS?UsXil&BO8^t;Eg& zUGS}ZX4IU}aS?KB*kT$;BfOPPkc#ZO^Q?a%n?km&)L~!0W~YJI1eUVPtKJgDtBudX zG+vlXGp%kObfWH7)u=CxsjZ|kq0Lh&(*;CvO!b(4B?l4`XiX8MEV|~fS_tCWG zCE9m&H8fS&pL!{Wp(S;kPw%#Dn+%uKh0taUH6w9$@*q zP^D;V`^{>oudqc873*A_raQK0pIX1g*@zC>;2~s)MDAVA68;%qHxp5~z~+aQPeA~1 z^x#eBgOpIG;g@irY^%*NM4kM*MT|aQ=*8dGH{VLhnG+ylFO(0ds`!8^1Y97mBG{oW z@G9kD!)%b5wTkLfrH4pf!#P>+koZb~i&k zMeeXcD=dvtI7TYjjvVSF;waZ^lQTUdgUqjC%C@Z8BnaXiKwsc8y&;g^EG_n{*a6a`*+a#~uEN4nMv;F3v_hO`p%o+u>%Y7aRSr|k$z zF-4ul@~$b(Cz{a_S69Xk7C6=QeZKu@#(jNZR1;Y;Lw%gRP0{6jxM1MedamDj2NuV} zU=~`@T7jNRYy{WftB+F^Ls{g*FrK1^8;=9pX^yhFo~K$xL?%*AzhB~4G}`+ZjH#v} zyV(1y_wl-RS))YNW%Xi=Xd-^Zopg~u0Us_ro77IGv_4?m3 zf!sq*PEOmz#)ngm#HRZXhRU0Y@3ag^DvOEcnjnSp@(;0nIx|N05T$T>+uvUMg|TFp zXr=-M;j{Pf%r~9kVMk4nj_3}fFn!V7wn|+UOcpbQuVG{k2yk&4g$-*mz z3_iq$SKmc>(_8(@22{tZH=nkdukPZhzj(At<~;n?O#Pmj{Hx)g(s8YC zf8zE~^QIrO@X6MbR*mZ>qh{cay|rJ@O)?Ddr0KSCg$q*gYXkS9l3i+o@y}F{Ornv8 zz_EBjeBrM=w?K5dfP?CtkwB9Yh0P}nYg>7gyGJt_a_nOz`b5P%JulFc9gIIzw8nPu zXz`fy7nF4_(+^rEpL+dTUS6vJ8!>XQVdRTF-p-t~cRY=A>wx&VAO1wUoP%@AjJx}= zHSw8%u!QvvM2o_*ql1T+~|MD0&*) zy)XJTzloiF8hjC}mDD(?bj0zt_K99QeT+U-BXoth|Gn=vW!&=wzZXf1g{+pNFS3LO z%>NC8i71LTwMlFL+{vLOeiQI+WeA^uD%8tz{CetlvF^BuOo_@EN<&8S~f&I2u-h91Bh{P?MH7yZ~)mg)PIi`{L_H zpl*FZ&)41!{x(s8v(XX?TZJDez*nCWVhS9?US-;^0^MBDFiT)Cy=XTh;%9pL^#fTL zR=HhMdz0!oZayj|WSIxFr1PB1OukX`c&Rw4=fOM8_e)WR#K?<$>sg-irpMv_{->%q z_vD{VeZ;-GqHsLf=1v`U;F|A=0i>%Xz{4Ppx*z$+kaOAgOyM&E!(Y~?x@Ze6ck#V@m!~%Oc=nvcu%p%-pb#J>F8kT7Di@x=^)^xEMH~noI5eFoSa#tXlAb zv2h@&=#F?DY$br@;V(aqEUf}5B4Xkcm;)k~GD=D`)n7w-bExU!n60J8(|lE&7JW#P zvqg~C)^5(_M9DI}Jf9FSUYwLIB*f~Ob&8Eyc+vPy=2+*a{!UNfMubkG^&Eym;SO@{ z>~RXU$Ko#juUVa@5liFR5?l<@BdoaYtW4&ccV8tWB^|&)gOyBM6u**zbkU!P+m>N> zL2WD)aWqhKG(Bfg?<-)vIn_YXHA_BJUBH0-IiB-e2YIFE&GoKS-OuQ7Ou-=0H8)HTtcp=MjTnTu!d|E;?26uiASuc?&I8}?es1FYZ`^F&q-^vs*|q2y@kOOD zyRR#~*w`{P;^DNPi{R)||MR;}V{^ceo}j7YW{inXneXC_jgT9Xo9iyN0e5b=JqgmW zfh{iEdfeClK^UUc;>*J&O1>sZQXlh&bLwV>7`%22>BV7DZsAmyxm0!qzAxulsU*;v zCrkUbGM;x(OS!nLV`e$&fdOqjBD-2IQ7;(usITfp_sSQe)Q{2k5-rrsh=g}+w{iYwNeJ8`)6O>+L;sa05 zl7vQh8E?vjrZ%W*U&9wxmjMb@=rd3A*x#PSk6{@^>`g6wOxsnB4b-$(9#u&XQr`B* z+){@5flU?27HjqACiRSXcN_O&is62*s!;&oBGL`L@Qmx;{2=v!JTR6d4 zZQ7fgk-41xa@Z;4`*FwC@90WB3)fKpifv-SNQ0*(hb>-Gt}As@^9(&Yl(N%{1c786|p2d?383Gg|O5X&)Bij={x)3rYRh$Q%chcLUN8kRp&`Igk@wE_J%=Hs`Wo z?zd!>u^o?!dW%RHdu=@QD-!n2NG+g!|JaL(vOr5NpMMy4?j^SMhpzhHo;kG_#k7r- z2l8<4yT+fK`;XwIe4?7N`Zod%`CsV;?kD81CO@x`Em;T z<*z_61Y%OX#90o#?|Xm$jSaLmpSJC-zQII$*Y(yIlHpwHQCT5p+w8sH|FcN&@P^in zW?CUdg7HDy*SCyafbvH(Hw6w>4SoR1e^ezUhjw$kOyBJpaYD`9OeH?=WO#zFZ;64Z z)}}d#CI4wV?WmjNn@&jaE1$k6Bl9jbb%seK-s1aYn~uR!NMes%h4l-Z(`Ebfdzo>U zB-A)F<2663Uf>I*l>5FlT`XSC>>ezCSk4W&|8r=%^pbmkAKxvB#C`u_{GQ2})9KkhkP6ya)?peE^O?fQ6gxJGLmo&a+*i6 zu|^A0#3Tf%L@odAn}^j@hHrc0dx>j?{5Thx;j5zMB1crhU9|a`r+sv?XBC#??49cQ z-7tToe}cmD>~P}S42|1W(1}&I8D#{KnGv5qna!a1FZ<*qz?8fsPqW6W%If#Gd){}^ zn9tRV@7ZFJflE>S#sgt6IyPGAW+Q z6$#hl!)?cLum{`iL{!#VW%em&Y3hdAFV#&+sAY?}|M3=1POgM)CYUOFd76#`l)U$d z5ZVQ`f9=A(Uu4k3$gGuc1%VV-PbfAW;~%ZHo(_j=9yW7dI!>qW`mMl*2sltuZaJq$ zaLQqe)2U1(hn%5Kz!jaSI1_KF?6;*5qpfE;NR1dHfh%TnN5?M%UteE&u#d~qDNzgn zU3hEa13zXSo(@9;$EuH)=s6!nKWfB|*VJ&YsJm@1amRTdm1WyI9PF$jeXr|$dg_ML zt=BL{WXY;BPL=-~^L$$%I`tA8Iz0KcZNS~Omw$qj+cB3mm0Ld1e=Ay0X5{Q-aQq*r z1r&m+OrIGzp+t??3&Vhgy$O3E&EBxsMRW#5^>=+!qUduS^@HM%8e4Kg-+`k>I?(XgNT!o%ygx21^ zg;BP?InQ`ohb@Sc*8f$s5RgTa`;GM=ElW7Ozi79&IZgi_<^>a+x|~X5@XbOkL3MaF zAD%U&(YFDN7C+{GN90Xu5dCC`X<+%Y4D(@vg9LyObR!vPu+PI^PNc1NZ6H}<0TRym zG42oU-~TPDq^S4{X}IWfQa_bh4c?&tl&OAMfK=PIk8}BcUS*AWT>TbJ_{QfudHuYv zT`p}V`p?g0Nlk@}ziTiF3VNjjh2Z=D2Z~;L&oN9syC7*<@hMF-iTV27A?p(0ES%NE zrO~D*RiKmr2YpU~&;kaCnPP|wo&`(6JkDVgQvjyAD?_N<2}MsvmPmC0-4AE@<>RC6 z8pN}4G9Mfzz+CpShm9Pn{p#yG#|O;v912e>ddV+PUUSx+7wPYuH9m9Z^Z%%gAyZYB z*3d2Ggc_xDJ>S9MreqiVCL6$HPm3m}yIh5GNmR*w&>KKKT!u&NT33OivL#cF9n0ZbzvPjhLH2nQEUNxGH&m>?~t-l zbafrAJ_?Iy+nQx#y{hRYnw_c?b44vXDrc!`C(aMU z1nw^x8=d|586%p+B2=~Kp0-7w!Ep)Y6Q)ww3Bxx%-+UB6DwKzqPfkWgP-?83+_rQq z;DY4jWPz$N$XCgr+IWm(1H)UFSFl1XykX+>b;k5r+{~Hy&~Zq@6ME+bRr_y(FGCM8 zc?KmTGQa~jM4t9Fk%Jc*8O}ehzIL}?pI&For|)QMi`xuKTY0i-ievbQjNc@KLC$oN z9iV0*eoqEfA?N=t;ySG){h50nytTkxTx{6?HzxF=6|Vs$!P>+t()o-T9K6#_NEj{R zK8sz;&%c2fkU3@&Qe51E2|N-GxO)0z7hX-+zr3$9Yeuwv@@te$PZB#Vtw%bMje)A& z+Uxo|MLtQnV`FdKw#2kZ9@N+c-n(J(#NRtMs)I^4f>zA^VMYf-X#S{}%wP^((E3$EtXG5RWZNRy$)LV{}SU1rk?b(z5>3ZalY6m-sW zSJH9}D!1Uf)xPd?Ff1fW;q4g*^Ovt&;+laK?S}5QLnuX#e>c8(OGdxMCT7Vjiz7yj zqVQRBf}3b25hBE`I)vrx3(sK>(X`~Uq24onbqB^}n|qU*S7IVCQCBo&cV&0eBjt)} z<1fcmRh?VisnjDG9%}S$j?p+5_W22}Il+8hkm$qvUa4_P84rA)^(W{!nQ(VgbmpW9 zg)%+J9P5`PR9G^#9Jmk@dj$@~k2MaH7l7tE_h+=Lk zk3%tOW#sYJpV89UWUfJF;UKErO$txKxJ%+yPmlc<=t zS0q6%;bfY$6TWwvY|LT;%;e%8m7Bp1ts}crKFaVuf8ELKT?>`XrB|+sjOy{|EY+5B z^eRk1e~R!}%?p%s^8or`d7Uq+9o!jMAP(EkO~f&BQ*CZW+vj}ydj`7O?MF}i>S^R` z$glJ^=6y3YrpCo(PF#2*BBZ+-ACbwC;jgUE&jrDcT08 zdz1Rg&+|hUL4K)HIWCJD1#S$n)weOd$eV~ef<)4=K8%RZe4|#hPG#V6MQnetGdUwe z4(ZKkgq^bVSn~Ll{fPC!fqritLUjbHJ`wd$@#vgxTi&yvS> z{8d9Q%1mxe&+bX{h5?k<6^hOH%G0EsD7{n>@Sg9M3Sit{{oV8KubpWxpH9-KI!%bO zWjgGLdKE=qEbTaZl395L{8eUgj{{0^QT* z5du#UL}Z{;G^;#3`KF`bOj3;|?5k!CyVe=*SM~O?T=%KXPdW8gT7IO7CSo7)x>z*~y)skzoZ$P@NKoA|sGi;V5? zt$)Yc(f22IL(cKj=?pE9=(q-ptFmC2F<+?<)XxZT;%i`Cv+ynXspbeBk{<~$^zQPe z>cm}0j<2wWAcOJ;@Ak~C+Y@ljZ=0ER#ot55QDCeP{Huvp(Y0Cf>mUW=Z)VU{nv`%? z8hg(jJ2yVNVvw>lY{_t_4(kTiB~{YT{5@7n*o|(HDZBZ7==Du`vOuOs4-<9kccP5%Yg(@^;pNn zNxit10_o|#wd45&1RvAOq-Kh{WgU3|bD8N92earC8WZK6;bY?ugaa7fuWsBrc-yHV zzC3k7mEukeLY5^}&u0a8H+X|BI=eok&6>j&uucU@ zKf#cym3K`!2Uki-wuBIDzz}|rGcVrV!vlS*%g`^ev*A-BVnfyotNQ^gM-t0gkMYI- ztRi)m!b=Bdlo|#B>K(CYFIAi>*!=Ka(JEUWI(priy+S4H@{Cc0EOm^VGFTQm@i^in zD*1mn)agINIsW6mPQMjJJZr2FyfQ(s?~csJ#CxeOtJV-a^LugvMGJk@>BaHP89+}JWlgDGHSRt=v`WCxPLgXnY1@ZK=ZH0iJ0sq z!Q?BnPLHz_zsE1dd|CTK0Wopqmy-g=;L3QYb@tvBvi^muI+wFK7W=kOpQ85oHWslamh6mPMnSA9g$w*PDXsEwU6YNZpc?$YE4ipWNCQDUo&cBzLj`J$;4?-NJ3fE=cD)ROuQn5jkZ>q) zqeBwGK}piy`*_P067syLe(ohHg)N0g|9PP3E>BvruD|*R9ylm|j*OGd1Jp_;$_>2Hg*+HUJxUR$n!5UWM-)9dITKKD zy#tT10&tSDAfgzOug5Ubb9Eb9Veu+GV+c}w{au%+dZ+$vSDDUYpzte|fJl<1wku+g zYe>J_+^MZXFY{cBnSzb93|*O#*o(Ljz)+x(Ro%DyuyG+E0V`BC?F2$M9&s02zxu5+ z9P!8)@j1>0P?{;66Gv;|HJFvbNbsrL9Uebo1!=-bSQ;W6g>nrV)g6xG1)Rg9DNkhC zwSEg;_u0VRc*iq}GGWEWHoqi`BsqXk_?08+nRVB_E#;3|tO!SH*s%sFOR}fgd&S(G zPoESSp(o~gO^Eua_NAa5P`EXaa&Ll(;2s?l#cyMWQ}HlhIS1ZGh7Q(9{4Ys7(%XP( zOrTtA=HH@Dy39B>{m)kq%UNypyD!hrI++_$i3}+mydg;j2bXBefjpo@!;rmQd9%m( zLu{X_*x4x6cyC+^>lyx)!&LtKEo|mHEThEnD3%>T67rt`GASMgq7_DMn69s+vZu?L z_pg`T>1Wc`b8e+G1NouYq=TN)gXT=pxn`0F=f*AqC^nPmlu5MN%#V3V!FO z0kD+6a2^3mpCdRqlUB(@ds7rC{X*z|r!@w48l0CDkH3$;^siK$ zH?~a^eoh8D(uVzC!pODA>&|6% zFQ|6{85@iaO;I8iJbVK*7y6UfueU4oCQw8yE_UcN#2!J_dWW!s6`gEl-Ikxd||Oa~2g_XDbN&6@87e{o z|F5<0j%#Y`wv7exfPw|YgA}o$AYcKcNKr%-q!W4(5kaa*k2DLSpkM_>dJz(O@1P>0 zAe{uIOED0tKxm=7xq|21^WFEpci+9gxBpTIJIUH>%`)bgW6D01-YlO|wYJooOnK!u zL=8%>oB4GzajO1<3`>XOt47slJ3h#q;E%zJXxu;K))e3M(QHV*Sk#JEB#+Btv#Vv( z{$+s9lniZK_Uc4eB3_`cF}KjpWmkX zED3{qjbWv;-^HQZ>bpHN^B+&7MJD>8s8Gj!As~rL-0##bgK{T{ThF1K+F5=HI-Z5O zb41+(c&f`6l6;iTog<&gcjImJIxm30nLsE!OYt}<+n{7fcL>H?emj$`J7iC z=?Ww;+t7YYz2S}dn!c-^{DFgI6LXSw6_z(yukYFO6ZkLV*B~(XoUQ=)dzC^lXC@qX z3S}s@N~_R0Q9(jmBuFuE&atuo^J)~w@54UA$4zv;xBIR_m*;a#%ymG+|Li;I3q@0) zeN5_Oy71abhgR36&Sns@C#;x71$h_v$IPM~o-059?qGCIKL)XSfCkps3?=p*MA|-? zf%fW;#-;-P(!R+o2?fHTao)CyUuPShFX~^yLY|WdVSvnjXa#*paeXTEBp8YSEckWr zY!~I+*S2Tw>5vtx)J4}1w_u3s1G8UE)&1||{imFM`UH#>sE-|V(&Pi**1Po)pE>Gl zgw_OyqeE(2;+qKm%R2$`fzZqdh zm^6V`*n=}@a1tma%?{{bx)|Cy1Y=g$V59<}-!M}3&y61O?;h-axCe&^Nap=TfMim~ z`5+6fnQ|*~ISmYf2$R*9Jb}mesWCMG)j~r`w20r)pRsq>&zhT1T_A*>gT?gg+1Lol z?NdbNWZ{z7t1i2cfaX>DfJvxLE9e&Z^5tl7-&XqBpKXY`okkO%_)-_v;QQI_+e%O^ z?xK!ICYYn#U1yr?510>#A$WZ1fSm0^HamlalmtPw;{4FtMe~m$Oqa6&9Cx}1Tgv8M z{>VUR$GUf`FNPR43nC$lL&_RnjK6c1+RMK#^FBgQac7Y;nrEC1Bks9xa6UedsJn;{ zha`zrZXDx#U68q{vAkXCpq!w#+i6*n!y6pUtH&x-+17Pplzpm>0$I$jCcf`=KlHYp)z;2jvrXK=;=b3I0c|-$ZEPs6PFP4%L z7uR(t9zN6+X8}lgRD{6N?Hqm+!Su!MDDO{vUb;T6p1dEdV*d0o2sq?Mj9>ovdNIuH zdi^YLa0H%ohaP#`mm|y#K|nQeer0ty185W%pgL3%JXCIR;Fv%ZZL9oUsBuH)eP@00laDi!Lm6`Nlr$_WW2DSh5G{kfdd}6aGx(^!I%Q!r`+m~ayoVa zdIK=(Q_@g9IrK{()zb|?Apb9OA|?W{?|Hn({+RTHEFm!G6qXUyFHxF+KWx9WZY!{x zPk%-M$sCFc?COtfnB;>TXI~ZV)HnN)h3bZb<9BLL1J7|>kwFym?#)Tr4CWYHBx9K# zxYFRevL6H#76(wpatfqGaX1^FFLN7)sFhkHgHZFKz|%YZyw7WJWba|d)0XL#Fx?UIf>cLE3DUJ z3?#jqJ>8jEsCTbScWY0;Q^PhgZYuOP#ng-JJz#ug)1^iQyg+@}2ku&Mtrusng{j3S#$964NeK}OB82en zO61VY(}h@IRAf{9(~?b-8&LFQ7rzRGU(D$*DeEC_gw|VL@0q8NVvRIP3Ja^D2|y-) z$bho0y88C=RQ9}Gnm?u=C2`b}mXr3BHdL$3)w0J+s{n)2h`i~YD>TaPI5nds^P*;H zCyr0ScCU$>M}bQ9$(OrozEjPH!Slb|+vI)HWjA+=Z|Oa^A@f_c>=-$Hzf5Ld1+)So z8-_|tPb1L_Xa!fA^U{sYfm+PNo&3;r4NYh-dV4Q}?1<01>pgo!wph2G#t!BP{>XWj zxB94iB9BpN`{{M2!iV3c9e5Ox9-!rOLJxvz9TA&CR{$Em0xt*HN&_>L5Ncy}$b<$U z5VC5%Tduc3SOl^*f;Y9+rL|ANQeb>Izb6mB40wmohT?;wv{Fo=`h_=a*WWk=@lfV07qXV~ zDb~o^l(>}Ev;+(eG`r6`1Z=r z-NMFKzO3K?HB7d#5L%^k5Q2%u{S%c*Rol1(AQO1@YK9_j$j~E@ zu&I}uy>Yt^on%%C4HwQw)u5kHkHO`f{&a5LgU$Q7)-Pe@FBE><@+u}k3mYKY%} z-0Jpp2iH|Qtv$g{VlWr4tE}o`i0k)XIRM6ZMnhm)9Wh*u*3M*74e*CRk024!eVse! zl0szK6mSu6B+oDlz1!Lvq(tFn5@V=xf6`lcuKpA<5P=v@YegZ6)(^K^&L@0Hs?FZx z{RqFm8Hzjdk$0}ND}VSrj5|EY@0_^T_Dc~@b5KIx)nFB?48r7#2vh1)@cX~G2`TX{ zDI12LxN#Sq^1G&y8?we#1_iv^xb@bK#zRw|9(KK9lZQ5sTRHWNevZu7pYH>OoF)Y! z9%COnU!ZG*0+j~KkS=t<*mE&6!gw(lZah{3nX{(M9gB_HSnZ!LrGoo~wB?Gx7k31Q$)it3=6;8Hq)z zoD1&?6k>rtkQDNqeq46TjNWelI+(cVslA6@VV&Hxu~FVzCZjMb5efx7)o&VyUXzW={Tt`keeVg`}=_OX(T8yW}{=op4gT7{ghoJ5F zBLC?QcD1ULn2GJ%b_i!};HSjuu#;7O%vutII2}xm9%L*iguq+zb$Yrk&;n589gK$< zSyt_X^z6>NO+%p_>ph4| z={-UBa9Wu{IRFe~U#JNl1@H2<`K)BgxC$KE_#MzuZRg55?9+cBSAn&A>-uP&(tnto z?%Qyd5``o%@7jP{{_@__7T{v2|Ge;?bnsa;mivNUL=EsU=-!GD0)8T7;&Mu`F0Vqn zs|%#8hPZVfcC6tq9Q*ywN3O-?3p^jvG8q^#^Vv_t@y%|dTV!-NgUIu2jhMt zRJVX@ONE>i;|REu^xKiAqGP5Ack;R%KaisMM-H5W*+C#pyZ*;Hm|=dh-DH#AT!jqAvl3<`$ zebmB!pWI-0RO*{{|7WX)hkScAptNOe)TrU@u!YHqthzp2V_n@^e$KP8@tq`Ptdj*; z-I%mz#}DfR0N%WRu$cd&j_E#sX`e=7lfV;FY~=UO-hKl}84E4^A!Yo!L*Yj9xw+y( z;MOrEl(01Mi8W3!Z*y?}#`E1D6!Bn!)!o$>ez^Ii4?&9qBCp zO3IEO=+y| zpq*9F3K@lHw%yzJJ8>Al4bbwOEdQq^OP;8^bzKH(%^iG-zBy)g706d#?(?rS?`{RgY z8D+oBi{AOGkVIh*ayN_DK@hc4^?m(40;?4IWU9z$G{PkJC9Ml@bbv!~pU390AM8gL zR~OgX;DcJ&8y^5;ji`VATxB&LKG~JFwdMo;KZ5Ni-Wshp(>({N1K@h=;28(PGxn1r zZP=_zd8-6+-)uqOkAl5~y{$9C0)oFPWHoxCG`@A&Ro|wEA`h)T59vGcN|A_$NM7c( z>*@W_vGm*Jcf(VWP{I~U#sI}pgQj=4DVgI>o&CejJ%QlRA&G(0VE{NJ?dEa47sW2! zIx%UVK=p<1im38A@_Wc$p#d}0uW|D@~J;zN`Ul+pWf`+c||}l^h2Z|RMYlxU*MQ9o6smS zE~`gEPgvAd+U(Evb_C8z zZfaZ$aQ1*O0yJ9>@S1JU+95Y;1$9fSkP-haX+fN!AXnb}t9AVYE0T9CJZYCb57l^$ z%9711oOXn_MsnzQKOzv;+K6(2T9&L0ZRoy{+`IUo0wb?sCk;YXhB7avo=fJhG*K+7 z@Wx@mde|FUY8gu!tOq4zzw2_RogIFXf8C*odvX#md1+<90ml(0?7@xp4xGSVhO_Mn z${kq+6!R?ogwfC4(d&m(tE4WwDOL@i2Ae1cxFElqj2^d)X`21`)HC(I`+E@ti%eiI z`E2{!i)i=7`iKw$t!-DWv3zOkujwUYXamiwL$P3rpAXFd82V}^OA4b@`C2(BL9`8w z>0QakaH~l}dk5#P9-~Pub6kw z82Oi-r%}*1s>))k3&`|utlI_*%F7%kc&;({Duznw*_Qpo*|d*J;nnJ(iHL|JD``&>%AF*+c1U{j}zq-Ov&sw=91*`G!RCgtg4MC^l__EVl8b_ap^F zkti$iQ}cF&$zm|tGE=f1%8%Zg1LWIBVM0MzTX24El1sOU*+t*(w;wsq*WlAeQCuTltR^pm&c#j6<|P>je>5De%0p z+h7ea9w#BofYxoan&IS-dT4y&bt{4*#CHpzSR0hZn3JxQNyG~N!vY$C1++!%QW|0O;Aq>ijTq4znbHH2R)^P7j2yWr#ab z55#mV1OLEG4J!nR>KC#Qmsg=ZA+SM6{+sIqrJqQlq3{m)y6V2GU}rFmC;r-qUpqP0fDh*$kBhzoR5M~^(TR>B()XV#3HG1Veuj>);bVsW z%>?RwL7iJ6OLaZNPve7md9=Ab;F-`KKhJC}FIIg}!X`bk+PT*1+b9L=N7hGY!%tl+ zO`tEp1i&w+*;i~=EppcT$)mjR>+ZAYya8kXnSJ=v-rpd6V~=>! zA}H4gc@yWD+0 zJS_7VMu_udML^vC`l)eonO5mI$fsa0AO39kpLXU!Q9KQ(d_7yhK$9~=n4*~rgzP8h z4!1AdRdGg}{WJu2t~IDjzF4wHnG(raE&;PuCRIUhD3e8I2FAzeA)`50sJsfFBHrGH%75P*{|7hAX~rU5=?Dq3HRjb;$@c^68C66iTTgiukD zMp|XNFk#FGF;ZB)5x}{)zxQau)Tm~?4E(v*4mdWv(HV;nlBA=cP$%RE?ts~{(s8{| zm8NkbaZD;-m=vAzr1(8Dh$W%YXl-PXTgyOCAcp4?RHPu9Wi47c0(~48p@$X7q(4AsWgpn#Vy1*3WnlsybYY4V79i zeIHFD;}>l)GqVws!4iXq_y~LTm(+JV!hq#a$M~@}o)s6b#cOK^` zB54V9iS7cQtB8&jx?&8_i(}DtKv$zRO2>B5(;NB^IDu%Bd2@&S9({Da(Pb_A*D<31 zms2S7SIPQvlo4Xyo!lq2btcdP{i2Tkhos*Nv|g&~_?v0eh97;7nI%RRRRho^9}jO& z){nrVNETh_?&#dx_b#pd0Q2}9nB_|Ozdvvj%D)V6{Qhkngqq1BYscoVzmGAUK?{Q} zPV_|GksYRAj}_5li~W0PQsG{|f8}rn(^b<2c=MHGyD3~Jt?FlH*o+X7oWg~6Ib7Be z@6L}iia5b!jGB9Y>;$m{z{^3H^+3y8)#=ZrUP+9ibdwD)MQfhc2^|+R$zSZmYk61a zm4GC)^{-!JLSh*LSQl4l6VNN5=TN?I4Ty46(9*<;0&w~j&b}^}6$jGO)M^;kLyn44 z9#KLeA-ZB<32YC8(z0Xu!ucrApMMrn2q_5`)xr5yOHn|*+Z5RdtR(L)pXJ4iSFa`@ z?Ksq%AW}McM@L8JLCGMQTx%8WedsPDYv{@{tz?8GsDm?q2YTlL-bAXnL+7`ciTGe6y}~6# zcQi2gli2Y1!#>oZ4wt<1KEHHD|4IVxo78y4st+ja4|%F5faXT6aLJ{B2B7Ytp^&K1 zXu55(l2s0AXHfhWfPVh$?-mw`qlfgvfKI5as~g%<3!H)d@87>itXJ_N{p4tZ4+@^) zYLwX6DE5d`Oj#{j*53UNw5|Tw6nTMt$)T$l>$?K|Y8@Z0!VHZz6Zc4#IPBq6L$73k zA*A0kmhq2f6N#>J+csf?GF+aqbDh`@Uf!MVUxKOIYi&nO?Cwb}fnkm)>p)uD!@ORu%8 z1Ot$QiSG(D*C{(|vy&!Dr-RcIqE36>%;D|vZGxDVepw$~KXfhAqMC&pBM975SJMXE zYswLm5~7keUPBtXsjq!~)s&#@u=>;PpWD7`3k$ET9qqKFBvjOhbZ-9{YK*Q!$yok1 znRGdN%)EAg-5EcJwKjmt%F2;a)4E!l@fI$4rZO_&XY>BJz8U*|EGRH2kSibfUJIjR z42?ITR$Ms@Cyve$Re|JG9C=y#6~2?bv2L$`K#jwh4{JRIrd@YFT=aJVDWmzgzz3VZ zWaJ462((){iui?AH|zu>(gXI_SAoIh8J;M-bQQq@`uoTuA^q<#{U~Rx&}@Ql?*x-Pe^nRHc{NjFOuDz?|HuqJ;Wnzj+|(Yfz|aWYh>IgI#CG_y6jTij z_g5CVv6U}Ak(8L@lY~mlSbk#H)T!YcgA+*nru+v?n9K(fkOZ#dGIji( zt%zvt7uD6<+^>O@Kk5^R@?G(So$s2mHF|j$%JR_VcRrrcMTkTh{1g}%NN;{&%{?J& z_#J`%aCtDXH_Df-6Gd4j{X(|3H2}E65N7qoQwO%YXQb$N%YF*w@BJbhT|Vb5uI>w-W-dSt(*!;Tg?l z<(S=}>*f(sNe%X*K8rAC83s>HUo8Itl2Xm`;cf>Hh#g1fZJV|HKq*c4;vG!Mw((`~AbT-GA8^45k2Z zwzaSf#EWj^|9*w&KVJc1`|lqYGW_kD-U*jy-(PIf za%1D93L1`rCfU(1*^s{i#ee-oe{C=bWzZm}wd=pR82{t1vay()ImrOKAS&wjv+~n@ zgtAKdf6V#MSNzvc>fe00^T^C#}U@BMp)tQkO+{P%kl zGO#yV9|L8o=T?H?eSN=v{mSM`wk5}jb@4d9N#Q5AH20JoERUu}?k`_-W=p$UupIG= zKjK!8EJxa)xM#;{8JXzNr4=yAzCVV^d2;=owAqEvleaq*{xWu$fBUHK`rijqcp*Fe z$$(fdW{LeD8|&YEoqulRzfReI^O4HKY23ZyIrOVtaB<1aweQe9AHpL9 zno6&el3;zj-oYRx`gkMLGTd;5*uMSy6CHcXk#xNBnKQSOk{B_Qe4B!*uKb!g0B5NT z)UiyVTTmK>NmoBV;nRYtRv_9Pc4r%lJT6R!uLMW3|F#Ut%@33?S} zq?F*q-io?4F1Y|bDJO+pIZ-3$9p?xKj2AVG~6805CgOS@<`ygE? zwTD>bY_RL^G+c!(DafyT=r&68JGVB58eGBP{TrY-p$d2r`ucT)t{$1^8HHWnyWm@!Snv z6g>WJYO!sUOV%^>Ok7c#0?mnDQ{Z_qdQ|psqS>l5(3dzhJ#9Hy^D^7x*A8|F8uucpNcMDkyHwHL_kdFkQ0LK49K5bgoF zhm&1^7gxli9LaUJn9Yj_S?_j-4v5@3W2VKRW#@abYs$(J@SQ_J0CA|{@%eaT_byl9 zundBlMXO^8_T@ziuez0hpx|J;Z3b~w2ZRUpge#VAj*Gx%zISRVP4f+|$w=s4YP%2q zGE8%N8wWY4pVY>U8$}m8lQZ>E&)FF7hw$`@6_XE~~@PNwFdr znO>9uSvOzQic#fj0zp8=x)QXmg^|AyJxL|CA+DUSEW{g0ZIDl!-28HIaT?e*q#rl2 zc`hJZS);`3eIUoGAr#{7n{ohaT%#GGIh$&8w7H7av<=pIL&qVhyN&$0Xb!iu+YrV8 zVup-q9H|yj^m=^u@bWsAT%z9AVP?b!zP)#EBN`z%I?FA26A>HzX^?QrwQ~8|BHhX- zV;u#xA0O^ags@plN9O?az`g`M&a=?zaHL{+%+aVf6pYR~7nj@OVw0 zc^Dk*)-ME4dMEO-;RfdBv18Mdlao>zdT8wPVQAdwgvasD9zNlym0Sz>F?aeSr`zK- zQ)zuoQ&ZC|Xd^N4r={2&1T4LdUp z;#%3Qxn1#Di+Xmty1J2VIi{hnURf6$-UhEi2G)!2fa@*^%{#;F`QZkRqx!uwQoDi_ zL5`5#Rk-8tw^K(Y-FI6Z%BZ@_PQTSD0&3+za1^z6u6V%!yH1FG{a}&+M)o1Z4t_Ww z;h$HFI(>c5wVdzyewXXUVD>S~I+qHcWtYThBt{&xOQ^YnxFX;cCF%Tj1y}Rp2vT$# zhc#GYqkp_en4xwOOpY2H>WeHqr#-rJ;@6YmJbtsUg;A@(rKg0Deh*u1F=OQvy};Pw z%b@{6}d$S5E61_(p)0;x8dfO>z+pY*P=RMWC>q-;j-T}{&bM_}|n z^e6ukFZ~<0_1}ti{!J+Mf4g<>N#F`pT|PFnuCN6pC1(2VD6M#wFoZNLo}g~8bA+5> zamfvSXb@rqM**hCzl5Ax5)^ZDcz|T;i#!cxsr|XS>~-d&L=PB>TVK)ueVwDTAY0+^ zv}k?z5V(m&&xHGnoxHE=u6g#}W}G2YK;Yxt`STPSRwmf0E_`r!7S}?Q1cH0GWhwE>YY&zfBf~Ox)qWmCapTBK{iEhWwEVdc1EZ7zQE8M zhvJK11TCSRDZ&iSApLy%Jz$B>DEi5{Ou}KiJj$7_<4V@AbXa*I!B)M=fiKTxWVj38 zg9A&IE1#~H*3#1YAPL9qA^oe=FMx=-gHdnIl`M&fFm^%R&!8EEI_ggVzOX3(i)8#0 z5)&^%M?Rn+Lj|KLOf`LdSCVKT!&yU)_7w)4=LIMpQDOuS95^TCGPD-k^mdHT535(ybI*HZ-R{1rmjt51z~1B2JkoW zf<1Td-lb->7MWyTJ`+j9(*_sF(P7A|ENo(681Cpwg4vGHxH||3XqO(Ib}>@jwSs-u zt{Q|e4S&x`y65jdT!fcQGsyW_TCGW*hJjdN&|t0X;Zd00$93$o9};`NIW_QQQ&*VH z3S_{~Amn)1WNoM6Xp!d#Yi@6r(qKEQRKU~m-NG5H%i=7)`kK7FUuBTgUM!%tm2lL9AR(wwEw$zNatPTBQ8D{c!PBVm^`pR?Q zsnX0ooT^ETdfx70h{FyXJg68w4cf2ET-qU9sq=E)i@7b!tqnQQZ{6hPi|9FY1 z>xu%!7^6FP?##5o7_wdQP1YS15Rrz0n%(*63>XilALTu35cK|X<9=YM`{wfa!jlgL zH=R^H4};_#+)c!wr<^U;Q z1qM>sw`opm^i=#TT85^-u}m`ve9=%0HyXZ!Yxi#Jj<@wiDXp?LJ+Pjday+eMfXYH) zxjUSQ%d>EElIJ5(sO=p0B2))T-KW~Rn!+?$jS26ClAh&yx01fTzL|a<6CYU=nIQ?j z{E0+a_(o;+$+eL(;%GQ$M?~7(Ds(Ee576*<@_<)s!vk>o-j9&Es2{<1#HDLbixXXd z%`1I6)MmwQ_bG=6pRbbB(D)>M+BdZz4F#r|^d7wRQliA`7aez&#qM_gR90ebH^IsTn)REo%Lw z+za{zc0K=cwO|5dJZu&~P!*ryJ~eQ$!;QU$;P;u6v@+qlnsV}$=KH5i&N`dZf2IIm z+X&=d**jK1z(ha3A&lv3Ytei^@2a&VsCAbgm6J1L^d(LA9Ti=Myuc2Q_`+c5GHq^T z4938Rf?AIO1f(;T&~M?gIn((jOj=82!ndUw#22JYC*2u-@q$ezqj>!gSSWo!g03>p zgR<+jBZ7i0VUaSobOYXzGVQ^Ty3q`IIY1OFG(>LYTYmInHw+ELjl&<=xnR;h9B2RL z4gGG5Rm_&O*)b_D-LRAxBY&?vgUa)+-N-T<26IWq6{I-BoDnH+Y2qLhe|_G+$C(*A zPQ&-TRSU2v)siwRdCTeJpVaw`i6RYJ7T)xjhB-9TbzQq1;+rpogQy0*iZ~$KS>i z`pkt8P}=1KdWi&$>e{<5G#*L{c2eNYX5 z3%Z`bIZtzgHDY7`a1OME{Iy9nhU|-nLEBc~hUSllOt+z#y>Es_pqyv-!!KidM+3=G zx6~$1`7F)jQj$gp<+-{gZX=|+K0-J)$QAq96vnbFPQlQ531bqNY47i#WH6e0b`j>$ z7jT;~*off+74uB6F_0d91yQMPTV1WTSrBGvKYPUXvrz}gX35mK&%E=z%Vy^!Nnedqj)IETt7d3K4BBfS*lsLAtc0j+@-&zkU=9;bWmD`amvnSqj4RrV6;YU6jV*};EOj;z zTjmgnCU{Y+!GDV5ZVM56sP7qhQ|IQ5^6j!Z(#yNcy-QY>vZ{GnU;w?gw>>t*Zm`R} zC(mAw+|nLIMe4)j+QvpWT(Rke=0EdRjU3Ao<$NLjp^SE=jPE2)>ryxCF6X3Ov9O?4 z!$Er>Z48nf#jB3?go+6TR+$7jJ#SmAx?u`f?-80hHN_RJX%gj(X;1$p>aCSZ-j%&V z7pcW=)yu8M^&g++$4e&cWL}I=vb>>;FkJpEFM5MbMO6^TNCVO0tPG>= z9?00B1VPYr&XN6u6`3ktY-xL&ZslzXg8?cR$yUx;D-qgdb9)H*kmzum?rIW^8}cf( ze!V(K2qu42ZLV0JHR8US{LX+OuQp7iZQWxr(ex0pW12w>1jP(W+^#L!5QPujcx{+D zVz#;%78PJPufY(KS9y&+7hkn8(%;G@B$vx=jw=Y}RT#$9VPSX=F{qg6zSJJ{ zG7V(Te05@<817CL-pQ}_0wgk;+`I?h-QJMj|9H=-^m=cirI>=WAiI~*oL*2DzA=w) zt`=Mql^kY@!^@7tkb`~vY7tM6W)}$4^_^#YTL06#y!Pw09HbK$@JNi;%D}*YDqG*$ z_mI@l{|FZ$!93HY=uZJazzt`hJ(2WNir-apu8g`s+)b*^WwN<7GxTFdisJvLYjn!7 zvY*cKLIi6QuDo(`h~d2Ea_P;NxF1R~QN*C56fXh?t~JQX=C(p!vbjk?Qg(LpY~Rxq z=z`Aht;aJQL0#dMj#P<^Bd1Okysx&U^e4RypUa}63ejUUOcN40h2X{3^s~Az>yVf_xm7_gVMBHM+mQ6x*TCaDy2JD`QpW_5}z@Yy*89M+KUP9-otr_ zSZdblopJV;?@~W6k_H(z-|EP(8lGU5y!*t}2co*(1$am=8}3rY@yCdq$>KA!t*y_X zEc>x_zl#J+3m0Hc=trBKwGLF^a)Lk$oL0CfC8q;w+#!}nP<9mSzV4!*FW8Z z6!XVY&Kgn6u=A&H^vW-Q*-eB|N_v|J1Z(zr$nx6Vpa=A-t(Kmcr9wtq2=%&X)Je!fCRQS05oA520g_F#EGB?B#u`m-tUZp zKKP_+ClXUtB!%Xq3L)x9L@P&>puleXVX}N38RtzqHE4B*;GsJqt|2##vSu_v*k=x` zvlky~2;Tz9>eSR!0$wdrDskczyKIV$BlV;P!+Xjt1K)b@`5ys1o>4|bFUEj>?4RCy zVs^T*@i^u}ib(3&Pxmv+slHGY6^=H-E~j?yNIzRFQn94P7@3#y*jAT2{}4&p2>V(; zhms{eVl>{7Z$FahSXUc~7vzGuZn4Mbf?mQ0Tn-{KZxA@AJr|ZIRI4IVb^tU6J)xr7 zV^07)VY=}}_3Hu~%JRUt^A+z)BnOce4H8WR(ys22$O%aeGxff=vx3179yFRyL%u&F zE#GPkl4BLaH3Q)0ewrHZ8D-NTb_80E3y4V*f^MLX9~PiJ`Zww1z&9lsHJ zP8FeHo?|GlQMR=-Hi(sEtf965g)7e1JMo8Xk0q`TDehK=kglhohW3@SH$`9qvMrLQ z%*a>?bpG3QNV$*OjeH?2R*|ZTFBsWV7&cO8eX81JG`C547(YHbfWiQPV7BhvpIXZHoBG>nBetYfDTZWN>o zn2vaxPD{p&-?ANau5(vy)$Or-9P7if48nVDL5oD6M=2dUxwum0^v-0b$dPq?hl_EI zr+$)+on;%a2MhBOpAa@8E1*QT0*qK{(pe5MsKEc+6rF@h&M_DAnmmSL11M$bl=sFb zKWi=PvQx!f#B8+R6Kh2(It3nIwKVP%4_hm^K9v z-*?>QHdDV#{_pFPw|!aMcZV`;Z<3lWoF`~JlJZTLNFi-Y&l)@0>J`BdK$shaWr>lC zKJA(9Kr>Gy=wOM-QXv%!b&e5H>VkQhMs-nr5D3g4QC;SN?cM3E2BV?z6OgUdF|6OC zER=cOo;(T2w^Ug@pn;A??9O#$Kis-c#h66gC4DollaFt>FZ_zpsh|HXHeW_EoTJUl5{flyFFVEV^O5m)*k7j3J2S#UX} zyN=}ZPT*)HsIalWnZGgU|Ul2z<`jhB?oG`jb)C|oM%}u0iJ0POevAisZOy;M5@vXp}I)!0XC&Olh zS(U(;Z8*@pn4=&xuXoG57U=@Yt>$EB4_L|3UV}ALa?~SJ0KaJp;UkjbvKmyWKiA)MhDtL-l1W0T!oL6Ct2$a){B#`^Kz>U+qqBVafDEh}DyCS;I(ZX)Ru zfE--H86X&f(5jt=tC)l1BLs3^nbH*$nv#-IhdtQMH(fpyFGCS+XFVE<#mAh% z#FV%Q+w$l|J_nUQN&|5$=;6aGnh$V6aFNv>v)`?RV60UBNQyI_$Lh?PGj7wj0Btrd z7uiD@PKwU=<`r9o$>iQ0=_TV*VLWe;U%&x$0Xa=cw>uy*uBK#?kt4B9lQ&W`g| ztMRw0Ae0sYSo3yzRm~wrKo{wT*52Q`X8@dD;)6v=V@N%CcB zkG+th?1g=p%5#d~uLu*g2bz?4iDemS6S>LX(-Zkx8}Myrz9tnAr3Iw8@6_jD(s}v8 z=8P`2Go`38nh4XuEUSaLix-V$LWrHj<(JQ%#rX!~JB(;-OJ8Jh|$eUFC5G7keExm zM>&}~ev%sZ5dq%a=kjtjmY);3dc3Ap)zy=*oPda!Zq}kcjXG`tLjX5mK$F3Rf|HkvYluA8&D)~E(-@9`~3dujoG+Y-3zHlfoxBM+clDtWZ;;eAgq5Hr<40n;r|g|eb5$G0uj z@acipIE8d=qnfvG-!8Q1jnv#$0iJTfq%Yd1_|y_A^28qLF885YgZH1t@i?ax9ufg( zO#KGQblqJZ0QfO}0Whf}-LTjNpYEm`w3kQn<~g8Yr72AE$l!(;=34JW*sN25*E#;P z)Q^gxYuCRcJN(}ecKTobf1(b|tU9)_*kCZ1_uh;2C2GI7xF~q{82t^H)~`_{|5!(7 zWBA=e0R#YzXAzPBP&nV(cX+Z5sy%RB*o;;KgE_Dfs!ee1_<#HJy=!-w4A}7BdY9B2 YXA%qaJmd@I(0g4}R6Cb`_VS(o1xfr80{{R3 literal 0 HcmV?d00001 diff --git a/quadratureshap_n8_speedup_vs_nodes.png b/quadratureshap_n8_speedup_vs_nodes.png new file mode 100644 index 0000000000000000000000000000000000000000..b7c94b7de26f3b30a55c1cc6f73406b857c1f5c6 GIT binary patch literal 71642 zcmdSBcT`ht_bnPlKtaF)qDWJ$AfT@z(gIQh3!TuL1*rj~69`2_tbhej5kirc06`%1 zZb3x|H332usUd_G0)!;ziN4==erKHj?zm&zaWjU)h+${%XRl|ix#pZ}KQ=Ma7vK}) zgFqkxR}FN{AQ0{h2!!jwP9E?RE5qb&@Rw?Uo^^n^&)tCFn|>}3!SjqeZ0>roI9&<>5Q~nK!C5ms-hzNzn@U>@pDydwhQb4N7?0TVB-&g9L93~Y;zQQ zAr9FFfn3$SY!UK)ewg>oeiz2tcvwmfeeV9wqdSipn46mH*dbyXZ8gvBwX?sIcwgXS zR@Tu278Y8jA2ZFq2p*NqI3;~_dv|J7H$%H&0x`n=-FCNZVE(1o%gJC#{ckUCoOt#9 zPH9-X2L08qm5p%D1#wTuW`K+O^E!7KBJl6qN4vKF|LV}+2r)uXx6`LjFZ|>3 zLzo~OW$)gT`#e%=`?2##UIeMM;jj>))TM1}rFIUD$tr#JNN5e*+23d6@4G|B5|>Fn zjRP}X*%S=6p>(n>h4l6L;i8bGUqSsh;|7*mrFYHdH>`tqwjKdyYB%$4-! z-0dN}LruWP6Nc~lGw*D!&aSPId`JnJteN}IitH+FUmVpd)Q;t|tPO&D4pn_j3t#?D zXh4Qx?5q90h3%&s#usN6xmJBHR-0Q2F_y}S$>*CG^~v&?tZpFbFq=R zMoOg5mgN(XL-$$-Y+TbOTi!2Ew&~Z|L-q*7@CsG5rz+c3`%TUh?0gKXlF-x}$lxc* z#%;SGkVGkPbH~_)4z&dDMrKeLwVq!3;9l6;{JlsE*2XmIqDSwkTKY&Be5@%tXz7b+ z8S9sBq^{;_r_RE(OoIAy;;xKJuYp@chu}0dgMDXYWIhEg47p=0i~MdZC7^pd2KlOnzjDh2?K|H>TXV_hb>&jABOnt z*CICsw!c%_e=#azrCoU#uWF*t3B6^05SMQTsEurIR}`|M$iBL}NC@^aJ#^}A(B^EO zT^}0eke|1>F_EBI@#U!~>4}Kk{LAYKAFKVRYfAXP2{G0v{vr+M@BZ{=xhIY;nwFziItPbqauF4&@a6C=^=|vUMV;) zDy&MTvBBjnoQeJZ>ZDy;vb+#tV4Iff&0{SM!jIz)xPZWU1`%sILIo!p1m_dBe>`~L z)Hk#h)-gG7Wql%n^i;9IEqLf&96uND?t{)QKVE-qKr$qk$$LL4-}$yDa=j2uz543J zz`0iO{eHIG$2GSWW9pZG$R)oM8$BXb;WN5`?YGRSNrrd4S6S=Qm#W6Ug6dRfHaXlh zIg1qRJE{-Ay0$RvSvee(8j1X~d4I)R-nmVDxh9ypAcV3jcN>@0*NMe@=e>RBQ90!I zw4Sk0%NoJaMiTE6rg zTubgIHB1KG?wCa^dX|LqbWM_l`K@$~1!{z?1;}4LszK?s^Y)R*hdo9(5wK39u!^~c zCX+>Hr?{z{P)8)Kk40K7s}vv66KRQFWt&vSz3eHUOPke5;DHF^Tus$>*GuAjzkB;? zz^1{WMhtVQ3<;mGIaPD1bf`dre6b8ghz?D*_@R5XXiV7DTYrD$a~~mOY$S>u1vKT8Ka=Z+Z|lBcl5aS zlc06T2Z9Il{p`53J?w~n9+HkEl);Su#weRbguU?W)RXT4YrIgQMyVAPzqF?-+(H8C z)qj?L2{~0R;9=+0oBx6wLxuOK41ByDr^f%mrtI!QnPY{QjdB8u$*66%li6EWEO^(J z5>v;hH~*!{Ts0>TRhPlAj>AQY4o-;KoCW`bhI2jCjMOL>!f1U~ zOh)PZwWVgPaY7Hvh?FrhU0zKX+xs+zR~rpO)@K{xtb|SPo~c1 zp#)Xh8`&(fpmbQztl9}--11UxHAn=0TMhiI4imO(SKwl>Sw6MJd2`5ZM9c&iFx$P5 zg-;pLug8Z@B3(n7sQPHw(USoanbWBON$?P;_=ItAV(xzRA3X z)#Ej!G4p8RrzLdjs11yv2rGuMPi@zgg}1%;DyAM@GI7tL|(0G6Hp#X#IzPW)(@s6 zCr?SHzI^%1uFCgjvU^=&D~-3fLkus1rkzAO!@s9q5RcPz!fW54~Hla>}ys z4YjCpWSK@C9gQUY{{E`$r+#H9-E&ZRG`z>EE@bJ<%HeajbbB4+rbNC^coGWV&feRM z%qQu5cOP6Bu4SNHo9}Zsu(zf)kE9&wT%(YXDfWI>L&Qgu&z?jS`;u;DB+J`v<{_ukg2StEAxpQMK+!om z*b%{`5lh-OK}Op&Qki`1CFN8T;6D2K@%~Q=c#yopIcY>uNl9tG^n%aGP-PHhuw(_K zWK=lLqmNGwbyV-kx?|{MQprjeOJOJ+z$! z$;09s({XJ24%jm)5#HOKMm5=66Kb?xbN?YM6D(eXM;|&CsVQe!IK$Xl!RtV$aYRQl zY5|;FhaVe6KN8evx^XtK+7+LEre~HGN}A3|aeU_@2`ZI?E-DX--2ClFVGJ$5K4sdX z(Qvyp{z%>iIPiH5yb{rVzhBKXDsn{Aa8Z??)xb|nd(m@zvDO!qE*6QFmW7`J`!bz! zJN1J5uZpL2JGVOcg1Z7T9aD-^&e;6&?990QjiMI9E^rZ`s}EXk(#Rv;x`T5uq#BFx3y|!#AX;huB4v5Pd#+1Oj<<5~0O)>CWj$ z#wHodWYY(?>!NLO9>oZ2k=}R26sL%{14gOp zfk?dD{%%s1(95=p>5cX4(i0sJbqXJku!MDk?!UbRf zM+Cm*|FCQ~Zp;564%5Zhp0>s6aOp%We?Pg_t~}Zg!hOuUc4I1am=e8=p*+~!joD|} zH{qd}IR}NI-8B$v^8*vIym`i`1s2xhIPB&{x6US*oQHGU3mN^Iz#P64uxQUPzJ=k! z=|x!tZ4iZ+JAFMMS|rj|Ga6NihcbR@K9}9fa{2U7fQ9lWIoYBn!B=+HpUm>O4Rb~F z8Ii`pTFnCx<;x}Bk*=ZEx-skj0sj!)XTby+(9mkK;39O+xE@ZrsK(w}aS+La#$+bz zqpaX7S$tY>HN(`4fv%qU-OEh+XeBC=HGmWEf?k!6)`qju?|Xt~U}he+0u5+cH`v9w z2zXpgBPiP5q}|1jgp0V4BbG(BlPoNse!UTey{bYOfOxp9qS#xlu$eEP{P7yDx4%BS z=NfDuJRDg3j~(_p={+I=qivq_SVYdUJk`7AG#;N#-2*2&CY7O2;_CD7nJ-M6HxH0a;^?r-A0i=wW0a(Qc`o1Kfj+Zimge(G?m3KSCF( zr)!WAzBn{G7bc*HuIa5fV1Uo0ld`9~BX0B!N3V+2^KFeq>*26wfn{!Cot{0V=v$7| z$Fd7Fn@(3*MT-J79}Mn4ilnEhO=yjti@radHtZi-NiHj<<{1=pJ7D!(ueh_b@de)e zxgJv{0Vh7IOuwIiYJ$M4wu?jWLup1fkeOJz2{pDuq1*LBWv}XuQ8~2Zn&U)grp|DX zm;uS?)TUrh#5LEO`I|Pb(X+wQ^_I`G_lBH^`r5VF5p`+y!>b{B)!N?8XEXCBDO8$j z;DOf-6Qyb~H%5y9Ze3%iXGNo6+(nctq)O=MriOD@mY#Q5@8+&82N^TOXgSulF`}N) zIF*BcN{CH1#_EmMZWfZ&R4dDeHYIoIqt2yW&SaL%DDgZYQ1?I}pJDbBK9EP*8!OWc zxhobo%YvyR_IR0+cl6RWg*LxPBN3nKx0WKR!r7PcNEe7Td&9kov zSoH~e4VJq%ygYt=%KF8A>%Fp#-*-n3Q zQRQYHr4M8E%^k-KBoo>-huahz-@CkPl?*%6vjV{KfO5pvy1JDbcZb{okxq2D`maYa zTS*P*I{N%jb!onXd(;#`|F0?8Z+@Gr)o4x#9)k{COy!t#ZN5vRTgy+C)dMNZ9C>W zDm)L$K@c5QY(HQHS{}N8{*NV+cWt{SYOE}}Zt{#>g;)PI7@U;Zaq7|M>;&jsBx|-O z&$YS{An%2S1M;@(dEg^9x$dhQ=D4bb9vk827MRxbWY~$4?Y1d?lRskZgOK`UsG)hj zNv1g7m+$0at=voGZC@99#^spOG)t0G+1>W7HkI)8#l)W3^RqAm%s#|OU8qg&q4Rg2 zu~le}cj#)LK=3kq9Kx5rG}NsxeiCA?%na8C_q)?r^CR_)jJ4aD9lk06Br_8l0fxA> zrnUnDS<;0Fj3v^(h&ruDnrcqHxxIWLA$Sx?)Jy;ggKDF5|8(3Vfvzsp0{_rQ}VRG9^;EWZjq7G^CaeUxdzxO zr8~klWf~%wn+MBBGH(IJe_=Rbhzy# zV^>GY!9`b6onz#pt zo(u{VVgS09^;BWGvTG-9YlQy9)?mkq?Wq9Jor%hgMzWC=yxYs$(T2$j<|VT@@=n&s z8;!7_{yk?2+XAZaImsux@jivy-Ei;tra{;&8)LYn#%Hu1XnPovvh@?r(3L%UzYSXz znB|C*;%CB)dpsfla#);9t`PO9j@+0Ob#fHH0YQRmWJrP5f_Mfs>BbV=2GLT#=O`GZ zlWeq&_M3KLsi~E_L(#F9rJYyz#HdwB@{GQ`LS4l+v9q#6T0yFC4P~-k5qF0gRlO~N z_`;wUAwATTq$)6S_}Ephy4dWV2BpNrnety_&0`m;-WsV83ch>z{OT&8dhloVFgb}P z*tXbcm<=LMKK-GPlw>LMN`8fM^`U1bMY{qvalxi)`~qe<(TaV3S^<0!L0m6m_xINv z?QXvGsf)39-I2T7*&Ly$;g~9rFvtkWizN#qaAG_4iv)Cw#Olm(8iN;gxjxSdvweFB zm-C2EiRjkzH(PeF*9zCf#de=l3mys0`_jhY4xqdPrgB_9(m%&&k_8`cx88l{6tvpl z-dQr_tvqgovPyl`L=*Skqc4ztVD6qIRzA*X6l|;K>>%6?lk5DZ*LPxZ4=d!clsF=^ z$x~q8P=w6l`5dHe{$;i7@z48>SE+F^B+n48R(Dheoa9-w*F)7fpSbtB$3W)V>49Bd z=W;5XC|5_Z<7?-l85(eGG`#N@4v!bW9bB@- zWB*Nc~)@gh6Bz28GdfjIC9n-G6rBfw|&Outg^?4NtYFGPk86cJtEHMI<0y z(fqV?gz`Cmbb6i2_t}lWFh&Lu^^&G+0RQtKPL zM}k1BpFUBsze>Q3v_#kdz>s(;DzRXTDsx*xT$KCS zyG<4;6?U69q`4GPh)F4Ne7ZStmnb?&KAnat_Zr#u!bA|z7Kx*KH}u+a?;UPgTCqbTV%0}Dd;ZHZkXXz@m1}iB~B+o6C+&H)SoyY^*>1~+^ zx<2Diw-w7_ROCv~S8m!ko+|%Fc6P1K+17ZWk_%b|{ zENiLs=50%agE0EkYqPWY(F|e7D@XJ2iRs2<3r0p0-gU9UA9Nj)p`64X=gFX(Yes;Kbnis{nEm#qX9mtX1vFP!c zvb=3KU^eLUfvYzBg}Fkzp9emS71auhE^Ij`P%K+xnNyy-vVG>gY#b_l`KlqohG1%C$o2#?t5Rj?P$ z{MuYTZj!MYIbmJ(bo-%LJG6&|+WoqRc*)pJgdnU7(Ohj6nrogRVtwk;3fkn;_x9;C ziz|oP;nU7j`|F&ZO+t^o^Q-l0CPVk}VV0NeyZW&+KNE+35;1N<;e{v9Itv!^2?~01 zMG7i80M1IAhqCG2`&L1DIN%ifZ?$LElC*OsT_?Xt$;GtCH$?PhWb&U1oyS~!>@~EN(Yz}ua(YHQ?YxaD;v;h+{VMx9v3Zu;m zyCDgE9$NCJ%8>v93Mu@y`r=Zx#<%%?(js#X*|A*{)yyVdF0Hd1}u(qTU&02cX|7~B!hUA@{ zUUi%iYIhSnHJfIMk-j9>Xa2%WFOvtcyrVMsQBKUROF~m0Kk}`>Y>F>#_qdnk+u>=N zG3Shpb9L0z86?yl(27PG?@~X`aOP=!LJ0S(73y-m8h!Ue5w~EF=Al!{3}vL~%^rXS2 zz1&Vq#hYnYGq@5DU-a^gNkWjK5vF0$LEfRwrKgTrB;iCKpi7lX%rvTAl4=Ql?}4Tf zieTxy9VCWaE)YB_e&cu;2=35KJdS4>EX(xJHdCYz#<<+-x%l^kI|r&knP&0^t%~zy z?5AORaH*gR4f$9jCYWmCjm7r(KyhRm9Kd zKRILH*w@78151ycNINQF2%@<;?~ML(YcbvN)Mq}Tw_aauSE1$Z!jUHd8I5kccE4LT z8-FXhd7}>KC@lI6i$M*JyfYoJlnhIvJ`Nu&X^G-+a8lOjv>R1xCsaTPQ-z=cYPO_! z+n4$9N0oJ|ICZRFnULwDw>^*t(HdVqQBL+r9#I~@Zq$}LM(}LU;8F^mJ%@c_w`*SN zeVPG}OIsW0=tqIpAQ5B>gClPG1-WkQc-YE!n~UVJV*vyEJ`cbCZpv9Mj#h1jUuznn z@~+fR)CI3)x zzMgjI36D%~RO3j-W5=lXcB+pGsDf1ge8S=HRZx(J&VNj2LcWMt=?19c zc%FrBOr|i|k?Z4eI!4pX%wlIXd+s41c1W#lY?rI7yH;Ux;gM05FRcDj>P7fm1i`&C zIvc}l4zn%iEqP2J?-aC<+4ZcRuDv2auklAH`d#M|2pvbG8$?2MWt#AhZ7=Sy+=sww z5>6PspV_`saWxcZD_b$Dv94VX5R1z`BH{C-6)xSGaPY8eiLsS6{gizrsJH@G17IZ9?@d zH+GHBvFZKr^iZnPA|kZ7bE5rTIUE1J#;w;E8Z$fc1CMSl zbh2eMI`_8!CHpwHLml=ZAKixWGWEGdO&~b)`8?cx=oexwG;+}ia7wJjGjDjTmRBhx z2p+HpiqjEdA4ejS3SW$#oboT=GS1u%r#bdTKO5@YB%EIGSlZ*YwZ5cKP7?$* zbeOa(rWM9G!aWjw8LN+KKI~m}ghYKEqq1i#HUNE)FqWFRmM3O!h$WIlXPG|=5Z3U! z+5%cLOWx*6^}!~`YS#{Xz;Z4sIpRqs0i8t5?xdL6ZP>CGb@*&$V{T+KT*;z+5Of4L zdSb&QUMyGeQph;T{f6K+`|FIVj0D$p?Ch*KP^|F`-HTfo1-Jc1CFXei8n+}e9t7=5 z3BD-CFqA3oshPJe@sq?Vi$f~NNBPwv=e!h(_>oYYv6Qm6eX%V|cjXVzeQdfLBEGyD z7bWW@yif>eTyAZF!xsaZAF(7Fk^36jcNa_MObC>3pT)-y=4WB`IZZ}M7tl}o{0c<#9NPe%7p=Hu zfX8l|18ktI4Yw1~F{}uq+AZb0`N^~|SrukQ{@r7&`F!x=zII|JBXCTsuL2!lel|BT z6Nujn5e~fRHE3u9kWueNR?TVpAoo)4kc^A7Wd8Prh5BjmO@_*y3X{% zDXHbi8UO5(%gv9_!#Jcnsjr!b61@HNdZ!=xe5P;c;#jkSG@5Q<{ClvQHR+3#ux^NCq>WyM~+wZ>DCo$g^( zK%yPqtXgd0A9gjPrVvoj6~5_Ji9JI+N3a)!+U>Z=vt`8|{`IbdjbRM)`bNiEh&-ku z(-)Sq*|5@puj4wAx!ohNd=+RMuZZtp;Jge#C^Z5s~R6{b64{Sd@P^}6W z37xKvSy8RM!4RgbtTv=0*9I1}-X&WxctOid-95;jt;e^$<>C9qSe>0{hzz*$J{d*^ zn|xFUA>arpra&uQ6WtoAuvE`TObkl}7uMt2q{cY@u2BcrV632!s4|Z}*-<@Fhu_z) zUN!w^@0uCf;`C@!G#8RrQ2g6H3KA$(4Y$5LaZ$s9LPD&(-73gt+f{l?_(1`cze^4E zNl0@@*?;PW6MbQrSqYSt3%~0)`n%f1v-72CDe2Rc%F(s|OSCi4T?yPRp?ZO|(OYP9 zzHJqxfii6P#_|Ak>ccCe{iiiPT#M|y!a-1aM zR8}BIb{ySrtiIPzGPAd$ZDdE>r6ey`@Z)*3ubVp3R(3e~ik+;2&<nGN7p5%z=rQK-JjB>Y_a2772+jkGkn@?1G9up2j9Dm zz8_xZ#;fH~TTBRrHWLJwW7O&fAwYNFyry#)U=Q!_s`D#*7G2!1IrnJFsr1g((Qhwg z{Fws+l|F3?`{vXX#Qp@5G4M$3xdRBO1_rfpG#E%>3{VcHYMW1)`-IYkgu^s((ak;l;yj zgd_7@&ZLP}CPXh<2ecyLK=t(R`Fu1gevNEW(w%c%R5XN}Ct!ei4$UP1plBJ^C{1-i zNiNa5D=XILTjG(1`Y?Kp`>nlNVSpq5T?_JrLw^xPT3){0Dob~l6KEO3fjwze07#I^ z!3#rPQrHaF{*s&CVX8}?_YbR=8@g-EAuXyxC^#ZVPM9WD`Z)Ew>>4W0-m9hT)!!M( zQF)QoocLf*kcn8S6o@Kt`Y0ne1L)9zna<|eLg2LtroO630;CsXbB#hCX`SB@w|B6( z9EtN9=!TOAjXa6Pl!{M42m3AC-{q>iVr*<2Os%cB31?79m2?`Jh^GV7&eSC- zB6$+piinhhp?`BE=Hbi#AO(rQZsl~5!pIWQ3Zs4k>cP+iPIGasU=TaY?7Ez2Fq@H7 zxqC3-4v;yatl4t;xNNw6Go}JF5wF%^1vHAvdiTa)(5cjF1pu7V;MR4yvI$87?6{~O z$HwBSDqRqxKCZio%!6+dr1E3_Fu7DRwlRq6>UgTZ zuM`S^;sX7M#17&X&an>vQSb)vD5#P`f9pdeivA$a6k+E6Q86!wfuO1btspPmPm?Tc zlJ-wJ(Y$h-u)T+AMo&wuQ`vF?PniWn*i`LF3}yPT;oPU80`rsf6|gq;Lf*jXGiK5( z{Q$D(2+ab9cQ$21X>kHdO(GCTbId=8eWwrdf{!e70r%GL79g-tG}#*o44asf?oDC> zA9g@4T?p-4$m9rEfG~>wsT*T(5k#9*iXix{M~!TO(A|(nkFMB60y72$NESGt$kdV| z*2j63|51aCb@miHs9|83C}dak|FC+RY)gY&I?5w$Q)pM&;gFY?*BZw8_-H#M zG4axb#J6YvE{$EMH{%P%k4 z{cE}KKWmpLiCi^JJ?|P1Y+pZscf=bMtdjRYkRvJ7Eoy_Jb{`Zeyh82#KXMK=$e zqz8m{bYAtKj?{w!WC>KE6p2f|(j!l~jvb@81)?7t!A^thUnj~5YselwU;*?m;s)+I zZ(x?ha0*j`fNJ3Rxj=Zv4>%Li3r?wnx-~-e}nv!93Ge3Jf`-o50Uz#UgFyQJWA?d@elRuK4p3)4POtKnc z)CA0$0TMm|27G2>%HH#XcI7^!&@7_(I?E#+QI~uqdGN}fOT0j*X&my@0JVt%vb;Y@ zrv!w9o%$ZFlZR;cY%0Ce;3IV^BTu&!@f03lJgRh;}P*DZf}!Po_q3o>nO<_P)*B z3iFoI%;h%4+G!?WkaPs|gIgtntIjg(RyxYgSqhFP(_xu&!qYX2@f>_R6r=Xsl)MV z%9)w(go54AX_jI z3sL>PBMz_X_aZs@Nc1#1DSnQA+%DKZA!`FRl$;Gu>m7^gO&lTQ_tA~`(W_wyu zU2c0+Vqa(SI7yqp6V_#$X(I* zO{myP=FfBu(osNNsMOL3b`!?l$fpjg-tbQ%_qc$aVbuN;^k zC~H-YWQ7OY)P;QX0qf8J#EQ=WhobM~;^UwvH|s)B63F<(U~r5;P^S_Pu6y(Np0cY6 zC}@KTs?DDr1!g~gCjHjie_X;j*f)Ryvc-_PB7ntb5^PHEJg&upUuBUji|uElPNKZ|mC%tTn9Lk_n4#Pn%a8I7@uk=At0m2WUD>^j{MTUV6XBZC5<%r}~dyrrj zI93CfYHBmDQ~;e*`cgyzN(o3Y7!F@)w@G#gu*=kr9a7y*XpD>uZcOg$0YEjFR2RCU z4K;ps!Z18Z&PuSh5EwS@t+)-8x}>j^nNWQ)z@IUO$ZXDCL2}6Q@9zWNDe13_p#zQS zz;|+H6y)S7kW^AR_+_oHz`S)hFHl10t(vJ!W9Ec48UJ4rhYrx5 zZRdoYu=rdn@HBS;7kptqa8-11Jl(_}m-|qgcBUd#KWut8kG$S*bm>SNd`^Obd%3s; za?FD5iV)D#puNq#+jV;n9C_2>XXW{{DRjEr9=OQ;iRe{3FS$nNx*O<;@lv zRG)MHQTLD3mVE*}rmJYOM}`V*ujS2``jLl$&v`KeMdC*Sv5V@}y8L6|T;{?nV9B`q zPN8cRR8;`tei5)lpTQm==89?o7q=>3c>SggwF_8?!%W)H(ge&qLR7eUo>5@mcMwh$ z7**CnEj5e9`ikKVs7KK3M;A?ajkj}oV4IXQ_iCLyGUyEn=ipZeWL(~{uHPYeuJ^S0 z`kC{t9nY@q$-B&|i{1k}Br%kGS)_SD;|kpYbdPTr%=@rY$H9C;nXUs?Kbw#k5!F2nsjG!YN^PB^)UI+a=*IMNQ%L%OF zo8yE9ggKS!bEVD;Y%%p2h%MJ$H1`j%mr957o~&}t#$)+Sz`= z`4P!7+&kOC*A%#)X}^12hT)NMVAGs$L^3yNK@-oFuehjiytacT;h?D~CI+CO$SwMF z3AKxNlzdYZ9PBmcI7TJ;xE9NV!TV=yNbvb?k!*L`Im_B-z%%E{ zxWrwBWUK{Z?I@~ufEjiTV1U}PWNg(hK6W{0)_^}FY;|UQQ#1@e!SSSnNb-40a5zhQ zM5s~c7?{*8Cv;|2VBaMubSn#&qU#Z=paeN!-r_acL-N>U(h69(s-UBwOvhg*?fHYg z!EMof-ZI@@cH*6)3<4P!lk-8Og2H|vRAsEbq^mah+I*ff>%l3F(Z74I-R%Bwwbg$no_hhH;9c^WQ_8us=DG?Ui5_uTj@{9H zb>KuE3MdeL3+2ZdxcoD;`a-94mmf0h*z7ugsNxeJSK?xHLxc zGas8;Lsn2OYH_aT<;=P`1P!>@BxLFlQWhTku*FR!vZBKUef)i-vX1%pXv;OZ7#MNd!J3+~R)%dA8AlLN_0 zazjMO6I-XjVv8;bq0`X5Z!*P(%Q8s*fCsKX@390dNac$!pVrbCUJEZH=kX$`@Ydqp zayIJibA6{=pU0!!a<1vJuXEeX8l(hif!-2N?@@Vab2L1HZ#y;L^XT(cen?a1GwNN) zW4H09;tH1zt03vy*qo%z(I(ff*B_Cua&kpJmOdxh4c9_T@8H9t?#J`F>Hfy{*?G*c3L`iuE{ znSd7?)~iSM245wr!hkW6^V0ggGh;771N*b2%bZ2DJFzc}E>V&bI8Lyg%6fIYQJOVYG;Q5Z5}mAX_t=q;JL3 zD=Sj0AQQRijjqP(jbmkH9ja{(RM1z0UHf{H#cA=otN|Ab=ZyL=vz0ET#tZ-1P90Vv z1kUaTolffUxAn4()?19NaXtkAkEl6^Zt<1VLZ9WT?K11DtzoYl;} zpPVGlX;Ya_F^BHnxpU_oo1?;koR0&(Pdm_|55wKv-3fJ}{#F}9`KB_Q2|gr9dmHQM ze@|oPXw;Phk;VPueYjHRFS!q#+U7{}4!sGWmb?RgPi!?KUS;ad4fuuL~b5J8WUVj`gdWMbcEdQe7tQn6K*VfbpOGAH&(i%^{^5|92@!NlWt@aL96U-Hbjx>I zNedpCMOhY|QwQdUM{caS6>$TKGLNA6n}WG&!0%grFxVUETco9Gaq8bW*<<{+zS2(- zNd!@CPz1TI?RJYZx%8X*CbyujvrTJP9d8p&o;6N*Y~GXsr8Vb{VM0L>k_ z9{x!{xt{}yd8931xT(CS2m@Z8iN6u9^$%yMImt_pU-Ze3cU|G}YG`e<;z_`q^(R?5 z&T5ZwWCba8V6M;~zvo>Kn2CkW!m^IiBRUVO_&i}+yk?bj5=3 zQ$^mhYU4PDS2Cn2_EY56Qd~WtZx$|ce-*Pod-g`hb5M3_$Y2cPVVzTSu=5wCzh{(K z2ojAMzJpd|FdjovUtK4w^%>78*#oclpHRhvfze0cmf*X)yXW6|7T>Ep{W|?3+_pb7 zgzEqT@C@YQ>7Q@+`)yh|hJHytW3w;E>hFOepM|zv&_*LeY-%U(PY8NYW|rO=m(~Kl z!Cr9&Y{&tQZU_f}$A3Hkw4@vskVio5wYydeAVvBC<#QJ#lS&ZgF7HseDPXX&+Jto2 z%@^?O7oD3Rf2b*-0c~uG{TsyA(ot6`5J6Caf}6iZI-!#rp5T|DxU<(U8{o=?X~6pZ z6gbF{FqGdoPkepu5ETyEDm##-3PB7Mw^W1q-5^l+NkB}10|x2s6riD$BTp3_kjlB9 z60~AI6Aq>>s-pm1G;%NW@027&>jFrOg{@&}cY^H)`=5NsxiS=_)1x`E@?ksfuE_2- zAUJr~gv@ zj9q0hqj3`o^3ITB#D)u?rRT+KId;8p>$PyO#H?LO_=X>;+CMJ|G}9i0 zLideXP{ro^?nU;0ma_ccRZMv-0|6C%P;vNF$gho!(*~UR&5o{9QX0NsO1xy$4K zsFbo&v<+9U(K`k1dWBvSYnJrlgfjyxk{KD-pVGHWtDLfkD5ZZVU>Qqmq?eKK<|l>~ z{=pBEP9@)M?LFnwDgkT0mUK#-`sLq}mZ-HeZ4(JkfqC^0`?{e5+%{xZAsmeYT%`v>SD4%+orJPlKjL_fZij&>$CSEH1I%u~0 z3urOz*qlPb0cLi<(!CGVk2BTsFm z3u?l_jO4-}#sD;+kNMohUjnFcXQDMBSW^>B zptjZKJ^M>to@#*U_@M5qCv7&CCyRjZDCqNE{eCda=mDx2{-1hBjN?;%OO-9 z4@M{h1GT|e&WJ4`Q-`xKMeS};Gk!_nF0)=oY#^b_XWXF&ZK^{*Rcx@ zpwc;z<2kJa?!`_Qa0%YNIvg*aj?oHWh)G~5a)JkHp?RkZ4@LwVIP5T(=YLngjBA+y z?OILf)9nLVAQKC0s2gk0UehQfqULjL2kxP{BSpj|Ky=nzJl1^o=bQ1C(AeKu?&a@o zojB~9ol$HLz&Z)+b0V@9CD)Q<2LSdTX4Cw~>H;G_2rGbI{5y~or&-tx+cq!;!~@`9 z8h;aDBAnP!aeptwK;1%XA~65}rA;}XdKUi09H8B5EqB%edMvSrW9loesRHSp0B8g+ zxep1@O*am$>jgTs3M3;)ln2vdbs31ZMK#i-gyKwGVWX8msHdvqBruyAqXKEFZ<7(J{eW>0qisY1=mpxAl>1HqD} zQCr^RZu_Ywm)-U(-&J+jw7_J36fCr;#Hp?L>0ss-c*epS@RVBvEv|2e3(DeRU)Jh!lrq=ae0LH+7&qgNp?SYoP`Z zdBt$uQ)p+)vGA5%C)XK9l|G{*4+SOWSZn}fsy!xRGD~c3E=6tQvR5j7ecJphZnTq} z2xtK1wuyzh6(hQJ>~D@fV5e*g?eer@JdNh!tzlzcst#JH`hUfzW}bw=)z)WFQuXs^5BB z+3{n|R6=~eY7N%}u2|XdVzXm%Vjx*DJHT+~^SQde86q)qXU4s6W_noJP%y#?2l*(O zU#-u7fIqUY^cP1%GfyVoiBh1 zRuG#a%YVj>!{enQx4HmRH5LIRf*yxGQgd0MVabY4K#mXPC{N6A;A+u>H~d|RZ;?AX zFa0jSeG;*KQ=I0I%5~#Vrxo#kkrK&SN&ZvK+f45$b=_`7!0~Zqf#vETcFD zAZxW`N0#GH6mrexTl3C7UR!1cS0rI9#xY^?PFW;7BDiQx+QV?)_Z+Zd=WGu6WElKC z+hZreQJytPc5tYG)1w21xnYrE=*o_36wr6pbfd1h_4p$!LdE$3He0ol z3~NS2C}e9h=lU?UsMZ826uI^91xerA1NL<}(2O{KmiQhH1ZAO8fsi&CAz+{H_(SZ& zB6y(5bGOCj7*n7svhkqy#DDj<0P2!z@WOmV1Zd0r?2Vga4qmx^2nzUjF99r;M3tez-4RPs}P-m3Y|JcY_*M%oeLBK7)+pJrmN{0h?9?Ge0Yw-k_}pfg0yBa8r? z>OT_o^+4Ye*&No20aNvU@)w0`=4|^qIK?;BQZo{?6K`#=KM;Fna!X^~ z`SHHv^FQRU0h*(MM2BKM1CU}U9J)UfUM?QkYr4kC${XJOCC$b4YqoMu4Rx@5lV;!% zf^$eE@GP4eS{ChKA=o<$n;g=+Ex$tS9r1XI{l2ahvdV>-2_VjS0$IwhzN&tjIQ9sA zmO9$lcvyVj-=%#lCqKdu`TPltl({@TAn&(r6*O~RVAxXOr$yV&*gC+mmw^tV=IjPw z;(lHXPX>h6Fn6|x-V3Q1A;Clce2%~LYv*AwXvF~smW9^i*lY18>REt?teu+%99jQ+ z6@UAKg?wJHlc zx~vaKfl~mY@juU2X}o^}exmN0TZ`-8?;T^y03_yqyiH{0$)uYZe*!==B&K+~aEUr?MD!!z@W1N}K-CE@trkL-zph<(d>_G~xN*jnR%arfSVSoiPS z_=QkJi{|3 zG@T&-e>%xXLB2*Pz>rx)0gevfeigfq%$|k2f5_^Z7MT;`jS5FJx>NIyJM3 zh;P|dI7GHW*Mm!W8s1#zih1bAQEqCc9oO;AM31wEg)8M&!Bs|3OfQ?7Y z0!d|1VBkw(7P-{MKH*35k5;?g{Qbdz{qw_aA4h{6seve4H6zn*O892i5t;qJ|6H^h z=W*z62{PL>UD84)(U?-_};<{Pxmi~VKm_J4G=;q@p^Ap&A{{uy6 zEz6qO@`UpLo_*Hull(7CzvhPx)`M?&8tz(CbC*N#yeifB8w%A_LQ3R`n+*$;#aAej zm(cU9Ar=%lj=kwf!7n|SyKn(ajO~l{F19dFpaIm4O*sRA`;I7k2tj6E^SvmNMR**# zLGMT*_X!HTRc0Z-pG68~ZNTz(2QLQ-NITa6+v!4Wj*m5)jQfCQt^VT?T?!CAcx)ovBezZ@v4y zF9Ipx8!$BoMWq||70;XUTDDw=H5V zBQ!MbqFP;|dD7Qxviub7K5H1~>D?2s=x>|2=l_LG+>@Ub6)S8DqoXN-AF+Kl*gHEr z=khR9D1D6!DGRRVvi@HnCQ<$$@vHm&@cb7G?#%y>4E&yG@_{~71k$t>Q6zGU>+LId zyS?#bIx4ZEATeDtU}7W))X>y*jZi|~s2`Yna@XD?CWk?#%SAK{P)o#o43VmyEHHc* zxZ9?gO7ipx55Q~PmFoH*2Qo=Cfa6{~zCvyB6GBXr$YrJkY(!;yp}M1^BVLw5Svo_3 zyIADnDZyF>KqLh)cohn(+nX;|ULM5Sox%QLVFT_2D1`jLwtIlI2W8a@(_#(L7sw$h zl|X(eA0asuf{+0d%R(DkIfBdk-Zyg|jzZ4toWr(FfDpRjBtkmp1#Tdl6VS!O z-z2SRK4G?jj24{p3nhL~m&AG|YI4g?eOxAX!W5jjOzPA))>JLid4H5fq_4u2u;0Jh z_VtoAtK;5wB%?3js;M>`=BP`V%Sfen_XFzBk9Il2VYHQ^`1XZk8q$h9A7^71C7@mej@W~9bC(*uj$S6kA#xwAou#a*Q0kX=7Y@YdE3018P$Qw75TM|qw#|@lq zhv+l>lq2sc!a5=La4v>j5k*9_=|Kj_k3Jw z9DM7`wcn7p=ccmS?u;hqIONgq0c)LmM`X+ywkR5sr27}`KBa9sveM6 zYviYIb~z=1B|QhNTA5?_RSv#Qo_<~}u2uPow9cSds3bOi;CVblgWFJQe@K7h^RtI} z%*>thf~0zL%XeEh1siG6_{^JP=c#eq5GE_==!v{epj4f{XAfI}L{Z;5m5XaE6;n&F zT$@WIs|V8n%FqPg9s{;2_ebe@Eb?qRm>yuv0pLkDE)}SLxvS6K^kEkIHjSlh2@FD^ zu1TJp4Yk40R@62niKlD~y|`BY-tTPQFj6V)u)l!VPMX)BQ+nzWKS`lEC2u#sQxTqX3=xq#J84BQkqajw+`+qf8J?ESk3`9aLN|FDev`adBEoqzl0X;ej3qlcJYO%(m<8sAc8BD}A2X%}#Z(?$GrUZf}&OVf$dQxuiWVrEq~{xi{ChM#yjI0Ct+(AKkbn47(not z%K{fPQ;o6vMYHErGVT4dX_;@WW8}3h%3y^`}a1(W}@Y`DbjILXb!Y=Rw#;4F+`AatMiV$X)qa z*4ZXX2{=w{B*ahzZtH8ju0g1Zjb9R$Z8$#P$pai9S6?>n0n5hhU2mO0a(64JT3h3d`MG7=Q&IDlTasIzaWZt@ z-r`iagxVAfbfuOnUKnLfT$He`5vwFOptSMH(PjEz1+UX8A^rtUx}X}axku|&w>tiC z_eW{PV*cawG#DkVsrIV6d(ZUX`2~6zp%CdB-5&v%v2aWVs-v350_!jN0fT6MJ;?Du z5uSf>9YWe;#^0{Sm%a}(s0|O|83By|5jyAnS#(qr!~vzX3H#7$_kn+@-s%c0W%{gI zl3w?;rsc*8F*!?~2Qc(wxn*7%dzXHiFV>GnD~qk6?Kep3O~U^ftIKh;t{#=xA01_H zP`uE;?)F7d$=u!Rv@Ul#{vu|m3R~0ifiU>QjU;M3kI1nEz?{gadAC(71J|gyVnWh} zpT!LOcB0;8t;bjDp3YM-8im@z42*2`pb77ZGi`UredzWb5$RjZL?hV7L0BwIQ@xsc zIqf2SW2PA&_1Ah;iW|(gvSf~QuIA1htWcDw;QSp9U*D`|{Dv=MlWN3SLg2jXlDEp_ zjiew>-b|&*$41!EtOT>D1iLra4lf+6@O#O=p;B8@I*_0s)2W?I355$QjmJTibC?X` zc2ICn`x5YCH?P;{&r>`po9-;qxooy8oMGMtt;=j-+|6;L10U}1(ABp{C`ElW!P;H! z>Rlavm@c^|SKyNu{inf?d+KaADx2gJ6oBis)!j9DR=8le zXB6d{0*_@`cV&i(y{A;LYDoV9a)#x-Wv=Mp+>fM*pB`42!bYe=jXS1EJHE%Np=PN) ze`i|UTU#MQ?09MH`&?_07Z$~;-9;sq!%3r4;0=6P)2p9hrF-0kdigw%v>eaWRnlCP6qe`N(VD9i>@C!4W<-ChRz^a?RkPY<%0nBd*t#C}rPyXS3^a z-&o!*F}}2CnIOS*e^wArJ9~5QU)bt`Vs`mPSNqQ#Aq!Q3$Aq>}$2sEjom+(d-l7 zo2MP`&joX(HvNEfjX+t7aj8Snj?BAP(>JCyDZRPa$KX%=b!cVgSupIyHdJRwlH2{G zot5;kAa<(7-{;6BV;#Edlj}-&u<>m+^*zxa#`3HDg-2{tcUia(dw6BkMdWme+qEXI z@`z|O3tS-i8HA8$2rSFKXV;aGI;m(mnyj(S&)C8tuG|^d;BT>vT9rv`_bR7ah~vbm z1$2%>(G2vR*36CJEF#Cb*opx*F4&V9`f?BegBDI$r?d?OlbmB`5;oFC{N&ms?K^7- zk7sLRs;NpwVwxh?`_48K<>H&j2x^gmwg@L8rE*1Z|M9_=uHKuGq4Vhpn;Cx%5zkb{ zYnC&A;=QC~NgKjKRi5spY8iLRCtAw+aCL_Ha4K%9s204_$swS#l0$IY-&aC)EWXca zI+Q=?)TRq@*G&4lzZm<*KXC~qDOpZ*?+ULUhW3XdgDonlQ^l zEkl6jb)TjR3<5IG4y~muRq-or8`ShJYfs}nO*@>o56GQ;ROh^#g4#+KeVByGv?-L+ zPCfzOt4(UK@$JihxJZ|s)gevvLVhySN*B=t)v@T({bwgOD4f`G#%NVWMz*7y;nhRk zn^a@yzERY*DCxE`vio6~sQNS^s^(mNe8k z5!>!?QOylr9ceoXyYuJr#Pb|4wuQ{^Z_TTuc>W?9yWg*W>dEG~3KUA`Rf8)|W zcLCllr%vONBz|?mvJktdc=YJ=Yhu|&0xn(KP{%pUex>In!ZX>PZteHZRW#v9MH@Rs zV(7KBC)}*khzy*RpcA9Ljx85pHQ#-8W9Tyw+U9=ch(yerW17dbPJ@tpc5t4le8aEd zeSqAID0#rKSAk%Qg!DCxO}f3?P!=~amx360t@_N&^A^w-8|o65BlFC~nujC(}UoS>XI8t1M>Jt7g-6?`3GOF&4DqSj&PT;J;+-p zONzw1BKIfUY0kf};w|6ZTz7^N$Uy@VAhdiAJX}!5R9Bqq0eQX8xZvF5uv9&twTu)9 z|NcNg!7?zhc@FomeNp5wdr$uPi2vV))V)_yPr@KA^5i3d{vnY4?fmBsKq#H-5wO`d z#wU@HT4+;Q0uNqtLO)JT6hjoA8W|dDBood@wXHilMexku@DfOH|ALq1#d($}P!tU2 z(D19`YZ%)b(4V@@Ev5rD%G_^lI>Imn6xWm}1%W=io{?$Joc-P-DXzO6x-=XYq!Oy< zj|b*2(;I4Xol>Bxb4>JC1ao8H^wGKRIm%51v{O$i4jv(wykJL|LR(!sh&(&NV|r?v z-m%#$O8v}S2J*&pVi`U?4Hg3dxw~P#`F>XPVbM3%BpCt3+aV@1(-q2^7U~ zJyQRJZzJ11ki0CBayvvrS@W7aR98R%DbY1AK&bla5B&PbzVFi1;4NW3Cqnnt0(M&3 z`(|#9aivm#`&OZYGW=WFg!iZdv)*oW@bhDy715uE5AJYfonk`eYm z*W<2Ff!%HaEJ-cq)Owe`Le>Oeiv4;Pa|gwF%C+_`U)ybx)__{r<+mmA8F8wX`r1xODHWpyDE)68cTj``1_T5P zS=Oi;rO5n=jsIj#2kG6ZFCirU+U7!^LCdkZOK_BziY&7Z49R$Oq^Ih+j>N4${EQgB zl9>bG%;?F8&-VN%f*sCOp3SF(io?92A4*0=tpWBUdz3V4FX9_eukSI*eR)0b^_iCt zjoA)pmj#~Kbd;IXmS_e(pJ$j3}ZVXWcz+x?KX64IvoK~6T}KPTwS6&%t02MfMT5# zahVSMFs(VEa}O=jnODF(h9Rf65MvS=D(IT02XRew=S!nZBU(vpnoeTTL<-P_2lgg? z`&>9VAM{lv_F))7hhg)Yj<&THgk}j~N6*Xxk!|6~|MP`41H~{I#i?URXlTg;28tj9 zX*OKFoXdJ7f|4C}_VPTz+P=f6jc0lNUR=^0}?nehQr?X!T> zz})cD#O>(?~u(c}eCr?7(7-2x*Ijx-G{~;OG-Acft z%-C$8%eBykgjyF@fN$;s0!G){)gGzX#A`uL&|ywxY)7H;N?1K{MrSRin_UB9&449- zXXYh%ztx4AD84KZ;zl5qBP5IDhG}p{G!ZtsD^yOV#i&GStu0FqwXf3bX(L^CFSlf3s@XV!XFD0P;ll^2CU=ST_c@NdARiGO$w zHfu4}a`%8Zayt$G)3NT0bC|kwnk|et?IJ@|&Cuuqrh#8e6nm zfJg9~MKRN3;iGPh!&6&d5u?&5cnMJXwfFmK*DxH}{p%Jxg?gqXXC-CnBBwN@FOOEu-itux2kWmgvoEh9JJ9!{ zTj{pn4iy&_UC|2b=D7Z62k@9k2Sz4XzV6rN2IHwW~IN=do-Rk<)&Hrwy|-G#~)BfPANB$+rJNC}9H=E?7o;{I$W2xg zqP!Hw!4P5?=_J+Teh&UkB%o{y{5tqxTU!bSOVuGtS)n32e>%19eEVl`uJ0lF+&=E>Eq3yZ>S)X|Tu+KF#Um6b+ zwrN7jIN^jXTMh0+TTE)m!%nuOe`VSTrAy+`?{|2&{$33T+otH+q~V}z;V;qh(ilmu zJ#jfp1}am3;-_oU6qyFw{MR~MtL>!@?=~}-FUj#bnb22 z=ar#mFB-|mBp9-3S)l1MmVJEg-R%nf>y?h4C)qz_CFy9GNO5fy7WOnunN1}GdgT$5 z$wMr+_Mb=rzCiiEU_bn;NdG@*-yeL?nnD72{RBZK`J!%Nzb9j09_PzSZ?-z|8sI6+adOw4S%AYGYR6dTB?b%d0lD@KRUwwdO~w=k6y`n`E$2O~%hJZG+|5qypuVF)o%bp84_G}+7B&%B=RI88u#q$Fw6Blu(~06oKH8x>Ol zL6nwYIEJf^+1%m2RG&%KQ5BHXR~^)vGla$VF8=2R`1TIPg7p?m(_dI;zzZlJo#+|3_; zT|m0-L^+a6SFPVlQ7BeFtAgx|-_8tHdef+opzgB-M<%tO`KM2)=v;OF{wcjDKUJNtXDUf8 zX+fh*WaoHTR}>4c=Ts&_mkTjk>u#6{m40|GTO=pip)?r47hK6r9uLN>JSXTdvuS=gXzXjjc1pF+A zmJ28@MMOo-Pdh;U(Ty=R7s{_gyDvs&H_p6J#1Q(jqt|&7xX~_8$Bv|h4Lq46)(G0v zvE09x^b_{5!`D}|D4w{8Tsj`9$>}JRR#^{6koXp7w9X|pLZ*~S%&5tv<5+)4SdwNo zy!g@qU%P5-pZ!(pDb_OL>=B;)vh@gDy99sy6JJba;ZdCr?Q9QUlrDG&WUUTg`}=W7 z&txq_=g*;|C04P8DkYpY#Dvuvjx*y`23cuBG}3(?r@ z;{_usYfxX(zzF)nap*Vg^30dwbd+89tq4@o*bNroUPJZ$bb!s9JoNyH&G1J6Qh^FF zXff3ONz8(?mB>2_hpr&sQr-*XIm~=5 zxg(Vnq{K{;Z}nqgxG<2_4|`&%ayQA_!Ov4Ee4j-n2&Ww3I0MkEWJJdj zS?+|`KWI`c+~@2uRf?A8ClFOnoQRA>)|Ubqrn40c{d(Tz(KURn3G_39@2UaoRX>QH zBNG`dSo1O!`);<9b~!z*0cZ@U)jvea6@lS>oWOk6G>py7+!u~8Kzzw+u`9h2qeCO8 zzpp^TeztxRmC#=H_I*Xe2dvGv5d^}K? zT6fw+lk3wyL~t`GnmYj{j+Z@aM=XUjj>WDU{dW@~%)gZEY8Y4Kq(7 z(r4!%9N){w-|?ZS$VSR-=E{9FLdul*S@Uix39gX=C%I$TylLI>FKQ9+Mr+V-NuKwF z*KVfI*S(Awby}%VW7QV2-QYgXy8j1qv;JwKIsfn2E-5ep%IcHY>RI*WC)><8*_S^c zUL5Nk)H>NdMLhOAa;lU`ca&xwMrsk#hF#c#d9nvyK?JyYZjX!`ty_E-m6jx?1H4jFxK^H$K1un8bQh&ZJ z1%}Z%6-q2|izag5=!QOLfHeZm-V!=2g7fjj|AY{LUSJ8Q1gj~6h>f;=zqCxT@9ER0 zM&r7o*yPp0=F^9+(hVGs!B*d(aFnI@t>{p~36P#y)+d1Cr*%%C6*Tf*#xhCq+C zkE)IyrS)}4G?|J^o{v}2JqD`K&EE|1#o~Ia4QTsaxTC!)7-jFijXgH3r5~N}n8lMa-+ay?Cu$uxxurQMZH@WEk(zAgdQE{WZ~j8E5HUp zF{Ad)k_&Y8#;oX`gtkA8eJnHYkFa6S=N2nqbokss!+SO7)k1mhFpW?xTikOL$MyI3kCMoAg#RJ@qCs?BT!%_ zzXk-Gyt}m5+&?p(k5kRJ9vHq4lrKlOz{s~TP#+Urp4~Vk$Q-^@g5^$Cc%6kAr7*Y( zM%>IlJM;FT7_R2grl(-(h`nC^apOgP=hVatujorcbf*I9$KC{vdv1yD#g-FPCG?(F z;jeupj3(77MFv_KE$I6b$P-B`eIQk>;#> z;NDg9T+FrVe)-=t8cxo0>oj+N3Ud#K=cw3eXKFPJIhj9%zx3e0QZ|wWB(1vi0U717 zihAv4MMqJvobiqJI6IupeD?GV}P<26%s=EoNpT_BE^jtLI) zy;@Q>snR4?+|<$_&W;XXOV>g8@NLS9)8fP$v$XJIW;B^Cb|s~Stb^H{8GW0qz*5)KCQORh35O zwaHK8MAXiOzo)-Qd0u&Fr|Pt^b?v}I&g0R?^jb2Uy;j3@smFnKkLdq;jMFUp>%u5h z_iGbP0Lgau+*$0FHIp^jZ2cFLCVj2SXSRnA_)T8yzhU&Gvysd#TCC;frw#$0@Z(Ce z^F%?v4aCz3sT|_h?9dX)@?6C}wuDiwY$ad5nTQZ`5*T-mm6@@gP1*Kw?6XXo{zX4{ zjxc;DWZNGJVNw;IzE|Z=k3lok@|X9iivYO}6R$%0ST5Wr)Q`pwKT%s7QI>@k1E$9c z@*FoWlTI*CSVjw{|94cH5yXdyE%AA=)-(xo>QQCp+;LYI)!KNUHJwa}Ov|wUsC&=8SRJ*kNyFh~w93EUFG#gYgb8C# zS90;Ma*os%pfyZwi{vS(K1pY zsvpnSCWW7n)K)Vo=-=ecQY(q#bcc zWtg`!%#+2sEJhBEGh6B8(@mxYh zebXeAFrDcKH~=yXdsfXaM2q~w<0`*=uF6##ZMNm=?QQ8oQ5MKA?FHm>{5WV503k~{ zm3kTYO8`>DV>vE45$Cn2KGk&Dpb;b6TJ&~HxXXwHcUA6t_AuHc-_d3qFJ1$R+#knu zd{`*dU)nYRJl8i}oB4PA#}%*m6XAdVAI3SWJIcRdZHkN9QAO9`9y==etTySBoBaUw ztDgcT4{RY~H4(b{QDESil`B^&>u#9)J6JTW2C~d6P<;yBk9iX# z<=vV82(%jp@ea+AODGZlt%!>21I3w`1lf10PM~;9Aydt@a+1B#1Z)8MhJiNwrAN2h4^Mq>6u;n$F$cTgOr)2=bUCgQ>sU8>gUf z);M;|j~Ku>5D(&d?CxhV4gM++w>qorj>t?AT=bcj;C6E5jBTGxk!L&Tc*Z?({U-2o z%}_5V15?PZuW){hNiT`3haf%@soiXFPXfsD;^)Y6v*^gQtMJ?`AkD(L#(7PiO9pYu zF1I8DB-m%{Havccpv3NHEuZ~W&)3UGDcOK3iL2T4EtfHzPlG*j?3CjLASQTJ{*9N~ zzLEBnAwvp=0Rc3my=*7WP$7wBfToU@!^Xajxy+09>F=-Q!q;LVMk4C~skpEYWdIkd zL$YQCST!9;(EjpN;LDtQ4c5$lMz9~;Eg5=zN*A%Yy*4J#8xY4P8GgkZnDiq*e(i$B=kFJKmNGEv?k0_LK(95#7z-lm z*}rn#S-3ZyT26dZ@Tt3zL?I|P9=V6;J?(du68xRHkULVtqj6R7a=th ziYZaRZ`nlr=4+LVnE%;_eZi1w458`>0)O^at!kjg{T>$~+-uXa!w{?L|0tv*{iRgU z;aorl=q9pR+`p!2S+V(+duO^||8uMN!TL<(&$V!lWW{F*oGXhHEbToeWi zv%@a1Gt@cRj?`z&T7AV_#3~=YT!KO_ zosf8R&Cr|bm7nWJ3!SA*f^(Qd2auct#Ha){=JnCvd_R0wptR@6xFkIr-M#*Q>vE#$ zys8&Xo!_t5R5Outb67p)Z?pUVRS4_J*nMM9(=$RH3AuiA-@p;ay&77q;95O5)Tlbp zF$t#7rgoxM1`zO^czMFJ+0P(E3*a3~dq>GLLghP-7BE35Ml)my zmyA?g?Bcm)upv9#1Qw>}{3r6;m1XZ?HDHsPn&v@|m?-rj^LMz1@xpbmVzDIjou%uK zy|zcw4=|WNIOBDwz!SiUN!o0i&aZ@D?>(clf5Aya?V_B*0yEPR!K`7O@D*=+i)OD1sGEA6NTOGvwtllh_ckvlE+i_|E?K`?g|tt1rsx{a3}8_{7-t9>rbAWkzSveD)hM><+OBITXC z%(7IU^kEOluCjC%9S^a^_*$`mI}lKnaqlr%x$_IL0|dF$@)Kbc&Jbz5tdw% zG}2F;%U`1IWs~KKW1i8 ziK$+mA{$j^X|N>-WIt^IY2m@CTq*qdsGRYw&C$ffP8W4w&-Lxhe-TIjtZlnM#IXd* zpMh{z58c=qG)vAK-;Y^LJ+1eQ$W64vLES8-g}Z}>Isyk;=AXF}H|^9;_qn2ZzO z`xmDymkmuT!6v?x?WTBUGb4}8mH_q!404pK|^?fOG z0$UAeT2`aEsLzs#YK@HQ(@XiWN|dc-IGEr)?5*7IAq9FVdabGhQanOT(-eq$p7C%{u+ckQ9#|2&f7Y~2-JZ|rk;4>pY z?Z`OsT}ZsQxA3#Fk)}{!Y5((y|%MoPa2a4>kZEskg3zyQhczwJGXV7FyX6Z-SkD>L}9db*1zPlLmXM zdp@yWTyGP7>=dnIxv=V{@@|s%5I5npT}ny1{=$#AZ<;yNzJP03;_)aHUWQ;(IA9Vo z!tdhvXO560CDzqF7OZ7ou&X`wrIK>aF5hA{L+%Y*L`F=S*lv|dd|Nnyr}wG@{wt?KkM!thW@(U93Zwe8%XEz30B1 zGH7*o~2_JPo0Ur@^DP;)vTORdf94VjZOr_$TeD1)Z;o>H{z z_7$H+a`3f>#D{4@hs>_WcGC(5Q_;YmXkFz@?9wkKAdgiJ;w6$TOyBrItmt~Gdt*WY zb?4OUp)DZdpG8EA#6Z0G{1-A@p@#%9adGI8I>c^e193|c$!Sl=c!w*(kNni1_y>#;)5R9np&I z({X>?_DJwe#|Cl{F5VrS45?f#!A0wg4Y?m{RVjI(U^*rv@o+(}Z$wJ7w^KqHqsw8s zO~9bd!`OL1Q)7fg*bmq>cwFe zz~|n#1f@3j2~T0IWE2XhGVe>KPwTyL?vr$G55N42ero3zP=RxHQ0x~(wy7-oZbSQE zRvNCh-fTfc`bsmWh%YgfiSm_SZxt0!&}pXh^IPE_YXWXTQsenz(ZqWhC6rTlGllyr z(+_f2Js?ff+H2qppaGLrzP(bt2ff}>J3k{q-!e&ErSi@S#Sq?==Hsu9Fr}I-=AD!s zhMCg$jyu^bm9wAjA zoTXX6ka2mA2{JF8#_~=MF4Q@vtbk^|bLMdSu$Nj`Tskl2zx%NJ4@+6}4R$#8C>1v| z=Q!`ro8T5PHZ;sMkPIgJGM|ODqH%KPhgsBD+YH|xsmck`oUR^OtcZujq|8~>e8upe zBXxNe=ppl4DDAgq{z|&|?3HPjZ;43(Meq|!gevif@o!(@zW8pp>vEcm`2!-{kb$G4 z>w>UKeMO)0A_eih@4_y)nv2K09BEgCT}Q@SV9ui6lBd~L8jazaS__3;r(a6uB4(4 z7s>{xA<3;=un!s6*j~jr+4;2ATaBgqTe(G(UniEG^l3U^{(nOOVgD~Hg^&S48!%IU5~Ok2>O_vid# z+?s#@hSWoEsyePWZE9#~zjq=ev2Ai-^%luDic#h7fc@V!+G&BE>|pxYcS-3EFAbz- zlrZbkG&aXM_avN)X?>;G#vFp~^mE{?wH`Ub{s3kEiWX5o63~5pM~Mx0oUmY-riB2l z`j5^<4P>rf8G-7p_A1JZ8BV(q!iD8}m;ya|dii`rnX@kXj>OOJi-*>mMP~QHW$J5{ zuPscwTSgyopVq@f7LxOb+2nz1_^Z4LvPv;ewK#uRr%pWD_$=*8tOrxIZgi(%g2J#X zr+Yl3j&u~~hJzE6Hx4Z_8IG+QBw9?;4)+kZR&GskwxL@uV;;t3IMPhfDtH#d@fKku zC0J&~@$%@5hwcnFxI>bEB43+0X{ov%1u}KJ^3PnSs)K_@zmmkQMV>sFOuBnWE2?kO ziws7ne%VaW*@DrS0gY}^u4Yr4wq~UrNfqx8ROSftL47H%Lr+B;I_JxdJlQ_)EkTzT zt?v12#V602Y_*zrVQoB2PhtZURoO1P40jr#X0n&QLX(fDs(c-KqScIQd{#roND+;`xcGLq4%1_KUwl1bV(WDP>!lZuI)ALHb?=RVOn$ zEpdJRY}Ik#J4^Ij6TV}Atg^C+$}l?YlV`$vBJS;L@0UMb^l6?Z*C!cQ^<~?ni$jgq z^*rSQll%uBLflo7K9-PoCANCU+S5#>Fa&8v_>YXUv|`z~!6reP!Xn@2Y-e`$g^aWN zuiUYrUdbGdO73cJhiik8-l4^`ieLF z2EC>>b$~BB@kX^<91dlUeh=NVD#hc*eW>f4=sXs*y|350j?`JTXOR(S*LEeb+Rb!9 zm(b!CxNY!yoo-{GUcbF<3(xYXyKG<9t`mo@Ex96fty;RROPovfVdY2rJ&XG1T6VaDGq0t&j9T5hzILF@yYF5VCedP2MjW#QlLTD*3a><#-}it{>9jmK z*%%Vyqrde(@ltLt1hne;D?)Z`?WFYG8JmZ9Vm&h@ug#|LpQL)c9GE#eH+v5H4cjC_ z3p5KAa0GHJ?>5KLr!a5w1l|1x0K@N{+aAb(uyyXA940%x@#ar^kU7u1mg+aSP*(%i z>M{ZzI06cxHrm7q$bd#r!)4e1Mq~P>>v9(LZF-+}69<$QjYF^XO(<( zs(@Z+e!KC0ZS8&0chaT%#3`@S&nyZXL^aGU1x(Bmt;iYILr|nZI?T;fM_+s9rSzGp z{Y8E$ajUiblHW+W{a^L?Xa7hwT@dyZJ|l^1#}OgKdT76ookF>DNp}9LyFo$cfLNqK ze8D{$JNpsJF%{+^kbAz#wId_t`NzLIGuuJqlK>#D$#w8V4$>H1@pfcl+@of5UwcN{ zQ}M`=`7T<(83`hX_;}bpWbT`vhB053YqcTi@NUDR#c=Qp1;&t+sXj-^UrgsNgHrn4 z@YL(q2MLRZ|9sJ!o>}8W3iX{**HtSt-27bT?A9nxw~@O~wfnjKvxr-DRkTR;(%b8P z6KtNsMjn6KyzJ4bcVCy@7AX`y)W)#%cH_YCjL+HP9SlRDD6DvH1e<@SCE74#qne%? zy|bi8^1_$IXz4#*wq?5=KJ;6xb?mYPL^)exS+Du~XJlq(%84H?J_3nFU`hRQ`Al|7 zm=c)W&587im{&pbD1s~m`Fjr4(WbxH+EGrouAo{&a)o<1SbA*z1}%`78%n`D&)9E4 z+#?_aPnlo6V)imADfO#ef(Pd5;(Owy4p|^SL00k+CZ`sp+CzLIii}1hA_qtZJd&{*otlk$Gl* z)6Y+IUKyWwT)txG+}agP=XRmE;W)HW3}ZdG&4~7h4rlLwAM$t|<{9}Pfd|RIuURtoM;fp)q;|yXxg@~B| z9G++Zfau%a93=S%w@3wA>G*;+&Lh{sxCcJ3^@L7NPtf&^%=~cGIJJ*WkW0_dUV-(K zkW@!}wA~2m`1q#edkjM`g!dX^p)gucYX~+$ME+pfHGb9O{{9LbP1406j^S}CQ8(Yb zeLDlE5YComHDC{ZK(^{ZPQM=@h0TJB_mH5Uulk0EnW6-zrfQuC_NJW$eR5N=k8E<@P& zSnQ@bVTNN+z#ItuL?r6<7~NqA)Ic|#G7cT4P0Jv*6!iu((FC`@$ru56$4<+ajc1IgcTbZo|ou$DN=n!6LP-^ zfcnY&L*N>GN~@~Pndi{?kOd0En z2NBS5Tdi!V!<+_oTbL9#r)LRYdwXLQ&p`j$^-&IH1Xkd;dBFgKxj>N&aqaXEmMZ*P z^U$qIX^3)*T`mA%2w^@tVFxal=P4zI1P?p->a;t{^kP0Ca+%z(dh5eJ_*Iy8v-uPA zr0pu<(WP5MBoi&8RNf5da_eB3d?b}C=J43AIcn4F04`=N2o9$Sw08QTTmrF@&vxiG zBhZahBi|tcKqt4OP4GLV*=G=qqICiTnFCm#&Ao=4Of1Al5Bs0p>au@G}JsvHoqs2v5d6M zCVwGK&I$diNp=r&oU@qoesjt>f2(_`sXOWQ8A-%k%xyU|!F$D`X@4gA=}Wsex4k9( zB0@PfKOH_sU=HC+JQ$^2rOExusTPtq_GgtWCzzxcfdGUa-Z{rc`+HIfPohm-=>7E= z4cj^bEGWmEb~sM=a2OVQ@n28`s~w51nAD$Bi3 z`yFE!o{|rXWny?eMJ~1K33atiM>s2!4T-Gmj@lO$=A&NH-IWu1Eh*mHIo@~DSoEN4 zuFAS#%wxchu7;Q)!?7K11zTLL;A%$irn^;t0n?B88&vPegbnyjMJg- zKSJ=X`*WYc3yKTk?_^72s5!v-10{plM>3f#$E2S%Gk1HXt|s&(;^IqsA&I*#kK_pF zGdVFW6Tnu#=AH*g$2l0-`Fr@?nd{;#I!>*3^MC%lsIQ|FoP-sff9cY^QI}^#tkEcA z`dLWdH0X&j13zL7z9;ODNOnqMf62D4>mOgqByI2w{_-+K$tT*-VJywB-;P(2|YT0N3> zaY%P6ksj4?{DLjP^|=A!aS!9cSLh@Jf1|r0&eY0m92k#hgQf9IMMocR%(1zPn zwL|21k4HsxR;Ub1J&|Tr+831;Cwi&rTIVijGReF(#;hOS=og}_FP{bdDbZPx<^s9+ zjRN749JRZTS%hen&&CA-Vkth-&v(t5Sg2%gfU_EK;D6_*Zp_DIa*CFLo z+G~2bNYb*exTCkuy>l=&WuLd{VlipskMv=G=3{XPLnXR#muS+V9a#CEwe{$-C))|>RlNl~n+m%tJo3M$o{?hkD^#}8UDx>shZH20D5~ki4 zs^;nbS!l}Cv2d~OkW?y||O*Om^jXUk^nY*yNw zIc>a2E59lR?jB+l0|MX3te_AupslDcf~!RLbCtC(Jsy6u&a%B?jHr5u)`B>NtcpL* z#KEE%Wy&ep9v0B@;N>RqV8&xo)?u<~@$(}cxLWuw4mfX)v%DeGyR$FsrKWGPWZwq1 zz4m<@?(v{d3^-_I%d6+peqkVij?{uMS`a@DpzX0zaM(5Wrcn=}F~_$QD<%&8}+xKd=Z>hd?h{WS#GN-r6!m&02s8C-6!KqoV#PcYUf2v4<~tKnRPTbm4yrJNNR)U zds*R-W{f}ymhbI}Rm##y@;=*tIBZocbhDCosqg%0Gt7JdEV7fBSCk;i7j>}VIeQWD88O=(cFtN6Tt|vUNu!S_L zc{3la9ckyD#8G{{X|7|~g0Go$dwyPXDk_ccc`GAW>3nvcpsGRy(5c7{qY5@y=Q6% z?4`OAN`vZ$#^NkW^X-a13~4$?mLA`@^!5Gqf4$X|a}A~)Ihw5Zc6Vxd_U^RE;i)mc z1@gv{n=Vzl=(2`XxEHS4N}Fuh)ov<5lnG7RXtiFDZ6ggTIep)UaCKd=*CdnoM064$D;~57)+1H6Jm@4$vf${s(h!9aVL^wu??dKtZrT6j8C3 zNZBYTVPFfWOj5!?BqklwC5nj&CQJ~dyIXkiRY3)$gh?uh#H1U6eLYkC)>_~G&e-dW zvBw#Qe_mf=*6;T`&wXF_70G*Bk&tIit(6s5>I$dVOg=m6E*^Q!TCZ}+Vdi$*i&w?U z?Eb!9Xzx0WnR^xQPhn%iW0*YP7hY;4cSvAqM3S~Zb6qOCuo-tnhy8L@Y7o60*uR)1zk10MNXXLZXlYc~vGI#wcIr?96 zv1h6tI_cHT&ze2@El}PyE1S;^O-x5u}#;<#zgxSkK(Xgtg=k! z#cxx2HWp7$OkIe~9Uiyd9c^T@ygEJX(Xr|FZ6c4I#Ke!QJ&K+vjyBPBy@%0pqgigd zO^CM|vF?*=wtIc3SJHK4V{_GOS8ffPt=6rFM=x_<-|<4sirOFjc79KI%&Y;@T(H@2 zWSFGBEZzR%Z17#pf&q-XdxG>lqh3|C-*Os^HqjshbTH%NCtNBhuR3*OU!Mtz6;e!u zz>AEHiBZl}$!U=|X{8AlVwIq~G#N}s-1X;ZzIjpK8a7oYGq zTp5aDx9^}*REzN}V}7S8n^%qALbeBImYS4_fFslvBIZj5>%b4JX|k2))n{B<`PBl} z>1|@1Yvk~?;(r+KC5JR=&Zi(pq*3KCMr@E*`lp%ur!o@DEOK~XPQG+ zs()njy33mr&>%LMtIGQvCTSnGfbGwP{RHkC0(IC2qzgL+EiEWdk3hu>MXclRaR6- zD{fNOYrA|hb@SeCt#rq9-O;g{v~yF+jMSYUPgQJ{*toIa)cr*gTedWQ{Wd-BR^fc? zO<4zb1b!kZ`Pcmii(*cVuQNTBo^*SK+w-dEBM>EsN|1T@2(ksXsLzTmeny-phKEmc z+$j`PF&GPVzaS78xyitIPLOBFo>Vz(PvVrSjb%~Qr!_lYgv4#*x2?;Vu#5V->-lIz zp(aNe_qkMg?i+TU6CheP7&D{XDhF(Za;r~rMcN+ClHYdl*3x_J1GyD#c21dM>0G=* ztzKuSJe_L2YwzqTT+w`Edu;yn!b)ik&G@4hJ}ps|r!AZE-Dk|IzRBL!^UPqgkBzS5 ziddt%Iy-73DRCd;Wu-o|X9%3nw_H8+NzS-u+YgW8Hi<2(Rn2@>x!v+8o#Q>rtLHJ16Z8t-VR;&Cy+2>#P66BDUwUf?R zC8An)psRA^@bhY^sLHBI)o?Fy5voW`+R{#&&fy(=8ziVfAT2&hFG!!Z;M zcGr&XIUI~AJ$1|ImBHL50;1vv=3o01p{f{1*Qu%j3&7rQ5`(vl_JzE93RK>6uM%`h zkFkeG`TR{(f%}|$FJ&Cc2N5DTk8@jvCmVIIX?ExKv7Cu->`KG6wo{Asv-=$125uJ* zdTtNEf42Ce07>J>_+vYa=ew{?Ynm0ufnAy?rr1GTD|EXDjEgP@8iKs+8gwDZO9&d zRhw5iFIqkGIj?+&!E*V{Io?Mp9^HS?$RYy{+s(Z7-){HuJrr2qR%xFae^^;L4ZQj> z3v3SCmV%(&#+k>aI-$+h5i@Va(#h~)m6mCjvY*BsPs|-p8%ZXfmX)^WDkYbM5~gyT zvrJLz^nGPojfYQXcuXF%GdTw&REKAEKen5FDT6X?ars+z)t`;iE}5otLzB1K8c5Wa zzIuLX`l!HtGsdSW^GplZv=4D|;y=|D$wPYFUwiT3AaT=>!vvbBHfyZ+v-5G*<+)}r ze|+5al;`JzazR!0=i6rrjb79RHhCPXFqQup-*(BdRC!YJYgZlg%dB8?j4`d)7eyKM zLmb7T3#C^fbn&a-T>4nV{HYmm(?8LyKn54A_aVRKcnfMF%)i#5N#_ zPYtf3#Bbg3{c58}{`r@OrFVyD-S(AQMc$w4&l|aH6L=VVYAMO;!9iHNAEs1=-wn7! zH)Z`;BdD5>R{Tv;oMMU!t!87#J=s*$8_3lsID@tpf_hZYsNb5g4Y z1mo+MlpB&Wk#cT-B3Bvm7}Ei>>A=vAMtt1hvaJp7%zou)x;8Q$PZycaJ-`*7#_nrp zD&3PbbUbKxhJb#?xAV0{Z{K9LnT$v8?77ih$ik1e1b<=wf%7BNU7onl1FA1g9yJ3j zjZmr3ch&PnqfRqU-(R%G%_Q5u>qQ+Og(G;|#t-4i;z{Xm#Uvz4B|nemzhynrL)kYg z6*nPe=>CHTvEZ+rJrO@5EYvqoly_I^3dNXX>WtgFw(*uX->z7&Q9DrQLjBB2mZC(p zlYi3P`N@+f{*NCoRi{3B)d0*65zL8)t9gB;@u4i}BIm>P{+S_@Pq)mb&7MekjAXX$ zjM3eg#KU@oqAQsCpiws^o(o`heV1WE7bgKzCU6^1t?S}Eu$=W_kRxVp|4dhvaEtV^ zj*gij|LOsq?eZcaC4@ynDY}Y>fzqinZo`SE4~>Qgq1~-lh!RRz@cS*8mPbU9`m8nd zG9$o$VYy-Nv5%G|Rd|ebtZ|yh=Lme5#G6-_U`&M5d-eH6%+^W7>@!q0!0icP^yELU z&&K!HtGm8j?UBWS@Oj&s(;TsFpZ0e>sgb+Sua((YfE`zS$l%IT z{gUzlwOMlX} zms+spc^w~p-6ie#8@#f?nN+c$3(KC!Gm&tI{Tp+dLjQUv4&K-Y6sU5a17);2M ztliIgYd7AEb33n;14@0Zes#4u*m|6N2#!er@$LF%tD0DO5Crg9_$;1i+d?VOs*ydwu#UNbdXbd*3 z;)NFMV152RWt-|h{}uRRM>2m zWDkE;szY-gnX1rRD|h;Ra$f6)Mc=-`*^ztWk+@Y;9_C!-i{`K_n@;)VaQPNcoIrjd zCX)!EaEhC{yLjUNnvHmykkd?Cp>JOeeC$Hw4Ntzww<7t_+5zDa2a zym-FUg4a|dY02&R2}^kA)4wjGgk1SEPP@fXhPb-~BH@o}$BOx4p3%a&8)8^uVAaE$ z4Os(;P9O_`=t?wrr=Sb2CCO3iSFAexZsWVT@~Lt=B)8R&9dWTt>Ix+c>Tm`CaxyR>dAs3(PEonwkE(y}_q@3Nyi*B#jYX9XHQ}!(rd_khhuRAsRvN-AWNK#Ga$}-3IP|8yS|GU#N1P!^bs_f6#A0>l(zBu< zY-JLA4MK$`A3T5;ts}(G<*1PIAP2UEKLcOiM9KtETwCu2jaF}0AXkPDGvNmWyjRb6 za~-{^ixc|elWlLI*fS-jFc^?$%a;W|#|T>eexK)TdT2|~861s-?zX@gb~HW^nEz{B;DfNb2Q$Au1Q*oC0RI;J-}wcX%}7KqHsDryT}( znU*qNU?*dWfVULlT#=az((?$hT!f#~4Kvq2Q{DBU9B%d3u8G@2osJy zPmrs;zJJNj@j=+JxGCip|9)v6_ZKJO@kBv_5Q>> zreO;ggS%qMs#U9!WVK1%fb*ud2a$8B(RcLVa&j+*b%NkE(r0s&K^=jcP5($pL{GA2>NH5h z$aIysz7QLr5g6{WdQ5W-I~&Z=_kWrM5oXeP@1MKtoX$NiSVWT|qt3G4w&Daa2q11R z-7w0qX6__8qtOU$eCLQ+J$2`ah*{i!J}<*P9{xBHYLua<(dh5iZ6yO+Kyz}fdrZpV z*Eny~21waWTXF+=e5HTANsKu%X&)YM=@~)JN6LpD{xtkZ@%=qb;rX~mR6=n#dw+lZ zbY7{@RCkqa*;;8Cxil9-he4uN>jqB}$~z=OY4_2-4(0Sv(;Od(daia#1@OH><#Td}Vl zHdz@X9wvgj$o&K<0ZHMphq)hC4yvAqOWz0)_9x|C5CrL2JZpMMrQebq^&CM325=ne zYuah`^+0xCArBLM;y+0FXIEVJ0aH_Dq~DN1JxCKem@Ko>B_LNad}a^(4|~vj zHPPh9ATAwQnCD2tsf&pv(!Q$ADv09>8K9G;jL})<=2J$kWY#5ZS}n2j_V)@VkkUav z(?!S62SpequT*$C<(KPLj)j(n=<=dYl%Yg0dp4@D;`s+mQg0`Q5W|?vdLlzXS!`DE z30s+@4*PWt#PAX`A(%s_kt8S#%mcczqMS0YA;5MyETPn_Qkopex&lcndts`PKbzEjcO~T%^vfdfE2pD&}g{Z6e}AZ5wv)52v+y305Csx_Dj@l;x0sgdghDE>sef5z0Tmx5Fbg)QINp@nr5WlCT=WI zPuU6dn)*uhiyUT&W?f;fA)6=Rtu(L8G2-c&;LLr^IQHJ^yGr=&8SsU@^Splb<3YJr=_xv{pp9Ses=L(Pv(~czd>36tM>>fDTu#ROy5L@|oL9S=6*d=PW^5e|~=9z6vewns> zgI?aj)$gX4lQvh{hrS58>op;NnQjZAJqBk3>oY5R_USI3;oa(~4!Dl>G~~D13%rYx zs7XUxYvkWp$pc zTc%E{8rs6~yqi#LKs-oN^}~&<>d!LEq!;yT8|=ZfIa4{EIF_>vS%#2l7A>9` zPu351X_h{&0s{Y>@2R{3isG*|dfQ+DoS`mI7YN#b_3E@sG6jsXm$VnF^!$Fjrk#1* z=3@%$_Yoq=NYI)rfB9O6Ywb{7In%`Olo}UlL5|9@%BY9CP|gi=Jqi|c^LE9S1iuuD zb=2{ZeWymw)%6byr#0S9?lk+d;L*ym zMn!KS(Jg8}V)M%2dzOnrF5@ugBkUNWysVfK)vv-Jf~8ru>=R@|_{xUTjsPf#;;^{} ztp6QhwA1A5mdkwxyP<~Cog?GWQ!f#_@j$G!+L72|2v(J3#tinRw-6Z{=NBTXENr}L za%j=-3gHx4g|3h#tH65ED2T0I8+|r_i*1P+YhBpN&xaV~-x~W-Ca$cCE zMM>m_>>nPamSsSNsZk(&Snh7ssHUW(*Z#w?hkzpK3dACE!g zqO7R1h_$$&a@5pc=-`;EK5V+Yxsb354aI-(?JzhSqpImfc!vmq>@FO{NudLm2;I_l zq6a&AjzoFMo1B{FHwemKydba^fq!M*zbR&#gC+? z>)i{T+6i<6hOYOV|`W39w28u5P2UoIDqbOcx-BSw|M!BeuJx_E`f3yAq z&1T`;yj$CkCES?i?jlQGU69xd%Gi9*Xe`0TZhJO~lY3r_yt=^{2yV5s^oH+FiV7%Z zNpqA9p+W<#0LFGBRF(78N|Rbrh~Tni6AKm274wul4NIwyMXSc)$HfT?O8oLXy1iYlUr*2M;_{D$Vlj*I(%K!;QJ zlWj%^*LzgI_QJUm##3vTu#a6e>Zu+V2r}R15!+(sA0PNt7H{Ni8~BwIK#g%7bz4)`G@7zqp!l_rGmuX)Okhd zl5bO3`wYE|wsZU4AWq>*x+5YUL*KkLQYg0s*%gL)V+sC_*jcjWEAiWkN8Ym1gA`ZW z_BP7JY>{I{4Os?|s3D5SXGF-8r()xNxXgw8%aGT50SlwUzyH-ML*TcQ2mU7F7by$B zKsms{TE^s`B4WQIknrz+HACk9XLqUgPc!8ID-V66=!XR8+7!b_o7R5B$9=olT5^lb z*Agh&DGTISFWA=?JIa!P#~0?RZ;{p6zNJ=DpByz5`UTdTP&`m6XLu$1Q+>L)i4J5g z4S8!{rn%tflzRg0kcBVQ`$u8Jn3Yw}Q`8rs7@c{uf@-be(KR*0cmCWW_|6Ob-~Yt6 zkx9adAo&K8VM`){b|B5}>({UKhG>4uYX`_H#FuLE`X!^>Md0Hu(IlOc^NwHNr_j|_ zdrSvmf6_u<_A`LkI6!!P!=d;0&lTq$q|h!RGbs_?0~cyH6L%gf=AUj;RN%)f+C>yi zZ*p1SKmLebx%$8475{e8f6Q8;WMyS__QcMMA0D6Sjn{||1dLxgGl_PU!r;$eI3bI)SHca}Kmzb*O|dRKa@iX)nr&;(AU_lA+h@p6qS_r`900pyZh?TR<*U22^dH1>)2 zcBya3C7vJBdueZ^7Srwu-Nn{axUt~*{EaPb7DC=gUtbc>+zi+x9OXCLD zA51A`>6Hb)Z-@o|!TMhGGWpAWVlVE0?_SG{+_r5Mx`a549;`cG4pew>+0`Vve!0g| zDC>_rM^u}@=+PoCbk77y8hmb!0n`Y@ss~!PEE5n=Ji-E{b-|kjA)P`YQsAO>fSSd@ z<^KH@;zKoGETHbB>Y8AF)$fDh-+E)%tftB)mf&L`vmPNHEWIlSt)x#F7e>PpG9cC$7VAe26s1equ(KGeZlU=FsT_d@^u_#%4 zG9%ji!Sd+ZchnI?M{?#_bT;c_*xD6ht%CK!m#b2lKw)c~`IZw2EZ4(?=<$^_V8Np;q%f8r80ldrIf&7b{- zC1MhSZ1#>=gkG8gDnKKe1ixS4t?H6>*zzcWSl8Ou_*Qh^-Ora;;LSyvV_gFYND5*I zEQy7^b2l6gNOmsNp2WKQIoVFXmj?B+eK$a7o|g*|Xmzy4CJj3v()o1+*ao}x_N10) zL_6p?DX@R|1bok8Co%cdH2@*pebgCz%0eTKw?MYCc!C5D6LW(eE2PJi+-2`1=DJd6 z2)g(J#?~XGupw5XUg{j(glg?vPdrq>lQtujlk+ES^H9+%HRlPV$Y+4tkdW}Il^C|z zGVNg+W%NwNGvABB&bR9|bx7=h8^)pa@;ce}u{{3l4ZqQc(HsJt32JZpjXtDr%A1}r zopJkg-D{FqCJ|Wz@w6eX5v4D7*IgA(NamE_Ep*h`o3+j+wgWG9@cgzu;LJI+I49E_1x|@yK_8Ph z>56l$8i_v|_hLHr8k*o8;Lrrs5)&waZI?4Kw4&@>l;Yx(;kh~oDyhGz`hLPPTQ5*z zht&o85e%dIDCds2*aYq!iO>AZLdE;@U zD*qHJljk|)HX1)VF%D={;*66UiH5L#rnf3;i3K+7cQw+iwW}EUeSc#4cO$_u7YMN& z7%iYxz{V(jlRjfJwm=OC^Yr~z$|cpy1HDWCfP2yCnGwOW@{n&cVWERIFPlWd0I(l2 z8w8H2lnVXRT{ z8oLJ25OfX#>1>#m^7ZO&dOao3v%XTgdis{1-ze##YtyP=kKHA1Oj7GHP7sm)Gs$>& zgZom^ykQ4*_hwbtBy22nlk+>UCM8N}CbQOb1`C5XRGzA>>TW8jQx)$R3m@I#`cyGjZ^uWUqm=KgOP%8g zPu?XA+MeA3b-l}b(=J!&l%3!8+4F4tvz;m^RrY`S*09oq zo(qe#?-_cT4dEj4r&8pq(zU%s0xy5PZf>B+d#9auIzmyp(*C3Aaz};|zeF^Lk5@;1 z#ET=!vr=-PEsBf1&MB)_!dLEOI#qQ{Hr!6^!?dJC{P0z$r^K*xp*~AGf}&lKa{KW@ zK|#TR5fb6z6DD*?+4TzbWY;egLG{Pmk$p|zkBYZWEtfWXXBDVA`W{7*6<$uVjh5_~wynxraD%GRxAD z^RPPoUb=28@fs(@Fj(*ij%wbcFA;+$;-t=k)Y-=6AskGZ-%D1Ba$A_GHa1-`*PX?UReC5BjWyl=?njNUgR`tWcLPKVXeo=CZ{Aof>eJB__osO}e~ z6(FheJu1!HS@bP8t%mewBp#qwAvSSQrPO}cG@B60D7H!4QXQu31weYl!Y@l5J@CAK zmEB-T3Mxhyj{;8aQ@CRU@g*@RGOV>Ps9dYfI0aATb^vGL&c~_Nhn!}Xq;aK0r3GOx zzFoJQ5%8^3R_B~vOiF{ObryM0j8?^d`g1@sb-u;dzhS?)tqk-WTIJP(gwqNnYeJy zeQJuaM`_dt(X4Q-5gZiR)yp6!OQk}=^YwROM)4U{8DF!&3ob+QM%_)cCmoRs^Dkv; zyM3Q-$TW9PJ$4ygMK^&ntFod}sNx*dJJ8T<)6u!(bxKjzqH&Ml-(tS5WQ)l^n|gNX zW&_1{j@v~-UQrXG?8-eZgB$vc2TFg_vLeke`BGAKc(0H8ScK%@&?is?&c}AG$k-?|jtK~w z6ETv?^j_$6ocHKb^WIjkZhh8ur0^?0S81<*smaxFvr4z`Ejv4H(l7ap=5x(R?V^C7 zJk#hm8~&kFCB5Ox>KV6+;$nG?R9l16v#IA*Uq@M{Lei(cB$4+hpT+BgD@Z;6hU9Vj z$|VMlQj~6UU!oWmV-nOhy|l~_xbvG_mWTo*IFz!gAFjdyqRw8}=EIgZxng0BfEU|g zch~!C2C6u;WzK5V4=e|uahle`J2;Vvo|Y%$Vps{>+Pdp)N>ho{^yy9ubHpB;vWoxZ z{HYjuh-3S?8=S9qLz1BDUfbGwVW(&yXGuk~cfs~|O~kd2;F?4&4A08tmxx$B8N-{4 z&*U+e7({%>R!y{QKC#?sm^qjtm(8+7b+ao;yRbVc>IyN}MXk4(u4foyRMaa~{;Dj4 z1SR})J~g`WFw$`@`F9|YLz$EVJgou`3D1UjpUsyWy6cJpNUEFK*OiG85)~KnZ(HaJ%@0YU#s=55G&(BWN9V!Y`oYn_^UwfO@BW{0zjg;_0{TL=50D7sox0 zlG$=K8NgeVhtj8rAQ<2I!Y9*uH$tAkGWeQag|e~`bFLP#7t|w@&}?f#;x;w<5wW!# z1yT0u&BB`HWb}4{=PoTgW|fjb#-Oo{G!dPfN)=gDFGU68~bf2zw8R0zZ0@j@w><3q~Us`c@PC!(P zH@JRrS;Z?1R25eVr?5I(ibxiDFFEqBxt9~)eX~&uBo}bJj(RwvtuLCOZkQnUj7%Ri zI-Wjx(h)@HBGcez?PIRtKL_9yMR3*)&Ed`J@0K5@mT@O-K)^+Mz4guP6Vx{*1y*Pg zv-936?UjpV*SUDPGO)Wd0$ZI@tn_n?TYhK>an_EW?C#8 z^YwolJL$obvc(B2XFq3IfotL*d_Nnly?LWf$%iNWcEKnorAsr+{?W$TyfDYom=2;o ziEdUa2#GSGM#gX(E&J_+T9Si3KFU-lxL~mI#y$+4oW!FK4G=wSkh&{<$K&9r>lRrZ z|IpYGo1k+eYU=82jjhfYEID?GJ)B>EwWFiMoKldJ#xnemlG0Bw@a(!7%hUn`O7PrG z|HV1Ha(dLu896J&Ive_vs20k3A392`*ks40^FXA8NZ(LQpmnjBz$XdZNlfJ#w zj(SR6 zpNg|vRAPUbZoYwGzy5^J1TXkohrrU(nwU_@KAw4V{@~FVTU%Qt zMvdj1M!Jm&WY85KW4|<*v-PGxojo`G8t?Lp!~mQ}-8-lY2u`b4XEQBg`pZh%28+JG z4|jY?e3^QwypSV5Du++lw-5W|{MwmXopi6yd#}PpQL5FIJ5oT=G~p<<{P#kWokPF( z(zPq#@us!9Clkj%?;B;45)K;Q>e6FXx56o6d7ldVOr+B&ghlqNQzrPMA9=x@KWGd) zxr5D0*P(L4JKg#2dEp7(bKxU=q}tnSWH}a9FvOn&r`pfdZ(`5uku1rw=Z6vH2<-Xy zENwwMU7sXidrc@|v%086{_kSVzJ>T2lg2f!o%mSju1`#RpnctUUpt9cgUbBOW>QFH z(PC*KVzyJl8%=8affZ!?3r1zE1Yqslja|Q-rq8sn$o+K|s5&iBTze?*gL`57%+!)F z$+pR+Z;wfeaB9}l-S#nr4L7VuY*2U}>`*l@7`EEC&kKN*Gd#A^!Q(?k8&L({N8(uQ zpo6nKY0r%)*QyFlfmH1@Cw9bAH~3U>({}tY5H;yPnSAlYPFIQaroISkEiA=rb=nbi zno|L(G*%31`*Rz(>@%Vik~p<0?O7)Khv3d4=Dp+6`E|y*LraMh1w=SNweL)EGD(66jhquu7Ua+)M#>Tog@DPW!zx`IN zsvg;mqd|9-$*&KeE&=@Bi!}j~Kfz7kk#IJnW{>{YZ>_>Y{rNTGvd5bZ0qGxx)Q-idNq*H4bFERosdC(&HIyOmeCYb7 z9u}jQJoQ86|IemQ17~5wLYRVpPUIQ9Cu;zsJN!D#MI*5;H|QUH8rEAU)PK|e=9d9r z*cl$WHKMu1K6p3M-D81%s^j232GAD9HQM=ZQ{5Q&zPHLFX8ve=E)heq!Ag{4#5Nhk zfTkO^Q>@1)^tF0%k1qoMgFn%YPh{8{M#9a3)9TG z-((Vhof5PAg1=lEOy)W24;$lbSCWU^1R&%4ZgB2g@fHS2pji#}t0tHRCV$)p>};oB zT)0Y55Afq>@q|}9;;OCvwDGWQ$xO;NPSI;L-8vT1uDbyBS2@`>FbH|m1`;{xj(%ZimCC*p9) zZ%k*g^*S9pz+JX)A~qeR*RNmC139=o^Y+$l+Zx=evrLx*kSKK^M$OExlIfzuy-iKD zCUU}Ry#@4>$!G`0t%EtbzavtazxIItZGSHgoYTI{-nM3RhKl{ju1yMI6B6~nwPAps z-0t*451&mMtbv_@1;%189l^1j9aUyP4me_yal!_i_sm(m>p?goO~IcGZ{*m2o9H6+ zzc`4SpMj!5EReMP?NHc}O_~s-;*JjE^h_g`)WgHNa5(ZGnX65r@apWlznsPTNJEhb zq10r(O_?>ZB%Gj_8oVLUccbg5F1cff_Ga4f$H>pJQ6`7r83X7rlR%=Z^O(g>qB+t) zb05ta+T$nNDY`Ruf~yH>Vm+B?u-+1=&Vp-;>x6C1qOUkNKaDI!|C$WGg%&Q=3$zhE zGRDiJ7imxlL-FI*DY$D8wLl7Pz|0#BKs4)c^LL}PPa?^mb2mJ14pOf$o*-<58~Vet zB&Ci_?#*`-cL;U(<4csNoH${}l)!;|Zo>=P``P-gj|NG?tTT-N8r*VP?;nGP@;#HC z=^%N!G(N%%#k8C`tfA}W2PB|0Vm?5TQY5s)M@NvhZ%BH%9<&XLwjZH zx9jViY_yxnv&&edrbhyE#6^eGPBfo)5tNb9PX+Y0_M z1o@s(X|ojlsXi9tnrTqLCV1I7elAC*_C;dF&Cy#IFdjU$?6P9PAo0Bdb?OM6X0Yg7Vt)|@h!dnNwGwM6%d)PxtA3Vt=FD@Yr%_SVVB-E(hvt7-^&X*m<$^eEod{X&s``S`4ZA$5rFr7E z3Z_=~6{=dS!p>VNSEKr-*@qWCJ%q|T44YT9n zElu=uTaQnNhiSG%p3)PHUtaz7p-Mo)C{2;Vo%t`5AJUe zJiPkVPhi`b>Zv`58k`wK(#e^}x$xl;JQdZHAf`&48`*4E9rnu7#LbQDwzn6Vz1KE%OjxdRPlmjcN^^5WU@Q>u3GFO zugUFX)+Xh__sa65_c}36YHurHgQf9z&W%mb$bGo>4bN5f=meW>Fcz(mKfn}83H03l z`X$?s!!Zxio`}5}pJA1l|C;`7T&`=OIxAU z5Ip=Ir{>q5wLK-`F;qKCI8Zc%0a&U|HPvv&T=M5E+h&R{N0?$ZPQ5s^rT1D>C5K#) zjGJKAlDw~(L>jLneWrPW9Mwa-yVSJq&x9aduiU&Xq~`tzG-A^~s$jld@6Iqy4yenBRH$=( zknd14p_|?{Z5b(003XOFZPL^}6H8fY>UVfz^AG4^+D4AY8t%0j0m(n|fbi}(_IT-1 zd*{lgaUgyr()o23SE9a*m{%CMt&DH;-uHK5U!7BgmrMrD+sjZc{B`?F?mE0r!;XzH zcX#wT3SLc1JOh6IYVl-ti{&!bq1h{?%{2Y(xVx=l**WhuQxk~XQcEnZ>H#np3qBdY z8g^NQfaar)9F2AJ3t-CES*?Wk_aH0_hB9s72o!Ja1zyxuXg$n7zp8{#xlljT)D&M< z5{+Q)aIN(-dG|-m+5qz>Pvno44m~lhPuZ1RG`8!QuU|6MZbY+buU+%1Iz?}!fXNnV zt$DT=dYlFiH*U46ulov~*9l$`65f=)gx}<5@ZKgY+wW0_tL*oB_iub$ z>8C4JcAbc!>{OU?#VqQaM($6HJF*Jch^CoD#qrn@HUNo{?>fTd$m8Bg-5A^cQp^%= z7^R^2*mY@Xc0Pz@#`LK@sx~^(k`BBPp2blybb*9{qQMWFI1l zKAAfU`s?kkOEb9^5@nT|Q#Pm(5j~{_!Ol%i=6q_S18Ptj7gf7!M{s*+P^xn0+Z8|R zOyljSty!0zT~a)>BBD(3w$sPsn1v5Bf7QTq4p45NGIck(Dz}Zf)7h?i(XkVywgR6b zji1D=Q4@G!QrmT!Zz{jZq31+$WROH^y1c%*(Xg5MUlVC5*6(LhLg&0vJmuMcVrN5Z z+z!iUbsOv9$u|=3J|(W-75VxkZ8Mo6?^ES);odFqrPGT4DIbjL6s7=vo+YCOhI#f! z`Ja9J1(!lz_xJE!^@ydyip0{ue2U<>H9dQ&)!ebAkg{kjjA^cTS%#9g$+qe(cP5o7 zj&ap{IE^qP^(wjDf*ijub0!1?Sy9!h*Vh5y4yr_p8FRF7=kD-x+HNW&msh=h(X5n+ zbGmzpPh)3VV&F4QzCE20FS~OFKb34H`7=KX&z^|((DqCS%uizw0swmOJ4!b=r%(4@ zSh23GfQK3!?KKV3gx%y0p%izJLDy=$k7Y*ti_3nY8|ZP(-u zn_cqhp^`zqyR18%GW*8hM%Vp`eVmf#1rIc4eavQNz0FS(=yIrw%Qg0Y>VNjgk-7f& z9~xZ0-oIqkXz55zn#LAe_+&w&d8{O%HFS2gn*95xm9b{$udOnI0QOd)5Zn3(JIgK- z37az&PeNEz51_=0*`+TO_NnefHgn)wUc!3QG)DD{N!^m$>|z1QQ34Nim6Gddm-RWb zs(Z6^M@?l<#E);!_kX$hx3{h0^g=Aj)P{`d>+!mma?Yzf4HMOjduVojZcA?Eju)?L zT%I;XhgL&`$@*%YFGoKa715qtXWm`^;9}c@#WCl`LtQrQryU53kNZ9jUut;;@R7FX4v7Hx-ED1T2|?0CiJZ4CpjK53r| zw>9gY*cYJ09kiqDjGoX=^W%gTO1>1dE$Nk)CkNa7p9Xet3dg9ee&5udNhj~6&6wqW zP#i@_3A2_DW{ewDWrQ678}CPXE_d>s)O|96!4Z3f%TCle<{(pGppZ^ zyt??MCwDylO7+lFl|fS(?M6t|U#9&QU@97841Y>V?&(`Tg-@MTF_)-f>yYA@-g}fc zdqc{rir;^;GWU08p~p{hrC*kfz3uxwf|?+^P0?0jr8&HpanjhREgxp38C zCH;lDFxJq?v3?fYm{CGi+OXkooxkcH#VCDak5T%IJ>oLwz-xs}7&k2WW16G2o^y)t zp|(l5)q!N8p5N4Lk%|9z)Xo2h8u@z#{9mDF{|Bt^>*ZP!18b$E?(qIPF2OMlX8TpQ zfW9rDrYN`;?&0f+MRmh47Qi_oNrarN73uOnMolM<~8?!K0h{)#h!Qv1My#9ivLe^bSrrw zM>a~~)Mq*uirp>5yPC3~>pvi!&j05_tIqnp&ry8;gOB3RGG}r772gk`2_k9*LZD70 z4$;t)5g=s*$OQi=Cf2IoW3*F?;hc5u+DfO0Nxe8k7p}k*Mf&20r3|n=tm0RK({K3I zMLsA#LxYW?n)Sl2T8m|@Odu1KN{~3pI|D-^-Wby85wg+4-gFk2MG@VWH~xKjcFWb} z-SFc~1_GiD9NrbpGLcD>xjE_Mplb%%d=^~Q_u{*ufl0iRGku||V;2w*?XZz$_i!&+wZP?j&npD0cM9gDHk0#*-YhjGM zrf(k3@9Di9+bIth3iZV!AcZ54(EF-$ogH+LAUT-}vTK_Aj9blLS^*nRzarUs=KzcH zbOrTPDDVC_W%CmV$db!1@Klj_s!YfjS=QU&;xY*uMf9K`9`WQT<7N&fcblQ_^jv0b za`^J`X((M2W$ikz>rYC!Y~rezb&gjkW{DQ}D?HU2Pb^r`wLr4C_rw>T>nMMDZ6>H*(ddPp z{|d`&G!FbV4{;GGJ`QT$24i&JNEo$O{@J86*xch$nE7z;cjroWB| z(!0d9!ghf(8^vSIm|h-KXG{n;Qwea7(#s55jU-}VqH6iV?8>mNIvSOlG@>h`9R&f%XJ31+LWgdRKz0Scz* zxEe4;vj;?}1IqdrEFzI} zKCq)gPy#8&A<)pczkkL>_b0}MGRo}@8kmepu&r`vq9_{Jj_HDfoY5y?;LmPplZ1W+p5v3radaQ&Z^DQ~$z>Be zEV@vbQi|_lW2nOnb1Hz)S1Dee>Af(iWhi8Uypj=ZyApJHlG@v2|1rML5jQ~3C`KJx zkeS?Ud4ipHfc0LAy>8N%d(63FyLInmwXCOP!L7eRgz` zxLJC2K#rqi%)}O}JEn&t&v2FiKCgtCBLqZlKyOzKlK;({HrZ}TPWo{+GYLG11uO7=iz zVK77g0Oj*D=T#jcK_y6V(w znphojr(JNT%)CtKxwE^+G9YP@qRP|l?@6|)!{$y1BR2L_?VU}GAi#_>CgM97i(R3? zA41|pmmY;8D0}34@42KK{;%SdH#;7MUSKCgxs+VP!#=ZJ-NFu5N#$bTd9(Jq0X z00g2lG`LP~Mv{tyoa4X+NXOqn*SCH6BQWitV7ClHD}SU^Jd znsy_#Mmpd?NkG5@OQ%ldS`CgioD%y#CBrg{WvGEgP@P885ws=hB}_hyNRTizs@LRp z)qotzgsEf-rX*R0mm1<>_5rO*w7@GPd(>@H?1_=Qp4BHy<8e;J3H*f_kG1Y6Y)%$L zmq0XaAH(^8qOI`gB`6TjoY99n<(%^i$ zG!gJsk4V6$q<0@NW;Q2~)!>iRkeUxfD8gAT-c4~e*|Oc>HN_MuY>e+D=e~DKuJq6Z z5k$NjA99p7uirH@{n_U8)*<3vlboxNZqd}^)YFyg0&<@Y2;B{nT0wIOwnEA@?w6b6 zg9t&BPkziG{SNW<53a_pK`-jVb{nx2B*FU^^)XR#g>wKgwa;vVg+#Jhg&tH=J$Vp% zRP4mII+`p+GH9e}WFU+4c0}dnF^sxgpgfpvf;PAXC5=*=;|AsDj}7x+nm;L~7IC*N ztt&2Ds&%mcd(|K!%x`SM)N2aFBC5WNOOE?A3D!tQoycYpkJ@CMECk~0pDm04i=H|i zH%Ka6Pz$aO=qFR5C+c0_W5cD0@q`6V+STnZ;BgxdO`cYoF&LzinRpB`bRFyEX~X^9 zw*qle)5a+)oqoQi5gC%vGYiay^Q38Y7oMn;%0PwS{w&SW&3NU<-S|XDoytnVn9sx3 zFcuI!HG*BJJ>wQJqA3}vG*v3;bE~l*N7-oI5KyD%96RHP5%0%bWjl90<=xEf`p`5@ z`Go}`c}a0bY4n;CO@83r+G)Fykr@!Sd2)levri0!l=@<(mtjZ@?qrn1+e3`mf=<&x zR=nse$>Am~Oh(W<1dD;5J!MXPlS>CBENfI_uf889Jcs;gG0az^n$5I{ZXTlA*C$%` zoQuP<+-Fo`H6m*Co1JC)a$_BbW#_K%OXWX$R$svy-22Q9Pz^q~(DvhqzJCK}pU0Xx zbOIG~FB8Uyzl38Qw>qY$kBlg;aF5BD#hJ;j|RFu*Zbc#xJ(5^?h& zlFMeKG70r=bdOzr+KIkdTo(FHrQ=OA)9DsiGClHg`Knxp_LX-jq>t|FGd;~Uk3TIC zfsWI_&>b-R@M%=>Y50fZI9}aUsU29B9NhF(&fQyPwhqaUR;vVU(RS#i|BdQ6=_4sf zC+4qv0ToFe$>(DXKP4i(hOX1`FTG}$)ipIH*(vUeH}y-KL>^L0Ho^z%rRRmZ{#bEE zb&edW`BAO4br!@u-U3ssJdCQ1o)Aiq?cSfDA=k@~vbnkbF!OB4l1 zhQB$Y0{LGyS0qB{?A z7(CVYv~eT;9Oij}a?aniyv6h~UOaowc8RItQRCryGmn0JYCOoEsQD9^6)p%KwFFyykEgZ(HSQ z6}y^a{t_GeC_0_JPWUL=vwq^VGR)M?pTYgv(+(ziW?p_lWsSOA654jX{-!u)$MCt_ z?xd14L0=6})7rBixf(JKdTp~?9{BXu-3`&MA%W?g3uC5h0|}Wz7kZt>kYF%g;$|#1 z50B=e$FDE#)QJvE2}BE;F%|BX8vI>APGwl6lxg?M?SD0QUQtb_X&6V57KAuT1RZ5W zMXF#yFan7{OahL6M0x-!3Byu27&^r=Q6 zM5_z}aoJO49(hO=jvJNFGY|hEK?#dbx}DWn{|flIG5tG~kZALQ)Kfwqg5n!1~!O&+To!^GY$>?r9YO&`6+@99z1I@pnL(}lCXI>K>!ey3P?!u^4Nf2AsnJ@+S4-`ob*1!Cu&?UiPwW42 zxWLK{n}|J3at#Y_6Lb+5j_oS&9>1kdq&kb2WLFjIkMsxRX|8XhXX2d;phBE#yfF!# z)MaJf+tge*>zMa!KM@V8%z2|fg!$eyBgr=Jk*EBm)5RBH!Vdpudb&SIBU1)>~-zVolO82igClx&mmpU*BhLqbXgLeuZO! zInD=>l;6;{2E`T#Xj$;S_H_}CLK2s*K));Mr%MAGjQ50sjjjs_I}oycwx(`%!BW=F z$mw*OYkaYTL)X`)&=k>#v9`dNrgb(=$QsP+SG%fzx1aSDa!>YHBpqnO)Ou0)4l;cq_HzaiUo?d*4=P^t>*YoYz zS^>e4y7XrFOGj6Ki5GFGAfy6B{vXqnD|wt$X$UrkKPMz zGY)GIz!NR=7@};xxExO*XFC3TU#5jAG3MFNK2(1@ZA}ZBE&yFG7W_&+aL4TpC?1HX zgfoPXfBA0Nms-E~PQ`grS^jxT%H!h%XlS$_13#J!&POhomeii zPuWM9=lQwhp5Dl16kLRodOF(;0RVFkcE~)0Sw+ra+&eI4N=C0fIz&kjuO+!u7&j^N zkUlV5m?YJ4N`K5pbckCErmuoObg$wz3>bvV4ZU6%=?}F{fxOS68!muD*XEX;kwS@p zw8gB50PenRA1ED+HLEKIR$q#Hm4xc=*WbsdnBy}7X%ZvMU4JJ=yw}wJ#YUuGic~^Y z5OR19!bM=!5*d}?PE%+cJNc-lc|T>Jf?UZyrMNQtnC(29uqhB2Zs2|TLdEvJJAc1I zl?8{;{|JSxG+vu=Jq9^geh!KZ2&#;)Wytu=PaG?X*6uVo#=NyP;Aw;lQM6P}z;->S zDPinGul&~ESsZ<*mbO99NzkaJu9bTpv;<2^Al$SjU^{J#d+{3%goSKhYTMv_L2;-c z>Eg%eF4;7df{n&Z-RCK){ZSwE-pqH?niEZ}kC&{)>Ws7<{ ziy^P=$gtQA_jz0rv}hYmVu$fP>;I?9Koa@}%(zo7xs!El^=@@c3{KkT`BW^CH7HU? zYrV%8#lPEJA&^>WC+VvoSw1#>LZkkfJP~HCogB`t`T~uP_Bbp#87}hDzK9?HNmUj%g5$0JSpoKl6bhNG6ng@dV}-- zBs-m}Y4E@(jWRktTDw+2aTl$G9gqGTBOASnPpl}TwFli^1m`DD1gGGldJ;Bj1=P$i zmtPsy|4m~8%5hygwS9i&5uYt{)%`zWa?sW~rKzySvO1k8GI(@ypnMXsf8i}~)KxX1 z)0?D!hh5D;H1O??s0+`cb@)5MI7ND{fLy_?ZWn8nRE(f-Xj+!{Vg_jtF0@LR5Hvw2 z*e8`mO9ky2i?4XmnBJy$jvs8!3x>2pTN1W+H%9Ji(hde_Y55w@IYa;>rSufhIU%9% zcZqdb%XW>GEaB*?u}YEgf?XfUD9T0Gwxkq;q`>XKBm6yxzD8bf4dANcBt8%ftYv^? zZ!Ckh%y6!jepV5kv{I){*T*jYfQ$JV1a8o6scrl|1<1WHG0&;?>W_oI`Cy0xy@kW9 z;@P)0T=wf{NUk53J5d%eF50xxNdo4(g7p#ms~XK@%vDN@4*uqHVpzpCTlyc^jQf7W zB{Qblm>rHSH;rE`?0usgY)MUNRQE-QTB&LZ+-sm4RSm)o7R?UvMb=U2x|>z=HIl*x zoI%xT8x*X(9xwrmiF-i)TGbmenV%qyP|s$Vp0;dRSL&d4vk#?bJI*Nz-;y(V;c}-NCZaB*I@niJ!(j!JEefHi+~O6g)_1`c$L(_Ky$pf6h`9Z zz^;&rpwgLG;dbeaI1m{^crcH(=6mP(BY7tqRToW(9r;~JK5?(7Yh~H5fFkhcuL*z9 zu>u7l62;@NMOfGL(HW9f@irg9tuqXm^s!=IRht;ZpB7CMJM*3J-O4pc0rdM+MCDh# z@9m>Gp+E!~r-sgtCE{YQp7;n;?SWLG~AhS1D`qShF5YUwO#H;fU3#`l6y^(5+P z9}tpzVn27yg9>o8yPQW(jxX05x~`zWDBxa=H-d*DEpXLwL&#!5XvmZep$Q;II?nQ= z9fSNL>zra<4Lb#48j-m1w;^GT-;RtV)lrBLp1Q1rW9kiYgk4OoC!!FN9^KwQokM~6 zn#f7P6MI4;kUK7mVc=Yl*PG4(&ANSPTS_9Ibu_8;$t z1w~mwrR5fGWFbhm%hJ3ftT#b{h#*nKF zN@bvqrokyQ%Q8;-Y*_?Q&vwnvo8$kMV=O5{aSc3s9k|j;wodQ*K(#&v!ERnW@Q>%lPi)!0Vs2i8n z6M&5BEh%xC!z=d>gQp_GWX$B`q7Ryax2bsx8<1=H1@_-SWi9G|Eu#Omoc;d+pZ&)` w9X|j6U=5)5*NMGEyUuU_{%a6iKb|*bQ$X530Aj&>z5oCK literal 0 HcmV?d00001 diff --git a/quadratureshap_n8_summary.json b/quadratureshap_n8_summary.json new file mode 100644 index 000000000000..c7ec09f2ef9b --- /dev/null +++ b/quadratureshap_n8_summary.json @@ -0,0 +1,92 @@ +[ + { + "dataset": "breast_cancer", + "family": "real", + "depth": 30, + "points": 8, + "mean_nodes": 37.37, + "speedup": 0.4245682082044919, + "max_abs_diff": 4.76837158203125e-07 + }, + { + "dataset": "diabetes", + "family": "real", + "depth": 30, + "points": 8, + "mean_nodes": 191.01333333333332, + "speedup": 1.5975975642047238, + "max_abs_diff": 1.52587890625e-05 + }, + { + "dataset": "digits", + "family": "real", + "depth": 30, + "points": 8, + "mean_nodes": 46.516, + "speedup": 1.5759131851301158, + "max_abs_diff": 2.5033950805664062e-06 + }, + { + "dataset": "easy_linear", + "family": "synthetic", + "depth": 4, + "points": 8, + "mean_nodes": 30.61, + "speedup": 0.645007986369304, + "max_abs_diff": 1.9073486328125e-06 + }, + { + "dataset": "easy_linear", + "family": "synthetic", + "depth": 8, + "points": 8, + "mean_nodes": 370.47, + "speedup": 1.1889959234572578, + "max_abs_diff": 2.86102294921875e-06 + }, + { + "dataset": "easy_linear", + "family": "synthetic", + "depth": 16, + "points": 8, + "mean_nodes": 1467.43, + "speedup": 2.3738298777126716, + "max_abs_diff": 5.7220458984375e-06 + }, + { + "dataset": "easy_linear", + "family": "synthetic", + "depth": 30, + "points": 8, + "mean_nodes": 1928.48, + "speedup": 3.459982033859825, + "max_abs_diff": 0.0033998489379882812 + }, + { + "dataset": "random_labels", + "family": "synthetic", + "depth": 4, + "points": 8, + "mean_nodes": 30.54, + "speedup": 0.7424122662547481, + "max_abs_diff": 5.960464477539063e-08 + }, + { + "dataset": "random_labels", + "family": "synthetic", + "depth": 8, + "points": 8, + "mean_nodes": 321.22, + "speedup": 1.8998626925981703, + "max_abs_diff": 1.7881393432617188e-07 + }, + { + "dataset": "random_labels", + "family": "synthetic", + "depth": 16, + "points": 8, + "mean_nodes": 4273.45, + "speedup": 4.594703258820682, + "max_abs_diff": 2.8759241104125977e-06 + } +] From 16a5a592d7c7ee223a42daaca8a8f0576ca83b34 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Fri, 20 Mar 2026 04:05:22 -0700 Subject: [PATCH 05/13] Add CUDA QuadratureSHAP prototype --- src/predictor/gpu_predictor.cu | 37 ++- src/predictor/interpretability/shap.cu | 438 +++++++++++++++++++++++++ src/predictor/interpretability/shap.h | 4 + tests/cpp/predictor/test_shap.cu | 60 ++++ 4 files changed, 536 insertions(+), 3 deletions(-) diff --git a/src/predictor/gpu_predictor.cu b/src/predictor/gpu_predictor.cu index f4c0d00c6aeb..cbdcf1da9318 100644 --- a/src/predictor/gpu_predictor.cu +++ b/src/predictor/gpu_predictor.cu @@ -8,6 +8,7 @@ #include // for proclaim_return_type #include // for swap #include +#include #include "../collective/allreduce.h" #include "../common/bitfield.h" @@ -669,6 +670,28 @@ class GPUPredictor : public xgboost::Predictor { } } + void Configure(Args const& cfg) override { + for (auto const& kv : cfg) { + if (kv.first == "shap_algorithm") { + CHECK(kv.second == "treeshap" || kv.second == "quadratureshap") + << "Unknown SHAP algorithm: " << kv.second; + shap_algorithm_ = kv.second; + } else if (kv.first == "quadratureshap_points") { + std::size_t points{0}; + try { + points = std::stoul(kv.second); + } catch (std::invalid_argument const&) { + LOG(FATAL) << "Invalid quadratureshap_points: " << kv.second; + } catch (std::out_of_range const&) { + LOG(FATAL) << "quadratureshap_points out of range: " << kv.second; + } + CHECK_GE(points, 2) << "quadratureshap_points must be >= 2"; + CHECK_LE(points, 64) << "quadratureshap_points must be <= 64"; + quadrature_shap_points_ = points; + } + } + } + void PredictBatch(DMatrix* dmat, PredictionCacheEntry* predts, const gbm::GBTreeModel& model, bst_tree_t tree_begin, bst_tree_t tree_end = 0, std::vector const* tree_weights = nullptr) const override { @@ -766,14 +789,20 @@ class GPUPredictor : public xgboost::Predictor { void PredictContribution(DMatrix* p_fmat, HostDeviceVector* out_contribs, const gbm::GBTreeModel& model, bst_tree_t tree_end, - std::vector const* tree_weights, bool approximate, int, - unsigned) const override { + std::vector const* tree_weights, bool approximate, int condition, + unsigned condition_feature) const override { xgboost_NVTX_FN_RANGE(); if (approximate) { LOG(FATAL) << "Approximated contribution is not implemented in the GPU predictor, use CPU " "instead."; } - interpretability::ShapValues(ctx_, p_fmat, out_contribs, model, tree_end, tree_weights, 0, 0); + if (shap_algorithm_ == "quadratureshap" && condition == 0 && condition_feature == 0) { + interpretability::cuda_impl::QuadratureShapValues(ctx_, p_fmat, out_contribs, model, tree_end, + tree_weights, quadrature_shap_points_); + } else { + interpretability::ShapValues(ctx_, p_fmat, out_contribs, model, tree_end, tree_weights, + condition, condition_feature); + } } void PredictInteractionContributions(DMatrix* p_fmat, HostDeviceVector* out_contribs, @@ -832,6 +861,8 @@ class GPUPredictor : public xgboost::Predictor { private: ColumnSplitHelper column_split_helper_; + std::string shap_algorithm_{"treeshap"}; + std::size_t quadrature_shap_points_{16}; }; XGBOOST_REGISTER_PREDICTOR(GPUPredictor, "gpu_predictor") diff --git a/src/predictor/interpretability/shap.cu b/src/predictor/interpretability/shap.cu index 50d680f10a77..306844ff5b3f 100644 --- a/src/predictor/interpretability/shap.cu +++ b/src/predictor/interpretability/shap.cu @@ -10,6 +10,7 @@ #include #include +#include #include // for proclaim_return_type #include // for swap #include // for variant @@ -25,6 +26,7 @@ #include "../../common/cuda_context.cuh" // for CUDAContext #include "../../common/cuda_rt_utils.h" // for SetDevice #include "../../common/device_helpers.cuh" +#include "../../common/math.h" #include "../../common/nvtx_utils.h" #include "../../common/optional_weight.h" #include "../../data/batch_utils.h" // for StaticBatch @@ -53,6 +55,370 @@ using ::xgboost::cuda_impl::StaticBatch; using TreeViewVar = cuda::std::variant; +constexpr std::size_t kMaxGpuQuadraturePoints = 16; +constexpr std::size_t kMaxGpuQuadratureDepth = 64; +constexpr double kQuadratureShapQeps = 1e-15; +constexpr double kQuadratureShapUnseen = -999.0; + +struct QuadratureRule { + std::size_t points{0}; + std::array nodes{}; + std::array weights{}; +}; + +double LegendrePolynomial(std::size_t n, double x) { + double p0 = 1.0; + if (n == 0) { + return p0; + } + double p1 = x; + if (n == 1) { + return p1; + } + for (std::size_t k = 2; k <= n; ++k) { + auto kd = static_cast(k); + double pk = ((2.0 * kd - 1.0) * x * p1 - (kd - 1.0) * p0) / kd; + p0 = p1; + p1 = pk; + } + return p1; +} + +double LegendreDerivative(std::size_t n, double x, double pn) { + auto n_d = static_cast(n); + return n_d * (x * pn - LegendrePolynomial(n - 1, x)) / (x * x - 1.0); +} + +QuadratureRule MakeEndpointQuadrature(std::size_t n) { + CHECK_GE(n, 2); + CHECK_LE(n, kMaxGpuQuadraturePoints) << "GPU QuadratureSHAP currently supports up to " + << kMaxGpuQuadraturePoints << " quadrature points."; + + QuadratureRule rule; + rule.points = n; + std::vector> nodes_weights; + nodes_weights.reserve(n); + + for (std::size_t i = 0; i < n; ++i) { + double theta = M_PI * (static_cast(i) + 0.75) / (static_cast(n) + 0.5); + double x = std::cos(theta); + for (std::size_t iter = 0; iter < 64; ++iter) { + auto pn = LegendrePolynomial(n, x); + auto dpn = LegendreDerivative(n, x, pn); + auto dx = pn / dpn; + x -= dx; + if (std::abs(dx) < kQuadratureShapQeps) { + break; + } + } + + auto pn = LegendrePolynomial(n, x); + auto dpn = LegendreDerivative(n, x, pn); + auto w = 2.0 / ((1.0 - x * x) * dpn * dpn); + double s = 0.5 * (x + 1.0); + double ws = 0.5 * w; + nodes_weights.emplace_back(s * s, 2.0 * s * ws); + } + + std::sort(nodes_weights.begin(), nodes_weights.end(), + [](auto const& l, auto const& r) { return l.first < r.first; }); + for (std::size_t i = 0; i < n; ++i) { + rule.nodes[i] = nodes_weights[i].first; + rule.weights[i] = nodes_weights[i].second; + } + return rule; +} + +double FillRootMeanValue(tree::ScalarTreeView const& tree, bst_node_t nidx) { + if (tree.IsLeaf(nidx)) { + return tree.LeafValue(nidx); + } + auto left = tree.LeftChild(nidx); + auto right = tree.RightChild(nidx); + double result = FillRootMeanValue(tree, left) * tree.SumHess(left); + result += FillRootMeanValue(tree, right) * tree.SumHess(right); + result /= tree.SumHess(nidx); + return result; +} + +std::vector MakeTreeRootMeanValues(gbm::GBTreeModel const& model, bst_tree_t tree_end, + std::vector const* tree_weights) { + std::vector mean_values(tree_end); + for (bst_tree_t tree_idx = 0; tree_idx < tree_end; ++tree_idx) { + auto weight = tree_weights == nullptr ? 1.0f : (*tree_weights)[tree_idx]; + auto const tree = model.trees.at(tree_idx)->HostScView(); + mean_values[tree_idx] = static_cast(FillRootMeanValue(tree, RegTree::kRoot) * weight); + } + return mean_values; +} + +template +struct QuadratureFrame { + bst_node_t node{RegTree::kInvalidNodeId}; + bst_feature_t split_index{0}; + int path_len{0}; + std::uint8_t stage{0}; + double w_prod{1.0}; + double child_p_enter[2]{}; + double child_p_up[2]{}; + bst_node_t child_node[2]{RegTree::kInvalidNodeId, RegTree::kInvalidNodeId}; + double child_weight[2]{}; + double c_vals[MaxPoints]{}; + double h_vals[MaxPoints]{}; +}; + +template +XGBOOST_DEVICE void CopyQuadratureValues(double (&dst)[MaxPoints], double const (&src)[MaxPoints], + std::size_t points) { + for (std::size_t i = 0; i < points; ++i) { + dst[i] = src[i]; + } +} + +template +XGBOOST_DEVICE void AddQuadratureValues(double (&dst)[MaxPoints], double const (&src)[MaxPoints], + std::size_t points) { + for (std::size_t i = 0; i < points; ++i) { + dst[i] += src[i]; + } +} + +template +XGBOOST_DEVICE double ExtractQuadratureDelta(std::size_t points, double const* nodes, + double const* weights, + double const (&h_vals)[MaxPoints], double p_enter, + double p_exit) { + auto alpha_enter = p_enter - 1.0; + auto alpha_exit = p_exit - 1.0; + auto has_enter = p_enter != kQuadratureShapUnseen && fabs(alpha_enter) >= kQuadratureShapQeps; + auto has_exit = p_exit != kQuadratureShapUnseen && fabs(alpha_exit) >= kQuadratureShapQeps; + if (!has_enter && !has_exit) { + return 0.0; + } + + double acc = 0.0; + for (std::size_t i = 0; i < points; ++i) { + auto weighted_h = h_vals[i] * weights[i]; + if (has_enter) { + acc += alpha_enter * weighted_h / (1.0 + alpha_enter * nodes[i]); + } + if (has_exit) { + acc -= alpha_exit * weighted_h / (1.0 + alpha_exit * nodes[i]); + } + } + return acc; +} + +template +XGBOOST_DEVICE double FindPathProbability( + int path_len, bst_feature_t const (&path_features)[kMaxGpuQuadratureDepth], + double const (&path_p)[kMaxGpuQuadratureDepth], bst_feature_t split_index) { + for (int i = path_len - 1; i >= 0; --i) { + if (path_features[i] == split_index) { + return path_p[i]; + } + } + return kQuadratureShapUnseen; +} + +template +XGBOOST_DEVICE void QuadratureShapTree(tree::ScalarTreeView const& tree, Loader const& loader, + bst_idx_t ridx, std::size_t points, double const* nodes, + double const* weights, float tree_weight, float* out_row) { + auto const cats = tree.GetCategoriesMatrix(); + QuadratureFrame frames[kMaxGpuQuadratureDepth]; + bst_feature_t path_features[kMaxGpuQuadratureDepth]; + double path_p[kMaxGpuQuadratureDepth]; + double ret_h[MaxPoints]{}; + int stack_size = 1; + bool have_return = false; + + frames[0].node = RegTree::kRoot; + frames[0].path_len = 0; + frames[0].stage = 0; + frames[0].w_prod = 1.0; + for (std::size_t i = 0; i < points; ++i) { + frames[0].c_vals[i] = 1.0; + } + + while (stack_size > 0 || have_return) { + if (have_return) { + if (stack_size == 0) { + break; + } + auto& parent = frames[stack_size - 1]; + auto child_idx = parent.stage - 1; + out_row[parent.split_index] += static_cast(ExtractQuadratureDelta( + points, nodes, weights, ret_h, parent.child_p_enter[child_idx], + parent.child_p_up[child_idx])); + if (child_idx == 0) { + CopyQuadratureValues(parent.h_vals, ret_h, points); + parent.stage = 2; + } else { + AddQuadratureValues(parent.h_vals, ret_h, points); + CopyQuadratureValues(ret_h, parent.h_vals, points); + --stack_size; + } + have_return = (child_idx == 1); + continue; + } + + auto& frame = frames[stack_size - 1]; + if (tree.IsLeaf(frame.node)) { + auto leaf_value = static_cast(tree.LeafValue(frame.node) * tree_weight); + for (std::size_t i = 0; i < points; ++i) { + ret_h[i] = frame.c_vals[i] * frame.w_prod * leaf_value; + } + --stack_size; + have_return = true; + continue; + } + + if (frame.stage == 2) { + auto child_slot = frame.path_len; + if (child_slot >= static_cast(kMaxGpuQuadratureDepth)) { + return; + } + auto child = 1; + auto child_node = frame.child_node[child]; + auto child_weight = frame.child_weight[child]; + auto p_old = + FindPathProbability(frame.path_len, path_features, path_p, frame.split_index); + double p_e = 0.0; + double p_up = 0.0; + auto satisfies = child_node == tree.RightChild(frame.node) + ? !(predictor::GetNextNode( + tree, frame.node, loader.GetElement(ridx, frame.split_index), + common::CheckNAN(loader.GetElement(ridx, frame.split_index)), + cats) == tree.LeftChild(frame.node)) + : predictor::GetNextNode( + tree, frame.node, loader.GetElement(ridx, frame.split_index), + common::CheckNAN(loader.GetElement(ridx, frame.split_index)), + cats) == tree.LeftChild(frame.node); + if (p_old == kQuadratureShapUnseen) { + p_e = satisfies ? 1.0 / child_weight : 0.0; + p_up = 1.0; + } else if (fabs(p_old) < kQuadratureShapQeps) { + p_e = 0.0; + p_up = 0.0; + } else { + p_e = satisfies ? p_old / child_weight : 0.0; + p_up = p_old; + } + + path_features[child_slot] = frame.split_index; + path_p[child_slot] = p_e; + frame.child_p_enter[child] = p_e; + frame.child_p_up[child] = p_up; + + auto& child_frame = frames[stack_size++]; + child_frame.node = child_node; + child_frame.path_len = frame.path_len + 1; + child_frame.stage = 0; + child_frame.w_prod = frame.w_prod * child_weight; + auto alpha_e = p_e - 1.0; + auto alpha_old = p_old - 1.0; + auto has_old = p_old != kQuadratureShapUnseen && fabs(alpha_old) >= kQuadratureShapQeps; + for (std::size_t i = 0; i < points; ++i) { + auto v = frame.c_vals[i] * (1.0 + alpha_e * nodes[i]); + if (has_old) { + v /= 1.0 + alpha_old * nodes[i]; + } + child_frame.c_vals[i] = v; + } + continue; + } + + auto split_index = tree.SplitIndex(frame.node); + auto fvalue = loader.GetElement(ridx, split_index); + auto is_missing = common::CheckNAN(fvalue); + auto next = predictor::GetNextNode(tree, frame.node, fvalue, is_missing, cats); + auto left = tree.LeftChild(frame.node); + auto right = tree.RightChild(frame.node); + auto parent_cover = static_cast(tree.SumHess(frame.node)); + if (!(parent_cover > 0.0)) { + return; + } + + frame.split_index = split_index; + frame.child_node[0] = left; + frame.child_node[1] = right; + frame.child_weight[0] = static_cast(tree.SumHess(left)) / parent_cover; + frame.child_weight[1] = static_cast(tree.SumHess(right)) / parent_cover; + frame.stage = 1; + + auto child_slot = frame.path_len; + if (child_slot >= static_cast(kMaxGpuQuadratureDepth)) { + return; + } + auto child = 0; + auto child_node = frame.child_node[child]; + auto child_weight = frame.child_weight[child]; + auto p_old = + FindPathProbability(frame.path_len, path_features, path_p, frame.split_index); + double p_e = 0.0; + double p_up = 0.0; + auto satisfies = next == child_node; + if (p_old == kQuadratureShapUnseen) { + p_e = satisfies ? 1.0 / child_weight : 0.0; + p_up = 1.0; + } else if (fabs(p_old) < kQuadratureShapQeps) { + p_e = 0.0; + p_up = 0.0; + } else { + p_e = satisfies ? p_old / child_weight : 0.0; + p_up = p_old; + } + + path_features[child_slot] = frame.split_index; + path_p[child_slot] = p_e; + frame.child_p_enter[child] = p_e; + frame.child_p_up[child] = p_up; + + auto& child_frame = frames[stack_size++]; + child_frame.node = child_node; + child_frame.path_len = frame.path_len + 1; + child_frame.stage = 0; + child_frame.w_prod = frame.w_prod * child_weight; + auto alpha_e = p_e - 1.0; + auto alpha_old = p_old - 1.0; + auto has_old = p_old != kQuadratureShapUnseen && fabs(alpha_old) >= kQuadratureShapQeps; + for (std::size_t i = 0; i < points; ++i) { + auto v = frame.c_vals[i] * (1.0 + alpha_e * nodes[i]); + if (has_old) { + v /= 1.0 + alpha_old * nodes[i]; + } + child_frame.c_vals[i] = v; + } + } +} + +template +void LaunchQuadratureShap(Context const* ctx, Loader loader, bst_idx_t base_rowid, + gbm::GBTreeModel const& model, ModelView const& d_model, + common::Span root_mean_values, + common::OptionalWeights tree_weights, std::size_t points, + common::Span nodes, common::Span weights, + HostDeviceVector* out_contribs) { + auto const ngroup = model.learner_model_param->num_output_group; + auto const ncolumns = model.learner_model_param->num_feature + 1; + auto d_trees = d_model.Trees(); + auto d_tree_groups = d_model.tree_groups; + auto phis = out_contribs->DeviceSpan(); + + dh::LaunchN(loader.NumRows(), ctx->CUDACtx()->Stream(), [=] __device__(std::size_t ridx) { + auto row_idx = base_rowid + static_cast(ridx); + for (bst_tree_t tree_idx = 0; tree_idx < d_trees.size(); ++tree_idx) { + auto const& d_tree = cuda::std::get(d_trees[tree_idx]); + auto gid = d_tree_groups[tree_idx]; + auto out_row = phis.data() + (row_idx * ngroup + gid) * ncolumns; + out_row[ncolumns - 1] += root_mean_values[tree_idx]; + QuadratureShapTree(d_tree, loader, ridx, points, nodes.data(), weights.data(), + tree_weights[tree_idx], out_row); + } + }); +} + struct CopyViews { Context const* ctx; explicit CopyViews(Context const* ctx) : ctx{ctx} {} @@ -370,6 +736,78 @@ void ShapValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* ou }); } +void QuadratureShapValues(Context const* ctx, DMatrix* p_fmat, + HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, + bst_tree_t tree_end, std::vector const* tree_weights, + std::size_t quadrature_points) { + xgboost_NVTX_FN_RANGE(); + CHECK(!model.learner_model_param->IsVectorLeaf()) << "Predict contribution" << MTNotImplemented(); + CHECK(!p_fmat->Info().IsColumnSplit()) + << "Predict contribution support for column-wise data split is not yet implemented."; + CHECK_LE(quadrature_points, kMaxGpuQuadraturePoints) + << "GPU QuadratureSHAP currently supports up to " << kMaxGpuQuadraturePoints + << " quadrature points."; + + tree_end = predictor::GetTreeLimit(model.trees, tree_end); + auto const ngroup = model.learner_model_param->num_output_group; + CHECK_NE(ngroup, 0); + auto const ncolumns = model.learner_model_param->num_feature + 1; + auto dim_size = ncolumns * ngroup; + out_contribs->SetDevice(ctx->Device()); + out_contribs->Resize(p_fmat->Info().num_row_ * dim_size); + out_contribs->Fill(0.0f); + + bst_node_t max_depth = 0; + for (bst_tree_t tree_idx = 0; tree_idx < tree_end; ++tree_idx) { + CHECK(!model.trees[tree_idx]->IsMultiTarget()) << "Predict contribution" << MTNotImplemented(); + max_depth = std::max(max_depth, model.trees[tree_idx]->MaxDepth()); + } + CHECK_LE(max_depth + 1, static_cast(kMaxGpuQuadratureDepth)) + << "GPU QuadratureSHAP currently supports trees of depth up to " + << (kMaxGpuQuadratureDepth - 1) << "."; + + auto rule = MakeEndpointQuadrature(quadrature_points); + dh::device_vector d_nodes(rule.nodes.begin(), rule.nodes.begin() + quadrature_points); + dh::device_vector d_weights(rule.weights.begin(), + rule.weights.begin() + quadrature_points); + + auto h_root_means = MakeTreeRootMeanValues(model, tree_end, tree_weights); + dh::device_vector d_root_means(h_root_means.cbegin(), h_root_means.cend()); + + DeviceModel d_model{ctx->Device(), model, true, 0, tree_end, CopyViews{ctx}}; + dh::device_vector d_tree_weights; + auto weights_opt = common::OptionalWeights{1.0f}; + if (tree_weights != nullptr) { + d_tree_weights.assign(tree_weights->cbegin(), tree_weights->cbegin() + tree_end); + weights_opt = common::OptionalWeights{common::Span{ + thrust::raw_pointer_cast(d_tree_weights.data()), d_tree_weights.size()}}; + } + + auto new_enc = + p_fmat->Cats()->NeedRecode() ? p_fmat->Cats()->DeviceView(ctx) : enc::DeviceColumnsView{}; + auto root_means = + common::Span{thrust::raw_pointer_cast(d_root_means.data()), d_root_means.size()}; + auto nodes = common::Span{thrust::raw_pointer_cast(d_nodes.data()), d_nodes.size()}; + auto weights = + common::Span{thrust::raw_pointer_cast(d_weights.data()), d_weights.size()}; + + LaunchShap(ctx, p_fmat, new_enc, model, [&](auto&& loader, bst_idx_t base_rowid) { + LaunchQuadratureShap(ctx, loader, base_rowid, model, d_model, + root_means, weights_opt, quadrature_points, nodes, + weights, out_contribs); + }); + + p_fmat->Info().base_margin_.SetDevice(ctx->Device()); + auto margin = p_fmat->Info().base_margin_.Data()->ConstDeviceSpan(); + auto base_score = model.learner_model_param->BaseScore(ctx); + auto phis = out_contribs->DeviceSpan(); + auto n_samples = p_fmat->Info().num_row_; + dh::LaunchN(n_samples * ngroup, ctx->CUDACtx()->Stream(), [=] __device__(std::size_t idx) { + auto [_, gid] = linalg::UnravelIndex(idx, n_samples, ngroup); + phis[(idx + 1) * ncolumns - 1] += margin.empty() ? base_score(gid) : margin[idx]; + }); +} + void ShapInteractionValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, bst_tree_t tree_end, std::vector const* tree_weights, diff --git a/src/predictor/interpretability/shap.h b/src/predictor/interpretability/shap.h index bcbc0be02e5c..c61c24c05401 100644 --- a/src/predictor/interpretability/shap.h +++ b/src/predictor/interpretability/shap.h @@ -39,6 +39,10 @@ namespace cuda_impl { void ShapValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, bst_tree_t tree_end, std::vector const* tree_weights, int condition, unsigned condition_feature); +void QuadratureShapValues(Context const* ctx, DMatrix* p_fmat, + HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, + bst_tree_t tree_end, std::vector const* tree_weights, + std::size_t quadrature_points); void ApproxFeatureImportance(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, bst_tree_t tree_end, std::vector const* tree_weights); diff --git a/tests/cpp/predictor/test_shap.cu b/tests/cpp/predictor/test_shap.cu index ea9039409be7..5f3119bb02b1 100644 --- a/tests/cpp/predictor/test_shap.cu +++ b/tests/cpp/predictor/test_shap.cu @@ -85,6 +85,66 @@ TEST(GPUPredictor, ShapOutputCasesGPU) { } } +TEST(GPUPredictor, CompareCPUQuadratureShap) { + auto ctx = MakeCUDACtx(0); + Context cpu_ctx; + bst_feature_t constexpr kCols{10}; + bst_idx_t constexpr kRows{256}; + std::size_t constexpr kIters{8}; + + HostDeviceVector predictions; + HostDeviceVector cpu_predictions; + + auto dmat = RandomDataGenerator(kRows, kCols, 0.0).Device(ctx.Device()).GenerateDMatrix(); + dmat->Info().labels.Reshape(kRows, 1); + auto& h_labels = dmat->Info().labels.Data()->HostVector(); + for (size_t i = 0; i < kRows; ++i) { + h_labels[i] = i % 2; + } + + std::unique_ptr learner{Learner::Create({dmat})}; + learner->SetParams(Args{{"objective", "binary:logistic"}, + {"tree_method", "hist"}, + {"max_depth", "8"}, + {"min_split_loss", "0"}, + {"min_child_weight", "0"}, + {"reg_lambda", "0"}, + {"reg_alpha", "0"}, + {"subsample", "1"}, + {"colsample_bytree", "1"}, + {"device", ctx.DeviceName()}}); + learner->Configure(); + for (std::size_t i = 0; i < kIters; ++i) { + learner->UpdateOneIter(i, dmat); + } + + Json model{Object{}}; + learner->SaveModel(&model); + + std::unique_ptr learner_gpu{Learner::Create({})}; + learner_gpu->LoadModel(model); + learner_gpu->SetParam("device", ctx.DeviceName()); + learner_gpu->SetParam("shap_algorithm", "quadratureshap"); + learner_gpu->SetParam("quadratureshap_points", "8"); + learner_gpu->Configure(); + + std::unique_ptr learner_cpu{Learner::Create({})}; + learner_cpu->LoadModel(model); + learner_cpu->SetParam("device", cpu_ctx.DeviceName()); + learner_cpu->SetParam("shap_algorithm", "quadratureshap"); + learner_cpu->SetParam("quadratureshap_points", "8"); + learner_cpu->Configure(); + + learner_gpu->Predict(dmat, false, &predictions, 0, 0, false, false, true, false, false); + learner_cpu->Predict(dmat, false, &cpu_predictions, 0, 0, false, false, true, false, false); + auto& phis = predictions.HostVector(); + auto& cpu_phis = cpu_predictions.HostVector(); + ASSERT_EQ(cpu_phis.size(), phis.size()); + for (auto i = 0ull; i < phis.size(); ++i) { + EXPECT_NEAR(cpu_phis[i], phis[i], 1e-4); + } +} + TEST(GPUPredictor, DartShapOutputGPU) { auto ctx = MakeCUDACtx(0); CheckDartShapOutput(&ctx); From 2b8adee0c60decd8fc1bed09b05710e47bfe0be1 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Mon, 30 Mar 2026 03:44:04 -0700 Subject: [PATCH 06/13] Refine GPU QuadratureSHAP kernel --- src/predictor/gpu_data_accessor.cuh | 17 +- src/predictor/interpretability/quadrature.h | 97 ++ src/predictor/interpretability/shap.cc | 82 +- src/predictor/interpretability/shap.cu | 949 +++++++++++++------- 4 files changed, 733 insertions(+), 412 deletions(-) create mode 100644 src/predictor/interpretability/quadrature.h diff --git a/src/predictor/gpu_data_accessor.cuh b/src/predictor/gpu_data_accessor.cuh index 8fee7a149f08..4fcdb58c9662 100644 --- a/src/predictor/gpu_data_accessor.cuh +++ b/src/predictor/gpu_data_accessor.cuh @@ -32,14 +32,17 @@ struct SparsePageView { num_features{n_features} {} [[nodiscard]] __device__ float GetElement(size_t ridx, size_t fidx) const { - // Binary search - auto begin_ptr = d_data.begin() + d_row_ptr[ridx]; - auto end_ptr = d_data.begin() + d_row_ptr[ridx + 1]; - if (end_ptr - begin_ptr == this->NumCols()) { - // Bypass span check for dense data - return d_data.data()[d_row_ptr[ridx] + fidx].fvalue; + auto row_begin = d_row_ptr[ridx]; + auto row_size = d_row_ptr[ridx + 1] - row_begin; + auto begin_ptr = d_data.data() + row_begin; + if (row_size == this->NumCols()) { + // Dense rows are laid out in feature order, so this is a raw-pointer lookup. + return begin_ptr[fidx].fvalue; } - common::Span::iterator previous_middle; + + // Binary search over sparse entries using raw pointers. + auto end_ptr = begin_ptr + row_size; + auto previous_middle = static_cast(nullptr); while (end_ptr != begin_ptr) { auto middle = begin_ptr + (end_ptr - begin_ptr) / 2; if (middle == previous_middle) { diff --git a/src/predictor/interpretability/quadrature.h b/src/predictor/interpretability/quadrature.h new file mode 100644 index 000000000000..34469622372e --- /dev/null +++ b/src/predictor/interpretability/quadrature.h @@ -0,0 +1,97 @@ +/** + * Copyright 2017-2026, XGBoost Contributors + */ +#ifndef XGBOOST_PREDICTOR_INTERPRETABILITY_QUADRATURE_H_ +#define XGBOOST_PREDICTOR_INTERPRETABILITY_QUADRATURE_H_ + +#include +#include +#include +#include +#include +#include + +#include "xgboost/logging.h" + +namespace xgboost::interpretability::detail { + +template +struct EndpointQuadratureRule { + std::size_t points{0}; + std::array nodes{}; + std::array weights{}; +}; + +inline double LegendrePolynomial(std::size_t n, double x) { + double p0 = 1.0; + if (n == 0) { + return p0; + } + double p1 = x; + if (n == 1) { + return p1; + } + for (std::size_t k = 2; k <= n; ++k) { + double pk = + ((2.0 * static_cast(k) - 1.0) * x * p1 - (static_cast(k) - 1.0) * p0) / + static_cast(k); + p0 = p1; + p1 = pk; + } + return p1; +} + +inline double LegendreDerivative(std::size_t n, double x, double pn) { + auto n_d = static_cast(n); + return n_d * (x * pn - LegendrePolynomial(n - 1, x)) / (x * x - 1.0); +} + +template +inline EndpointQuadratureRule MakeEndpointQuadrature(std::size_t n, + double convergence_eps) { + CHECK_GE(n, 2); + CHECK_LE(n, MaxPoints); + + EndpointQuadratureRule rule; + rule.points = n; + std::vector> nodes_weights; + nodes_weights.reserve(n); + + for (std::size_t i = 0; i < n; ++i) { + double theta = M_PI * (static_cast(i) + 0.75) / (static_cast(n) + 0.5); + double x = std::cos(theta); + for (std::size_t iter = 0; iter < 64; ++iter) { + auto pn = LegendrePolynomial(n, x); + auto dpn = LegendreDerivative(n, x, pn); + auto dx = pn / dpn; + x -= dx; + if (std::abs(dx) < convergence_eps) { + break; + } + } + + auto pn = LegendrePolynomial(n, x); + auto dpn = LegendreDerivative(n, x, pn); + auto w = 2.0 / ((1.0 - x * x) * dpn * dpn); + double s = 0.5 * (x + 1.0); + double ws = 0.5 * w; + nodes_weights.emplace_back(s * s, 2.0 * s * ws); + } + + std::sort(nodes_weights.begin(), nodes_weights.end(), + [](auto const &l, auto const &r) { return l.first < r.first; }); + for (std::size_t i = 0; i < n; ++i) { + rule.nodes[i] = nodes_weights[i].first; + rule.weights[i] = nodes_weights[i].second; + } + return rule; +} + +template +inline EndpointQuadratureRule MakeEndpointQuadrature(double convergence_eps) { + return MakeEndpointQuadrature(Points, convergence_eps); +} + +} // namespace xgboost::interpretability::detail + +#endif // XGBOOST_PREDICTOR_INTERPRETABILITY_QUADRATURE_H_ diff --git a/src/predictor/interpretability/shap.cc b/src/predictor/interpretability/shap.cc index 4e04d5e5fc61..66efb5b48a0c 100644 --- a/src/predictor/interpretability/shap.cc +++ b/src/predictor/interpretability/shap.cc @@ -19,9 +19,10 @@ #include "../data_accessor.h" // for GHistIndexMatrixView #include "../predict_fn.h" // for GetTreeLimit #include "dmlc/omp.h" // for omp_get_thread_num -#include "xgboost/base.h" // for bst_omp_uint -#include "xgboost/logging.h" // for CHECK -#include "xgboost/tree_model.h" // for MTNotImplemented +#include "quadrature.h" +#include "xgboost/base.h" // for bst_omp_uint +#include "xgboost/logging.h" // for CHECK +#include "xgboost/tree_model.h" // for MTNotImplemented namespace xgboost::interpretability { namespace { @@ -242,84 +243,19 @@ constexpr std::size_t kDefaultQuadratureShapPoints = 16; constexpr std::size_t kMaxQuadratureShapPoints = 64; constexpr double kQuadratureShapQeps = 1e-15; constexpr double kQuadratureShapUnseen = -999.0; -struct QuadratureRule { - std::size_t points{0}; - std::array nodes{}; - std::array weights{}; -}; - +using QuadratureRule = detail::EndpointQuadratureRule; using QuadratureBuffer = std::array; -double LegendrePolynomial(std::size_t n, double x) { - double p0 = 1.0; - if (n == 0) { - return p0; - } - double p1 = x; - if (n == 1) { - return p1; - } - for (std::size_t k = 2; k <= n; ++k) { - double pk = - ((2.0 * static_cast(k) - 1.0) * x * p1 - (static_cast(k) - 1.0) * p0) / - static_cast(k); - p0 = p1; - p1 = pk; - } - return p1; -} - -double LegendreDerivative(std::size_t n, double x, double pn) { - auto n_d = static_cast(n); - return n_d * (x * pn - LegendrePolynomial(n - 1, x)) / (x * x - 1.0); -} - -QuadratureRule MakeEndpointQuadrature(std::size_t n) { - CHECK_GE(n, 2); - CHECK_LE(n, kMaxQuadratureShapPoints); - - QuadratureRule rule; - rule.points = n; - std::vector> nodes_weights; - nodes_weights.reserve(n); - - for (std::size_t i = 0; i < n; ++i) { - double theta = M_PI * (static_cast(i) + 0.75) / (static_cast(n) + 0.5); - double x = std::cos(theta); - for (std::size_t iter = 0; iter < 64; ++iter) { - auto pn = LegendrePolynomial(n, x); - auto dpn = LegendreDerivative(n, x, pn); - auto dx = pn / dpn; - x -= dx; - if (std::abs(dx) < kQuadratureShapQeps) { - break; - } - } - - auto pn = LegendrePolynomial(n, x); - auto dpn = LegendreDerivative(n, x, pn); - auto w = 2.0 / ((1.0 - x * x) * dpn * dpn); - double s = 0.5 * (x + 1.0); - double ws = 0.5 * w; - nodes_weights.emplace_back(s * s, 2.0 * s * ws); - } - - std::sort(nodes_weights.begin(), nodes_weights.end(), - [](auto const &l, auto const &r) { return l.first < r.first; }); - for (std::size_t i = 0; i < n; ++i) { - rule.nodes[i] = nodes_weights[i].first; - rule.weights[i] = nodes_weights[i].second; - } - return rule; -} - QuadratureRule const &GetQuadratureRule(std::size_t n) { static std::mutex cache_mutex; static std::unordered_map cache; std::lock_guard guard{cache_mutex}; auto it = cache.find(n); if (it == cache.cend()) { - it = cache.emplace(n, MakeEndpointQuadrature(n)).first; + it = cache + .emplace(n, detail::MakeEndpointQuadrature( + n, kQuadratureShapQeps)) + .first; } return it->second; } diff --git a/src/predictor/interpretability/shap.cu b/src/predictor/interpretability/shap.cu index 306844ff5b3f..cf123813e2a5 100644 --- a/src/predictor/interpretability/shap.cu +++ b/src/predictor/interpretability/shap.cu @@ -7,10 +7,13 @@ #include #include #include +#include #include #include +#include #include +#include #include // for proclaim_return_type #include // for swap #include // for variant @@ -38,6 +41,7 @@ #include "../gbtree_view.h" #include "../gpu_data_accessor.cuh" #include "../predict_fn.h" // for GetTreeLimit +#include "quadrature.h" #include "shap.h" #include "xgboost/data.h" #include "xgboost/host_device_vector.h" @@ -55,79 +59,13 @@ using ::xgboost::cuda_impl::StaticBatch; using TreeViewVar = cuda::std::variant; -constexpr std::size_t kMaxGpuQuadraturePoints = 16; +constexpr std::size_t kGpuQuadraturePoints = 8; constexpr std::size_t kMaxGpuQuadratureDepth = 64; +constexpr std::size_t kGpuQuadratureRowsPerWarp = 4; +constexpr std::size_t kGpuQuadratureTreeBlockThreads = 64; +constexpr std::array kGpuQuadratureDepthBuckets{{16, 32, 64}}; constexpr double kQuadratureShapQeps = 1e-15; -constexpr double kQuadratureShapUnseen = -999.0; - -struct QuadratureRule { - std::size_t points{0}; - std::array nodes{}; - std::array weights{}; -}; - -double LegendrePolynomial(std::size_t n, double x) { - double p0 = 1.0; - if (n == 0) { - return p0; - } - double p1 = x; - if (n == 1) { - return p1; - } - for (std::size_t k = 2; k <= n; ++k) { - auto kd = static_cast(k); - double pk = ((2.0 * kd - 1.0) * x * p1 - (kd - 1.0) * p0) / kd; - p0 = p1; - p1 = pk; - } - return p1; -} - -double LegendreDerivative(std::size_t n, double x, double pn) { - auto n_d = static_cast(n); - return n_d * (x * pn - LegendrePolynomial(n - 1, x)) / (x * x - 1.0); -} - -QuadratureRule MakeEndpointQuadrature(std::size_t n) { - CHECK_GE(n, 2); - CHECK_LE(n, kMaxGpuQuadraturePoints) << "GPU QuadratureSHAP currently supports up to " - << kMaxGpuQuadraturePoints << " quadrature points."; - - QuadratureRule rule; - rule.points = n; - std::vector> nodes_weights; - nodes_weights.reserve(n); - - for (std::size_t i = 0; i < n; ++i) { - double theta = M_PI * (static_cast(i) + 0.75) / (static_cast(n) + 0.5); - double x = std::cos(theta); - for (std::size_t iter = 0; iter < 64; ++iter) { - auto pn = LegendrePolynomial(n, x); - auto dpn = LegendreDerivative(n, x, pn); - auto dx = pn / dpn; - x -= dx; - if (std::abs(dx) < kQuadratureShapQeps) { - break; - } - } - - auto pn = LegendrePolynomial(n, x); - auto dpn = LegendreDerivative(n, x, pn); - auto w = 2.0 / ((1.0 - x * x) * dpn * dpn); - double s = 0.5 * (x + 1.0); - double ws = 0.5 * w; - nodes_weights.emplace_back(s * s, 2.0 * s * ws); - } - - std::sort(nodes_weights.begin(), nodes_weights.end(), - [](auto const& l, auto const& r) { return l.first < r.first; }); - for (std::size_t i = 0; i < n; ++i) { - rule.nodes[i] = nodes_weights[i].first; - rule.weights[i] = nodes_weights[i].second; - } - return rule; -} +using QuadratureRule = detail::EndpointQuadratureRule; double FillRootMeanValue(tree::ScalarTreeView const& tree, bst_node_t nidx) { if (tree.IsLeaf(nidx)) { @@ -141,282 +79,614 @@ double FillRootMeanValue(tree::ScalarTreeView const& tree, bst_node_t nidx) { return result; } -std::vector MakeTreeRootMeanValues(gbm::GBTreeModel const& model, bst_tree_t tree_end, - std::vector const* tree_weights) { - std::vector mean_values(tree_end); +std::vector MakeGroupRootMeanSums(gbm::GBTreeModel const& model, bst_tree_t tree_end, + std::vector const* tree_weights) { + auto h_tree_groups = model.TreeGroups(DeviceOrd::CPU()); + auto n_groups = model.learner_model_param->num_output_group; + std::vector h_group_root_mean_sums(n_groups, 0.0); for (bst_tree_t tree_idx = 0; tree_idx < tree_end; ++tree_idx) { - auto weight = tree_weights == nullptr ? 1.0f : (*tree_weights)[tree_idx]; + auto const weight = tree_weights == nullptr ? 1.0f : (*tree_weights)[tree_idx]; auto const tree = model.trees.at(tree_idx)->HostScView(); - mean_values[tree_idx] = static_cast(FillRootMeanValue(tree, RegTree::kRoot) * weight); + h_group_root_mean_sums[h_tree_groups[tree_idx]] += + FillRootMeanValue(tree, RegTree::kRoot) * weight; } - return mean_values; + + std::vector out(h_group_root_mean_sums.size()); + std::transform(h_group_root_mean_sums.cbegin(), h_group_root_mean_sums.cend(), out.begin(), + [](double v) { return static_cast(v); }); + return out; } -template -struct QuadratureFrame { - bst_node_t node{RegTree::kInvalidNodeId}; - bst_feature_t split_index{0}; - int path_len{0}; - std::uint8_t stage{0}; - double w_prod{1.0}; - double child_p_enter[2]{}; - double child_p_up[2]{}; - bst_node_t child_node[2]{RegTree::kInvalidNodeId, RegTree::kInvalidNodeId}; - double child_weight[2]{}; - double c_vals[MaxPoints]{}; - double h_vals[MaxPoints]{}; +struct CompressedNode { + bst_node_t left{RegTree::kInvalidNodeId}; + bst_node_t right{RegTree::kInvalidNodeId}; + bst_feature_t split_global{0}; + float split_cond{0}; + float leaf_value{0}; + float left_weight{0}; + float right_weight{0}; + std::uint8_t default_left{0}; + std::uint8_t is_leaf{0}; + std::uint8_t prev_same_offset_plus1{0}; }; -template -XGBOOST_DEVICE void CopyQuadratureValues(double (&dst)[MaxPoints], double const (&src)[MaxPoints], - std::size_t points) { - for (std::size_t i = 0; i < points; ++i) { - dst[i] = src[i]; +struct CompressedTree { + std::uint32_t node_begin{0}; + bst_target_t group{0}; +}; + +struct CompressedModel { + dh::device_vector trees; + dh::device_vector nodes; +}; + +std::size_t DepthBucketIndex(std::size_t path_depth) { + for (std::size_t i = 0; i < kGpuQuadratureDepthBuckets.size(); ++i) { + if (path_depth <= kGpuQuadratureDepthBuckets[i]) { + return i; + } } + LOG(FATAL) << "GPU QuadratureSHAP currently supports trees of depth up to " + << (kMaxGpuQuadratureDepth - 1) << "."; + return kGpuQuadratureDepthBuckets.size() - 1; } -template -XGBOOST_DEVICE void AddQuadratureValues(double (&dst)[MaxPoints], double const (&src)[MaxPoints], - std::size_t points) { - for (std::size_t i = 0; i < points; ++i) { - dst[i] += src[i]; +CompressedModel MakeCompressedModel(Context const* ctx, gbm::GBTreeModel const& model, + std::vector const& tree_indices, + std::vector const* tree_weights) { + std::vector h_trees; + std::vector h_nodes; + auto h_tree_groups = model.TreeGroups(DeviceOrd::CPU()); + static_cast(ctx); + + h_trees.reserve(tree_indices.size()); + for (auto tree_idx : tree_indices) { + auto const weight = tree_weights == nullptr ? 1.0f : (*tree_weights)[tree_idx]; + auto const tree = model.trees.at(tree_idx)->HostScView(); + CHECK(!tree.HasCategoricalSplit()) + << "GPU QuadratureSHAP prototype does not support categorical splits."; + + auto node_begin = h_nodes.size(); + h_nodes.resize(node_begin + tree.Size()); + + for (bst_node_t nidx = 0; nidx < tree.Size(); ++nidx) { + auto& out = h_nodes[node_begin + nidx]; + if (tree.IsLeaf(nidx)) { + out.is_leaf = 1; + out.leaf_value = tree.LeafValue(nidx) * weight; + continue; + } + + auto left = tree.LeftChild(nidx); + auto right = tree.RightChild(nidx); + auto parent_cover = static_cast(tree.SumHess(nidx)); + CHECK_GT(parent_cover, 0.0); + + out.left = left; + out.right = right; + out.split_global = tree.SplitIndex(nidx); + out.split_cond = tree.SplitCond(nidx); + out.left_weight = static_cast(static_cast(tree.SumHess(left)) / parent_cover); + out.right_weight = + static_cast(static_cast(tree.SumHess(right)) / parent_cover); + out.default_left = tree.DefaultLeft(nidx); + out.is_leaf = 0; + } + + for (bst_node_t nidx = 0; nidx < tree.Size(); ++nidx) { + auto& out = h_nodes[node_begin + nidx]; + if (out.is_leaf) { + continue; + } + + std::uint8_t prev_same_offset_plus1 = 0; + std::uint16_t distance = 0; + auto ancestor = nidx; + while (!tree.IsRoot(ancestor)) { + ancestor = tree.Parent(ancestor); + ++distance; + auto const& ancestor_node = h_nodes[node_begin + ancestor]; + if (!ancestor_node.is_leaf && ancestor_node.split_global == out.split_global) { + prev_same_offset_plus1 = static_cast(distance + 1); + break; + } + } + out.prev_same_offset_plus1 = prev_same_offset_plus1; + } + + h_trees.push_back( + CompressedTree{static_cast(node_begin), h_tree_groups[tree_idx]}); } + + CompressedModel out; + out.trees = dh::device_vector(h_trees.cbegin(), h_trees.cend()); + out.nodes = dh::device_vector(h_nodes.cbegin(), h_nodes.cend()); + return out; } -template -XGBOOST_DEVICE double ExtractQuadratureDelta(std::size_t points, double const* nodes, - double const* weights, - double const (&h_vals)[MaxPoints], double p_enter, - double p_exit) { - auto alpha_enter = p_enter - 1.0; - auto alpha_exit = p_exit - 1.0; - auto has_enter = p_enter != kQuadratureShapUnseen && fabs(alpha_enter) >= kQuadratureShapQeps; - auto has_exit = p_exit != kQuadratureShapUnseen && fabs(alpha_exit) >= kQuadratureShapQeps; - if (!has_enter && !has_exit) { - return 0.0; +template +XGBOOST_DEVICE constexpr unsigned ActiveSubgroupMask(int row_slot) { + constexpr int kSegmentWidth = dh::WarpThreads() / RowsPerWarp; + static_assert(kSegmentWidth >= static_cast(kGpuQuadraturePoints)); + return ((1u << kGpuQuadraturePoints) - 1u) << (row_slot * kSegmentWidth); +} + +XGBOOST_DEVICE inline float PreviousPathProbability(std::uint8_t prev_same_offset_plus1, int depth, + float const* q_vals_row) { + if (prev_same_offset_plus1 == 0) { + return 1.0f; } + auto prev_depth = depth - static_cast(prev_same_offset_plus1) + 1; + return q_vals_row[prev_depth]; +} - double acc = 0.0; - for (std::size_t i = 0; i < points; ++i) { - auto weighted_h = h_vals[i] * weights[i]; - if (has_enter) { - acc += alpha_enter * weighted_h / (1.0 + alpha_enter * nodes[i]); +template +struct IsSparsePageLoaderNoShared : std::false_type {}; + +template +struct IsSparsePageLoaderNoShared> : std::true_type {}; + +// Encapsulate the tail-tile versus full-tile differences so the traversal code can focus on +// probability updates instead of mask plumbing. +template +struct SubgroupOps { + static constexpr int kRowsPerWarpValue = RowsPerWarp; + static constexpr int kSegmentWidth = dh::WarpThreads() / RowsPerWarp; + static constexpr unsigned kFullMask = 0xffffffffu; + + int row_slot; + int point; + unsigned subgroup_mask; + unsigned warp_mask; + bool row_valid; + bool is_leader; + bool is_warp_leader; + + XGBOOST_DEV_INLINE SubgroupOps(int lane, bst_idx_t valid_rows_in_tail) + : row_slot{lane / kSegmentWidth}, + point{lane % kSegmentWidth}, + subgroup_mask{kFullMask}, + warp_mask{kFullMask}, + row_valid{true}, + is_leader{point == 0}, + is_warp_leader{lane == 0} { + if constexpr (kHasRowMask) { + subgroup_mask = ActiveSubgroupMask(row_slot); + warp_mask = __activemask(); + row_valid = static_cast(row_slot) < valid_rows_in_tail; } - if (has_exit) { - acc -= alpha_exit * weighted_h / (1.0 + alpha_exit * nodes[i]); + } + + [[nodiscard]] XGBOOST_DEV_INLINE bool Participates() const { return point < MaxPoints; } + + [[nodiscard]] XGBOOST_DEV_INLINE bool RowActive() const { + if constexpr (kHasRowMask) { + return row_valid; + } else { + return true; } } - return acc; -} -template -XGBOOST_DEVICE double FindPathProbability( - int path_len, bst_feature_t const (&path_features)[kMaxGpuQuadratureDepth], - double const (&path_p)[kMaxGpuQuadratureDepth], bst_feature_t split_index) { - for (int i = path_len - 1; i >= 0; --i) { - if (path_features[i] == split_index) { - return path_p[i]; + [[nodiscard]] XGBOOST_DEV_INLINE bool ShouldWrite() const { + return is_leader && this->RowActive(); + } + + template + [[nodiscard]] XGBOOST_DEV_INLINE T Broadcast(T value) const { + // Each row uses an independent kGpuQuadraturePoints-wide subgroup inside the warp. + if constexpr (kHasRowMask) { + return __shfl_sync(subgroup_mask, value, 0, MaxPoints); + } else { + return __shfl_sync(kFullMask, value, 0, MaxPoints); } } - return kQuadratureShapUnseen; -} -template -XGBOOST_DEVICE void QuadratureShapTree(tree::ScalarTreeView const& tree, Loader const& loader, - bst_idx_t ridx, std::size_t points, double const* nodes, - double const* weights, float tree_weight, float* out_row) { - auto const cats = tree.GetCategoriesMatrix(); - QuadratureFrame frames[kMaxGpuQuadratureDepth]; - bst_feature_t path_features[kMaxGpuQuadratureDepth]; - double path_p[kMaxGpuQuadratureDepth]; - double ret_h[MaxPoints]{}; - int stack_size = 1; - bool have_return = false; - - frames[0].node = RegTree::kRoot; - frames[0].path_len = 0; - frames[0].stage = 0; - frames[0].w_prod = 1.0; - for (std::size_t i = 0; i < points; ++i) { - frames[0].c_vals[i] = 1.0; - } - - while (stack_size > 0 || have_return) { - if (have_return) { - if (stack_size == 0) { - break; - } - auto& parent = frames[stack_size - 1]; - auto child_idx = parent.stage - 1; - out_row[parent.split_index] += static_cast(ExtractQuadratureDelta( - points, nodes, weights, ret_h, parent.child_p_enter[child_idx], - parent.child_p_up[child_idx])); - if (child_idx == 0) { - CopyQuadratureValues(parent.h_vals, ret_h, points); - parent.stage = 2; + template + [[nodiscard]] XGBOOST_DEV_INLINE T Sum(T value) const { + for (int offset = MaxPoints / 2; offset > 0; offset /= 2) { + if constexpr (kHasRowMask) { + value += __shfl_down_sync(subgroup_mask, value, offset, MaxPoints); } else { - AddQuadratureValues(parent.h_vals, ret_h, points); - CopyQuadratureValues(ret_h, parent.h_vals, points); - --stack_size; + value += __shfl_down_sync(kFullMask, value, offset, MaxPoints); } - have_return = (child_idx == 1); - continue; } + return value; + } - auto& frame = frames[stack_size - 1]; - if (tree.IsLeaf(frame.node)) { - auto leaf_value = static_cast(tree.LeafValue(frame.node) * tree_weight); - for (std::size_t i = 0; i < points; ++i) { - ret_h[i] = frame.c_vals[i] * frame.w_prod * leaf_value; - } - --stack_size; - have_return = true; - continue; + XGBOOST_DEV_INLINE void Sync() const { + if constexpr (kHasRowMask) { + __syncwarp(warp_mask); + } else { + __syncwarp(); } + } +}; - if (frame.stage == 2) { - auto child_slot = frame.path_len; - if (child_slot >= static_cast(kMaxGpuQuadratureDepth)) { - return; - } - auto child = 1; - auto child_node = frame.child_node[child]; - auto child_weight = frame.child_weight[child]; - auto p_old = - FindPathProbability(frame.path_len, path_features, path_p, frame.split_index); - double p_e = 0.0; - double p_up = 0.0; - auto satisfies = child_node == tree.RightChild(frame.node) - ? !(predictor::GetNextNode( - tree, frame.node, loader.GetElement(ridx, frame.split_index), - common::CheckNAN(loader.GetElement(ridx, frame.split_index)), - cats) == tree.LeftChild(frame.node)) - : predictor::GetNextNode( - tree, frame.node, loader.GetElement(ridx, frame.split_index), - common::CheckNAN(loader.GetElement(ridx, frame.split_index)), - cats) == tree.LeftChild(frame.node); - if (p_old == kQuadratureShapUnseen) { - p_e = satisfies ? 1.0 / child_weight : 0.0; - p_up = 1.0; - } else if (fabs(p_old) < kQuadratureShapQeps) { - p_e = 0.0; - p_up = 0.0; - } else { - p_e = satisfies ? p_old / child_weight : 0.0; - p_up = p_old; - } +// Wrap the shared-memory layout in semantic accessors so the task runner talks in terms of path +// state instead of raw multidimensional indexing. +template +struct QuadratureSharedState { + bst_node_t (&nodes)[kWarpsPerBlock][DepthCap]; + std::uint8_t (&stages)[kWarpsPerBlock][DepthCap]; + std::uint8_t (&goes_left)[kWarpsPerBlock][RowsPerWarp][DepthCap]; + // q_d(t): path probability at depth d for one row-slot evaluated at quadrature point t. + float (&path_prob)[kWarpsPerBlock][RowsPerWarp][DepthCap]; + // G_d(t): multiplicative basis carried down the path before the leaf value is applied. + float (&basis)[kWarpsPerBlock][RowsPerWarp][DepthCap][MaxPoints]; + float (&q_prev_cache)[kUseQPrevCache ? kWarpsPerBlock : 1][kUseQPrevCache ? RowsPerWarp : 1] + [kUseQPrevCache ? DepthCap : 1]; + + [[nodiscard]] XGBOOST_DEV_INLINE bst_node_t& Node(int warp, int depth) { + return nodes[warp][depth]; + } + + [[nodiscard]] XGBOOST_DEV_INLINE std::uint8_t& Stage(int warp, int depth) { + return stages[warp][depth]; + } + + [[nodiscard]] XGBOOST_DEV_INLINE bool GoesLeft(int warp, int row_slot, int depth) const { + return static_cast(goes_left[warp][row_slot][depth]); + } + + XGBOOST_DEV_INLINE void SetGoesLeft(int warp, int row_slot, int depth, bool value) { + goes_left[warp][row_slot][depth] = static_cast(value); + } + + [[nodiscard]] XGBOOST_DEV_INLINE float& PathProbability(int warp, int row_slot, int depth) { + return path_prob[warp][row_slot][depth]; + } + + [[nodiscard]] XGBOOST_DEV_INLINE float const* PathProbabilityRow(int warp, int row_slot) const { + return path_prob[warp][row_slot]; + } + + [[nodiscard]] XGBOOST_DEV_INLINE float& Basis(int warp, int row_slot, int depth, int point) { + return basis[warp][row_slot][depth][point]; + } - path_features[child_slot] = frame.split_index; - path_p[child_slot] = p_e; - frame.child_p_enter[child] = p_e; - frame.child_p_up[child] = p_up; - - auto& child_frame = frames[stack_size++]; - child_frame.node = child_node; - child_frame.path_len = frame.path_len + 1; - child_frame.stage = 0; - child_frame.w_prod = frame.w_prod * child_weight; - auto alpha_e = p_e - 1.0; - auto alpha_old = p_old - 1.0; - auto has_old = p_old != kQuadratureShapUnseen && fabs(alpha_old) >= kQuadratureShapQeps; - for (std::size_t i = 0; i < points; ++i) { - auto v = frame.c_vals[i] * (1.0 + alpha_e * nodes[i]); - if (has_old) { - v /= 1.0 + alpha_old * nodes[i]; + [[nodiscard]] XGBOOST_DEV_INLINE float LoadQPrev(int warp, int row_slot, int depth, + std::uint8_t prev_same_offset_plus1) const { + if constexpr (kUseQPrevCache) { + return q_prev_cache[warp][row_slot][depth]; + } else { + return PreviousPathProbability(prev_same_offset_plus1, depth, + this->PathProbabilityRow(warp, row_slot)); + } + } + + XGBOOST_DEV_INLINE void StoreQPrev(int warp, int row_slot, int depth, float q_prev) { + if constexpr (kUseQPrevCache) { + q_prev_cache[warp][row_slot][depth] = q_prev; + } + } +}; + +template +struct QuadratureShapTaskRunner { + Loader loader; + SubgroupT subgroup; + SharedT shared; + CompressedTree const* trees; + CompressedNode const* nodes; + float* phis; + bst_idx_t base_rowid; + bst_target_t n_groups; + bst_feature_t n_columns; + std::size_t row_tile_begin; + std::size_t row_tiles; + int warp; + float quad_node; + float quad_weight; + + [[nodiscard]] XGBOOST_DEV_INLINE bool EvaluateGoesLeft(bst_idx_t ridx, + CompressedNode const& node) const { + auto fvalue = loader.GetElement(ridx, node.split_global); + return common::CheckNAN(fvalue) ? static_cast(node.default_left) + : fvalue < node.split_cond; + } + + XGBOOST_DEV_INLINE void AddContribution(bst_idx_t row_idx, bst_target_t tree_group, + bst_feature_t split_global, float contrib) const { + if (!subgroup.ShouldWrite()) { + return; + } + auto out_row = phis + (row_idx * n_groups + tree_group) * n_columns; + atomicAdd(out_row + split_global, contrib); + } + + XGBOOST_DEV_INLINE void InitializeTask() { + if (subgroup.is_warp_leader) { + shared.Node(warp, 0) = RegTree::kRoot; + shared.Stage(warp, 0) = 0; + } + // Start each row with G_0(t) = 1 at every quadrature node. + shared.Basis(warp, subgroup.row_slot, 0, subgroup.point) = 1.0f; + subgroup.Sync(); + } + + XGBOOST_DEV_INLINE bool HandleReturn(bst_idx_t row_idx, bst_target_t tree_group, + CompressedNode const* nodes_for_tree, int* stack_size, + bool* have_return, float* ret_val) { + if (*stack_size == 0) { + return false; + } + + int parent_depth = *stack_size - 1; + auto const& node = nodes_for_tree[shared.Node(warp, parent_depth)]; + int child_idx = static_cast(shared.Stage(warp, parent_depth)) - 1; + + float p_enter = 0.0f; + float q_prev = 1.0f; + if (subgroup.is_leader && subgroup.RowActive()) { + p_enter = shared.PathProbability(warp, subgroup.row_slot, parent_depth); + q_prev = shared.LoadQPrev(warp, subgroup.row_slot, parent_depth, node.prev_same_offset_plus1); + } + p_enter = subgroup.Broadcast(p_enter); + q_prev = subgroup.Broadcast(q_prev); + + float contrib = 0.0f; + // Extraction uses + // H * w(t) * ret_val * + // [ (p_enter - 1) / (1 + (p_enter - 1) t) + // - (q_prev - 1) / (1 + (q_prev - 1) t) ]. + // The two rational terms are the "enter current feature" and "rewind to previous same + // feature" adjustments from the quadrature recurrence. + if (p_enter != 1.0f) { + auto alpha_enter = p_enter - 1.0f; + contrib += alpha_enter * (*ret_val) * quad_weight / (1.0f + alpha_enter * quad_node); + } + if (q_prev != 1.0f) { + auto alpha_exit = q_prev - 1.0f; + contrib -= alpha_exit * (*ret_val) * quad_weight / (1.0f + alpha_exit * quad_node); + } + contrib = subgroup.Sum(contrib); + this->AddContribution(row_idx, tree_group, node.split_global, contrib); + + if (child_idx == 0) { + auto child_weight = node.right_weight; + auto child_node = node.right; + float p_e = 0.0f; + if (subgroup.is_leader) { + if (subgroup.RowActive()) { + auto goes_left = shared.GoesLeft(warp, subgroup.row_slot, parent_depth); + p_e = goes_left ? 0.0f : q_prev / child_weight; } - child_frame.c_vals[i] = v; + shared.PathProbability(warp, subgroup.row_slot, parent_depth) = p_e; + } + p_e = subgroup.Broadcast(p_e); + + if (subgroup.is_warp_leader) { + shared.Node(warp, *stack_size) = child_node; + shared.Stage(warp, *stack_size) = 0; + shared.Stage(warp, parent_depth) = 2; + } + // Push the sibling subtree with + // G_child(t) = G_parent(t) * child_weight * + // (1 + (p_e - 1) t) / (1 + (q_prev - 1) t). + // This preserves the basis after swapping the active feature state from q_prev to p_e. + auto alpha_e = p_e - 1.0f; + auto v = shared.Basis(warp, subgroup.row_slot, parent_depth, subgroup.point) * child_weight * + (1.0f + alpha_e * quad_node); + if (q_prev != 1.0f) { + auto alpha_old = q_prev - 1.0f; + v /= 1.0f + alpha_old * quad_node; } - continue; + shared.Basis(warp, subgroup.row_slot, *stack_size, subgroup.point) = v; + subgroup.Sync(); + shared.Basis(warp, subgroup.row_slot, parent_depth, subgroup.point) = *ret_val; + (*stack_size)++; + *have_return = false; + } else { + *ret_val += shared.Basis(warp, subgroup.row_slot, parent_depth, subgroup.point); + (*stack_size)--; + *have_return = true; } - auto split_index = tree.SplitIndex(frame.node); - auto fvalue = loader.GetElement(ridx, split_index); - auto is_missing = common::CheckNAN(fvalue); - auto next = predictor::GetNextNode(tree, frame.node, fvalue, is_missing, cats); - auto left = tree.LeftChild(frame.node); - auto right = tree.RightChild(frame.node); - auto parent_cover = static_cast(tree.SumHess(frame.node)); - if (!(parent_cover > 0.0)) { + return true; + } + + XGBOOST_DEV_INLINE void Descend(CompressedNode const* nodes_for_tree, bst_idx_t ridx, + int* stack_size, bool* have_return, float* ret_val) { + int depth = *stack_size - 1; + auto const& node = nodes_for_tree[shared.Node(warp, depth)]; + if (node.is_leaf) { + *ret_val = shared.Basis(warp, subgroup.row_slot, depth, subgroup.point) * node.leaf_value; + (*stack_size)--; + *have_return = true; return; } - frame.split_index = split_index; - frame.child_node[0] = left; - frame.child_node[1] = right; - frame.child_weight[0] = static_cast(tree.SumHess(left)) / parent_cover; - frame.child_weight[1] = static_cast(tree.SumHess(right)) / parent_cover; - frame.stage = 1; + // stage == 0 explores the left child first. After the return path updates the parent state, + // the second visit uses the cached go-left decision to push the right child. + int child = static_cast(shared.Stage(warp, depth) != 0); + if (child == 0) { + if (subgroup.is_warp_leader) { + shared.Stage(warp, depth) = 1; + } + subgroup.Sync(); + } - auto child_slot = frame.path_len; - if (child_slot >= static_cast(kMaxGpuQuadratureDepth)) { - return; + auto child_weight = child == 0 ? node.left_weight : node.right_weight; + auto child_node = child == 0 ? node.left : node.right; + float q_prev = 1.0f; + if (subgroup.is_leader) { + if (subgroup.RowActive()) { + q_prev = PreviousPathProbability(node.prev_same_offset_plus1, depth, + shared.PathProbabilityRow(warp, subgroup.row_slot)); + } + shared.StoreQPrev(warp, subgroup.row_slot, depth, q_prev); } - auto child = 0; - auto child_node = frame.child_node[child]; - auto child_weight = frame.child_weight[child]; - auto p_old = - FindPathProbability(frame.path_len, path_features, path_p, frame.split_index); - double p_e = 0.0; - double p_up = 0.0; - auto satisfies = next == child_node; - if (p_old == kQuadratureShapUnseen) { - p_e = satisfies ? 1.0 / child_weight : 0.0; - p_up = 1.0; - } else if (fabs(p_old) < kQuadratureShapQeps) { - p_e = 0.0; - p_up = 0.0; - } else { - p_e = satisfies ? p_old / child_weight : 0.0; - p_up = p_old; - } - - path_features[child_slot] = frame.split_index; - path_p[child_slot] = p_e; - frame.child_p_enter[child] = p_e; - frame.child_p_up[child] = p_up; - - auto& child_frame = frames[stack_size++]; - child_frame.node = child_node; - child_frame.path_len = frame.path_len + 1; - child_frame.stage = 0; - child_frame.w_prod = frame.w_prod * child_weight; - auto alpha_e = p_e - 1.0; - auto alpha_old = p_old - 1.0; - auto has_old = p_old != kQuadratureShapUnseen && fabs(alpha_old) >= kQuadratureShapQeps; - for (std::size_t i = 0; i < points; ++i) { - auto v = frame.c_vals[i] * (1.0 + alpha_e * nodes[i]); - if (has_old) { - v /= 1.0 + alpha_old * nodes[i]; + q_prev = subgroup.Broadcast(q_prev); + + float p_e = 0.0f; + if (subgroup.is_leader) { + bool goes_left = false; + if (subgroup.RowActive()) { + goes_left = this->EvaluateGoesLeft(ridx, node); + // p_e is the path probability after taking the chosen child for this row. + p_e = (child == 0 ? goes_left : !goes_left) ? q_prev / child_weight : 0.0f; } - child_frame.c_vals[i] = v; + shared.SetGoesLeft(warp, subgroup.row_slot, depth, goes_left); + shared.PathProbability(warp, subgroup.row_slot, depth) = p_e; } - } -} + p_e = subgroup.Broadcast(p_e); -template -void LaunchQuadratureShap(Context const* ctx, Loader loader, bst_idx_t base_rowid, - gbm::GBTreeModel const& model, ModelView const& d_model, - common::Span root_mean_values, - common::OptionalWeights tree_weights, std::size_t points, - common::Span nodes, common::Span weights, - HostDeviceVector* out_contribs) { - auto const ngroup = model.learner_model_param->num_output_group; - auto const ncolumns = model.learner_model_param->num_feature + 1; - auto d_trees = d_model.Trees(); - auto d_tree_groups = d_model.tree_groups; - auto phis = out_contribs->DeviceSpan(); + if (subgroup.is_warp_leader) { + shared.Node(warp, *stack_size) = child_node; + shared.Stage(warp, *stack_size) = 0; + } + // Same recurrence as the sibling push above: reweight G_d(t) by the child weight and replace + // q_prev with the new path probability p_e at this depth. + auto alpha_e = p_e - 1.0f; + auto v = shared.Basis(warp, subgroup.row_slot, depth, subgroup.point) * child_weight * + (1.0f + alpha_e * quad_node); + if (q_prev != 1.0f) { + auto alpha_old = q_prev - 1.0f; + v /= 1.0f + alpha_old * quad_node; + } + shared.Basis(warp, subgroup.row_slot, *stack_size, subgroup.point) = v; + subgroup.Sync(); + (*stack_size)++; + } - dh::LaunchN(loader.NumRows(), ctx->CUDACtx()->Stream(), [=] __device__(std::size_t ridx) { + XGBOOST_DEV_INLINE void RunTask(std::size_t task) { + auto tree_idx = task / row_tiles; + auto row_tile = task % row_tiles; + auto ridx = (row_tile_begin + row_tile) * SubgroupT::kRowsPerWarpValue + subgroup.row_slot; auto row_idx = base_rowid + static_cast(ridx); - for (bst_tree_t tree_idx = 0; tree_idx < d_trees.size(); ++tree_idx) { - auto const& d_tree = cuda::std::get(d_trees[tree_idx]); - auto gid = d_tree_groups[tree_idx]; - auto out_row = phis.data() + (row_idx * ngroup + gid) * ncolumns; - out_row[ncolumns - 1] += root_mean_values[tree_idx]; - QuadratureShapTree(d_tree, loader, ridx, points, nodes.data(), weights.data(), - tree_weights[tree_idx], out_row); + auto tree = trees[tree_idx]; + auto nodes_for_tree = nodes + tree.node_begin; + + this->InitializeTask(); + + int stack_size = 1; + bool have_return = false; + float ret_val = 0.0f; + while (stack_size > 0 || have_return) { + if (have_return) { + if (!this->HandleReturn(row_idx, tree.group, nodes_for_tree, &stack_size, &have_return, + &ret_val)) { + break; + } + continue; + } + this->Descend(nodes_for_tree, ridx, &stack_size, &have_return, &ret_val); } - }); + } +}; + +template +__global__ void __launch_bounds__(BlockThreads, 9) + QuadratureShapTaskKernel(Loader loader, bst_idx_t base_rowid, bst_target_t n_groups, + bst_feature_t n_columns, std::size_t row_tile_begin, + std::size_t row_tiles, bst_idx_t valid_rows_in_tail, + std::size_t n_trees, CompressedTree const* __restrict__ trees, + CompressedNode const* __restrict__ nodes, + float const* __restrict__ quad_nodes, + float const* __restrict__ quad_weights, float* __restrict__ phis) { + static_assert(MaxPoints == kGpuQuadraturePoints); + static_assert(DepthCap <= static_cast(kMaxGpuQuadratureDepth)); + static_assert(dh::WarpThreads() % RowsPerWarp == 0); + static_assert(BlockThreads % dh::WarpThreads() == 0); + using SubgroupT = SubgroupOps; + constexpr int kSegmentWidth = SubgroupT::kSegmentWidth; + if constexpr (!kHasRowMask) { + static_assert(kSegmentWidth == MaxPoints, + "Full-tile specialization assumes every warp lane participates."); + } + constexpr int kWarpsPerBlock = BlockThreads / dh::WarpThreads(); + constexpr bool kUseQPrevCache = IsSparsePageLoaderNoShared::value; + using SharedT = + QuadratureSharedState; + + __shared__ bst_node_t s_node[kWarpsPerBlock][DepthCap]; + __shared__ std::uint8_t s_stage[kWarpsPerBlock][DepthCap]; + __shared__ std::uint8_t s_goes_left[kWarpsPerBlock][RowsPerWarp][DepthCap]; + __shared__ float s_path_p[kWarpsPerBlock][RowsPerWarp][DepthCap]; + __shared__ float s_c_vals[kWarpsPerBlock][RowsPerWarp][DepthCap][MaxPoints]; + __shared__ float s_q_prev[kUseQPrevCache ? kWarpsPerBlock : 1][kUseQPrevCache ? RowsPerWarp : 1] + [kUseQPrevCache ? DepthCap : 1]; + + int warp = static_cast(threadIdx.x) / dh::WarpThreads(); + int lane = static_cast(threadIdx.x) % dh::WarpThreads(); + auto subgroup = SubgroupT{lane, valid_rows_in_tail}; + if (!subgroup.Participates()) { + return; + } + + auto shared = SharedT{s_node, s_stage, s_goes_left, s_path_p, s_c_vals, s_q_prev}; + auto global_warp = + (static_cast(blockIdx.x) * BlockThreads + threadIdx.x) / dh::WarpThreads(); + auto warp_stride = (static_cast(gridDim.x) * BlockThreads) / dh::WarpThreads(); + auto n_tasks = n_trees * row_tiles; + + auto runner = QuadratureShapTaskRunner{loader, + subgroup, + shared, + trees, + nodes, + phis, + base_rowid, + n_groups, + n_columns, + row_tile_begin, + row_tiles, + warp, + quad_nodes[subgroup.point], + quad_weights[subgroup.point]}; + + for (std::size_t task = global_warp; task < n_tasks; task += warp_stride) { + runner.RunTask(task); + } +} + +template +void LaunchQuadratureShapTasks(Context const* ctx, Loader loader, bst_idx_t base_rowid, + bst_target_t n_groups, bst_feature_t n_columns, + std::size_t row_tile_begin, std::size_t row_tiles, + bst_idx_t valid_rows_in_tail, CompressedModel const& compressed, + common::Span quad_nodes, + common::Span quad_weights, + HostDeviceVector* out_contribs) { + static_assert(BlockThreads % dh::WarpThreads() == 0); + constexpr int kWarpsPerBlock = BlockThreads / dh::WarpThreads(); + if (compressed.trees.empty() || row_tiles == 0) { + return; + } + auto trees = thrust::raw_pointer_cast(compressed.trees.data()); + auto nodes = thrust::raw_pointer_cast(compressed.nodes.data()); + auto d_quad_nodes = quad_nodes.data(); + auto d_quad_weights = quad_weights.data(); + auto phis = out_contribs->DeviceSpan().data(); + auto n_tasks = compressed.trees.size() * row_tiles; + auto grids = common::DivRoundUp(n_tasks, static_cast(kWarpsPerBlock)); + QuadratureShapTaskKernel + <<(grids), static_cast(BlockThreads), 0, + ctx->CUDACtx()->Stream()>>>(loader, base_rowid, n_groups, n_columns, row_tile_begin, + row_tiles, valid_rows_in_tail, compressed.trees.size(), trees, + nodes, d_quad_nodes, d_quad_weights, phis); + dh::safe_cuda(cudaGetLastError()); +} + +template +void LaunchQuadratureShapBuckets(Context const* ctx, Loader loader, bst_idx_t base_rowid, + bst_target_t n_groups, bst_feature_t n_columns, + CompressedModel const& compressed, + common::Span quad_nodes, + common::Span quad_weights, + HostDeviceVector* out_contribs) { + auto full_row_tiles = static_cast(loader.NumRows() / RowsPerWarp); + auto tail_rows = static_cast(loader.NumRows() % RowsPerWarp); + LaunchQuadratureShapTasks( + ctx, loader, base_rowid, n_groups, n_columns, /*row_tile_begin=*/0, full_row_tiles, + /*valid_rows_in_tail=*/RowsPerWarp, compressed, quad_nodes, quad_weights, out_contribs); + if (tail_rows != 0) { + LaunchQuadratureShapTasks( + ctx, loader, base_rowid, n_groups, n_columns, /*row_tile_begin=*/full_row_tiles, + /*row_tiles=*/1, tail_rows, compressed, quad_nodes, quad_weights, out_contribs); + } } struct CopyViews { @@ -744,9 +1014,9 @@ void QuadratureShapValues(Context const* ctx, DMatrix* p_fmat, CHECK(!model.learner_model_param->IsVectorLeaf()) << "Predict contribution" << MTNotImplemented(); CHECK(!p_fmat->Info().IsColumnSplit()) << "Predict contribution support for column-wise data split is not yet implemented."; - CHECK_LE(quadrature_points, kMaxGpuQuadraturePoints) - << "GPU QuadratureSHAP currently supports up to " << kMaxGpuQuadraturePoints - << " quadrature points."; + CHECK_EQ(quadrature_points, kGpuQuadraturePoints) + << "GPU QuadratureSHAP currently uses a fixed quadrature size of " << kGpuQuadraturePoints + << "."; tree_end = predictor::GetTreeLimit(model.trees, tree_end); auto const ngroup = model.learner_model_param->num_output_group; @@ -758,43 +1028,57 @@ void QuadratureShapValues(Context const* ctx, DMatrix* p_fmat, out_contribs->Fill(0.0f); bst_node_t max_depth = 0; + std::array, kGpuQuadratureDepthBuckets.size()> tree_buckets; for (bst_tree_t tree_idx = 0; tree_idx < tree_end; ++tree_idx) { CHECK(!model.trees[tree_idx]->IsMultiTarget()) << "Predict contribution" << MTNotImplemented(); - max_depth = std::max(max_depth, model.trees[tree_idx]->MaxDepth()); + auto tree_depth = model.trees[tree_idx]->MaxDepth(); + max_depth = std::max(max_depth, tree_depth); + auto path_depth = static_cast(tree_depth) + 1; + auto bucket_idx = DepthBucketIndex(path_depth); + tree_buckets[bucket_idx].push_back(tree_idx); } CHECK_LE(max_depth + 1, static_cast(kMaxGpuQuadratureDepth)) << "GPU QuadratureSHAP currently supports trees of depth up to " << (kMaxGpuQuadratureDepth - 1) << "."; - - auto rule = MakeEndpointQuadrature(quadrature_points); - dh::device_vector d_nodes(rule.nodes.begin(), rule.nodes.begin() + quadrature_points); - dh::device_vector d_weights(rule.weights.begin(), - rule.weights.begin() + quadrature_points); - - auto h_root_means = MakeTreeRootMeanValues(model, tree_end, tree_weights); - dh::device_vector d_root_means(h_root_means.cbegin(), h_root_means.cend()); - - DeviceModel d_model{ctx->Device(), model, true, 0, tree_end, CopyViews{ctx}}; - dh::device_vector d_tree_weights; - auto weights_opt = common::OptionalWeights{1.0f}; - if (tree_weights != nullptr) { - d_tree_weights.assign(tree_weights->cbegin(), tree_weights->cbegin() + tree_end); - weights_opt = common::OptionalWeights{common::Span{ - thrust::raw_pointer_cast(d_tree_weights.data()), d_tree_weights.size()}}; + auto h_group_root_mean_sums = MakeGroupRootMeanSums(model, tree_end, tree_weights); + + auto rule = detail::MakeEndpointQuadrature(kQuadratureShapQeps); + std::array h_quad_nodes{}; + std::array h_quad_weights{}; + for (std::size_t i = 0; i < kGpuQuadraturePoints; ++i) { + h_quad_nodes[i] = static_cast(rule.nodes[i]); + h_quad_weights[i] = static_cast(rule.weights[i]); } + dh::device_vector d_quad_nodes(h_quad_nodes.cbegin(), h_quad_nodes.cend()); + dh::device_vector d_quad_weights(h_quad_weights.cbegin(), h_quad_weights.cend()); + dh::device_vector d_group_root_mean_sums(h_group_root_mean_sums.cbegin(), + h_group_root_mean_sums.cend()); + auto compressed_16 = MakeCompressedModel(ctx, model, tree_buckets[0], tree_weights); + auto compressed_32 = MakeCompressedModel(ctx, model, tree_buckets[1], tree_weights); + auto compressed_64 = MakeCompressedModel(ctx, model, tree_buckets[2], tree_weights); auto new_enc = p_fmat->Cats()->NeedRecode() ? p_fmat->Cats()->DeviceView(ctx) : enc::DeviceColumnsView{}; - auto root_means = - common::Span{thrust::raw_pointer_cast(d_root_means.data()), d_root_means.size()}; - auto nodes = common::Span{thrust::raw_pointer_cast(d_nodes.data()), d_nodes.size()}; - auto weights = - common::Span{thrust::raw_pointer_cast(d_weights.data()), d_weights.size()}; + auto quad_nodes = + common::Span{thrust::raw_pointer_cast(d_quad_nodes.data()), d_quad_nodes.size()}; + auto quad_weights = common::Span{thrust::raw_pointer_cast(d_quad_weights.data()), + d_quad_weights.size()}; + auto group_root_mean_sums = common::Span{ + thrust::raw_pointer_cast(d_group_root_mean_sums.data()), d_group_root_mean_sums.size()}; LaunchShap(ctx, p_fmat, new_enc, model, [&](auto&& loader, bst_idx_t base_rowid) { - LaunchQuadratureShap(ctx, loader, base_rowid, model, d_model, - root_means, weights_opt, quadrature_points, nodes, - weights, out_contribs); + LaunchQuadratureShapBuckets( + ctx, loader, base_rowid, ngroup, ncolumns, compressed_16, quad_nodes, quad_weights, + out_contribs); + LaunchQuadratureShapBuckets( + ctx, loader, base_rowid, ngroup, ncolumns, compressed_32, quad_nodes, quad_weights, + out_contribs); + LaunchQuadratureShapBuckets( + ctx, loader, base_rowid, ngroup, ncolumns, compressed_64, quad_nodes, quad_weights, + out_contribs); }); p_fmat->Info().base_margin_.SetDevice(ctx->Device()); @@ -804,7 +1088,8 @@ void QuadratureShapValues(Context const* ctx, DMatrix* p_fmat, auto n_samples = p_fmat->Info().num_row_; dh::LaunchN(n_samples * ngroup, ctx->CUDACtx()->Stream(), [=] __device__(std::size_t idx) { auto [_, gid] = linalg::UnravelIndex(idx, n_samples, ngroup); - phis[(idx + 1) * ncolumns - 1] += margin.empty() ? base_score(gid) : margin[idx]; + phis[(idx + 1) * ncolumns - 1] += + group_root_mean_sums[gid] + (margin.empty() ? base_score(gid) : margin[idx]); }); } From 5e96ce9c76decac84e18ec8b0de1a22b949b5da6 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Mon, 30 Mar 2026 05:03:34 -0700 Subject: [PATCH 07/13] Refactor QuadratureSHAP CPU structure --- doc/parameter.rst | 6 +- src/gbm/gbtree.h | 7 +- src/predictor/cpu_predictor.cc | 5 +- src/predictor/gpu_predictor.cu | 5 +- src/predictor/interpretability/shap.cc | 315 ++++++++++++++----------- src/predictor/interpretability/shap.cu | 14 +- src/predictor/interpretability/shap.h | 15 ++ 7 files changed, 202 insertions(+), 165 deletions(-) diff --git a/doc/parameter.rst b/doc/parameter.rst index 0500d232fda4..e1f4e5f26279 100644 --- a/doc/parameter.rst +++ b/doc/parameter.rst @@ -197,10 +197,10 @@ Parameters for Tree Booster - ``treeshap``: Existing exact TreeSHAP implementation. - ``quadratureshap``: Quadrature plus telescoping SHAP implementation for CPU prediction. -* ``quadratureshap_points`` [default= ``16``] +* ``quadratureshap_points`` [default= ``8``] - - Experimental number of quadrature points used by CPU ``quadratureshap``. - - Valid range: ``[2, 64]``. + - Experimental fixed quadrature size used by CPU and GPU ``quadratureshap`` variants. + - Current supported value: ``8``. * ``scale_pos_weight`` [default=1] diff --git a/src/gbm/gbtree.h b/src/gbm/gbtree.h index c93ae99d48ad..b361b2a0ceae 100644 --- a/src/gbm/gbtree.h +++ b/src/gbm/gbtree.h @@ -64,7 +64,7 @@ struct GBTreeTrainParam : public XGBoostParameter { TreeMethod tree_method; // CPU SHAP implementation used for pred_contribs. std::string shap_algorithm; - // Number of quadrature points for CPU QuadratureSHAP. + // Number of quadrature points for QuadratureSHAP variants. std::size_t quadratureshap_points; // declare parameters DMLC_DECLARE_PARAMETER(GBTreeTrainParam) { @@ -88,9 +88,8 @@ struct GBTreeTrainParam : public XGBoostParameter { .set_default("treeshap") .describe("CPU algorithm used for SHAP feature contributions."); DMLC_DECLARE_FIELD(quadratureshap_points) - .set_default(16) - .set_range(2, 64) - .describe("Experimental number of quadrature points used by CPU QuadratureSHAP."); + .set_default(8) + .describe("Experimental fixed quadrature size used by CPU and GPU QuadratureSHAP."); } }; diff --git a/src/predictor/cpu_predictor.cc b/src/predictor/cpu_predictor.cc index 2591453c7b24..f36c683bb2c9 100644 --- a/src/predictor/cpu_predictor.cc +++ b/src/predictor/cpu_predictor.cc @@ -761,8 +761,7 @@ class CPUPredictor : public Predictor { } catch (std::out_of_range const &) { LOG(FATAL) << "quadratureshap_points out of range: " << kv.second; } - CHECK_GE(points, 2) << "quadratureshap_points must be >= 2"; - CHECK_LE(points, 64) << "quadratureshap_points must be <= 64"; + CHECK_EQ(points, 8) << "CPU QuadratureSHAP currently uses a fixed quadrature size of 8."; quadrature_shap_points_ = points; } } @@ -911,7 +910,7 @@ class CPUPredictor : public Predictor { private: std::string shap_algorithm_{"treeshap"}; - std::size_t quadrature_shap_points_{16}; + std::size_t quadrature_shap_points_{8}; }; XGBOOST_REGISTER_PREDICTOR(CPUPredictor, "cpu_predictor") diff --git a/src/predictor/gpu_predictor.cu b/src/predictor/gpu_predictor.cu index cbdcf1da9318..ea0813462791 100644 --- a/src/predictor/gpu_predictor.cu +++ b/src/predictor/gpu_predictor.cu @@ -685,8 +685,7 @@ class GPUPredictor : public xgboost::Predictor { } catch (std::out_of_range const&) { LOG(FATAL) << "quadratureshap_points out of range: " << kv.second; } - CHECK_GE(points, 2) << "quadratureshap_points must be >= 2"; - CHECK_LE(points, 64) << "quadratureshap_points must be <= 64"; + CHECK_EQ(points, 8) << "GPU QuadratureSHAP currently uses a fixed quadrature size of 8."; quadrature_shap_points_ = points; } } @@ -862,7 +861,7 @@ class GPUPredictor : public xgboost::Predictor { private: ColumnSplitHelper column_split_helper_; std::string shap_algorithm_{"treeshap"}; - std::size_t quadrature_shap_points_{16}; + std::size_t quadrature_shap_points_{8}; }; XGBOOST_REGISTER_PREDICTOR(GPUPredictor, "gpu_predictor") diff --git a/src/predictor/interpretability/shap.cc b/src/predictor/interpretability/shap.cc index 66efb5b48a0c..b85e2551f5d2 100644 --- a/src/predictor/interpretability/shap.cc +++ b/src/predictor/interpretability/shap.cc @@ -3,15 +3,13 @@ */ #include "shap.h" -#include // for copy, fill -#include // for array -#include // for abs -#include // for uint32_t -#include // for numeric_limits -#include // for mutex -#include // for remove_const_t -#include // for unordered_map -#include // for vector +#include // for copy, fill +#include // for array +#include // for abs +#include // for uint32_t +#include // for numeric_limits +#include // for remove_const_t +#include // for vector #include "../../common/threading_utils.h" // for ParallelFor #include "../../gbm/gbtree_model.h" // for GBTreeModel @@ -239,153 +237,197 @@ void CalculateContributions(tree::ScalarTreeView const &tree, RegTree::FVec cons condition, condition_feature, 1.0f); } -constexpr std::size_t kDefaultQuadratureShapPoints = 16; -constexpr std::size_t kMaxQuadratureShapPoints = 64; -constexpr double kQuadratureShapQeps = 1e-15; -constexpr double kQuadratureShapUnseen = -999.0; -using QuadratureRule = detail::EndpointQuadratureRule; -using QuadratureBuffer = std::array; - -QuadratureRule const &GetQuadratureRule(std::size_t n) { - static std::mutex cache_mutex; - static std::unordered_map cache; - std::lock_guard guard{cache_mutex}; - auto it = cache.find(n); - if (it == cache.cend()) { - it = cache - .emplace(n, detail::MakeEndpointQuadrature( - n, kQuadratureShapQeps)) - .first; - } - return it->second; +// Keep the CPU quadrature recurrence on the same fixed 8-point rule as the GPU path so the hot +// loops stay small and the compiler can fully unroll the basis update and extraction work. +constexpr std::size_t kQuadratureShapPoints = 8; +constexpr double kQuadratureShapBuildQeps = 1e-15; +constexpr float kQuadratureShapUnseen = -999.0f; +struct QuadratureRule { + std::array nodes{}; + std::array weights{}; +}; +using QuadratureBuffer = std::array; + +QuadratureRule const &GetQuadratureRule() { + static QuadratureRule const rule = [] { + auto const rule_d = + detail::MakeEndpointQuadrature(kQuadratureShapBuildQeps); + QuadratureRule out; + for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + out.nodes[i] = static_cast(rule_d.nodes[i]); + out.weights[i] = static_cast(rule_d.weights[i]); + } + return out; + }(); + return rule; } -void ScaleInPlace(QuadratureBuffer *h_vals, double scale) { +void ScaleInPlace(QuadratureBuffer *h_vals, float scale) { for (auto &v : *h_vals) { v *= scale; } } -void AddInPlace(QuadratureBuffer *lhs, QuadratureBuffer const &rhs, std::size_t points) { - for (std::size_t i = 0; i < points; ++i) { +void AddInPlace(QuadratureBuffer *lhs, QuadratureBuffer const &rhs) { + for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { (*lhs)[i] += rhs[i]; } } -double ExtractQuadratureDelta(QuadratureRule const &rule, QuadratureBuffer const &h_vals, - double p_enter, double p_exit) { - auto const alpha_enter = p_enter - 1.0; - auto const alpha_exit = p_exit - 1.0; - auto const has_enter = - (p_enter != kQuadratureShapUnseen) && (std::abs(alpha_enter) >= kQuadratureShapQeps); - auto const has_exit = - (p_exit != kQuadratureShapUnseen) && (std::abs(alpha_exit) >= kQuadratureShapQeps); - if (!has_enter && !has_exit) { - return 0.0; - } - double acc = 0.0; - for (std::size_t i = 0; i < rule.points; ++i) { - auto const weighted_h = h_vals[i] * rule.weights[i]; - if (has_enter) { - acc += alpha_enter * weighted_h / (1.0 + alpha_enter * rule.nodes[i]); +float ExtractQuadratureDelta(QuadratureRule const &rule, QuadratureBuffer const &h_vals, + float p_enter, float p_exit) { + float acc = 0.0f; + if (p_enter != 1.0f) { + auto const alpha_enter = p_enter - 1.0f; + for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + auto const weighted_h = h_vals[i] * rule.weights[i]; + acc += alpha_enter * weighted_h / (1.0f + alpha_enter * rule.nodes[i]); } - if (has_exit) { - acc -= alpha_exit * weighted_h / (1.0 + alpha_exit * rule.nodes[i]); + } + if (p_exit != 1.0f) { + auto const alpha_exit = p_exit - 1.0f; + for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + auto const weighted_h = h_vals[i] * rule.weights[i]; + acc -= alpha_exit * weighted_h / (1.0f + alpha_exit * rule.nodes[i]); } } return acc; } -bool GoesLeftQuadrature(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, - bst_node_t nidx) { - auto split_index = tree.SplitIndex(nidx); - auto const &cats = tree.GetCategoriesMatrix(); - auto next = predictor::GetNextNode(tree, nidx, feat.GetFvalue(split_index), - feat.IsMissing(split_index), cats); - return next == tree.LeftChild(nidx); -} +struct QuadratureShapTreeRunner { + tree::ScalarTreeView const &tree; + RegTree::FVec const &feat; + QuadratureRule const &rule; + std::vector *path_prob; + float *phi; + + [[nodiscard]] bool EvaluateGoesLeft(bst_node_t nidx) const { + auto split_index = tree.SplitIndex(nidx); + auto const &cats = tree.GetCategoriesMatrix(); + auto next = predictor::GetNextNode(tree, nidx, feat.GetFvalue(split_index), + feat.IsMissing(split_index), cats); + return next == tree.LeftChild(nidx); + } -double ChildWeightQuadrature(tree::ScalarTreeView const &tree, bst_node_t parent, - bst_node_t child) { - auto parent_cover = tree.Stat(parent).sum_hess; - CHECK_GT(parent_cover, 0.0f); - return tree.Stat(child).sum_hess / parent_cover; -} + [[nodiscard]] float ChildWeight(bst_node_t parent, bst_node_t child) const { + auto parent_cover = tree.Stat(parent).sum_hess; + CHECK_GT(parent_cover, 0.0f); + return tree.Stat(child).sum_hess / parent_cover; + } -void TreeShapQuadrature(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, - bst_node_t nidx, QuadratureRule const &rule, QuadratureBuffer const &c_vals, - double w_prod, std::vector *p_vals, double *phi, - QuadratureBuffer *out_h) { - if (tree.IsLeaf(nidx)) { + void HandleLeaf(bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, + QuadratureBuffer *out_h) const { *out_h = c_vals; ScaleInPlace(out_h, w_prod * tree.LeafValue(nidx)); - return; } - auto split_index = tree.SplitIndex(nidx); - auto left = tree.LeftChild(nidx); - auto right = tree.RightChild(nidx); - auto left_weight = ChildWeightQuadrature(tree, nidx, left); - auto right_weight = ChildWeightQuadrature(tree, nidx, right); - auto goes_left = GoesLeftQuadrature(tree, feat, nidx); - auto p_old = (*p_vals)[split_index]; - QuadratureBuffer child_h{}; + void ExtractContribution(bst_feature_t split_index, QuadratureBuffer const &h_vals, float p_enter, + float p_exit) { + phi[split_index] += ExtractQuadratureDelta(rule, h_vals, p_enter, p_exit); + } + + void VisitChild(bst_node_t split_node, bst_node_t child_node, float child_weight, bool satisfies, + QuadratureBuffer const &c_vals, float w_prod, QuadratureBuffer *out_h) { + auto split_index = tree.SplitIndex(split_node); + auto p_old = (*path_prob)[split_index]; - auto visit_child = [&](bst_node_t child, double child_weight, bool satisfies, - QuadratureBuffer *out_child) { - double p_e = 0.0; - double p_up = 0.0; + float p_e = 0.0f; + float p_up = 0.0f; if (p_old == kQuadratureShapUnseen) { - p_e = satisfies ? 1.0 / child_weight : 0.0; - p_up = 1.0; - } else if (std::abs(p_old) < kQuadratureShapQeps) { - p_e = 0.0; - p_up = 0.0; + p_e = satisfies ? 1.0f / child_weight : 0.0f; + p_up = 1.0f; + } else if (p_old == 0.0f) { + p_e = 0.0f; + p_up = 0.0f; } else { - p_e = satisfies ? p_old / child_weight : 0.0; + p_e = satisfies ? p_old / child_weight : 0.0f; p_up = p_old; } auto c_child = c_vals; - auto alpha_e = p_e - 1.0; - for (std::size_t i = 0; i < rule.points; ++i) { - c_child[i] *= 1.0 + alpha_e * rule.nodes[i]; + auto alpha_e = p_e - 1.0f; + for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + c_child[i] *= 1.0f + alpha_e * rule.nodes[i]; } if (p_old != kQuadratureShapUnseen) { - auto alpha_old = p_old - 1.0; - if (std::abs(alpha_old) >= kQuadratureShapQeps) { - for (std::size_t i = 0; i < rule.points; ++i) { - c_child[i] /= 1.0 + alpha_old * rule.nodes[i]; + auto alpha_old = p_old - 1.0f; + if (alpha_old != 0.0f) { + for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + c_child[i] /= 1.0f + alpha_old * rule.nodes[i]; } } } - (*p_vals)[split_index] = p_e; - TreeShapQuadrature(tree, feat, child, rule, c_child, w_prod * child_weight, p_vals, phi, - out_child); - (*p_vals)[split_index] = p_old; - phi[split_index] += ExtractQuadratureDelta(rule, *out_child, p_e, p_up); - }; + (*path_prob)[split_index] = p_e; + this->RunNode(child_node, c_child, w_prod * child_weight, out_h); + (*path_prob)[split_index] = p_old; + this->ExtractContribution(split_index, *out_h, p_e, p_up); + } - visit_child(left, left_weight, goes_left, out_h); - visit_child(right, right_weight, !goes_left, &child_h); - AddInPlace(out_h, child_h, rule.points); -} + void RunNode(bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, + QuadratureBuffer *out_h) { + if (tree.IsLeaf(nidx)) { + this->HandleLeaf(nidx, c_vals, w_prod, out_h); + return; + } -void CalculateContributionsQuadrature(tree::ScalarTreeView const &tree, RegTree::FVec const &feat, - QuadratureRule const &rule, std::vector *mean_values, - double *out_contribs, std::vector *p_vals) { - out_contribs[feat.Size()] += (*mean_values)[0]; + auto left = tree.LeftChild(nidx); + auto right = tree.RightChild(nidx); + auto left_weight = this->ChildWeight(nidx, left); + auto right_weight = this->ChildWeight(nidx, right); + auto goes_left = this->EvaluateGoesLeft(nidx); + QuadratureBuffer right_h{}; - if (tree.IsLeaf(RegTree::kRoot)) { - return; + this->VisitChild(nidx, left, left_weight, goes_left, c_vals, w_prod, out_h); + this->VisitChild(nidx, right, right_weight, !goes_left, c_vals, w_prod, &right_h); + AddInPlace(out_h, right_h); } - QuadratureBuffer c_init{}; - std::fill_n(c_init.begin(), rule.points, 1.0); - QuadratureBuffer h_vals{}; - TreeShapQuadrature(tree, feat, RegTree::kRoot, rule, c_init, 1.0, p_vals, out_contribs, &h_vals); + void Run() { + if (tree.IsLeaf(RegTree::kRoot)) { + return; + } + + QuadratureBuffer c_init{}; + c_init.fill(1.0f); + QuadratureBuffer h_vals{}; + this->RunNode(RegTree::kRoot, c_init, 1.0f, &h_vals); + } +}; + +struct QuadratureShapModelData { + std::vector trees; + std::vector> trees_by_group; + std::vector weights; + std::vector group_root_mean_sums; +}; + +QuadratureShapModelData MakeQuadratureShapModelData(gbm::GBTreeModel const &model, + bst_tree_t tree_end, + std::vector const *tree_weights) { + auto const n_trees = static_cast(tree_end); + auto const h_tree_groups = model.TreeGroups(DeviceOrd::CPU()); + auto const n_groups = model.learner_model_param->num_output_group; + + QuadratureShapModelData out; + out.trees.reserve(n_trees); + out.trees_by_group.resize(n_groups); + out.weights.resize(n_trees, 1.0f); + out.group_root_mean_sums.resize(n_groups, 0.0f); + + for (std::size_t i = 0; i < n_trees; ++i) { + out.trees.emplace_back(model.trees[i].get()); + } + for (bst_tree_t i = 0; i < tree_end; ++i) { + auto gid = h_tree_groups[i]; + auto weight = tree_weights == nullptr ? 1.0f : (*tree_weights)[i]; + out.trees_by_group[gid].push_back(i); + out.weights[i] = weight; + out.group_root_mean_sums[gid] += + static_cast(detail::FillRootMeanValue(out.trees[i], RegTree::kRoot) * weight); + } + return out; } template @@ -500,32 +542,29 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, CHECK(!model.learner_model_param->IsVectorLeaf()) << "Predict contribution" << MTNotImplemented(); CHECK(!p_fmat->Info().IsColumnSplit()) << "Predict contribution support for column-wise data split is not yet implemented."; + CHECK_EQ(quadrature_points, kQuadratureShapPoints) + << "CPU QuadratureSHAP currently uses a fixed quadrature size of " << kQuadratureShapPoints + << "."; MetaInfo const &info = p_fmat->Info(); tree_end = predictor::GetTreeLimit(model.trees, tree_end); CHECK_GE(tree_end, 0); ValidateTreeWeights(tree_weights, tree_end); auto const n_trees = static_cast(tree_end); auto const n_threads = ctx->Threads(); + auto const n_groups = model.learner_model_param->num_output_group; + auto const n_features = model.learner_model_param->num_feature; size_t const ncolumns = model.learner_model_param->num_feature + 1; std::vector &contribs = out_contribs->HostVector(); contribs.resize(info.num_row_ * ncolumns * model.learner_model_param->num_output_group); std::fill(contribs.begin(), contribs.end(), 0.0f); - - std::vector> mean_values(n_trees); - common::ParallelFor(n_trees, n_threads, [&](auto i) { - FillNodeMeanValues(model.trees[i]->HostScView(), &(mean_values[i])); - }); - - auto const n_groups = model.learner_model_param->num_output_group; CHECK_NE(n_groups, 0); - auto const &rule = GetQuadratureRule(quadrature_points); + auto const &rule = GetQuadratureRule(); auto const base_score = model.learner_model_param->BaseScore(DeviceOrd::CPU()); - auto const h_tree_groups = model.TreeGroups(DeviceOrd::CPU()); + auto model_data = MakeQuadratureShapModelData(model, tree_end, tree_weights); std::vector feats_tloc(n_threads); - std::vector> contribs_tloc(n_threads, std::vector(ncolumns)); - std::vector> p_vals_tloc( - n_threads, - std::vector(model.learner_model_param->num_feature, kQuadratureShapUnseen)); + std::vector> contribs_tloc(n_threads, std::vector(ncolumns)); + std::vector> path_prob_tloc( + n_threads, std::vector(n_features, kQuadratureShapUnseen)); auto device = ctx->Device().IsSycl() ? DeviceOrd::CPU() : ctx->Device(); auto base_margin = info.base_margin_.View(device); @@ -538,25 +577,23 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, feats.Init(model.learner_model_param->num_feature); } auto &this_tree_contribs = contribs_tloc[tid]; - auto &p_vals = p_vals_tloc[tid]; + auto &path_prob = path_prob_tloc[tid]; auto row_idx = view.base_rowid + i; auto n_valid = view.DoFill(i, feats.Data().data()); feats.HasMissing(n_valid != feats.Size()); for (bst_target_t gid = 0; gid < n_groups; ++gid) { float *p_contribs = &contribs[(row_idx * n_groups + gid) * ncolumns]; - for (bst_tree_t j = 0; j < tree_end; ++j) { - if (h_tree_groups[j] != gid) { - continue; - } - std::fill(this_tree_contribs.begin(), this_tree_contribs.end(), 0.0); - auto const sc_tree = model.trees[j]->HostScView(); - CalculateContributionsQuadrature(sc_tree, feats, rule, &mean_values[j], - this_tree_contribs.data(), &p_vals); - auto const weight = tree_weights == nullptr ? 1.0f : (*tree_weights)[j]; - for (size_t ci = 0; ci < ncolumns; ++ci) { - p_contribs[ci] += static_cast(this_tree_contribs[ci] * weight); + for (auto j : model_data.trees_by_group[gid]) { + std::fill(this_tree_contribs.begin(), this_tree_contribs.end(), 0.0f); + auto runner = QuadratureShapTreeRunner{model_data.trees[j], feats, rule, &path_prob, + this_tree_contribs.data()}; + runner.Run(); + auto const weight = model_data.weights[j]; + for (size_t ci = 0; ci + 1 < ncolumns; ++ci) { + p_contribs[ci] += this_tree_contribs[ci] * weight; } } + p_contribs[ncolumns - 1] += model_data.group_root_mean_sums[gid]; if (base_margin.Size() != 0) { CHECK_EQ(base_margin.Shape(1), n_groups); p_contribs[ncolumns - 1] += base_margin(row_idx, gid); diff --git a/src/predictor/interpretability/shap.cu b/src/predictor/interpretability/shap.cu index cf123813e2a5..3862ea3f435b 100644 --- a/src/predictor/interpretability/shap.cu +++ b/src/predictor/interpretability/shap.cu @@ -67,18 +67,6 @@ constexpr std::array kGpuQuadratureDepthBuckets{{16, 32, 64}}; constexpr double kQuadratureShapQeps = 1e-15; using QuadratureRule = detail::EndpointQuadratureRule; -double FillRootMeanValue(tree::ScalarTreeView const& tree, bst_node_t nidx) { - if (tree.IsLeaf(nidx)) { - return tree.LeafValue(nidx); - } - auto left = tree.LeftChild(nidx); - auto right = tree.RightChild(nidx); - double result = FillRootMeanValue(tree, left) * tree.SumHess(left); - result += FillRootMeanValue(tree, right) * tree.SumHess(right); - result /= tree.SumHess(nidx); - return result; -} - std::vector MakeGroupRootMeanSums(gbm::GBTreeModel const& model, bst_tree_t tree_end, std::vector const* tree_weights) { auto h_tree_groups = model.TreeGroups(DeviceOrd::CPU()); @@ -88,7 +76,7 @@ std::vector MakeGroupRootMeanSums(gbm::GBTreeModel const& model, bst_tree auto const weight = tree_weights == nullptr ? 1.0f : (*tree_weights)[tree_idx]; auto const tree = model.trees.at(tree_idx)->HostScView(); h_group_root_mean_sums[h_tree_groups[tree_idx]] += - FillRootMeanValue(tree, RegTree::kRoot) * weight; + detail::FillRootMeanValue(tree, RegTree::kRoot) * weight; } std::vector out(h_group_root_mean_sums.size()); diff --git a/src/predictor/interpretability/shap.h b/src/predictor/interpretability/shap.h index c61c24c05401..880ed8d2c489 100644 --- a/src/predictor/interpretability/shap.h +++ b/src/predictor/interpretability/shap.h @@ -14,6 +14,21 @@ struct GBTreeModel; } // namespace xgboost::gbm namespace xgboost::interpretability { +namespace detail { +template +inline double FillRootMeanValue(TreeView const& tree, NodeIndex nidx) { + if (tree.IsLeaf(nidx)) { + return tree.LeafValue(nidx); + } + auto left = tree.LeftChild(nidx); + auto right = tree.RightChild(nidx); + double result = FillRootMeanValue(tree, left) * tree.SumHess(left); + result += FillRootMeanValue(tree, right) * tree.SumHess(right); + result /= tree.SumHess(nidx); + return result; +} +} // namespace detail + namespace cpu_impl { void ShapValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, bst_tree_t tree_end, From d80dfa1f442cc998fecc646aa350725c70cc3b61 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Mon, 30 Mar 2026 08:22:39 -0700 Subject: [PATCH 08/13] Speed up CPU QuadratureSHAP extraction --- src/predictor/interpretability/shap.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/predictor/interpretability/shap.cc b/src/predictor/interpretability/shap.cc index b85e2551f5d2..5c5d0bbe7d35 100644 --- a/src/predictor/interpretability/shap.cc +++ b/src/predictor/interpretability/shap.cc @@ -242,6 +242,7 @@ void CalculateContributions(tree::ScalarTreeView const &tree, RegTree::FVec cons constexpr std::size_t kQuadratureShapPoints = 8; constexpr double kQuadratureShapBuildQeps = 1e-15; constexpr float kQuadratureShapUnseen = -999.0f; + struct QuadratureRule { std::array nodes{}; std::array weights{}; @@ -280,15 +281,13 @@ float ExtractQuadratureDelta(QuadratureRule const &rule, QuadratureBuffer const if (p_enter != 1.0f) { auto const alpha_enter = p_enter - 1.0f; for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { - auto const weighted_h = h_vals[i] * rule.weights[i]; - acc += alpha_enter * weighted_h / (1.0f + alpha_enter * rule.nodes[i]); + acc += alpha_enter * h_vals[i] / (1.0f + alpha_enter * rule.nodes[i]); } } if (p_exit != 1.0f) { auto const alpha_exit = p_exit - 1.0f; for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { - auto const weighted_h = h_vals[i] * rule.weights[i]; - acc -= alpha_exit * weighted_h / (1.0f + alpha_exit * rule.nodes[i]); + acc -= alpha_exit * h_vals[i] / (1.0f + alpha_exit * rule.nodes[i]); } } return acc; @@ -317,8 +316,10 @@ struct QuadratureShapTreeRunner { void HandleLeaf(bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, QuadratureBuffer *out_h) const { - *out_h = c_vals; - ScaleInPlace(out_h, w_prod * tree.LeafValue(nidx)); + auto const leaf_scale = w_prod * tree.LeafValue(nidx); + for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + (*out_h)[i] = c_vals[i] * leaf_scale * rule.weights[i]; + } } void ExtractContribution(bst_feature_t split_index, QuadratureBuffer const &h_vals, float p_enter, From 5d478c3823c51e5b46e8868b3b0d5d9b6c4ff485 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Wed, 1 Apr 2026 04:23:04 -0700 Subject: [PATCH 09/13] Add QuadratureSHAP interaction paths --- src/predictor/cpu_predictor.cc | 10 +- src/predictor/gpu_predictor.cu | 9 +- src/predictor/interpretability/shap.cc | 447 +++++++++++++++++++-- src/predictor/interpretability/shap.cu | 517 ++++++++++++++++++++++++- src/predictor/interpretability/shap.h | 11 + 5 files changed, 948 insertions(+), 46 deletions(-) diff --git a/src/predictor/cpu_predictor.cc b/src/predictor/cpu_predictor.cc index f36c683bb2c9..bef04f358177 100644 --- a/src/predictor/cpu_predictor.cc +++ b/src/predictor/cpu_predictor.cc @@ -904,8 +904,14 @@ class CPUPredictor : public Predictor { gbm::GBTreeModel const &model, bst_tree_t ntree_limit, std::vector const *tree_weights, bool approximate) const override { - interpretability::ShapInteractionValues(this->ctx_, p_fmat, out_contribs, model, ntree_limit, - tree_weights, approximate); + if (!approximate && shap_algorithm_ == "quadratureshap") { + interpretability::cpu_impl::QuadratureShapInteractionValues(this->ctx_, p_fmat, out_contribs, + model, ntree_limit, tree_weights, + quadrature_shap_points_); + } else { + interpretability::ShapInteractionValues(this->ctx_, p_fmat, out_contribs, model, ntree_limit, + tree_weights, approximate); + } } private: diff --git a/src/predictor/gpu_predictor.cu b/src/predictor/gpu_predictor.cu index ea0813462791..beba570fb11c 100644 --- a/src/predictor/gpu_predictor.cu +++ b/src/predictor/gpu_predictor.cu @@ -813,8 +813,13 @@ class GPUPredictor : public xgboost::Predictor { LOG(FATAL) << "Approximated contribution is not implemented in GPU predictor, use cpu " "instead."; } - interpretability::ShapInteractionValues(ctx_, p_fmat, out_contribs, model, tree_end, - tree_weights, approximate); + if (shap_algorithm_ == "quadratureshap") { + interpretability::cuda_impl::QuadratureShapInteractionValues( + ctx_, p_fmat, out_contribs, model, tree_end, tree_weights, quadrature_shap_points_); + } else { + interpretability::ShapInteractionValues(ctx_, p_fmat, out_contribs, model, tree_end, + tree_weights, approximate); + } } void PredictLeaf(DMatrix* p_fmat, HostDeviceVector* predictions, diff --git a/src/predictor/interpretability/shap.cc b/src/predictor/interpretability/shap.cc index 5c5d0bbe7d35..10055f7e54f7 100644 --- a/src/predictor/interpretability/shap.cc +++ b/src/predictor/interpretability/shap.cc @@ -20,6 +20,7 @@ #include "quadrature.h" #include "xgboost/base.h" // for bst_omp_uint #include "xgboost/logging.h" // for CHECK +#include "xgboost/span.h" // for Span #include "xgboost/tree_model.h" // for MTNotImplemented namespace xgboost::interpretability { @@ -263,12 +264,6 @@ QuadratureRule const &GetQuadratureRule() { return rule; } -void ScaleInPlace(QuadratureBuffer *h_vals, float scale) { - for (auto &v : *h_vals) { - v *= scale; - } -} - void AddInPlace(QuadratureBuffer *lhs, QuadratureBuffer const &rhs) { for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { (*lhs)[i] += rhs[i]; @@ -293,12 +288,277 @@ float ExtractQuadratureDelta(QuadratureRule const &rule, QuadratureBuffer const return acc; } +constexpr bool kQuadratureInteractionUseEdgeKernel = false; +constexpr bool kQuadratureInteractionUseLatestLiveIndex = false; + +// Off-diagonal interaction terms use the same return-edge delta as additive SHAP, but with one +// partner feature removed from the live quadrature basis. For an active partner with live ratio +// q_j, the weighted subtree return factors as +// H(t) = H_without_j(t) * (1 + (q_j - 1) t) +// after the zero-fraction terms cancel. The conditioned on/off difference is therefore the +// precomputed return-edge kernel divided by that partner factor and multiplied by (q_j - 1). +float ExtractQuadratureInteractionDelta(QuadratureRule const &rule, QuadratureBuffer const &h_vals, + float p_enter, float p_exit, float q_partner) { + if (q_partner == 1.0f) { + return 0.0f; + } + + auto const alpha_partner = q_partner - 1.0f; + auto const has_enter = p_enter != 1.0f; + auto const has_exit = p_exit != 1.0f; + auto const alpha_enter = p_enter - 1.0f; + auto const alpha_exit = p_exit - 1.0f; + + float acc = 0.0f; + for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + float edge_delta = 0.0f; + if (has_enter) { + edge_delta += alpha_enter / (1.0f + alpha_enter * rule.nodes[i]); + } + if (has_exit) { + edge_delta -= alpha_exit / (1.0f + alpha_exit * rule.nodes[i]); + } + acc += alpha_partner * h_vals[i] * edge_delta / (1.0f + alpha_partner * rule.nodes[i]); + } + return acc; +} + +float ExtractQuadratureInteractionDelta(QuadratureRule const &rule, + QuadratureBuffer const &edge_kernel, float q_partner) { + if (q_partner == 1.0f) { + return 0.0f; + } + + auto const alpha_partner = q_partner - 1.0f; + float acc = 0.0f; + for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + acc += alpha_partner * edge_kernel[i] / (1.0f + alpha_partner * rule.nodes[i]); + } + return acc; +} + +void WriteWeightedLeafReturn(tree::ScalarTreeView const &tree, QuadratureRule const &rule, + bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, + QuadratureBuffer *out_h) { + auto const leaf_scale = w_prod * tree.LeafValue(nidx); + for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + (*out_h)[i] = c_vals[i] * leaf_scale * rule.weights[i]; + } +} + +// Dense row-local output view for additive contributions. +template +struct ContributionVectorView { + T *data; + std::size_t size; + + T &operator[](std::size_t idx) const { return data[idx]; } +}; + +// Dense row-local output view for interaction matrices. Future formulations can target this sink +// directly instead of open-coding flattened indexing arithmetic. +template +struct DenseInteractionMatrixView { + T *data; + std::size_t ncolumns; + + T &operator()(std::size_t i, std::size_t j) const { return data[i * ncolumns + j]; } +}; + +// One active split on the current root-to-node path. Traversal owns the push/pop discipline, while +// formulations can inspect the live path without duplicating duplicate-feature bookkeeping. +struct QuadraturePathElement { + bst_feature_t split_index; + float p_parent; + float p_child; + std::int32_t prev_live_index; +}; + +// Read-only formulation view of the current root-to-node path. Traversal keeps ownership of the +// stack so different contribution formulations can inspect the same live path state. +struct QuadraturePathView { + common::Span elements; + common::Span latest_live_index; + + [[nodiscard]] auto Depth() const { return elements.size(); } + [[nodiscard]] bool Empty() const { return elements.empty(); } + [[nodiscard]] auto Entries() const { return elements; } + + [[nodiscard]] auto CurrentSplit() const -> QuadraturePathElement const & { + CHECK(!elements.empty()); + return elements.back(); + } + + // Iterate the active path once per feature, newest-to-oldest. Later duplicate splits are the + // live ones for path-local partner lookups, so older duplicates are hidden from formulations. + template + void ForEachUniqueFeature(Fn &&fn) const { + if (!latest_live_index.empty()) { + for (std::size_t i = elements.size(); i != 0; --i) { + auto const idx = i - 1; + auto const split_index = elements[idx].split_index; + if (latest_live_index[split_index] == static_cast(idx)) { + fn(idx, elements[idx]); + } + } + } else { + for (std::size_t i = elements.size(); i != 0; --i) { + auto const idx = i - 1; + auto const split_index = elements[idx].split_index; + bool shadowed = false; + for (std::size_t newer = elements.size(); newer > i; --newer) { + if (elements[newer - 1].split_index == split_index) { + shadowed = true; + break; + } + } + if (!shadowed) { + fn(idx, elements[idx]); + } + } + } + } +}; + +// Current additive SHAP formulation. It consumes the weighted subtree return and writes one +// feature contribution per return edge. +struct AdditiveContributionFormulation { + static constexpr bool kTrackLatestLiveIndex = false; + ContributionVectorView phi; + + void HandleLeaf(tree::ScalarTreeView const &tree, QuadratureRule const &rule, + QuadraturePathView path, bst_node_t nidx, QuadratureBuffer const &c_vals, + float w_prod, QuadratureBuffer *out_h) const { + (void)path; + WriteWeightedLeafReturn(tree, rule, nidx, c_vals, w_prod, out_h); + } + + void HandleReturn(QuadratureRule const &rule, QuadraturePathView path, bst_feature_t split_index, + QuadratureBuffer const &h_vals, float p_enter, float p_exit) const { + (void)path; + phi[split_index] += ExtractQuadratureDelta(rule, h_vals, p_enter, p_exit); + } +}; + +// First path-local interaction formulation built on top of the quadrature traversal. It keeps the +// traversal and weighted subtree return shared with additive SHAP, and only changes how return +// edges are written into the dense interaction sink. +struct InteractionContributionFormulation { + static constexpr bool kTrackLatestLiveIndex = kQuadratureInteractionUseLatestLiveIndex; + struct EdgeEffect { + bst_feature_t split_index; + float diagonal_delta; + QuadratureBuffer edge_kernel; + }; + + ContributionVectorView phi_diag; + DenseInteractionMatrixView phi_interactions; + float scale; + + // Traversal still needs a weighted subtree return, so the interaction path shares the additive + // leaf behavior and changes only the return-edge algebra. + void HandleLeaf(tree::ScalarTreeView const &tree, QuadratureRule const &rule, + QuadraturePathView path, bst_node_t nidx, QuadratureBuffer const &c_vals, + float w_prod, QuadratureBuffer *out_h) const { + (void)path; + WriteWeightedLeafReturn(tree, rule, nidx, c_vals, w_prod, out_h); + } + + [[nodiscard]] auto MakeEdgeEffect(QuadratureRule const &rule, bst_feature_t split_index, + QuadratureBuffer const &h_vals, float p_enter, + float p_exit) const { + QuadratureBuffer edge_kernel{}; + float diagonal_delta = 0.0f; + + if constexpr (kQuadratureInteractionUseEdgeKernel) { + auto const has_enter = p_enter != 1.0f; + auto const has_exit = p_exit != 1.0f; + auto const alpha_enter = p_enter - 1.0f; + auto const alpha_exit = p_exit - 1.0f; + + for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + float edge_delta = 0.0f; + if (has_enter) { + edge_delta += alpha_enter / (1.0f + alpha_enter * rule.nodes[i]); + } + if (has_exit) { + edge_delta -= alpha_exit / (1.0f + alpha_exit * rule.nodes[i]); + } + edge_kernel[i] = h_vals[i] * edge_delta; + diagonal_delta += edge_kernel[i]; + } + } else { + diagonal_delta = ExtractQuadratureDelta(rule, h_vals, p_enter, p_exit); + } + + return EdgeEffect{split_index, diagonal_delta, edge_kernel}; + } + + void AccumulateDiagonal(EdgeEffect const &edge) const { + phi_diag[edge.split_index] += scale * edge.diagonal_delta; + } + + // Walk the live unique path excluding the current split. A pairwise formulation can distribute + // the current edge effect across these partner features without reimplementing duplicate logic. + template + void ForEachPartner(QuadraturePathView path, Fn &&fn) const { + CHECK(!path.Empty()); + auto const current_split = path.CurrentSplit().split_index; + bool skipped_current = false; + path.ForEachUniqueFeature([&](std::size_t, QuadraturePathElement const &element) { + if (!skipped_current && element.split_index == current_split) { + skipped_current = true; + return; + } + fn(element); + }); + } + + void AccumulatePair(EdgeEffect const &edge, QuadraturePathElement const &partner, + float pair_delta) const { + auto const i = static_cast(edge.split_index); + auto const j = static_cast(partner.split_index); + phi_interactions(i, j) += scale * pair_delta; + } + + void HandleReturn(QuadratureRule const &rule, QuadraturePathView path, bst_feature_t split_index, + QuadratureBuffer const &h_vals, float p_enter, float p_exit) const { + auto const edge = this->MakeEdgeEffect(rule, split_index, h_vals, p_enter, p_exit); + this->AccumulateDiagonal(edge); + + this->ForEachPartner(path, [&](QuadraturePathElement const &partner) { + float pair_delta = 0.0f; + if constexpr (kQuadratureInteractionUseEdgeKernel) { + pair_delta = ExtractQuadratureInteractionDelta(rule, edge.edge_kernel, partner.p_child); + } else { + pair_delta = + ExtractQuadratureInteractionDelta(rule, h_vals, p_enter, p_exit, partner.p_child); + } + this->AccumulatePair(edge, partner, pair_delta); + }); + } +}; + +// Tree-walk engine for quadrature formulations. It owns feature evaluation, child descent, and +// the live path-probability state, then hands leaf/return events to the selected formulation. +template struct QuadratureShapTreeRunner { tree::ScalarTreeView const &tree; RegTree::FVec const &feat; QuadratureRule const &rule; std::vector *path_prob; - float *phi; + std::vector *path; + std::vector *latest_live_index; + ContributionFormulation formulation; + + [[nodiscard]] auto CurrentPath() const { + if constexpr (ContributionFormulation::kTrackLatestLiveIndex) { + return QuadraturePathView{common::Span{*path}, + common::Span{*latest_live_index}}; + } else { + return QuadraturePathView{common::Span{*path}, {}}; + } + } [[nodiscard]] bool EvaluateGoesLeft(bst_node_t nidx) const { auto split_index = tree.SplitIndex(nidx); @@ -314,19 +574,6 @@ struct QuadratureShapTreeRunner { return tree.Stat(child).sum_hess / parent_cover; } - void HandleLeaf(bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, - QuadratureBuffer *out_h) const { - auto const leaf_scale = w_prod * tree.LeafValue(nidx); - for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { - (*out_h)[i] = c_vals[i] * leaf_scale * rule.weights[i]; - } - } - - void ExtractContribution(bst_feature_t split_index, QuadratureBuffer const &h_vals, float p_enter, - float p_exit) { - phi[split_index] += ExtractQuadratureDelta(rule, h_vals, p_enter, p_exit); - } - void VisitChild(bst_node_t split_node, bst_node_t child_node, float child_weight, bool satisfies, QuadratureBuffer const &c_vals, float w_prod, QuadratureBuffer *out_h) { auto split_index = tree.SplitIndex(split_node); @@ -361,15 +608,26 @@ struct QuadratureShapTreeRunner { } (*path_prob)[split_index] = p_e; + if constexpr (ContributionFormulation::kTrackLatestLiveIndex) { + auto prev_live = (*latest_live_index)[split_index]; + path->push_back(QuadraturePathElement{split_index, p_up, p_e, prev_live}); + (*latest_live_index)[split_index] = static_cast(path->size() - 1); + } else { + path->push_back(QuadraturePathElement{split_index, p_up, p_e, -1}); + } this->RunNode(child_node, c_child, w_prod * child_weight, out_h); + formulation.HandleReturn(rule, this->CurrentPath(), split_index, *out_h, p_e, p_up); + if constexpr (ContributionFormulation::kTrackLatestLiveIndex) { + (*latest_live_index)[split_index] = path->back().prev_live_index; + } + path->pop_back(); (*path_prob)[split_index] = p_old; - this->ExtractContribution(split_index, *out_h, p_e, p_up); } void RunNode(bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, QuadratureBuffer *out_h) { if (tree.IsLeaf(nidx)) { - this->HandleLeaf(nidx, c_vals, w_prod, out_h); + formulation.HandleLeaf(tree, rule, this->CurrentPath(), nidx, c_vals, w_prod, out_h); return; } @@ -378,6 +636,7 @@ struct QuadratureShapTreeRunner { auto left_weight = this->ChildWeight(nidx, left); auto right_weight = this->ChildWeight(nidx, right); auto goes_left = this->EvaluateGoesLeft(nidx); + QuadratureBuffer right_h{}; this->VisitChild(nidx, left, left_weight, goes_left, c_vals, w_prod, out_h); @@ -386,6 +645,7 @@ struct QuadratureShapTreeRunner { } void Run() { + path->clear(); if (tree.IsLeaf(RegTree::kRoot)) { return; } @@ -564,8 +824,11 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, auto model_data = MakeQuadratureShapModelData(model, tree_end, tree_weights); std::vector feats_tloc(n_threads); std::vector> contribs_tloc(n_threads, std::vector(ncolumns)); + std::vector> path_tloc(n_threads); std::vector> path_prob_tloc( n_threads, std::vector(n_features, kQuadratureShapUnseen)); + std::vector> latest_live_tloc( + n_threads, std::vector(n_features, -1)); auto device = ctx->Device().IsSycl() ? DeviceOrd::CPU() : ctx->Device(); auto base_margin = info.base_margin_.View(device); @@ -578,7 +841,9 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, feats.Init(model.learner_model_param->num_feature); } auto &this_tree_contribs = contribs_tloc[tid]; + auto &path = path_tloc[tid]; auto &path_prob = path_prob_tloc[tid]; + auto &latest_live = latest_live_tloc[tid]; auto row_idx = view.base_rowid + i; auto n_valid = view.DoFill(i, feats.Data().data()); feats.HasMissing(n_valid != feats.Size()); @@ -586,8 +851,9 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, float *p_contribs = &contribs[(row_idx * n_groups + gid) * ncolumns]; for (auto j : model_data.trees_by_group[gid]) { std::fill(this_tree_contribs.begin(), this_tree_contribs.end(), 0.0f); - auto runner = QuadratureShapTreeRunner{model_data.trees[j], feats, rule, &path_prob, - this_tree_contribs.data()}; + auto formulation = AdditiveContributionFormulation{{this_tree_contribs.data(), ncolumns}}; + auto runner = QuadratureShapTreeRunner{ + model_data.trees[j], feats, rule, &path_prob, &path, &latest_live, formulation}; runner.Run(); auto const weight = model_data.weights[j]; for (size_t ci = 0; ci + 1 < ncolumns; ++ci) { @@ -609,6 +875,115 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, LaunchShap(ctx, p_fmat, model, process_view); } +void QuadratureShapInteractionValues(Context const *ctx, DMatrix *p_fmat, + HostDeviceVector *out_contribs, + gbm::GBTreeModel const &model, bst_tree_t tree_end, + std::vector const *tree_weights, + std::size_t quadrature_points) { + CHECK(!model.learner_model_param->IsVectorLeaf()) + << "Predict interaction contribution" << MTNotImplemented(); + CHECK(!p_fmat->Info().IsColumnSplit()) << "Predict interaction contribution support for " + "column-wise data split is not yet implemented."; + CHECK_EQ(quadrature_points, kQuadratureShapPoints) + << "CPU QuadratureSHAP currently uses a fixed quadrature size of " << kQuadratureShapPoints + << "."; + + MetaInfo const &info = p_fmat->Info(); + tree_end = predictor::GetTreeLimit(model.trees, tree_end); + CHECK_GE(tree_end, 0); + ValidateTreeWeights(tree_weights, tree_end); + + auto const n_threads = ctx->Threads(); + auto const n_groups = model.learner_model_param->num_output_group; + auto const n_features = model.learner_model_param->num_feature; + auto const ncolumns = n_features + 1; + auto const row_chunk = n_groups * ncolumns * ncolumns; + auto const matrix_chunk = ncolumns * ncolumns; + + std::vector &contribs = out_contribs->HostVector(); + contribs.resize(info.num_row_ * row_chunk); + std::fill(contribs.begin(), contribs.end(), 0.0f); + + auto const &rule = GetQuadratureRule(); + auto const base_score = model.learner_model_param->BaseScore(DeviceOrd::CPU()); + auto model_data = MakeQuadratureShapModelData(model, tree_end, tree_weights); + std::vector feats_tloc(n_threads); + std::vector> path_tloc(n_threads); + std::vector> path_prob_tloc( + n_threads, std::vector(n_features, kQuadratureShapUnseen)); + std::vector> latest_live_tloc( + n_threads, std::vector(n_features, -1)); + std::vector> diag_tloc(n_threads, std::vector(ncolumns)); + + auto device = ctx->Device().IsSycl() ? DeviceOrd::CPU() : ctx->Device(); + auto base_margin = info.base_margin_.View(device); + + auto process_view = [&](auto &&view) { + common::ParallelFor(view.Size(), n_threads, [&](auto i) { + auto tid = omp_get_thread_num(); + auto &feats = feats_tloc[tid]; + if (feats.Size() == 0) { + feats.Init(model.learner_model_param->num_feature); + } + auto &path = path_tloc[tid]; + auto &path_prob = path_prob_tloc[tid]; + auto &latest_live = latest_live_tloc[tid]; + auto &diag = diag_tloc[tid]; + auto row_idx = view.base_rowid + i; + auto n_valid = view.DoFill(i, feats.Data().data()); + feats.HasMissing(n_valid != feats.Size()); + + for (bst_target_t gid = 0; gid < n_groups; ++gid) { + auto const offset = (row_idx * n_groups + gid) * matrix_chunk; + auto matrix = DenseInteractionMatrixView{contribs.data() + offset, ncolumns}; + std::fill(diag.begin(), diag.end(), 0.0f); + + for (auto j : model_data.trees_by_group[gid]) { + auto formulation = InteractionContributionFormulation{ + {diag.data(), ncolumns}, {matrix.data, matrix.ncolumns}, model_data.weights[j]}; + auto runner = QuadratureShapTreeRunner{ + model_data.trees[j], feats, rule, &path_prob, &path, &latest_live, formulation}; + runner.Run(); + } + + diag[ncolumns - 1] += model_data.group_root_mean_sums[gid]; + if (base_margin.Size() != 0) { + CHECK_EQ(base_margin.Shape(1), n_groups); + diag[ncolumns - 1] += base_margin(row_idx, gid); + } else { + diag[ncolumns - 1] += base_score(gid); + } + + // The path-local return updates populate row-wise off-diagonal effects. Average the two + // directional estimates so the final matrix is explicitly symmetric. + for (size_t r = 0; r < ncolumns; ++r) { + for (size_t c = r + 1; c < ncolumns; ++c) { + auto const sym = 0.5f * (matrix(r, c) + matrix(c, r)); + matrix(r, c) = sym; + matrix(c, r) = sym; + } + } + + // Match the incumbent interaction semantics: each diagonal entry is the additive SHAP + // value minus the off-diagonal interactions in that row. + for (size_t r = 0; r < ncolumns; ++r) { + float value = diag[r]; + for (size_t c = 0; c < ncolumns; ++c) { + if (c != r) { + value -= matrix(r, c); + } + } + matrix(r, r) = value; + } + } + + feats.Drop(); + }); + }; + + LaunchShap(ctx, p_fmat, model, process_view); +} + void ApproxFeatureImportance(Context const *ctx, DMatrix *p_fmat, HostDeviceVector *out_contribs, gbm::GBTreeModel const &model, bst_tree_t tree_end, std::vector const *tree_weights) { @@ -690,9 +1065,9 @@ void ShapInteractionValues(Context const *ctx, DMatrix *p_fmat, MetaInfo const &info = p_fmat->Info(); auto const ngroup = model.learner_model_param->num_output_group; auto const ncolumns = model.learner_model_param->num_feature; - const unsigned row_chunk = ngroup * (ncolumns + 1) * (ncolumns + 1); - const unsigned mrow_chunk = (ncolumns + 1) * (ncolumns + 1); - const unsigned crow_chunk = ngroup * (ncolumns + 1); + const std::size_t row_chunk = ngroup * (ncolumns + 1) * (ncolumns + 1); + const std::size_t mrow_chunk = (ncolumns + 1) * (ncolumns + 1); + const std::size_t crow_chunk = ngroup * (ncolumns + 1); // allocate space for (number of features^2) times the number of rows and tmp off/on contribs std::vector &contribs = out_contribs->HostVector(); @@ -723,16 +1098,22 @@ void ShapInteractionValues(Context const *ctx, DMatrix *p_fmat, for (size_t j = 0; j < info.num_row_; ++j) { for (std::remove_const_t l = 0; l < ngroup; ++l) { - const unsigned o_offset = j * row_chunk + l * mrow_chunk + i * (ncolumns + 1); - const unsigned c_offset = j * crow_chunk + l * (ncolumns + 1); - contribs[o_offset + i] = 0; + const std::size_t o_offset = j * row_chunk + l * mrow_chunk; + const std::size_t c_offset = j * crow_chunk + l * (ncolumns + 1); + auto matrix = + DenseInteractionMatrixView{contribs.data() + o_offset, ncolumns + 1}; + auto diag = + ContributionVectorView{contribs_diag.data() + c_offset, ncolumns + 1}; + auto off = ContributionVectorView{contribs_off.data() + c_offset, ncolumns + 1}; + auto on = ContributionVectorView{contribs_on.data() + c_offset, ncolumns + 1}; + matrix(i, i) = 0; for (size_t k = 0; k < ncolumns + 1; ++k) { // fill in the diagonal with additive effects, and off-diagonal with the interactions if (k == i) { - contribs[o_offset + i] += contribs_diag[c_offset + k]; + matrix(i, i) += diag[k]; } else { - contribs[o_offset + k] = (contribs_on[c_offset + k] - contribs_off[c_offset + k]) / 2.0; - contribs[o_offset + i] -= contribs[o_offset + k]; + matrix(i, k) = (on[k] - off[k]) / 2.0f; + matrix(i, i) -= matrix(i, k); } } } diff --git a/src/predictor/interpretability/shap.cu b/src/predictor/interpretability/shap.cu index 3862ea3f435b..12561be9d46a 100644 --- a/src/predictor/interpretability/shap.cu +++ b/src/predictor/interpretability/shap.cu @@ -208,6 +208,32 @@ XGBOOST_DEVICE inline float PreviousPathProbability(std::uint8_t prev_same_offse return q_vals_row[prev_depth]; } +XGBOOST_DEVICE inline float ExtractQuadratureEdgeDeltaLocal(float quad_node, float quad_weight, + float ret_val, float p_enter, + float q_prev) { + auto weighted_ret = ret_val * quad_weight; + float edge_delta = 0.0f; + if (p_enter != 1.0f) { + auto alpha_enter = p_enter - 1.0f; + edge_delta += alpha_enter / (1.0f + alpha_enter * quad_node); + } + if (q_prev != 1.0f) { + auto alpha_exit = q_prev - 1.0f; + edge_delta -= alpha_exit / (1.0f + alpha_exit * quad_node); + } + return weighted_ret * edge_delta; +} + +XGBOOST_DEVICE inline float ExtractQuadratureInteractionDeltaLocal(float quad_node, + float edge_delta_local, + float q_partner) { + if (q_partner == 1.0f) { + return 0.0f; + } + auto alpha_partner = q_partner - 1.0f; + return alpha_partner * edge_delta_local / (1.0f + alpha_partner * quad_node); +} + template struct IsSparsePageLoaderNoShared : std::false_type {}; @@ -308,10 +334,18 @@ struct QuadratureSharedState { return nodes[warp][depth]; } + [[nodiscard]] XGBOOST_DEV_INLINE bst_node_t const& Node(int warp, int depth) const { + return nodes[warp][depth]; + } + [[nodiscard]] XGBOOST_DEV_INLINE std::uint8_t& Stage(int warp, int depth) { return stages[warp][depth]; } + [[nodiscard]] XGBOOST_DEV_INLINE std::uint8_t const& Stage(int warp, int depth) const { + return stages[warp][depth]; + } + [[nodiscard]] XGBOOST_DEV_INLINE bool GoesLeft(int warp, int row_slot, int depth) const { return static_cast(goes_left[warp][row_slot][depth]); } @@ -412,21 +446,14 @@ struct QuadratureShapTaskRunner { p_enter = subgroup.Broadcast(p_enter); q_prev = subgroup.Broadcast(q_prev); - float contrib = 0.0f; // Extraction uses // H * w(t) * ret_val * // [ (p_enter - 1) / (1 + (p_enter - 1) t) // - (q_prev - 1) / (1 + (q_prev - 1) t) ]. // The two rational terms are the "enter current feature" and "rewind to previous same // feature" adjustments from the quadrature recurrence. - if (p_enter != 1.0f) { - auto alpha_enter = p_enter - 1.0f; - contrib += alpha_enter * (*ret_val) * quad_weight / (1.0f + alpha_enter * quad_node); - } - if (q_prev != 1.0f) { - auto alpha_exit = q_prev - 1.0f; - contrib -= alpha_exit * (*ret_val) * quad_weight / (1.0f + alpha_exit * quad_node); - } + float contrib = + ExtractQuadratureEdgeDeltaLocal(quad_node, quad_weight, *ret_val, p_enter, q_prev); contrib = subgroup.Sum(contrib); this->AddContribution(row_idx, tree_group, node.split_global, contrib); @@ -563,6 +590,253 @@ struct QuadratureShapTaskRunner { } }; +template +struct QuadratureShapInteractionTaskRunner { + Loader loader; + SubgroupT subgroup; + SharedT shared; + CompressedTree const* trees; + CompressedNode const* nodes; + float* phis; + bst_idx_t base_rowid; + bst_target_t n_groups; + bst_feature_t n_columns; + std::size_t row_tile_begin; + std::size_t row_tiles; + int warp; + float quad_node; + float quad_weight; + + [[nodiscard]] XGBOOST_DEV_INLINE bool EvaluateGoesLeft(bst_idx_t ridx, + CompressedNode const& node) const { + auto fvalue = loader.GetElement(ridx, node.split_global); + return common::CheckNAN(fvalue) ? static_cast(node.default_left) + : fvalue < node.split_cond; + } + + XGBOOST_DEV_INLINE void AddDiagonalContribution(bst_idx_t row_idx, bst_target_t tree_group, + bst_feature_t split_global, float contrib) const { + if (!subgroup.ShouldWrite()) { + return; + } + auto out_idx = gpu_treeshap::IndexPhiInteractions(row_idx, n_groups, tree_group, n_columns - 1, + split_global, split_global); + atomicAdd(phis + out_idx, contrib); + } + + XGBOOST_DEV_INLINE void AddPairContribution(bst_idx_t row_idx, bst_target_t tree_group, + bst_feature_t split_i, bst_feature_t split_j, + float contrib) const { + if (!subgroup.ShouldWrite()) { + return; + } + auto out_idx = gpu_treeshap::IndexPhiInteractions(row_idx, n_groups, tree_group, n_columns - 1, + split_i, split_j); + atomicAdd(phis + out_idx, contrib); + } + + template + XGBOOST_DEV_INLINE void ForEachUniquePartner(CompressedNode const* nodes_for_tree, + int current_depth, bst_feature_t current_split, + Fn&& fn) const { + bool skipped_current = false; + for (int depth = current_depth; depth >= 0; --depth) { + auto const& candidate = nodes_for_tree[shared.Node(warp, depth)]; + if (candidate.is_leaf) { + continue; + } + auto split = candidate.split_global; + if (!skipped_current && split == current_split) { + skipped_current = true; + continue; + } + bool shadowed = false; + for (int newer = current_depth; newer > depth; --newer) { + auto const& newer_node = nodes_for_tree[shared.Node(warp, newer)]; + if (!newer_node.is_leaf && newer_node.split_global == split) { + shadowed = true; + break; + } + } + if (!shadowed) { + fn(depth, split); + } + } + } + + XGBOOST_DEV_INLINE void InitializeTask() { + if (subgroup.is_warp_leader) { + shared.Node(warp, 0) = RegTree::kRoot; + shared.Stage(warp, 0) = 0; + } + shared.Basis(warp, subgroup.row_slot, 0, subgroup.point) = 1.0f; + subgroup.Sync(); + } + + XGBOOST_DEV_INLINE bool HandleReturn(bst_idx_t row_idx, bst_target_t tree_group, + CompressedNode const* nodes_for_tree, int* stack_size, + bool* have_return, float* ret_val) { + if (*stack_size == 0) { + return false; + } + + int parent_depth = *stack_size - 1; + auto const& node = nodes_for_tree[shared.Node(warp, parent_depth)]; + int child_idx = static_cast(shared.Stage(warp, parent_depth)) - 1; + + float p_enter = 0.0f; + float q_prev = 1.0f; + if (subgroup.is_leader && subgroup.RowActive()) { + p_enter = shared.PathProbability(warp, subgroup.row_slot, parent_depth); + q_prev = shared.LoadQPrev(warp, subgroup.row_slot, parent_depth, node.prev_same_offset_plus1); + } + p_enter = subgroup.Broadcast(p_enter); + q_prev = subgroup.Broadcast(q_prev); + + auto edge_delta_local = + ExtractQuadratureEdgeDeltaLocal(quad_node, quad_weight, *ret_val, p_enter, q_prev); + auto diag_contrib = subgroup.Sum(edge_delta_local); + this->AddDiagonalContribution(row_idx, tree_group, node.split_global, diag_contrib); + + this->ForEachUniquePartner( + nodes_for_tree, parent_depth, node.split_global, + [&](int partner_depth, bst_feature_t partner_split) { + float q_partner = 1.0f; + if (subgroup.is_leader && subgroup.RowActive()) { + q_partner = shared.PathProbability(warp, subgroup.row_slot, partner_depth); + } + q_partner = subgroup.Broadcast(q_partner); + auto pair_delta_local = + ExtractQuadratureInteractionDeltaLocal(quad_node, edge_delta_local, q_partner); + auto pair_contrib = subgroup.Sum(pair_delta_local); + this->AddPairContribution(row_idx, tree_group, node.split_global, partner_split, + pair_contrib); + }); + + if (child_idx == 0) { + auto child_weight = node.right_weight; + auto child_node = node.right; + float p_e = 0.0f; + if (subgroup.is_leader) { + if (subgroup.RowActive()) { + auto goes_left = shared.GoesLeft(warp, subgroup.row_slot, parent_depth); + p_e = goes_left ? 0.0f : q_prev / child_weight; + } + shared.PathProbability(warp, subgroup.row_slot, parent_depth) = p_e; + } + p_e = subgroup.Broadcast(p_e); + + if (subgroup.is_warp_leader) { + shared.Node(warp, *stack_size) = child_node; + shared.Stage(warp, *stack_size) = 0; + shared.Stage(warp, parent_depth) = 2; + } + auto alpha_e = p_e - 1.0f; + auto v = shared.Basis(warp, subgroup.row_slot, parent_depth, subgroup.point) * child_weight * + (1.0f + alpha_e * quad_node); + if (q_prev != 1.0f) { + auto alpha_old = q_prev - 1.0f; + v /= 1.0f + alpha_old * quad_node; + } + shared.Basis(warp, subgroup.row_slot, *stack_size, subgroup.point) = v; + subgroup.Sync(); + shared.Basis(warp, subgroup.row_slot, parent_depth, subgroup.point) = *ret_val; + (*stack_size)++; + *have_return = false; + } else { + *ret_val += shared.Basis(warp, subgroup.row_slot, parent_depth, subgroup.point); + (*stack_size)--; + *have_return = true; + } + + return true; + } + + XGBOOST_DEV_INLINE void Descend(CompressedNode const* nodes_for_tree, bst_idx_t ridx, + int* stack_size, bool* have_return, float* ret_val) { + int depth = *stack_size - 1; + auto const& node = nodes_for_tree[shared.Node(warp, depth)]; + if (node.is_leaf) { + *ret_val = shared.Basis(warp, subgroup.row_slot, depth, subgroup.point) * node.leaf_value; + (*stack_size)--; + *have_return = true; + return; + } + + int child = static_cast(shared.Stage(warp, depth) != 0); + if (child == 0) { + if (subgroup.is_warp_leader) { + shared.Stage(warp, depth) = 1; + } + subgroup.Sync(); + } + + auto child_weight = child == 0 ? node.left_weight : node.right_weight; + auto child_node = child == 0 ? node.left : node.right; + float q_prev = 1.0f; + if (subgroup.is_leader) { + if (subgroup.RowActive()) { + q_prev = PreviousPathProbability(node.prev_same_offset_plus1, depth, + shared.PathProbabilityRow(warp, subgroup.row_slot)); + } + shared.StoreQPrev(warp, subgroup.row_slot, depth, q_prev); + } + q_prev = subgroup.Broadcast(q_prev); + + float p_e = 0.0f; + if (subgroup.is_leader) { + bool goes_left = false; + if (subgroup.RowActive()) { + goes_left = this->EvaluateGoesLeft(ridx, node); + p_e = (child == 0 ? goes_left : !goes_left) ? q_prev / child_weight : 0.0f; + } + shared.SetGoesLeft(warp, subgroup.row_slot, depth, goes_left); + shared.PathProbability(warp, subgroup.row_slot, depth) = p_e; + } + p_e = subgroup.Broadcast(p_e); + + if (subgroup.is_warp_leader) { + shared.Node(warp, *stack_size) = child_node; + shared.Stage(warp, *stack_size) = 0; + } + auto alpha_e = p_e - 1.0f; + auto v = shared.Basis(warp, subgroup.row_slot, depth, subgroup.point) * child_weight * + (1.0f + alpha_e * quad_node); + if (q_prev != 1.0f) { + auto alpha_old = q_prev - 1.0f; + v /= 1.0f + alpha_old * quad_node; + } + shared.Basis(warp, subgroup.row_slot, *stack_size, subgroup.point) = v; + subgroup.Sync(); + (*stack_size)++; + } + + XGBOOST_DEV_INLINE void RunTask(std::size_t task) { + auto tree_idx = task / row_tiles; + auto row_tile = task % row_tiles; + auto ridx = (row_tile_begin + row_tile) * SubgroupT::kRowsPerWarpValue + subgroup.row_slot; + auto row_idx = base_rowid + static_cast(ridx); + auto tree = trees[tree_idx]; + auto nodes_for_tree = nodes + tree.node_begin; + + this->InitializeTask(); + + int stack_size = 1; + bool have_return = false; + float ret_val = 0.0f; + while (stack_size > 0 || have_return) { + if (have_return) { + if (!this->HandleReturn(row_idx, tree.group, nodes_for_tree, &stack_size, &have_return, + &ret_val)) { + break; + } + continue; + } + this->Descend(nodes_for_tree, ridx, &stack_size, &have_return, &ret_val); + } + } +}; + template __global__ void __launch_bounds__(BlockThreads, 9) @@ -677,6 +951,120 @@ void LaunchQuadratureShapBuckets(Context const* ctx, Loader loader, bst_idx_t ba } } +template +__global__ void __launch_bounds__(BlockThreads, 9) QuadratureShapInteractionTaskKernel( + Loader loader, bst_idx_t base_rowid, bst_target_t n_groups, bst_feature_t n_columns, + std::size_t row_tile_begin, std::size_t row_tiles, bst_idx_t valid_rows_in_tail, + std::size_t n_trees, CompressedTree const* __restrict__ trees, + CompressedNode const* __restrict__ nodes, float const* __restrict__ quad_nodes, + float const* __restrict__ quad_weights, float* __restrict__ phis) { + static_assert(MaxPoints == kGpuQuadraturePoints); + static_assert(DepthCap <= static_cast(kMaxGpuQuadratureDepth)); + static_assert(dh::WarpThreads() % RowsPerWarp == 0); + static_assert(BlockThreads % dh::WarpThreads() == 0); + using SubgroupT = SubgroupOps; + constexpr int kSegmentWidth = SubgroupT::kSegmentWidth; + if constexpr (!kHasRowMask) { + static_assert(kSegmentWidth == MaxPoints, + "Full-tile specialization assumes every warp lane participates."); + } + constexpr int kWarpsPerBlock = BlockThreads / dh::WarpThreads(); + constexpr bool kUseQPrevCache = IsSparsePageLoaderNoShared::value; + using SharedT = + QuadratureSharedState; + + __shared__ bst_node_t s_node[kWarpsPerBlock][DepthCap]; + __shared__ std::uint8_t s_stage[kWarpsPerBlock][DepthCap]; + __shared__ std::uint8_t s_goes_left[kWarpsPerBlock][RowsPerWarp][DepthCap]; + __shared__ float s_path_p[kWarpsPerBlock][RowsPerWarp][DepthCap]; + __shared__ float s_c_vals[kWarpsPerBlock][RowsPerWarp][DepthCap][MaxPoints]; + __shared__ float s_q_prev[kUseQPrevCache ? kWarpsPerBlock : 1][kUseQPrevCache ? RowsPerWarp : 1] + [kUseQPrevCache ? DepthCap : 1]; + + int warp = static_cast(threadIdx.x) / dh::WarpThreads(); + int lane = static_cast(threadIdx.x) % dh::WarpThreads(); + auto subgroup = SubgroupT{lane, valid_rows_in_tail}; + if (!subgroup.Participates()) { + return; + } + + auto shared = SharedT{s_node, s_stage, s_goes_left, s_path_p, s_c_vals, s_q_prev}; + auto global_warp = + (static_cast(blockIdx.x) * BlockThreads + threadIdx.x) / dh::WarpThreads(); + auto warp_stride = (static_cast(gridDim.x) * BlockThreads) / dh::WarpThreads(); + auto n_tasks = n_trees * row_tiles; + + auto runner = + QuadratureShapInteractionTaskRunner{loader, + subgroup, + shared, + trees, + nodes, + phis, + base_rowid, + n_groups, + n_columns, + row_tile_begin, + row_tiles, + warp, + quad_nodes[subgroup.point], + quad_weights[subgroup.point]}; + + for (std::size_t task = global_warp; task < n_tasks; task += warp_stride) { + runner.RunTask(task); + } +} + +template +void LaunchQuadratureShapInteractionTasks(Context const* ctx, Loader loader, bst_idx_t base_rowid, + bst_target_t n_groups, bst_feature_t n_columns, + std::size_t row_tile_begin, std::size_t row_tiles, + bst_idx_t valid_rows_in_tail, + CompressedModel const& compressed, + common::Span quad_nodes, + common::Span quad_weights, + HostDeviceVector* out_contribs) { + static_assert(BlockThreads % dh::WarpThreads() == 0); + constexpr int kWarpsPerBlock = BlockThreads / dh::WarpThreads(); + if (compressed.trees.empty() || row_tiles == 0) { + return; + } + auto trees = thrust::raw_pointer_cast(compressed.trees.data()); + auto nodes = thrust::raw_pointer_cast(compressed.nodes.data()); + auto d_quad_nodes = quad_nodes.data(); + auto d_quad_weights = quad_weights.data(); + auto phis = out_contribs->DeviceSpan().data(); + auto n_tasks = compressed.trees.size() * row_tiles; + auto grids = common::DivRoundUp(n_tasks, static_cast(kWarpsPerBlock)); + QuadratureShapInteractionTaskKernel + <<(grids), static_cast(BlockThreads), 0, + ctx->CUDACtx()->Stream()>>>(loader, base_rowid, n_groups, n_columns, row_tile_begin, + row_tiles, valid_rows_in_tail, compressed.trees.size(), trees, + nodes, d_quad_nodes, d_quad_weights, phis); + dh::safe_cuda(cudaGetLastError()); +} + +template +void LaunchQuadratureShapInteractionBuckets(Context const* ctx, Loader loader, bst_idx_t base_rowid, + bst_target_t n_groups, bst_feature_t n_columns, + CompressedModel const& compressed, + common::Span quad_nodes, + common::Span quad_weights, + HostDeviceVector* out_contribs) { + auto full_row_tiles = static_cast(loader.NumRows() / RowsPerWarp); + auto tail_rows = static_cast(loader.NumRows() % RowsPerWarp); + LaunchQuadratureShapInteractionTasks( + ctx, loader, base_rowid, n_groups, n_columns, /*row_tile_begin=*/0, full_row_tiles, + /*valid_rows_in_tail=*/RowsPerWarp, compressed, quad_nodes, quad_weights, out_contribs); + if (tail_rows != 0) { + LaunchQuadratureShapInteractionTasks( + ctx, loader, base_rowid, n_groups, n_columns, /*row_tile_begin=*/full_row_tiles, + /*row_tiles=*/1, tail_rows, compressed, quad_nodes, quad_weights, out_contribs); + } +} + struct CopyViews { Context const* ctx; explicit CopyViews(Context const* ctx) : ctx{ctx} {} @@ -1081,6 +1469,117 @@ void QuadratureShapValues(Context const* ctx, DMatrix* p_fmat, }); } +void QuadratureShapInteractionValues(Context const* ctx, DMatrix* p_fmat, + HostDeviceVector* out_contribs, + gbm::GBTreeModel const& model, bst_tree_t tree_end, + std::vector const* tree_weights, + std::size_t quadrature_points) { + xgboost_NVTX_FN_RANGE(); + CHECK(!model.learner_model_param->IsVectorLeaf()) + << "Predict interaction contribution" << MTNotImplemented(); + CHECK(!p_fmat->Info().IsColumnSplit()) << "Predict interaction contribution support for " + "column-wise data split is not yet implemented."; + CHECK_EQ(quadrature_points, kGpuQuadraturePoints) + << "GPU QuadratureSHAP currently uses a fixed quadrature size of " << kGpuQuadraturePoints + << "."; + + tree_end = predictor::GetTreeLimit(model.trees, tree_end); + auto const ngroup = model.learner_model_param->num_output_group; + CHECK_NE(ngroup, 0); + auto const ncolumns = model.learner_model_param->num_feature + 1; + auto const n_features = model.learner_model_param->num_feature; + auto dim_size = ncolumns * ncolumns * ngroup; + out_contribs->SetDevice(ctx->Device()); + out_contribs->Resize(p_fmat->Info().num_row_ * dim_size); + out_contribs->Fill(0.0f); + + bst_node_t max_depth = 0; + std::array, kGpuQuadratureDepthBuckets.size()> tree_buckets; + for (bst_tree_t tree_idx = 0; tree_idx < tree_end; ++tree_idx) { + CHECK(!model.trees[tree_idx]->IsMultiTarget()) + << "Predict interaction contribution" << MTNotImplemented(); + auto tree_depth = model.trees[tree_idx]->MaxDepth(); + max_depth = std::max(max_depth, tree_depth); + auto path_depth = static_cast(tree_depth) + 1; + auto bucket_idx = DepthBucketIndex(path_depth); + tree_buckets[bucket_idx].push_back(tree_idx); + } + CHECK_LE(max_depth + 1, static_cast(kMaxGpuQuadratureDepth)) + << "GPU QuadratureSHAP currently supports trees of depth up to " + << (kMaxGpuQuadratureDepth - 1) << "."; + + auto h_group_root_mean_sums = MakeGroupRootMeanSums(model, tree_end, tree_weights); + auto rule = detail::MakeEndpointQuadrature(kQuadratureShapQeps); + std::array h_quad_nodes{}; + std::array h_quad_weights{}; + for (std::size_t i = 0; i < kGpuQuadraturePoints; ++i) { + h_quad_nodes[i] = static_cast(rule.nodes[i]); + h_quad_weights[i] = static_cast(rule.weights[i]); + } + dh::device_vector d_quad_nodes(h_quad_nodes.cbegin(), h_quad_nodes.cend()); + dh::device_vector d_quad_weights(h_quad_weights.cbegin(), h_quad_weights.cend()); + dh::device_vector d_group_root_mean_sums(h_group_root_mean_sums.cbegin(), + h_group_root_mean_sums.cend()); + auto compressed_16 = MakeCompressedModel(ctx, model, tree_buckets[0], tree_weights); + auto compressed_32 = MakeCompressedModel(ctx, model, tree_buckets[1], tree_weights); + auto compressed_64 = MakeCompressedModel(ctx, model, tree_buckets[2], tree_weights); + + auto new_enc = + p_fmat->Cats()->NeedRecode() ? p_fmat->Cats()->DeviceView(ctx) : enc::DeviceColumnsView{}; + auto quad_nodes = + common::Span{thrust::raw_pointer_cast(d_quad_nodes.data()), d_quad_nodes.size()}; + auto quad_weights = common::Span{thrust::raw_pointer_cast(d_quad_weights.data()), + d_quad_weights.size()}; + auto group_root_mean_sums = common::Span{ + thrust::raw_pointer_cast(d_group_root_mean_sums.data()), d_group_root_mean_sums.size()}; + + LaunchShap(ctx, p_fmat, new_enc, model, [&](auto&& loader, bst_idx_t base_rowid) { + LaunchQuadratureShapInteractionBuckets( + ctx, loader, base_rowid, ngroup, ncolumns, compressed_16, quad_nodes, quad_weights, + out_contribs); + LaunchQuadratureShapInteractionBuckets( + ctx, loader, base_rowid, ngroup, ncolumns, compressed_32, quad_nodes, quad_weights, + out_contribs); + LaunchQuadratureShapInteractionBuckets( + ctx, loader, base_rowid, ngroup, ncolumns, compressed_64, quad_nodes, quad_weights, + out_contribs); + }); + + p_fmat->Info().base_margin_.SetDevice(ctx->Device()); + auto margin = p_fmat->Info().base_margin_.Data()->ConstDeviceSpan(); + auto base_score = model.learner_model_param->BaseScore(ctx); + auto phis = out_contribs->DeviceSpan(); + auto n_samples = p_fmat->Info().num_row_; + dh::LaunchN(n_samples * ngroup, ctx->CUDACtx()->Stream(), [=] __device__(std::size_t idx) { + auto [ridx, gid] = linalg::UnravelIndex(idx, n_samples, ngroup); + auto bias_idx = + gpu_treeshap::IndexPhiInteractions(ridx, ngroup, gid, n_features, n_features, n_features); + phis[bias_idx] += group_root_mean_sums[gid] + (margin.empty() ? base_score(gid) : margin[idx]); + + auto matrix_offset = gpu_treeshap::IndexPhiInteractions(ridx, ngroup, gid, n_features, 0, 0); + auto matrix = phis.subspan(matrix_offset, ncolumns * ncolumns); + for (bst_feature_t r = 0; r < ncolumns; ++r) { + for (bst_feature_t c = r + 1; c < ncolumns; ++c) { + auto sym = 0.5f * (matrix[r * ncolumns + c] + matrix[c * ncolumns + r]); + matrix[r * ncolumns + c] = sym; + matrix[c * ncolumns + r] = sym; + } + } + for (bst_feature_t r = 0; r < ncolumns; ++r) { + float value = matrix[r * ncolumns + r]; + for (bst_feature_t c = 0; c < ncolumns; ++c) { + if (c != r) { + value -= matrix[r * ncolumns + c]; + } + } + matrix[r * ncolumns + r] = value; + } + }); +} + void ShapInteractionValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, bst_tree_t tree_end, std::vector const* tree_weights, diff --git a/src/predictor/interpretability/shap.h b/src/predictor/interpretability/shap.h index 880ed8d2c489..7dad71d7e91f 100644 --- a/src/predictor/interpretability/shap.h +++ b/src/predictor/interpretability/shap.h @@ -39,6 +39,12 @@ void QuadratureShapValues(Context const* ctx, DMatrix* p_fmat, bst_tree_t tree_end, std::vector const* tree_weights, std::size_t quadrature_points); +void QuadratureShapInteractionValues(Context const* ctx, DMatrix* p_fmat, + HostDeviceVector* out_contribs, + gbm::GBTreeModel const& model, bst_tree_t tree_end, + std::vector const* tree_weights, + std::size_t quadrature_points); + void ApproxFeatureImportance(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, bst_tree_t tree_end, std::vector const* tree_weights); @@ -58,6 +64,11 @@ void QuadratureShapValues(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, bst_tree_t tree_end, std::vector const* tree_weights, std::size_t quadrature_points); +void QuadratureShapInteractionValues(Context const* ctx, DMatrix* p_fmat, + HostDeviceVector* out_contribs, + gbm::GBTreeModel const& model, bst_tree_t tree_end, + std::vector const* tree_weights, + std::size_t quadrature_points); void ApproxFeatureImportance(Context const* ctx, DMatrix* p_fmat, HostDeviceVector* out_contribs, gbm::GBTreeModel const& model, bst_tree_t tree_end, std::vector const* tree_weights); From 2142c7b94fae5bd108ac5b4ee6af35f441c59959 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Wed, 1 Apr 2026 07:45:32 -0700 Subject: [PATCH 10/13] Support categorical GPU QSHAP and restore CPU additive speed --- src/predictor/interpretability/shap.cc | 130 ++++++++++++++++--------- src/predictor/interpretability/shap.cu | 55 +++++++++-- 2 files changed, 128 insertions(+), 57 deletions(-) diff --git a/src/predictor/interpretability/shap.cc b/src/predictor/interpretability/shap.cc index 10055f7e54f7..a50ed19de038 100644 --- a/src/predictor/interpretability/shap.cc +++ b/src/predictor/interpretability/shap.cc @@ -420,22 +420,67 @@ struct QuadraturePathView { } }; +struct EmptyQuadraturePathState { + void Reset() const {} + void Push(bst_feature_t, float, float) const {} + void Pop(bst_feature_t) const {} + [[nodiscard]] auto View() const { return QuadraturePathView{{}, {}}; } +}; + +struct LiveQuadraturePathState { + std::vector *path; + std::vector *latest_live_index; + + void Reset() const { path->clear(); } + + void Push(bst_feature_t split_index, float p_parent, float p_child) const { + if constexpr (kQuadratureInteractionUseLatestLiveIndex) { + auto prev_live = (*latest_live_index)[split_index]; + path->push_back(QuadraturePathElement{split_index, p_parent, p_child, prev_live}); + (*latest_live_index)[split_index] = static_cast(path->size() - 1); + } else { + path->push_back(QuadraturePathElement{split_index, p_parent, p_child, -1}); + } + } + + void Pop(bst_feature_t split_index) const { + if constexpr (kQuadratureInteractionUseLatestLiveIndex) { + (*latest_live_index)[split_index] = path->back().prev_live_index; + } + path->pop_back(); + } + + [[nodiscard]] auto View() const { + if constexpr (kQuadratureInteractionUseLatestLiveIndex) { + return QuadraturePathView{common::Span{*path}, + common::Span{*latest_live_index}}; + } else { + return QuadraturePathView{common::Span{*path}, {}}; + } + } +}; + // Current additive SHAP formulation. It consumes the weighted subtree return and writes one // feature contribution per return edge. struct AdditiveContributionFormulation { - static constexpr bool kTrackLatestLiveIndex = false; + EmptyQuadraturePathState path_state; ContributionVectorView phi; - void HandleLeaf(tree::ScalarTreeView const &tree, QuadratureRule const &rule, - QuadraturePathView path, bst_node_t nidx, QuadratureBuffer const &c_vals, - float w_prod, QuadratureBuffer *out_h) const { - (void)path; + explicit AdditiveContributionFormulation(ContributionVectorView phi) : phi{phi} {} + + void ResetPath() const { path_state.Reset(); } + void PushPathSplit(bst_feature_t split_index, float p_parent, float p_child) const { + path_state.Push(split_index, p_parent, p_child); + } + void PopPathSplit(bst_feature_t split_index) const { path_state.Pop(split_index); } + + void HandleLeaf(tree::ScalarTreeView const &tree, QuadratureRule const &rule, bst_node_t nidx, + QuadratureBuffer const &c_vals, float w_prod, QuadratureBuffer *out_h) const { WriteWeightedLeafReturn(tree, rule, nidx, c_vals, w_prod, out_h); } - void HandleReturn(QuadratureRule const &rule, QuadraturePathView path, bst_feature_t split_index, + void HandleReturn(QuadratureRule const &rule, bst_feature_t split_index, QuadratureBuffer const &h_vals, float p_enter, float p_exit) const { - (void)path; phi[split_index] += ExtractQuadratureDelta(rule, h_vals, p_enter, p_exit); } }; @@ -444,23 +489,36 @@ struct AdditiveContributionFormulation { // traversal and weighted subtree return shared with additive SHAP, and only changes how return // edges are written into the dense interaction sink. struct InteractionContributionFormulation { - static constexpr bool kTrackLatestLiveIndex = kQuadratureInteractionUseLatestLiveIndex; struct EdgeEffect { bst_feature_t split_index; float diagonal_delta; QuadratureBuffer edge_kernel; }; + LiveQuadraturePathState path_state; ContributionVectorView phi_diag; DenseInteractionMatrixView phi_interactions; float scale; + InteractionContributionFormulation(LiveQuadraturePathState path_state, + ContributionVectorView phi_diag, + DenseInteractionMatrixView phi_interactions, + float scale) + : path_state{path_state}, + phi_diag{phi_diag}, + phi_interactions{phi_interactions}, + scale{scale} {} + + void ResetPath() const { path_state.Reset(); } + void PushPathSplit(bst_feature_t split_index, float p_parent, float p_child) const { + path_state.Push(split_index, p_parent, p_child); + } + void PopPathSplit(bst_feature_t split_index) const { path_state.Pop(split_index); } + // Traversal still needs a weighted subtree return, so the interaction path shares the additive // leaf behavior and changes only the return-edge algebra. - void HandleLeaf(tree::ScalarTreeView const &tree, QuadratureRule const &rule, - QuadraturePathView path, bst_node_t nidx, QuadratureBuffer const &c_vals, - float w_prod, QuadratureBuffer *out_h) const { - (void)path; + void HandleLeaf(tree::ScalarTreeView const &tree, QuadratureRule const &rule, bst_node_t nidx, + QuadratureBuffer const &c_vals, float w_prod, QuadratureBuffer *out_h) const { WriteWeightedLeafReturn(tree, rule, nidx, c_vals, w_prod, out_h); } @@ -521,8 +579,9 @@ struct InteractionContributionFormulation { phi_interactions(i, j) += scale * pair_delta; } - void HandleReturn(QuadratureRule const &rule, QuadraturePathView path, bst_feature_t split_index, + void HandleReturn(QuadratureRule const &rule, bst_feature_t split_index, QuadratureBuffer const &h_vals, float p_enter, float p_exit) const { + auto path = path_state.View(); auto const edge = this->MakeEdgeEffect(rule, split_index, h_vals, p_enter, p_exit); this->AccumulateDiagonal(edge); @@ -547,19 +606,8 @@ struct QuadratureShapTreeRunner { RegTree::FVec const &feat; QuadratureRule const &rule; std::vector *path_prob; - std::vector *path; - std::vector *latest_live_index; ContributionFormulation formulation; - [[nodiscard]] auto CurrentPath() const { - if constexpr (ContributionFormulation::kTrackLatestLiveIndex) { - return QuadraturePathView{common::Span{*path}, - common::Span{*latest_live_index}}; - } else { - return QuadraturePathView{common::Span{*path}, {}}; - } - } - [[nodiscard]] bool EvaluateGoesLeft(bst_node_t nidx) const { auto split_index = tree.SplitIndex(nidx); auto const &cats = tree.GetCategoriesMatrix(); @@ -608,26 +656,17 @@ struct QuadratureShapTreeRunner { } (*path_prob)[split_index] = p_e; - if constexpr (ContributionFormulation::kTrackLatestLiveIndex) { - auto prev_live = (*latest_live_index)[split_index]; - path->push_back(QuadraturePathElement{split_index, p_up, p_e, prev_live}); - (*latest_live_index)[split_index] = static_cast(path->size() - 1); - } else { - path->push_back(QuadraturePathElement{split_index, p_up, p_e, -1}); - } + formulation.PushPathSplit(split_index, p_up, p_e); this->RunNode(child_node, c_child, w_prod * child_weight, out_h); - formulation.HandleReturn(rule, this->CurrentPath(), split_index, *out_h, p_e, p_up); - if constexpr (ContributionFormulation::kTrackLatestLiveIndex) { - (*latest_live_index)[split_index] = path->back().prev_live_index; - } - path->pop_back(); + formulation.HandleReturn(rule, split_index, *out_h, p_e, p_up); + formulation.PopPathSplit(split_index); (*path_prob)[split_index] = p_old; } void RunNode(bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, QuadratureBuffer *out_h) { if (tree.IsLeaf(nidx)) { - formulation.HandleLeaf(tree, rule, this->CurrentPath(), nidx, c_vals, w_prod, out_h); + formulation.HandleLeaf(tree, rule, nidx, c_vals, w_prod, out_h); return; } @@ -645,7 +684,7 @@ struct QuadratureShapTreeRunner { } void Run() { - path->clear(); + formulation.ResetPath(); if (tree.IsLeaf(RegTree::kRoot)) { return; } @@ -824,11 +863,8 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, auto model_data = MakeQuadratureShapModelData(model, tree_end, tree_weights); std::vector feats_tloc(n_threads); std::vector> contribs_tloc(n_threads, std::vector(ncolumns)); - std::vector> path_tloc(n_threads); std::vector> path_prob_tloc( n_threads, std::vector(n_features, kQuadratureShapUnseen)); - std::vector> latest_live_tloc( - n_threads, std::vector(n_features, -1)); auto device = ctx->Device().IsSycl() ? DeviceOrd::CPU() : ctx->Device(); auto base_margin = info.base_margin_.View(device); @@ -841,9 +877,7 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, feats.Init(model.learner_model_param->num_feature); } auto &this_tree_contribs = contribs_tloc[tid]; - auto &path = path_tloc[tid]; auto &path_prob = path_prob_tloc[tid]; - auto &latest_live = latest_live_tloc[tid]; auto row_idx = view.base_rowid + i; auto n_valid = view.DoFill(i, feats.Data().data()); feats.HasMissing(n_valid != feats.Size()); @@ -853,7 +887,7 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, std::fill(this_tree_contribs.begin(), this_tree_contribs.end(), 0.0f); auto formulation = AdditiveContributionFormulation{{this_tree_contribs.data(), ncolumns}}; auto runner = QuadratureShapTreeRunner{ - model_data.trees[j], feats, rule, &path_prob, &path, &latest_live, formulation}; + model_data.trees[j], feats, rule, &path_prob, formulation}; runner.Run(); auto const weight = model_data.weights[j]; for (size_t ci = 0; ci + 1 < ncolumns; ++ci) { @@ -939,10 +973,12 @@ void QuadratureShapInteractionValues(Context const *ctx, DMatrix *p_fmat, std::fill(diag.begin(), diag.end(), 0.0f); for (auto j : model_data.trees_by_group[gid]) { - auto formulation = InteractionContributionFormulation{ - {diag.data(), ncolumns}, {matrix.data, matrix.ncolumns}, model_data.weights[j]}; + auto formulation = InteractionContributionFormulation{{&path, &latest_live}, + {diag.data(), ncolumns}, + {matrix.data, matrix.ncolumns}, + model_data.weights[j]}; auto runner = QuadratureShapTreeRunner{ - model_data.trees[j], feats, rule, &path_prob, &path, &latest_live, formulation}; + model_data.trees[j], feats, rule, &path_prob, formulation}; runner.Run(); } diff --git a/src/predictor/interpretability/shap.cu b/src/predictor/interpretability/shap.cu index 12561be9d46a..3b62b023893a 100644 --- a/src/predictor/interpretability/shap.cu +++ b/src/predictor/interpretability/shap.cu @@ -93,8 +93,11 @@ struct CompressedNode { float leaf_value{0}; float left_weight{0}; float right_weight{0}; + std::uint32_t cat_begin{0}; + std::uint32_t cat_size{0}; std::uint8_t default_left{0}; std::uint8_t is_leaf{0}; + std::uint8_t is_categorical{0}; std::uint8_t prev_same_offset_plus1{0}; }; @@ -106,6 +109,7 @@ struct CompressedTree { struct CompressedModel { dh::device_vector trees; dh::device_vector nodes; + dh::device_vector categories; }; std::size_t DepthBucketIndex(std::size_t path_depth) { @@ -124,6 +128,7 @@ CompressedModel MakeCompressedModel(Context const* ctx, gbm::GBTreeModel const& std::vector const* tree_weights) { std::vector h_trees; std::vector h_nodes; + std::vector h_categories; auto h_tree_groups = model.TreeGroups(DeviceOrd::CPU()); static_cast(ctx); @@ -131,8 +136,6 @@ CompressedModel MakeCompressedModel(Context const* ctx, gbm::GBTreeModel const& for (auto tree_idx : tree_indices) { auto const weight = tree_weights == nullptr ? 1.0f : (*tree_weights)[tree_idx]; auto const tree = model.trees.at(tree_idx)->HostScView(); - CHECK(!tree.HasCategoricalSplit()) - << "GPU QuadratureSHAP prototype does not support categorical splits."; auto node_begin = h_nodes.size(); h_nodes.resize(node_begin + tree.Size()); @@ -157,6 +160,17 @@ CompressedModel MakeCompressedModel(Context const* ctx, gbm::GBTreeModel const& out.left_weight = static_cast(static_cast(tree.SumHess(left)) / parent_cover); out.right_weight = static_cast(static_cast(tree.SumHess(right)) / parent_cover); + if (common::IsCat(tree.cats.split_type, nidx)) { + auto node_cats = tree.NodeCats(nidx); + CHECK_LE(node_cats.size(), + static_cast(std::numeric_limits::max())); + CHECK_LE(h_categories.size(), + static_cast(std::numeric_limits::max())); + out.cat_begin = static_cast(h_categories.size()); + out.cat_size = static_cast(node_cats.size()); + out.is_categorical = 1; + h_categories.insert(h_categories.end(), node_cats.begin(), node_cats.end()); + } out.default_left = tree.DefaultLeft(nidx); out.is_leaf = 0; } @@ -189,6 +203,7 @@ CompressedModel MakeCompressedModel(Context const* ctx, gbm::GBTreeModel const& CompressedModel out; out.trees = dh::device_vector(h_trees.cbegin(), h_trees.cend()); out.nodes = dh::device_vector(h_nodes.cbegin(), h_nodes.cend()); + out.categories = dh::device_vector(h_categories.cbegin(), h_categories.cend()); return out; } @@ -390,6 +405,7 @@ struct QuadratureShapTaskRunner { SharedT shared; CompressedTree const* trees; CompressedNode const* nodes; + std::uint32_t const* categories; float* phis; bst_idx_t base_rowid; bst_target_t n_groups; @@ -403,8 +419,14 @@ struct QuadratureShapTaskRunner { [[nodiscard]] XGBOOST_DEV_INLINE bool EvaluateGoesLeft(bst_idx_t ridx, CompressedNode const& node) const { auto fvalue = loader.GetElement(ridx, node.split_global); - return common::CheckNAN(fvalue) ? static_cast(node.default_left) - : fvalue < node.split_cond; + if (common::CheckNAN(fvalue)) { + return static_cast(node.default_left); + } + if (node.is_categorical) { + auto cats = common::Span{categories + node.cat_begin, node.cat_size}; + return common::Decision(cats, fvalue); + } + return fvalue < node.split_cond; } XGBOOST_DEV_INLINE void AddContribution(bst_idx_t row_idx, bst_target_t tree_group, @@ -597,6 +619,7 @@ struct QuadratureShapInteractionTaskRunner { SharedT shared; CompressedTree const* trees; CompressedNode const* nodes; + std::uint32_t const* categories; float* phis; bst_idx_t base_rowid; bst_target_t n_groups; @@ -610,8 +633,14 @@ struct QuadratureShapInteractionTaskRunner { [[nodiscard]] XGBOOST_DEV_INLINE bool EvaluateGoesLeft(bst_idx_t ridx, CompressedNode const& node) const { auto fvalue = loader.GetElement(ridx, node.split_global); - return common::CheckNAN(fvalue) ? static_cast(node.default_left) - : fvalue < node.split_cond; + if (common::CheckNAN(fvalue)) { + return static_cast(node.default_left); + } + if (node.is_categorical) { + auto cats = common::Span{categories + node.cat_begin, node.cat_size}; + return common::Decision(cats, fvalue); + } + return fvalue < node.split_cond; } XGBOOST_DEV_INLINE void AddDiagonalContribution(bst_idx_t row_idx, bst_target_t tree_group, @@ -845,6 +874,7 @@ __global__ void __launch_bounds__(BlockThreads, 9) std::size_t row_tiles, bst_idx_t valid_rows_in_tail, std::size_t n_trees, CompressedTree const* __restrict__ trees, CompressedNode const* __restrict__ nodes, + std::uint32_t const* __restrict__ categories, float const* __restrict__ quad_nodes, float const* __restrict__ quad_weights, float* __restrict__ phis) { static_assert(MaxPoints == kGpuQuadraturePoints); @@ -888,6 +918,7 @@ __global__ void __launch_bounds__(BlockThreads, 9) shared, trees, nodes, + categories, phis, base_rowid, n_groups, @@ -919,6 +950,7 @@ void LaunchQuadratureShapTasks(Context const* ctx, Loader loader, bst_idx_t base } auto trees = thrust::raw_pointer_cast(compressed.trees.data()); auto nodes = thrust::raw_pointer_cast(compressed.nodes.data()); + auto categories = thrust::raw_pointer_cast(compressed.categories.data()); auto d_quad_nodes = quad_nodes.data(); auto d_quad_weights = quad_weights.data(); auto phis = out_contribs->DeviceSpan().data(); @@ -928,7 +960,7 @@ void LaunchQuadratureShapTasks(Context const* ctx, Loader loader, bst_idx_t base <<(grids), static_cast(BlockThreads), 0, ctx->CUDACtx()->Stream()>>>(loader, base_rowid, n_groups, n_columns, row_tile_begin, row_tiles, valid_rows_in_tail, compressed.trees.size(), trees, - nodes, d_quad_nodes, d_quad_weights, phis); + nodes, categories, d_quad_nodes, d_quad_weights, phis); dh::safe_cuda(cudaGetLastError()); } @@ -957,8 +989,9 @@ __global__ void __launch_bounds__(BlockThreads, 9) QuadratureShapInteractionTask Loader loader, bst_idx_t base_rowid, bst_target_t n_groups, bst_feature_t n_columns, std::size_t row_tile_begin, std::size_t row_tiles, bst_idx_t valid_rows_in_tail, std::size_t n_trees, CompressedTree const* __restrict__ trees, - CompressedNode const* __restrict__ nodes, float const* __restrict__ quad_nodes, - float const* __restrict__ quad_weights, float* __restrict__ phis) { + CompressedNode const* __restrict__ nodes, std::uint32_t const* __restrict__ categories, + float const* __restrict__ quad_nodes, float const* __restrict__ quad_weights, + float* __restrict__ phis) { static_assert(MaxPoints == kGpuQuadraturePoints); static_assert(DepthCap <= static_cast(kMaxGpuQuadratureDepth)); static_assert(dh::WarpThreads() % RowsPerWarp == 0); @@ -1001,6 +1034,7 @@ __global__ void __launch_bounds__(BlockThreads, 9) QuadratureShapInteractionTask shared, trees, nodes, + categories, phis, base_rowid, n_groups, @@ -1033,6 +1067,7 @@ void LaunchQuadratureShapInteractionTasks(Context const* ctx, Loader loader, bst } auto trees = thrust::raw_pointer_cast(compressed.trees.data()); auto nodes = thrust::raw_pointer_cast(compressed.nodes.data()); + auto categories = thrust::raw_pointer_cast(compressed.categories.data()); auto d_quad_nodes = quad_nodes.data(); auto d_quad_weights = quad_weights.data(); auto phis = out_contribs->DeviceSpan().data(); @@ -1042,7 +1077,7 @@ void LaunchQuadratureShapInteractionTasks(Context const* ctx, Loader loader, bst <<(grids), static_cast(BlockThreads), 0, ctx->CUDACtx()->Stream()>>>(loader, base_rowid, n_groups, n_columns, row_tile_begin, row_tiles, valid_rows_in_tail, compressed.trees.size(), trees, - nodes, d_quad_nodes, d_quad_weights, phis); + nodes, categories, d_quad_nodes, d_quad_weights, phis); dh::safe_cuda(cudaGetLastError()); } From b763e69c377d9777ea350c7d2416036b20ec5227 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Sat, 11 Apr 2026 07:07:40 -0700 Subject: [PATCH 11/13] Consolidate QuadratureSHAP benchmark script --- demo/guide-python/quadratureshap_benchmark.py | 484 -------------- .../quadratureshap_rapids_benchmark.py | 605 ++++++++++++++++++ quadratureshap_n8_accuracy_vs_nodes.png | Bin 78004 -> 0 bytes quadratureshap_n8_speedup_vs_nodes.png | Bin 71642 -> 0 bytes 4 files changed, 605 insertions(+), 484 deletions(-) delete mode 100644 demo/guide-python/quadratureshap_benchmark.py create mode 100644 demo/guide-python/quadratureshap_rapids_benchmark.py delete mode 100644 quadratureshap_n8_accuracy_vs_nodes.png delete mode 100644 quadratureshap_n8_speedup_vs_nodes.png diff --git a/demo/guide-python/quadratureshap_benchmark.py b/demo/guide-python/quadratureshap_benchmark.py deleted file mode 100644 index b164461c8184..000000000000 --- a/demo/guide-python/quadratureshap_benchmark.py +++ /dev/null @@ -1,484 +0,0 @@ -""" -QuadratureSHAP benchmark harness -================================ - -This script benchmarks CPU ``quadratureshap`` against ``treeshap`` on: - -- real datasets from scikit-learn -- an easy synthetic binary task -- harder synthetic noisy tasks - -It emits JSON results and two charts: - -- accuracy convergence vs quadrature point count -- runtime speedup vs TreeSHAP - -Example -------- - -Run from the repository root with the local package and library on the path: - - LD_LIBRARY_PATH=$PWD/lib \ - PYTHONPATH=$PWD/python-package \ - python demo/guide-python/quadratureshap_benchmark.py -""" - -from __future__ import annotations - -import argparse -import json -import statistics -import time -from dataclasses import dataclass -from pathlib import Path -from typing import Callable - -import matplotlib.pyplot as plt -import numpy as np -import xgboost as xgb -from sklearn.datasets import load_breast_cancer, load_diabetes, load_digits -from sklearn.model_selection import train_test_split - - -@dataclass(frozen=True) -class Workload: - """A benchmark workload definition.""" - - name: str - family: str - objective: str - rounds: int - num_class: int | None - build: Callable[ - [np.random.Generator], tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray] - ] - - -DEFAULT_POINTS = [4, 6, 8, 10, 12, 16, 20] -DEFAULT_DEPTHS = [4, 8, 16, 30] -DEFAULT_SEED = 20260320 -DEFAULT_THREADS = 35 -DEFAULT_TEST_ROWS = 512 -DEFAULT_RUNS = 3 - - -def _split_dataset( - X: np.ndarray, - y: np.ndarray, - seed: int, - cap_rows: int, - *, - stratify: bool, -) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: - X_train, X_test, y_train, y_test = train_test_split( - X, - y, - test_size=0.25, - random_state=seed, - stratify=y if stratify else None, - ) - if X_test.shape[0] > cap_rows: - X_test = X_test[:cap_rows] - y_test = y_test[:cap_rows] - return X_train, X_test, y_train, y_test - - -def make_real_workloads(seed: int, cap_rows: int) -> list[Workload]: - """Build benchmark specs for real scikit-learn datasets.""" - - def breast(_: np.random.Generator): - data = load_breast_cancer() - return _split_dataset( - data.data.astype(np.float32), - data.target.astype(np.float32), - seed, - cap_rows, - stratify=True, - ) - - def diabetes(_: np.random.Generator): - data = load_diabetes() - return _split_dataset( - data.data.astype(np.float32), - data.target.astype(np.float32), - seed, - cap_rows, - stratify=False, - ) - - def digits(_: np.random.Generator): - data = load_digits() - return _split_dataset( - data.data.astype(np.float32), - data.target.astype(np.int32), - seed, - cap_rows, - stratify=True, - ) - - return [ - Workload("breast_cancer", "real", "binary:logistic", 200, None, breast), - Workload("diabetes", "real", "reg:squarederror", 300, None, diabetes), - Workload("digits", "real", "multi:softprob", 250, 10, digits), - ] - - -def make_synthetic_workloads(cap_rows: int) -> list[Workload]: - """Build benchmark specs for synthetic workloads.""" - - def easy_linear(rng: np.random.Generator): - X = rng.standard_normal((40000, 50), dtype=np.float32) - y = (X[:, 0] + 0.5 * X[:, 1] - 0.25 * X[:, 2] > 0).astype(np.float32) - X_test = rng.standard_normal((cap_rows, 50), dtype=np.float32) - y_test = (X_test[:, 0] + 0.5 * X_test[:, 1] - 0.25 * X_test[:, 2] > 0).astype( - np.float32 - ) - return X, X_test, y, y_test - - def random_labels(rng: np.random.Generator): - X = rng.standard_normal((40000, 50), dtype=np.float32) - y = rng.integers(0, 2, size=40000, dtype=np.int32).astype(np.float32) - X_test = rng.standard_normal((cap_rows, 50), dtype=np.float32) - y_test = rng.integers(0, 2, size=cap_rows, dtype=np.int32).astype(np.float32) - return X, X_test, y, y_test - - def random_regression(rng: np.random.Generator): - X = rng.standard_normal((40000, 50), dtype=np.float32) - y = rng.standard_normal(40000).astype(np.float32) - X_test = rng.standard_normal((cap_rows, 50), dtype=np.float32) - y_test = rng.standard_normal(cap_rows).astype(np.float32) - return X, X_test, y, y_test - - return [ - Workload("easy_linear", "synthetic", "binary:logistic", 200, None, easy_linear), - Workload( - "random_labels", "synthetic", "binary:logistic", 200, None, random_labels - ), - Workload( - "random_regression", - "synthetic", - "reg:squarederror", - 200, - None, - random_regression, - ), - ] - - -def margin_shape(predt: np.ndarray) -> np.ndarray: - """Normalize output-margin predictions into a comparable array shape.""" - - predt = np.asarray(predt) - if predt.ndim == 1: - return predt - return predt - - -def contrib_sum(predt: np.ndarray) -> np.ndarray: - """Sum SHAP contributions across the feature axis.""" - - predt = np.asarray(predt) - return predt.sum(axis=-1) - - -def tree_stats(bst: xgb.Booster) -> dict[str, float]: - """Collect simple structural statistics from a booster dump.""" - - dump = bst.get_dump(dump_format="json", with_stats=True) - - def walk(node: dict, depth: int = 0) -> tuple[int, int, int]: - children = node.get("children", []) - if not children: - return depth, 1, 1 - max_depth = depth - node_count = 1 - leaf_count = 0 - for child in children: - child_depth, child_nodes, child_leaves = walk(child, depth + 1) - max_depth = max(max_depth, child_depth) - node_count += child_nodes - leaf_count += child_leaves - return max_depth, node_count, leaf_count - - max_depths = [] - node_counts = [] - leaf_counts = [] - for tree_json in dump: - tree = json.loads(tree_json) - max_depth, nodes, leaves = walk(tree) - max_depths.append(max_depth) - node_counts.append(nodes) - leaf_counts.append(leaves) - - return { - "mean_max_depth": statistics.mean(max_depths), - "max_max_depth": max(max_depths), - "mean_nodes": statistics.mean(node_counts), - "mean_leaves": statistics.mean(leaf_counts), - } - - -def evaluate_model( - bst: xgb.Booster, dtest: xgb.DMatrix, n_points: int, runs: int -) -> dict[str, float]: - """Measure accuracy and runtime for one quadrature point count.""" - - bst.set_param({"shap_algorithm": "treeshap"}) - margin = margin_shape(bst.predict(dtest, output_margin=True)) - treeshap = np.asarray(bst.predict(dtest, pred_contribs=True)) - - bst.set_param( - {"shap_algorithm": "quadratureshap", "quadratureshap_points": str(n_points)} - ) - quadrature = np.asarray(bst.predict(dtest, pred_contribs=True)) - - diff = np.abs(treeshap - quadrature) - add = np.abs(contrib_sum(quadrature) - margin) - - quadrature_times = [] - for _ in range(runs): - t0 = time.perf_counter() - bst.predict(dtest, pred_contribs=True) - quadrature_times.append(time.perf_counter() - t0) - - bst.set_param({"shap_algorithm": "treeshap"}) - treeshap_times = [] - for _ in range(runs): - t0 = time.perf_counter() - bst.predict(dtest, pred_contribs=True) - treeshap_times.append(time.perf_counter() - t0) - - quad_mean = statistics.mean(quadrature_times) - tree_mean = statistics.mean(treeshap_times) - return { - "points": n_points, - "max_abs_diff": float(diff.max()), - "mean_abs_diff": float(diff.mean()), - "max_additivity_err": float(add.max()), - "mean_additivity_err": float(add.mean()), - "quadrature_mean_s": quad_mean, - "treeshap_mean_s": tree_mean, - "speedup_vs_treeshap": tree_mean / quad_mean, - } - - -# pylint: disable=too-many-arguments,too-many-positional-arguments -def train_model( - workload: Workload, - X_train: np.ndarray, - y_train: np.ndarray, - depth: int, - threads: int, - seed: int, -) -> xgb.Booster: - """Train one benchmark model for a workload/depth pair.""" - params: dict[str, int | float | str] = { - "objective": workload.objective, - "tree_method": "hist", - "max_depth": depth, - "eta": 0.1, - "subsample": 1.0, - "colsample_bytree": 1.0, - "min_child_weight": 0.0, - "seed": seed, - "nthread": threads, - } - if workload.num_class is not None: - params["num_class"] = workload.num_class - - dtrain = xgb.DMatrix(X_train, label=y_train) - return xgb.train(params, dtrain, num_boost_round=workload.rounds) - - -# pylint: disable=too-many-arguments,too-many-locals -def run_benchmarks( - workloads: list[Workload], - *, - points: list[int], - depths: list[int], - seed: int, - threads: int, - runs: int, -) -> list[dict]: - """Run the full benchmark sweep and emit row-wise JSON records.""" - - rng = np.random.default_rng(seed) - results: list[dict] = [] - - for workload in workloads: - X_train, X_test, y_train, y_test = workload.build(rng) - dtest = xgb.DMatrix(X_test, label=y_test) - - for depth in depths: - bst = train_model(workload, X_train, y_train, depth, threads, seed) - stats = tree_stats(bst) - for n_points in points: - row = { - "dataset": workload.name, - "family": workload.family, - "depth": depth, - **stats, - **evaluate_model(bst, dtest, n_points, runs), - } - results.append(row) - print(json.dumps(row), flush=True) - - return results - - -def plot_results(results: list[dict], out_dir: Path) -> tuple[Path, Path]: - """Create summary plots for accuracy and speed trends.""" - - families = ["real", "synthetic"] - acc_path = out_dir / "quadratureshap_benchmark_accuracy.png" - speed_path = out_dir / "quadratureshap_benchmark_speedup.png" - - fig, axes = plt.subplots( - 1, len(families), figsize=(12, 4.5), constrained_layout=True - ) - if len(families) == 1: - axes = [axes] - for ax, family in zip(axes, families): - rows = [r for r in results if r["family"] == family] - for name in sorted({r["dataset"] for r in rows}): - group = [r for r in rows if r["dataset"] == name] - best = {} - for points in sorted({r["points"] for r in group}): - pts = [r["max_abs_diff"] for r in group if r["points"] == points] - best[points] = max(pts) - ax.plot( - list(best.keys()), - list(best.values()), - marker="o", - linewidth=2, - label=name, - ) - ax.axhline(1e-5, color="black", linestyle="--", linewidth=1, alpha=0.6) - ax.set_title(f"{family.title()} workloads") - ax.set_xlabel("Quadrature points") - ax.set_ylabel("Worst-case max abs diff") - ax.set_yscale("log") - ax.grid(True, alpha=0.3) - axes[-1].legend(loc="upper right") - fig.suptitle("QuadratureSHAP accuracy convergence") - fig.savefig(acc_path, dpi=180) - - fig, axes = plt.subplots( - 1, len(families), figsize=(12, 4.5), constrained_layout=True - ) - if len(families) == 1: - axes = [axes] - for ax, family in zip(axes, families): - rows = [r for r in results if r["family"] == family] - for name in sorted({r["dataset"] for r in rows}): - group = [r for r in rows if r["dataset"] == name] - best = {} - for points in sorted({r["points"] for r in group}): - pts = [r["speedup_vs_treeshap"] for r in group if r["points"] == points] - best[points] = statistics.mean(pts) - ax.plot( - list(best.keys()), - list(best.values()), - marker="o", - linewidth=2, - label=name, - ) - ax.axhline(1.0, color="black", linestyle="--", linewidth=1, alpha=0.6) - ax.set_title(f"{family.title()} workloads") - ax.set_xlabel("Quadrature points") - ax.set_ylabel("Mean speedup vs TreeSHAP") - ax.grid(True, alpha=0.3) - axes[-1].legend(loc="upper right") - fig.suptitle("QuadratureSHAP runtime trend") - fig.savefig(speed_path, dpi=180) - - return acc_path, speed_path - - -def print_summary(results: list[dict], target_error: float) -> None: - """Print a compact per-dataset summary table.""" - - print("\nSUMMARY") - print( - "dataset family depth points max_diff add_err " - "speedup mean_nodes" - ) - for name in sorted({r["dataset"] for r in results}): - rows = [r for r in results if r["dataset"] == name] - safe_rows = [ - r - for r in rows - if r["max_abs_diff"] <= target_error - and r["max_additivity_err"] <= target_error - ] - if safe_rows: - best = min( - safe_rows, key=lambda r: (r["points"], -r["speedup_vs_treeshap"]) - ) - else: - best = min(rows, key=lambda r: (r["max_abs_diff"], r["points"])) - print( - f"{best['dataset']:<20} {best['family']:<9} {best['depth']:<5} " - f"{best['points']:<6} {best['max_abs_diff']:<12.3e} " - f"{best['max_additivity_err']:<12.3e} {best['speedup_vs_treeshap']:<7.2f} " - f"{best['mean_nodes']:<10.1f}" - ) - - -def parse_args() -> argparse.Namespace: - """Parse command line arguments for the benchmark harness.""" - - parser = argparse.ArgumentParser( - description="Benchmark QuadratureSHAP against TreeSHAP." - ) - parser.add_argument("--seed", type=int, default=DEFAULT_SEED) - parser.add_argument("--threads", type=int, default=DEFAULT_THREADS) - parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) - parser.add_argument("--test-rows", type=int, default=DEFAULT_TEST_ROWS) - parser.add_argument("--points", type=int, nargs="+", default=DEFAULT_POINTS) - parser.add_argument("--depths", type=int, nargs="+", default=DEFAULT_DEPTHS) - parser.add_argument( - "--workloads", - nargs="+", - choices=["real", "synthetic", "all"], - default=["all"], - help="Which workload families to run.", - ) - parser.add_argument("--out-dir", type=Path, default=Path.cwd()) - parser.add_argument("--target-error", type=float, default=1e-5) - return parser.parse_args() - - -def main() -> None: - """Entry point for the benchmark harness.""" - - args = parse_args() - args.out_dir.mkdir(parents=True, exist_ok=True) - - workloads: list[Workload] = [] - selected = set(args.workloads) - if "all" in selected or "real" in selected: - workloads.extend(make_real_workloads(args.seed, args.test_rows)) - if "all" in selected or "synthetic" in selected: - workloads.extend(make_synthetic_workloads(args.test_rows)) - - results = run_benchmarks( - workloads, - points=args.points, - depths=args.depths, - seed=args.seed, - threads=args.threads, - runs=args.runs, - ) - - out_json = args.out_dir / "quadratureshap_benchmark_results.json" - out_json.write_text(json.dumps(results, indent=2)) - acc_path, speed_path = plot_results(results, args.out_dir) - print_summary(results, args.target_error) - print(f"\nSAVED_JSON={out_json}") - print(f"SAVED_ACC={acc_path}") - print(f"SAVED_SPEED={speed_path}") - - -if __name__ == "__main__": - main() diff --git a/demo/guide-python/quadratureshap_rapids_benchmark.py b/demo/guide-python/quadratureshap_rapids_benchmark.py new file mode 100644 index 000000000000..82b5aceed033 --- /dev/null +++ b/demo/guide-python/quadratureshap_rapids_benchmark.py @@ -0,0 +1,605 @@ +"""RAPIDS-style SHAP benchmark for TreeSHAP and QuadratureSHAP. + +This benchmark keeps the basic structure of the RAPIDS GPUTreeShap benchmark while +benchmarking four explanation paths from the current XGBoost worktree: + +- CPU TreeSHAP +- CPU QuadratureTreeSHAP +- GPU TreeSHAP +- GPU QuadratureTreeSHAP + +It supports additive SHAP values and SHAP interactions and emits both a model-metadata table +and a timing table. +""" + +# pylint: disable=missing-class-docstring,missing-function-docstring,too-many-instance-attributes,too-many-arguments,too-many-positional-arguments,too-many-locals,broad-exception-caught,no-member + +from __future__ import annotations + +import argparse +import gc +import json +import multiprocessing as mp +import statistics +import time +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +import numpy as np +import pandas as pd +import xgboost as xgb +from sklearn import datasets + + +@dataclass(frozen=True) +class TestDataset: + name: str + objective: str + X: object + y: np.ndarray + + def set_params(self, params: dict[str, object]) -> dict[str, object]: + params["objective"] = self.objective + if self.objective == "multi:softmax": + params["num_class"] = int(np.max(self.y) + 1) + return params + + def train_dmatrix(self) -> xgb.DMatrix: + return xgb.QuantileDMatrix(self.X, self.y, enable_categorical=True) + + def test_input(self, num_rows: int, seed: int) -> object: + rs = np.random.RandomState(seed) + row_idx = rs.randint(0, self.X.shape[0], size=num_rows) + if hasattr(self.X, "iloc"): + return self.X.iloc[row_idx, :] + return self.X[row_idx, :] + + def test_dmatrix(self, num_rows: int, seed: int) -> xgb.DMatrix: + return xgb.DMatrix(self.test_input(num_rows, seed), enable_categorical=True) + + +@dataclass(frozen=True) +class ModelSpec: + suffix: str + num_rounds: int + max_depth: int + grow_policy: str | None = None + max_leaves: int | None = None + + def training_params(self) -> dict[str, object]: + params: dict[str, object] = { + "tree_method": "hist", + "device": "cuda", + "eta": 0.01, + "max_depth": self.max_depth, + } + if self.grow_policy is not None: + params["grow_policy"] = self.grow_policy + if self.max_leaves is not None: + params["max_leaves"] = self.max_leaves + return params + + +MODEL_SPECS = { + "small": ModelSpec("small", 10, 6), + "large": ModelSpec("large", 1000, 16), + # "sparse" here means a LightGBM-style leaf-wise tree shape rather than sparse input storage. + "sparse": ModelSpec("sparse", 100, 0, grow_policy="lossguide", max_leaves=512), +} + + +@dataclass(frozen=True) +class Model: + name: str + dataset: TestDataset + spec: ModelSpec + booster: xgb.Booster + trees: int + leaves: int + average_depth: float + mean_max_depth: float + max_max_depth: int + mean_nodes: float + mean_leaves: float + + +@lru_cache(maxsize=1) +def fetch_adult() -> tuple[object, np.ndarray]: + x, y = datasets.fetch_openml("adult", return_X_y=True) + y_binary = np.array([y_i != "<=50K" for y_i in y]) + return x, y_binary + + +@lru_cache(maxsize=1) +def fetch_fashion_mnist() -> tuple[object, np.ndarray]: + x, y = datasets.fetch_openml("Fashion-MNIST", return_X_y=True) + return x, y.astype(np.int64) + + +@lru_cache(maxsize=1) +def get_test_datasets() -> tuple[TestDataset, ...]: + cov_x, cov_y = datasets.fetch_covtype(return_X_y=True) + cal_x, cal_y = datasets.fetch_california_housing(return_X_y=True) + return ( + TestDataset("adult", "binary:logistic", *fetch_adult()), + TestDataset("covtype", "multi:softmax", cov_x, cov_y.astype(np.int64)), + TestDataset( + "cal_housing", + "reg:squarederror", + cal_x.astype(np.float32), + cal_y.astype(np.float32), + ), + TestDataset("fashion_mnist", "multi:softmax", *fetch_fashion_mnist()), + ) + + +def train_model(dataset: TestDataset, spec: ModelSpec) -> xgb.Booster: + dtrain = dataset.train_dmatrix() + params = spec.training_params() + params = dataset.set_params(params) + return xgb.train( + params, + dtrain, + spec.num_rounds, + evals=[(dtrain, "train")], + verbose_eval=False, + ) + + +def tree_stats(model: xgb.Booster) -> dict[str, float]: + dump = model.get_dump(dump_format="json", with_stats=True) + + def walk(node: dict, depth: int = 0) -> tuple[int, int, int]: + children = node.get("children", []) + if not children: + return depth, 1, 1 + max_depth = depth + node_count = 1 + leaf_count = 0 + for child in children: + child_depth, child_nodes, child_leaves = walk(child, depth + 1) + max_depth = max(max_depth, child_depth) + node_count += child_nodes + leaf_count += child_leaves + return max_depth, node_count, leaf_count + + max_depths: list[int] = [] + node_counts: list[int] = [] + leaf_counts: list[int] = [] + for tree_json in dump: + tree = json.loads(tree_json) + max_depth, nodes, leaves = walk(tree) + max_depths.append(max_depth) + node_counts.append(nodes) + leaf_counts.append(leaves) + + return { + "trees": len(dump), + "leaves": int(sum(leaf_counts)), + "average_depth": float(statistics.mean(max_depths)), + "mean_max_depth": float(statistics.mean(max_depths)), + "max_max_depth": int(max(max_depths)), + "mean_nodes": float(statistics.mean(node_counts)), + "mean_leaves": float(statistics.mean(leaf_counts)), + } + + +def get_models(model_filter: str) -> list[Model]: + models: list[Model] = [] + for dataset in get_test_datasets(): + for spec in MODEL_SPECS.values(): + model_name = f"{dataset.name}-{spec.suffix}" + if model_filter not in {"all", spec.suffix} and model_filter != model_name: + continue + print(f"Training {model_name}") + booster = train_model(dataset, spec) + stats = tree_stats(booster) + models.append( + Model( + name=model_name, + dataset=dataset, + spec=spec, + booster=booster, + trees=int(stats["trees"]), + leaves=int(stats["leaves"]), + average_depth=float(stats["average_depth"]), + mean_max_depth=float(stats["mean_max_depth"]), + max_max_depth=int(stats["max_max_depth"]), + mean_nodes=float(stats["mean_nodes"]), + mean_leaves=float(stats["mean_leaves"]), + ) + ) + return models + + +def predict_with_algorithm( + booster: xgb.Booster, + dtest: xgb.DMatrix, + device: str, + algorithm: str, + interactions: bool, +) -> np.ndarray: + params: dict[str, object] = {"device": device} + if algorithm == "quadratureshap": + params["shap_algorithm"] = "quadratureshap" + params["quadratureshap_points"] = 8 + else: + params["shap_algorithm"] = "treeshap" + booster.set_param(params) + if interactions: + return np.asarray(booster.predict(dtest, pred_interactions=True)) + return np.asarray(booster.predict(dtest, pred_contribs=True)) + + +def _benchmark_case_worker( + queue: mp.Queue, + booster_raw: bytes, + x_test: object, + device: str, + algorithm: str, + interactions: bool, + niter: int, + margin: np.ndarray | None, +) -> None: + try: + booster = xgb.Booster() + booster.load_model(bytearray(booster_raw)) + dtest = xgb.DMatrix(x_test, enable_categorical=True) + pred = predict_with_algorithm(booster, dtest, device, algorithm, interactions) + if interactions: + additive = predict_with_algorithm( + booster, dtest, device, algorithm, interactions=False + ) + row_sums = np.sum(pred, axis=pred.ndim - 1) + metrics = { + "max_row_sum_err": float(np.max(np.abs(row_sums - additive))), + "mean_row_sum_err": float(np.mean(np.abs(row_sums - additive))), + "max_asymmetry": float( + np.max(np.abs(pred - np.swapaxes(pred, -1, -2))) + ), + } + else: + assert margin is not None + summed = np.sum(pred, axis=pred.ndim - 1) + metrics = { + "max_additivity_err": float(np.max(np.abs(summed - margin))), + "mean_additivity_err": float(np.mean(np.abs(summed - margin))), + } + + samples = [] + for _ in range(niter): + t0 = time.perf_counter() + predict_with_algorithm(booster, dtest, device, algorithm, interactions) + samples.append(time.perf_counter() - t0) + queue.put( + { + "mean_time_s": float(np.mean(samples)), + "std_time_s": float(np.std(samples)), + "error": None, + **metrics, + } + ) + except Exception as err: # noqa: BLE001 + queue.put( + { + "mean_time_s": None, + "std_time_s": None, + "max_additivity_err": None, + "mean_additivity_err": None, + "max_row_sum_err": None, + "mean_row_sum_err": None, + "max_asymmetry": None, + "error": str(err).splitlines()[0], + } + ) + + +def run_case_with_timeout( + booster: xgb.Booster, + x_test: object, + device: str, + algorithm: str, + interactions: bool, + niter: int, + margin: np.ndarray | None, + timeout_seconds: float | None, +) -> dict[str, object]: + if timeout_seconds is None: + queue: mp.Queue = mp.Queue() + _benchmark_case_worker( + queue, + bytes(booster.save_raw()), + x_test, + device, + algorithm, + interactions, + niter, + margin, + ) + result = queue.get() + queue.close() + return result + + ctx = mp.get_context("spawn") + queue = ctx.Queue() + proc = ctx.Process( + target=_benchmark_case_worker, + args=( + queue, + bytes(booster.save_raw()), + x_test, + device, + algorithm, + interactions, + niter, + margin, + ), + ) + proc.start() + proc.join(timeout_seconds) + if proc.is_alive(): + proc.terminate() + proc.join() + queue.close() + return { + "mean_time_s": None, + "std_time_s": None, + "max_additivity_err": None, + "mean_additivity_err": None, + "max_row_sum_err": None, + "mean_row_sum_err": None, + "max_asymmetry": None, + "error": f"DNF: exceeded {timeout_seconds:g}s", + } + if queue.empty(): + queue.close() + return { + "mean_time_s": None, + "std_time_s": None, + "max_additivity_err": None, + "mean_additivity_err": None, + "max_row_sum_err": None, + "mean_row_sum_err": None, + "max_asymmetry": None, + "error": "DNF: worker exited without result", + } + result = queue.get() + queue.close() + return result + + +def check_accuracy( + booster: xgb.Booster, + dtest: xgb.DMatrix, + device: str, + algorithm: str, + pred: np.ndarray, + margin: np.ndarray, + interactions: bool, +) -> dict[str, float]: + if interactions: + additive = predict_with_algorithm( + booster, dtest, device, algorithm, interactions=False + ) + row_sums = np.sum(pred, axis=pred.ndim - 1) + return { + "max_row_sum_err": float(np.max(np.abs(row_sums - additive))), + "mean_row_sum_err": float(np.mean(np.abs(row_sums - additive))), + "max_asymmetry": float(np.max(np.abs(pred - np.swapaxes(pred, -1, -2)))), + } + + summed = np.sum(pred, axis=pred.ndim - 1) + return { + "max_additivity_err": float(np.max(np.abs(summed - margin))), + "mean_additivity_err": float(np.mean(np.abs(summed - margin))), + } + + +def benchmark_model( + model: Model, + x_test: object, + dtest: xgb.DMatrix, + niter: int, + interactions: bool, + timeout_seconds: float | None, +) -> tuple[dict[str, object], list[dict[str, object]]]: + margin = model.booster.predict(dtest, output_margin=True) + details: list[dict[str, object]] = [] + result_row = { + "model": model.name, + "test_rows": dtest.num_row(), + "TreeSHAP": None, + "QuadratureTreeSHAP": None, + "GPUTreeShap": None, + "QuadratureTreeSHAP (GPU)": None, + "QuadratureTreeSHAP Speedup": None, + "QuadratureTreeSHAP (GPU) Speedup": None, + } + + for algorithm in ["treeshap", "quadratureshap"]: + for device in ["cpu", "cuda"]: + result = run_case_with_timeout( + model.booster, + x_test, + device, + algorithm, + interactions, + niter, + margin if not interactions else None, + timeout_seconds, + ) + details.append( + { + "model": model.name, + "algorithm": algorithm, + "device": device, + **result, + } + ) + if result["mean_time_s"] is not None: + if algorithm == "treeshap" and device == "cpu": + result_row["TreeSHAP"] = float(result["mean_time_s"]) + elif algorithm == "quadratureshap" and device == "cpu": + result_row["QuadratureTreeSHAP"] = float(result["mean_time_s"]) + elif algorithm == "treeshap" and device == "cuda": + result_row["GPUTreeShap"] = float(result["mean_time_s"]) + elif algorithm == "quadratureshap" and device == "cuda": + result_row["QuadratureTreeSHAP (GPU)"] = float( + result["mean_time_s"] + ) + gc.collect() + + if ( + result_row["TreeSHAP"] is not None + and result_row["QuadratureTreeSHAP"] is not None + ): + result_row["QuadratureTreeSHAP Speedup"] = ( + result_row["TreeSHAP"] / result_row["QuadratureTreeSHAP"] + ) + if ( + result_row["GPUTreeShap"] is not None + and result_row["QuadratureTreeSHAP (GPU)"] is not None + ): + result_row["QuadratureTreeSHAP (GPU) Speedup"] = ( + result_row["GPUTreeShap"] / result_row["QuadratureTreeSHAP (GPU)"] + ) + return result_row, details + + +def markdown_table(df: pd.DataFrame, float_fmt: str = ".6f") -> str: + headers = [str(c) for c in df.columns] + rows = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join(["---"] * len(headers)) + " |", + ] + for _, row in df.iterrows(): + formatted = [] + for value in row: + if value is None or (isinstance(value, float) and pd.isna(value)): + formatted.append("NA") + elif isinstance(value, float): + formatted.append(format(value, float_fmt)) + else: + formatted.append(str(value)) + rows.append("| " + " | ".join(formatted) + " |") + return "\n".join(rows) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="RAPIDS-style benchmark adapted for XGBoost TreeSHAP and QuadratureSHAP." + ) + parser.add_argument("--output", type=Path, required=True, help="JSON summary path") + parser.add_argument( + "--out-models", type=Path, default=None, help="CSV path for model table" + ) + parser.add_argument( + "--out-results", type=Path, default=None, help="CSV path for timing table" + ) + parser.add_argument( + "--out-markdown", type=Path, default=None, help="Markdown table path" + ) + parser.add_argument("--nrows", type=int, default=1000) + parser.add_argument("--niter", type=int, default=3) + parser.add_argument("--seed", type=int, default=432) + parser.add_argument( + "--case-timeout-seconds", + type=float, + default=None, + help="Optional per algorithm/device/model timeout. Timed-out cases are marked DNF.", + ) + parser.add_argument( + "--model", + type=str, + default="all", + help="Model filter: all, small, large, sparse, or a specific dataset-size name", + ) + parser.add_argument("--interactions", action="store_true") + args = parser.parse_args() + + models = get_models(args.model) + model_rows = [ + { + "model": model.name, + "num_rounds": model.spec.num_rounds, + "requested_max_depth": model.spec.max_depth, + "grow_policy": model.spec.grow_policy or "depthwise", + "max_leaves": model.spec.max_leaves, + "num_trees": model.trees, + "num_leaves": model.leaves, + "average_depth": model.average_depth, + "mean_max_depth": model.mean_max_depth, + "max_max_depth": model.max_max_depth, + "mean_nodes": model.mean_nodes, + "mean_leaves_per_tree": model.mean_leaves, + } + for model in models + ] + results_rows: list[dict[str, object]] = [] + details_rows: list[dict[str, object]] = [] + for model in models: + x_test = model.dataset.test_input(args.nrows, args.seed) + dtest = xgb.DMatrix(x_test, enable_categorical=True) + result_row, details = benchmark_model( + model, + x_test, + dtest, + args.niter, + args.interactions, + args.case_timeout_seconds, + ) + results_rows.append(result_row) + details_rows.extend(details) + print( + pd.DataFrame(results_rows).to_string( + index=False, float_format=lambda x: f"{x:.6f}" + ) + ) + + models_df = pd.DataFrame(model_rows) + results_df = pd.DataFrame(results_rows) + payload = { + "nrows": args.nrows, + "niter": args.niter, + "interactions": args.interactions, + "model_filter": args.model, + "model_specs": { + name: { + "num_rounds": spec.num_rounds, + "max_depth": spec.max_depth, + "grow_policy": spec.grow_policy, + "max_leaves": spec.max_leaves, + } + for name, spec in MODEL_SPECS.items() + }, + "models_table": models_df.to_dict(orient="records"), + "results_table": results_df.to_dict(orient="records"), + "details": details_rows, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2) + "\n") + if args.out_models is not None: + args.out_models.parent.mkdir(parents=True, exist_ok=True) + models_df.to_csv(args.out_models, index=False) + if args.out_results is not None: + args.out_results.parent.mkdir(parents=True, exist_ok=True) + results_df.to_csv(args.out_results, index=False) + if args.out_markdown is not None: + args.out_markdown.parent.mkdir(parents=True, exist_ok=True) + args.out_markdown.write_text( + "## Models\n\n" + + markdown_table(models_df, ".3f") + + "\n\n## Results\n\n" + + markdown_table(results_df, ".6f") + + "\n" + ) + + print("Models:") + print(models_df.to_string(index=False)) + print("Results:") + print(results_df.to_string(index=False, float_format=lambda x: f"{x:.6f}")) + + +if __name__ == "__main__": + main() diff --git a/quadratureshap_n8_accuracy_vs_nodes.png b/quadratureshap_n8_accuracy_vs_nodes.png deleted file mode 100644 index 4abfb87a3233bb2a08e82073371f4a4a8a1365ac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 78004 zcmd?RcQ~8v|34hn(%z-I?pvisTPCQMGql6@pMBcF^iB(Nfgjd$qP8p+gk0 zB?!_sc8nSkMEITQ{rP;4-|zR&^ZfDr_gu%}D0=0(uJgRk^F3a}t2>4|ynjLefoJg+TUYKp>n?_j7^YSQy0jfqztdZ(91Ac*1=B?|VBz4DS0ra`*Ighd(&w=j81J z_wFq4lVuS1gS2^(LmX!|#air!Rlwk(`T!M-EM-Qbtk2rNvi! zx`MrA_w`=9c>U3->kkvI*(g-`)^3KSQFp)Q=#Eqw1jVc|SRvQGB*#t=f(;F0rq!kc zzql(T8-PdnbNv4F$U*bee|~`M%>WPa=Lo&dCHJ3io2B+^asKD~?I)1`pZ@xZp{3;q zukUSbA2FAH_eJn&xHxzwCMF_&68*4M-FphYKW}VvC{=u8Ro#{Q0v2 zNi)|z$@_p@gJ_MmZ5o*7_ArA^T&aaOsclZg%8!dHDk}bXa@J?MD^6dk$rPMqkW2yT)KD$axjtEJ=>qJIJ0V)sxla0&pHT!T+_)J6oNpW z=85W{+e)k|=XYy|-COyFUkC@a^@V17^+2kqHB;ryOSm1nRQ=8tEmt>I>?qIr;@BmA z`+!Vx#x@qkd^p>e+v?Q*{=!O|Xmp#M|Cp1mCWmFmXxO0>SF~EAMJV^Wiyu!fkZbi{ z91?oim!r$O-Uxv-+u()jmCpyI+8UOymyFbwpRO%YD0QV(Ri=wrrbpoNH-3plc}PGW<-*#p z4p}=0Bz)<}pBp55V&U6SuV2S>NH%oVm01}$gl?V_MZY~PBI4OS)X~uqtHl@(Ui#i1 zcPWatFj5^$31u4Cqv&S!vw6uAG5V6y6{!iAtsYb*3J3h)5PihUUeCyz5Gdq_k4=sH zJ5TIpYn_Jg%$@p$8nW%4eyxwmN?ut_qn(b?>SB)Cs*7(qOv7`}3l{e*Jpu+O;Qw{-a*4PkomfSJB7$ z`N;{SqdYJixR7Ic8ujZjBe{4|=-leyGuV`?)tlaIos%>x*{JGrI2_(;IkWy_$jILj zMYM8O9+W|*DD^ZBka!hG@A3V@uC!m$9-Gyw31)aOk*OgcRq(>WB`gfr5KWK4nPi5d zpcvnqA9JSGG=OFN{Tsi-Q2sifhv7q_k(L&nI_6$c1rDvxWK{nTpf;u%iHwE%} zjMnuIkzDTKz?fS#ZZU3djh|Et__&OZ9@OLYs|>h>t=F^V z@lqPIRKqbl*E-K28P;fl#O1L1Ncl_-RFiy@_?k3fS|MSOl*K%c!qKS6`61)7`YEVY zt%qGHHrHIDn=JCt+U?WFY zxem4u)gVu5PN(=xnn)1|CH;D-nzC_jk8QW2@$@lr^~)nsYh| zhBHVfpTzKfVDz$L#1H;+`AUu5So9oLhr|GD6j6m@F4W!?2@0#^!z_*t@i(*ludH?7v>VqVT(?4R(rm#OQB&9t}H*+&gvIF%>Y&a zim!99T zu}mEs$7N(R`_1Mi5Cg)R+{rbBSuCI0?{HR4Id-dPB$QZHQj=Iqs{rxreqe} z6e(4d=@xF0wIWJW5WQkyw7#_mPd>?4*&DdKd>sklL{rR)0Q>v zi7YQk+_txk-ONzHDkfTE%Y}?V(u#y=Ns>tEd(99g%g5V05((Q08?E#5Sjz55?3iJd zX3@UR;}1$T-dU)~Pw{^J$@a#Lrw9rbg$y-Bc?MNVK0hSnstRINRCGS`-O$hwRK%#3 zQ6xxeDKaZ|2f@#$-eC~AvC<*cfGSt9H&pt$mrN|RQD5JzCfQfs)X^agO~)Fle4oTV zd-km6h^Tge{Bt8sRuzf%jo*#PUX%B0inc ztxAEF!pPmRYB!7M=)uhE+_mJ=TBZ@c8xs=~17(&aw6)zLgFw4fvos>&nMPXnshD%~ zGqtvMbpwge%f{L|sWdC=vw7;uinpV(o&8ciquiCTI9BtRlbbk%FD}Q9EU=m~9R)y2 zQ0w~r^UIjzDp=Uc>J&2zyGg%#umyl?MggCdp9U1J>B!9ysK$-xe;!9CNu+s$Z^D@?Gfo=Oftn?eshPp zv_Q^l(699vi(JZ)Z^A13Obs19cFb+wBv#D8b^aJc?&+=F=d~P=s3=$p{GSBO@6QLL zZp$DyClflF7~^gaW8eaZ2sMRWyE_a`(MzOG$$>}=m<38(dvB)*L2nz}A*y1}VuiT; zb@R%5gibz!cK>`;zmeux+o4|fQh7Jx2>}J%?oM!o0eWN1pCT{`J#*$vbtFydS?ktxPnaf?WBFhR(cqZ-R0=869|4g-*@leG`=L` z+~H#DykDiY<1%yS-Ly?k^(}nue9r_C`JAYB^ak zuT-J4iihok*XN~)+2}-%{v_i9XMLCT3h~NJ^`7pZ7)eBb1^YTMLR4j z>1$1Q#+ZR%7qwbGbaw6l>m$mm=dwCVQyCWOYTf|DC$6bycg(Hlp-8L`Sbq7AJC&b6 zdJq?N8+=z|7sjHuh8j`2F?{_CD(BAKY;SKrD=v<=)TTHt&S>~7j@Gv#3Hynx0vP$6 zn3&$xtB>7Bs&wNe?X1f?n=z458u_pb$%#P`T)Z;l(MkJCnf6Vnsm{dK1(oL*-huZV zh0eA!TO`C?*qx*e5J|Xe!`S#}DL?Yq8w)rh;^Vtw2!TxaD`d63!Emk2)l3Pjn5OrSpT&W2&&YqvbrhLYAHX}FDV4ng{{M)TS}OvM>*+(gDjkD1NI-*Xt^E<*TtK$ z7W@*XM)RRDj*TtSkuwkbDq#efF57;VFVdMWuwl306^w)CT!_`7%U-AKZlQ8&)|gQq zOqe-lu>ncRBELl!*U&=s7!utY%#!86NsnZcpk(X|-;;f)r)gsu)S_~OZu9=75{wb4 z({l_*6YyA^S6PkHp@ldvzPkept|H%6Jk>WXw9L)niRB1N)&OR7D$#`_rA zY0EhR!*BkPC>peS0rC~^(J3y7926~p*N?yKg+cUWTwN!RI+Z8YxDP+jzjfR2&hLrxYU_r@%tx3iZ)38PRQVb0Dctvb;f+rO_feP(DSk{xZGUzOK~91+NB z7y;_Y3eFPHGg7C&Cr2*KRdk_pUAcS*rJ=I0+o>`c(Sphkm>bk+ly@(Wod2474ojAq zMgD>$**OP;7lQ!ZN!r49@$@P>sO4v?XM#$j+12!9lNtT;K0=vnx~!~h|AIHWpjqgU zb0e0FFmhloK^`fWTtF{fMHy``h3&3xoJbOIg=x?#wzlP+u=bBy_DOaq!G2g=S`NaVR91xKHa-}i%`XD{YY6OyO>>Z497j&wioom`(E zHbHPz*>|bXF4vMreY(}Bt9E^BBdMX5QWyz~1hew=OH&m|feY~A;H4%Kb`(U;g;nGl z7{`#vZdJeGY8{0n`=WDF5;JqAw!@TgB3(VWrb8x}v50K9b-;!MR!2_~tWK&N3TlnX z?}pPmq;Mx_h9av4+31xIh&coE;#!t1#c$<&QxO8m8jvO^oVw8)!lbG!d~RC zF-dMdVRr{Ze2%0rYOU9m+1C{8Ppt`MZ3U{m(I?#mOJ<&IX0~P5+*qClpITIXrIAOi z`E?<Rn-O=)06 z_8|)esA)5nLSfs>t&V5UofBZ{`U~Q?9YB2~8C~f3h>WOJjw);*O>d~ryb@3QJtT!v zV+nGFYiY5YBw6sNRosX*9sNO(Z1m*cix`?@bTl( z*W~O;58Wyx@UQ9MHmP^Q2I_pel&1$UYp+CEUITdsC#gHo+bhTA0_cMl3*Fh8k=qNM zvfWh)zdou!N`n}S3(^@>1Bf+e8k&3yC7Gu8_yakTYhaDLBA>~aAMRJGuEpGok(%X0 zu7<~^VeI2Pe4uIti>BV5O4!!#vEmjDCF6?(tX@R?6r33Z^&ec>fBiX+v`7CbCj`FV ztD(353#(AUQP!#bII}|5xvRv_O1;imk~!oUZFVx8C|x8OUVPQU$I9PHU6Anwqi$xA zR8UaR79SJy;BxU*QS}UqRM{@2FP7#VCuUKodbFY1@(t=vkY1?4q(|qeLaxnt+GIuH!APbRR2^TP`D81c(Nia?9AX{K>)>6vIf{?wOVyE{)HX_qh3i2;p zROhXqyO0pHRi8FevNjOxAL|cKPFr_Rdu==RRXJ^jMbK?qt3s(usN9*Eoea8&lbV57 zs0g6@OcaXag?jow(AV^H^ULRM*jMM~@3`H%ylH!HkY~<+% zv+&i9c!||vl|GXRi*kE}EvXLz-kmfvGd_W}>|+71DNG`hJ)ql)J7un3VQB z&#QjWRQjv{xxMyc`GrWR%eLjfaF_hs+yygu13H<=gH$}E(q)}rAz8Ip#APWzllge6 zNYfmdnpj;!f8jW&Qisf>OHsmhQB*JaNBQO5V;l!7|H&dIR>cs%42sNJ=bLd}59&?& zG(I=<$uVN!G)aVRt$5e%n(Gw{zTY!fd3bYXmE1~=N{!3N=yeCuW8zu~A8)e@>Nf9= zOS3`59_YLmBdFT107vPzZm2ZLHx%|H{fOYQ9sux)DdLVPTx+yM(O!NgLta_lvS`U_ zEPb}+04h45vS}{ZDnc{pkuHS~vow(LO~5(j=cn-o9A3Njr0l&Sjv02D(25_hZquXX zSmft1*lqfQy*a*`&WoOEz0yT!B2j*}#?o%V%SXhqkCeG2N_AAF|Z zrl;DsFgQWrP`7JU`;p018k?e3>^{<|)nsiCgn#Pe9Jda0;L94&K{#YD7FXLhY(mG* z{WaqpL5w8Qv$#E~mS935o;99$mbn45A<2WZqrqD1Z>6txsdjH%CM8YG&dv^w^Fc~( zuye$*EvM1#&tY?g`R&qpL8_l}s{6O?Px&0QA`Tb4)Iv3@cd7&bn;$Q-Bj3cuYdZKI zF;U)0=rU_TW{QjY7HF73sK(8a17TMrxaeW>3bL0{Mi+nLqa&fGd^X? zY`Y!{A-;Mo*OM#Y5R?~Ay?THE=@Yi&7PwbbgOvJzc6gfOTesuF*e{<62ucs!s}{$kA4eY ztjE(LDbGhdX$sj(^1P4cre7-^rxUmLwTMJ3kuXMN)s}e)cuGR zYS+(U3URsh{s#micWb>;H>@C8^RC5KQPRKomc`AR@V)2Fm=WEtt;+k11J8PFc-;j(qzfe(6K{e+20V)*mC{598m z*EsKYtF1=jGjd_IQ6ie-b!CauXj+4U+^vEZMcIuKZW)ipWo>FbU8gb)?OtZY+jh}^ zn1vlh16**PgSN{2j?DY|Q``rUJn za&{H<{Q|cZf0ffsh49^evNI*zVT4ZqRll*FPSo4_lr;DQE^Ki;1G$#!jBQvThoG!Aa`P%Q$JcBa7q*0DH8tZx z$LFl}2mLnbI1I-XtGWQT>lKd+M5}3*Z$Dmy1ijt+Mv>zH>!*s>uN%S1jszTL=Y+J# z^)QG__yFaqi~rTiEn=Sn%{wXocktHfEb8_mbZVBT}^O@}a6 zwXv#MrL6p;yd{G$l88S~-^|s&@SM@~$yrlB9W*UlgIiQ^shFByy-CY1qqTh!Njj$q zaJvKAps~U&TTm#N-BnYQp(5+*AoHlu?w)%eLoK{YOx#2Ldl`k2)9m4-^heynNFnya z?pZOUpq4}#O<4Hw<=*?~dYBGPKr*Ny&B@q~R$^uv2o6D- zWl|1QHrceBmd~R5=2;kYJ7Otx)k=>y^7}H~g49^p+{dscmqPxr=^3mJUnu;PHI9?U z&_$3|M9|J<9N~Yn8JdA)qIVa8P}+|w=DH$CBMq9$4jgk(drrO3F7I?@Gw7(A?Ed1a zRtKcj6apjC0+#; zLwhqJ-hsKoF>IY0bG~vgZHq?uYKT5}x^Z*aer#(1aown!aXiGMZDHsnIyOGYMCGej zV$vQabeqI@2aBU!ScRUyBAYGm<~33U7hWpn?vwlA3+Mp>rtKIkE#TJy=2a0w%~-8x z`bI4E!aDYlSQ6bry|>egx8i(iNw>)+?CT4b=a zIM)ED>r}9LL>kP_UczXi(4(e4ZB3ngC--`1b46bSG$^B;M~gPBfOh8ja22-dw_1+X z>8!ZJTOD02fl}lBAi}0ccIEfmCA{Kc(V=u^x)rA_-(RK{u#&t5^0F6l&*a=p=x6X0 z{#Cw_Y0s7c=z5Y|opzO!t;5IkD|Y?~2_Iy$r*j19mnDg4m7hii{FT~OTy~wqLxZ5r za^ZVB8(lDG$XJ8zx#%hO-CM_Pss7-2J(XsSo9bMUA3QMd<9XWF0^#WkYg|1xaU{vf zk?TnHV&zVoH0`!)=~cW)$WF-ERz#2Vm*PB=o|6x}HFc5*PB4Yp8het-N7Q%enG#KH zw_h9At=)`?&Zwm7%j&%h9P#7YG_ziz3YM36qj77 zuh-??pC_7$7{b(LIy9uLICbS5zB6yG;-Rm5 zQE_pRz}KnyrKC(4>=J2O=hsYcb`;Gz+DeugBENYLVPEUvD8Fx?kP)KXjmy&xu3c;> z=gj3TL`yUG7vZhj`1Pv{R%v+KiR+ArnFE}`C5uofO~Xgzxg z%2Tq4DwY3jFrDge-;}0bd7toV3)Yn~%b0gGqTQm{z*y$$j?KwhR@bE&ng6~Rw|W6j zy_x`KV|6aKE`i5c!Y-GoSNSxq#*M2=JjrUV4(D3XeH_0cu(flg(ziH%u;*TOAn|pY zyc4Voa|w6fBN~<{X0WHgBOT*CsmJs`rG6!!=@{0t#E%J6WYAUWol#GERF0%82LfQ! zXW?AIXz%9!&QF7!4D>NZU;~Qg{1}Zw3TleTwJpvvlRc8<)y+?ZDnPWN+#!&GF96&w zh3mCCZEq~wQ>3J%y8jUnfRB~~K%@qsxl9BgY(D{%6FqG&?0`Wt|~|MZ6Fl+_I0q?y?U+e*~6V=0-zS3G4WP zjh`o&_*a5TAd=U$j$)9oa=@9CGg>6Mrys--HVqm zuWcw{773`7P;(5Er+0aFxH7*VLqsm2sln~Z3-t=+n;2SIrPMA!7Rdllqm`F0V*wF? z?RVBg(i%%jI@g!+B4qo3pZjc3?-LTpfPtE4o?i8_WvH7)2`cx&m779Xbds=$4esEF zj@;bbVPd(%Qbi=1P3I#V7B{EU{h5~4tdQcxrlRy>_QnOIq~!FgO>7@-QEmC_v330H zNOkeDP<|0mDtseWIO#LJeWp6StfoFVqCZh~z;)_J%b3}xuRm8pP|{j|%~y8UNcj-t zNhfG^bJJOi&VnKxW3=Fz%omYMj)N#h(8_o5g=nuqGmAjmLlEd+KxU%V^g;73y&_=w zm-U$c#_yl&OxN(jxoA;5q*x<*rzZ>`MSsxQWvX%5;Z{7pfpxlAvj+fd2pn$5rGHZH zQE@lD^`z&J2ae@u1Ed6MT%JM^j?lw6Dq7Wj3Ac+yv_=YOL$uOuB23YmA5cfai6ZyV zS3B73VL;Q?D6~yFA_MwT8F1G{yjqv=A|pY{ph1ihfvKROYbZedBECH5m3_i3Pd5q^ zRNgr?*uk!4R`>CGS0Tv6ojM&MH`f9lQnO zBtWk7I#_NEhwMW)o${xd9S%DW<~7$g0YGc)$dvQ!Uoht@3jFJ>@7%!m{cUbNCQF(= zQ@0(-#0bYJ$C<}U_>A~^N{{rwrv|oIgcZMts}fNWM$=Y7VZaC)8}N7c9TO85Cohoz z3@UoKl+}N)3BAi=P&`XS)n8_I%%~JeLQsqK)6g(_#<*y-+5>lS%ppNeRcJGqSfjD^ z!Ws@2dB1tfd_0krQC~jDLYGPLS|>5pShO|^+AVy$myIelQZ|y2wwR6#?q#KG5D$o_ z+zZ2QmEn`(Q$ZKf|CnP82GDY6O4uT~wtH6VO?^q3=UDlS(u@iTFU>clg09VnRE4$z zE;{rln|~vus*-jZhlhuep#;ELe3B9o5<=j1fUblX;Gq8Y0UEcdoKvntiYoI&l|p|v zZ2Wq&K--=xF0UC(pfYmi4iy+C%6x+>vpjw+o+``Mm#qH^547{dq#+yn=~ElRB0!OW zzY#vNsOcJ1KIdG#Io8Q@ts!_ZX-^Mybb*4y^>YZb#`DXKG#P76h{g1^ET)Tp8fx8& zDYLcd{q1jy6HA^fjiU5b%x*CT6%0M7t~IArwl!wXP~OrYAt>8C#E9QhR7@}mP2r?B z<)lBG@>gMn45D6AgZsv=T0vY~*upntjqBjWe8_D?eqVCCE9pE0sir;G{lX z8?-=4m~&+i7$uGh3Hfhez3{iySo@IWn+dm3_%~bvYuxqJI&&SqYsy?`H9NVwJ7#vo z$6N_T!6I|HfSwSh1VaPXVY}HJTy~0Rx!~Sp=G}kjySk&MG z`8sx01kWNp=RFxItD&a;yv%8IrZfN1e)1t+@$dHhFG`sZ5a~-jtYY+zV zBhIE$>*|#uZoD*+6AL|2TGkzO_zvHb!QUE~cq!L=gT<~4Kvw<`~ zwi-Ie4EXj&4ACb)e6yxsepNKQf$o^S8j<9ZR|ES=TvJ&KHbs&>5Dby=Mhp5qTZ5}J z;PnGc3-%%yGt(lj-b|fUEzwwVO0Y9uaGc@L6B6}o{RL!X#p?}-f7$oJCT!BU=hV&g zY0xN^!w?XyFQ*~AkOEnJVbtu%zb}-!Bw7tjqUL)=|;26&mt&dBe|HTl7kxe6iD3a6oGI$n8gt!1D!AgZJLY zOZCL;vA%d?Qxt+ob1ii;sQUU0>#%Vn4_Vy8qx+7|VGj8Q z4Xq( zayPq0AC*3L^(^lU&4D0D>Mtr*C5+o3g--Hzx@cD{z5tak^Ol{6ieb>ybJ$+Pbd9m| zuc+t-772*k5EKf{(k+k&l?r=zrBG_u`aEba>NJs1>+lorLa3iaPwMl>ZYmvi6q z#T7D*ghbTq@)Nf)o~wfzrutu$)-5XQ#(EK-m&5ZziNX4{2m+C~q_CB$rFn@iYl7ns z@PQ?3?&*OYwx(Q9d(tz$H^absL>qT-rebTHNGfDrW_iq%eN9qLS}u9)uIp0`IlX*Ds;P{5hfq>xZGQ+- zay)N$!1e+Z0{OxB2UGfl7nUsp>>l15qz@T6h${lr&wr?aEg<(u6(j6{K(uD|*J6N{ z_f)|9`1p2^I`7+hl50vMLCG^G`A7P@=L9G$AU~=T973(Ufi{VzI3AFFLt9%W2*?~E zkm!v;pm$Uo0o7}X<)>?UdU_AR0$l;A>Er^?EWWkzY>IpLM=WgmhOHF-?Lwa$lyM)r zl3(-j;}y0R3&<4{|9i$!K?tPzl7vmo`>PRL-z^+a=RlFBkKpxvph6CK}-y__CtpWY_*x1LO#XraYn_u6|2^0P&^OUIQmpRG*d~E-N z|MMXN-X|u`uI++J)CQ7Y_7P6nua(`xevd!DyvQvmxOMT;C4s=^sNcUIy?XN|c4Fe; zD>NDw`R81%JE9O?T;;HPOmgxiK;=HH;*dIi{CHcMdTX4 z@oMPYys7)};n}P5&4iXDAi275>5}Y@C)TsCukXaAKc^mkzhCRil9OWRsigR0E?^k} zP3hqfuA&3i65hXW>q=E!SW5p#!uk&SU-9_f+(*T#;XQfs=?2gcbX3&24Ib0$;>-cY zE3Ct9ARdks_;c0$dV4c|TB!WYu=2Yuo6ZrSLy_9T8Uv*o&&mJo|ACOQ)xlMRv%32! zkr72@f<+0AQ$vTtwIlZRLw;lbeW6hY|DWH(w)B_3e`srKb5WlJozx6%?ED~oy*28j zcz?c;^NoKm13e;>(R+S^KmI!g!9DSU-VlY`+P$~>BZ3bat8CuH*3u>`ARe~8J-q^Y z4Sg%C-%UW3$UdHm$z=RAhQadscjbo!ZUg20b_G1WvB4#~(=O>;u2bAPIV=0K(+%7q z?^)&4$($~UCyUk99%Cb+>YADk>Y)4FNi$*b@$zL@@B5;Pj;B8j*XYK%tYgcU2VD6T`!q<%!2Y!AfFiM=aec(C0<`goSMEP)q!Vr8>3r;491fA&NWrbU6*q94`g-RH=V^fu zVP;!`a74G+tT(1{T*|WAlVZ=7SPRIxyj|X9*XnmcSBR`-e;NYmX*&|GeXFIQ&!;t1W*Fnz4ZM1 z^6h~-1OjP!#^dtzcQ_@OQQ`6Rl2L;b+*TS7MPyzH3)L#k4mmWQBl#=5&h#s_N%ZceBPXAtS z|2YuNuHK?4G-OvN0m;eVucD%=YvIF@EavWq_g)}T9Ti^OtTO?+xn+NV3iQ=_I^>h% zFI%1I&ohX1>WJ6%^Q#dAHO%DB$f5tt3xM~ostsrC5D0`gK*`<)Sx-1o*(cAqe6ve+ zsY;3s$S~<(hynBs>*?WMP*O=4X>8`05Pbkla#lrUM7WL^Ue%ECA43aNO79Wm>TU6p zxApa7Y2`S34IqQ3?m{msxLqxN{rWE^lGog=2b1>(rk4!D+Cx4KHqNvH&V(L2>}p=E zk!D}@yvp$&$Z723&O@y)-q@6ar8546c09$sc$I}u8P-i}oHae5Eg&H9r*E<9sU7ux zbL2enwo~abL|+X^)TBADKdkuD+tb%4p7Gm1YKY{@{O>w%16!(oEyZyTx!b`-TG}yh z*xg}9Wvl{uv=h5kPmZ7H`Mysm;(Ikv9UY>2MhmNtj+a?=qXkI+K2`WDh}^Yy-Bg>l zW_tw9*P)5>N=Gr_o*xkSb=>5)duSUr@lGeFq~!J)QPIhw;xd^Lx$`X0=$SK5Og(*; zCuWdzA}nlklE1bCNPs&koI3syY5Eh9K&U$kg3kqO&cJpF*2L3nSN>bXNqT6|?mY%u zxgH<{3^1`E*9ahXH}IN~KYRlN>v@D!Oxb0)lEJNjnV!>DAMZVmJgI2dr9|b)$M;VT zvQ>{6HOjp|069Jh5Ss*0UYJ<}<}#!F$iL%hc?^*&D!nitaNr*h*<@BlIGehj5+{8F zn;g#f#GK65iB>4n1VXpgr;iEAii%>#)Mj&HBOxnbNRnXJ^#Q0%^h-T>EOw2O9qJ2* z7xq)5gj9*b?}Spzc+py1tv0vE7k!^<=Onv+tT{= zNw0BNa?owlrzwgVv0e?5{1;oCW4WH2ftA|`{1B~lw$*{{PdT*;JRs*JBnbPp-I}dD zPs+j~+3KJQ-`TtQcwp3sd3KP$os&=P6x%eh99GI41=wLAJA8mpDLZHQA1{{KvthZI zy#Xzkjf|5GU|`eSwX{6Hyu928q}3g*%D9iWD;USy-eq~~zI^k3&HPo&aS`jQ&QH^{ z0(UcZKn94y0Z@)d9MovaShr@28iVAsIyEeJ+w${Dr=F25RL6w>5fZOix%;mDul)57 z?+uTR6*YOOXHviSEzcE}96$B-K?wbbRe3Py)oT$1I~BP~%)beG?`3?mSJ$Ys6SL}- zVXfkls&-a#o?h4du2Qu{e8Q~+Ccj>Qsc!`RZ>`@Md;cvuASuTeET?78EH7YmX-8^b zKUIAZ+rGXO!_wEhx0j=iZPWR47{2x^K?=O9^Rqjrkee;HB-=V!lQL|iI5fKrdGbH0 zeh7Z5z6SxpBrhu>2hJvt(eCya_Tf<7=l49i`rj-0IdOYO>FyVLb`!7XPRov@>EhER z59lzw1wN%H-`3dQ{P%wghoQfn8i-+Nx8TO7H*cIE0SNRgV&Y+f)4J5|Ej>R^=d&D2 z>@M@qp;|ks*w>)K!fnF&^-L?}rk`EBbk2X=C-N*r{J++^dlv5}q2cJ=-L3BR@Vll} zIfojCo$ALN8YTa}xfv-2QQgz?@5Me|@=o9w8&`&u-sN)e9C+?$Y8-HDJw!gDckex& z|BizznIQJe!&S?I5N>Y#T_Sfd{F3d6&$NbUO}n1<@o=y^{=FCv7;kRYkMMpMmw2Ds zdg8UB?^PaT>Dzd9i7D5)N{ar~%$E86Inp{jtJLj3pE)y%z7xXq=$pv#0hgS*?R10V z#D9%|%|8&EygG&1VhvS7J9K^J3WR+2U*Z3~+0LE!lVY4IO>Wg+`qw5-A0@9VC9K<4 zWN^IY_;;ehUG*PTm1}USTvi)JNX#@*8bzJB{blR4M0F1QC%nHTz+$f{a?B*@Jt{C4 za6EVBQ=&<7X`|`NKiJN;JyC-HeLc5^AKMlxdH2B2e$H|{Czfjxm3{0;@~P79`mS7c z&z3iLt;|2q?1^G`HSxi}Te#0%zIL?hpi9g%~-jT(K}6q^^ApADc@ZG&$Gf@@NeRSL4cYJ0s(O0l5J7b_}T5*mgA_ zh&4?g>~a>1)4 z)YMt;%`ls5)0GtHer6LE|HTa#FE+->D6xR-T!%nU-;C(qblc_Wc^(5&2()DyWb;Hh zc#JJjz-v$W1O(Ipyu1ULSRS@K*|!23XzQ`Tw*w$RZGw)%UTVe6Og)J7daG!MrNtSSRZHHz^~BwtpucpZ16!8px&%8bMlKKta8- zu{o{T@p^kwDy+~eu#~qHVD_HrN}fP2KR7=9`cm1W#Qk^lT|O{KG*AS}0BytUKw<3H z)>b_lv2^m+8!5kJfzeNnI?A*X;Gt9$!8e*(oO!tX>zmAjulwPHMRCh&b{Jpj_J@E< zv0y?E38|coR$u?Hwm`N3G>e$Dv~*vYooYRB8$KMCna*C)b126C6Kn53pG~z4kUel8 z4(HIGMT1Uy z@Vw~HE+7-10D$|IG6(-Lq(_1k@Vwa~2hA#gEiYTRqL0R|*`0Q_H&=j_hiyB2=wKg# zR}sk4sObwY$p*>34wVL=b3L{>lPLoo7=+Xk-x5imTzb!B^XQs&BuFVfS%sdpz(lT_ z;AsHGR@^FVnZ>adJl6qMK5lDW<9-3uh4MS)Bq0$SPewdf0s!_UfCl^%&Y@}-7&zVm zOM)l_8@3yos3QM+?hQM)g_F0s5e>IKBvTyZ54?H#e&BxXEuu}>c8%lBWsXQJvEGHlWQtrGplH_vFbQ(Gw*CSf~pYbUvx^q)Cmf8Br!+{AmG!z|dlwMg?`T zbo-l`n-OGaBT;t`U`0Zy>%b(Y2kZRs(_53!MJ#dT)o=iU^$GwVQ|T_ass)kVjTc8O zvA~f!8Ed5RiDJaeXZg-oI*JG}9fVM6fb%)g;!J2H-ovfiNm(Cq0wjqHP$RPgi$(OG zl*mg!ZAOg@hWWks?y5d2ysEPjQ&>@_wH~smohoun>k)YS@iu<;_ngdWXP;x|`U^#d zLspcmyM86PmAwPrCWBBG-9mY1gE|S!hYEVCYjcAgZSIT2(q!^wwtUa%?%lhQ;bn$* z3S#-qbEAtv!|VJ$>XDA@2l*fF@r)i!Kd?*vsav$CmSWE7N4itj*49*ThP(=zK#Z$h zQ+WZ{PS4{qTkLrERY?)f5z*XPXeBrrA*={6|vw$8~zT!Mj(4oIxwt*BG^R!=@lLy9`i6FNbc zdEMns17_tUY^quy#7_=XcmNSmC`*nuv4A!=0gh{Q97e6YBmhYSVf-(T|SLAIbPgvcE} zD@V1I4fJ0ez2MnIk3TX$kkkSDZ6TWixG4DScvC~i(+H^LAwIqYk+ALFy1I@~qIEA` z$Ug-?O?t&rqwCYulUsgl^;};3`t?R_Z8f~Vb;0Md9mW)(I#D!E?n82WmKPNH#j&Zn#YfsEvi8+flE4;=cNw5|jn@u|}f?h>48<$xaB)1&pa_tgU1 zhA8?7Yyr&7j-oU_rj@Wa*s zXBnhb2d)ZEPEHywfC0f_K1hC(`}Z?9UK3v{7*pAxjn*WV& z`i&HPmqcvR;%~J~nbAYtA?!;1@Wca0$0+O0bkusH@fRz&#MHin|J0>M-l1vH&OSkt zkSkY9{}zh8c=2K+&RY0J^hmG|Fx>yvcNiV~@#hU_S$vqx+wmbwY83XTg#0>Ho2%yi z{RtP8SM2oD$q;$tbZcfbkK@snty^;nS{U`13oW&jd79cq>0K5Vg`+UIrmb3BE5g9QsOvvGvuPf|m zn)ups2bMGbMrzf&*oPhZB$etV<|3|d=e77*%%;fXKO9dbWilQg{$;6I@2X|xrsV)k z@9Q#xXnF?0&!K->4nFlslkMwY?{wrXQS=p&FGx>I^RNX56Ti}>KR|=|C2x~mWyz*c zdb&`x+N#W}+a9w6uuD?0jr*iF=anIQCBa)gJcL0Ov+aTMlFl&AVgTh~M$3-zyGl5v z<~#xheR~59OOfy2zpvA{`BX^9R^#5kKMt&ZI}dEx5?xop{G!+sTUpT-cF4G82B7&8 z*q#l;=s&F{huM=a2E&n#Z_Woo_KLA@ZV7UU?`+?<^V!m% zHq0J~&=ue5&zJ^4)rH;7?y$%jfH2(J-C>Qz7hj2}2CU(->q^U(64>tN*qwI@-Qu6M z_ZLheRfsBuS=dF;TYdSE(wlQkHWU;C+Nm@)NjQrIMNAw3HJ~I)$f`kt?x7|xuw~kS zx>)qwxpOiuJ*Tp(Uafz#)r1}o1Db_N%I=PTlfxf$awBHP&?kXop%_#R#Ot0hAKT97 z*Q|UAc3M(g+E@dl5N40!CFr)=+w~F^J=3In@8uh&Mjky0^&P||EvNzb$I+^32XvjP zaq*871u|C{6A_>U`_s3RmVo~y`^GKsw-DHVQ0*R~1zKh9yj7LUyX88zFLBOB(T#XJ z3HbLx57}2_c1any5~I9&y-MbupWIP8x(g~h0n|VRbjy<8A>@-qcX@YatcKshh}j`Q zrF#Rv6w)2Ubq07=xrGCoKL{RWuf9FH!;b|E&u_~<)_JCb*S%eg) z0QlIFusWc1@7~dCr}py*$lOry7zMN@5SiFm0Y&SP&ViUC&a6M-fJYW{bh+WY!HWCYy1IA zta?gqY89R=ZZ*QL2>%&a{kFC{nR@Q>na|VRaVp+f^Q zu78O7a{ITRqoc?`7R>=_UZEJ2vi_la+N%8!dPN)CiRrH})MN2WMv) z))DnZ8|ynA;lz=?LQ%20s-^EKrzba=EbwPJN;{A$5DJv6RbB;)s{7AtLVFvkK-L5^D=w=qj(vsju*L zS)ZsspDVgJ`V)lTApQdf8J^e&4?`-dpaF3cT}a7>cdARux3I4Yy1{XUjxXgPz+#H@QR$uk^Hum<4;$^pw8j#-r`u;OHkf_M6 zzaKgsI21QPzx`^-w?bdlsz=M^G$e|v@1J zPwa|{`#VN;X_IRKeEM_|2kw8-_1^JR_J92NF%q|wXdxmiyNqNXvX4zNqmaEt_Ng=! z*_2&KvU2P_Dj8*EWD_DXB71#b*Ew~6?%((K`}N;_KMt;QUGMAte!ZUS_4vsXp}p9c z7}y6W(x`5$;OPw5M?&xylEMu;efbLxW{+b`$60e9j!bsd_;zONX#1ttZHx~Q-Jbar-j71`>FLC1J@#U z2zjn>+4ocCBxA{BDWBKzN8pCwCcBoZ%QGD%H4h5($;Bb&gMmiOpM9uGbIee)VJMIA zx#7xE%bPdFixG3S+s}fJEs$En4uyUsL>fGb2{Cm1(v=RHL9p-(0DYwR>LLoEm3sYp zaSZxrB#{M89gm<)j~gy)IF5I^(nNK2;-uL?$2fphW^Nj1757bi6yM9X&Jf_A*H?TzT>XM$FbFHoe^A{{1j z-q?=TR3o(Rvl(ab!bT=1FO*J`;xD?Mvtw2!L72qYUiND7^jvBB&kiQc7HR6TiZVyuqI>Y`7IEf_^TeM&N)Lum;_ySkzHN?(gSarJqe7v8gYAS}nOE}rHGBW{6 z?O+W@&^+ax1`65@z+5C6VG0h5XsBJgH;W1Pt5NqZYmMnu3ewy*dI5=%-Adh1J7dG( zpVxlBDXJcm#%zRoXlRK3AP(6CA=8C4Ypmj{B5^7mX{dv*-0VNvAh(@nT3p3vWh=>L8tB8f*AC|;}) zK-9XMg$Ep{lCj~!t88dsr_>|jzu%K9h#6;0&wWlu6XX&SH4K0ZJDz%F>56_5;5H3_P$Cs zQJnFBD5_WqH6RONAl*G7bV4Du)=9Tw2i3lVzrg@eGW0_|7{O>Nf?}klW!>FgKWWA# zARx*gJ&wQks8=r?L|0S3RGwOmBT-OOJ!l&+yNTQ1Hq@9eFHDJr*ZAhAo&(IcN!v!) zn>Yn_;5UZM!J*?`8M1I1>Kr>0voe<#?a30L$93H1$*;OWnhK!vT;VkSXh7BTg||9i zQe_Zp&X{kCJ;NTjTN?ZOAbL20Xr5%QFJDyE;WhAHz9wx|>h`MkSz(~x>)S2|Aj=7% z!4knCi~!eM=gx%#;AI535FgSVsU^;c`|{$yfKB2*g?Cv|Nw-a8RAz>ZpE_*S0t7VB3jy`A1NK=b@S0LUhgoYVG8z5rj2jsTwKrIp{ zu~2pyw$`~_0%61KfpK|XdZY?fl&q3`(GKU}v!#={w#y?O@vSPCX0Rg4%AZ6_0{#ZG zjLc|&l(h87gxk|nv5`z^#V{sE6TkXFde!)PD%29w>IM-CEAz{gWCYjm;AXi^AGdb* ztG*5mwszG~X*KQkEzD3?JSPQw45DfTO(SM5rj+O;%c^)lhZuy>|UkG%t*1yaamkM zr1*{xDeA@jU4Zt-D#>y@&QG6=E7<_dJXf88LXWSl)-=HA{1tA@lb?8bfn1jtM=kh`4_^sBjEFOkGIvh zhly;?%yeZu33|AHA=xU znsQXs!?Gek6YW$2;jfFN`ddPH4i4V4O@IO{HM|`ywuj0^s>ac4_Bc9}7Sc~coIW)r!r8qOdSH)(~ zgU{2Fixv$~cnCBJ&jXks5;mncxOco*0aF5EBe z&-)E^7OlXATyfG5)S_bEZ{KHlx!Sg~P|}m%zJ4+ESwkN55I6|>qF0h9X#?GXmmO@o z35mCQ4I9l0g&&foK6&yabblSG+trXg@0^k(FYl*~Rt^$Waa2NreG0Oh)%9)U5jmgU z`Qg4fAMrxN5!j4K+GHa-VUlO-n>@JH7>2}whO=oIA9)7ZE9gdx@1R3}!OJ-@1`3_* zDcG&!;QeIa5rShMF(V)uE>_S4$THx;4}g!a?28$+I6pJGX~F^+P11nFR7T|bW^3#! z1^P3P)O}zy!d9^G0B^lPJ1R03`dPq))%S#^yPbXD2imTipG;*%u!hc~JeAU0nYzZ7 zZahCW?P7zJJ{t83Fh2_dERS&0_N~!gR28xsjGl$SxQG|WoEc&DjjOT-eS=;Yq8LK{ zv8c^eov_NM+@d+nU7=uX8!%vI3cQD!RU0=mTI`G<@o7s-OG0~rA}#Oo@7K}3{WXhE zlH=M9qg&zT(HEi1Fka5nH6CB#yF9p0 z6ZV5O_+!%du4D<{H&9t=N&V4v3=&V_FNpc;oXkNpAVr`MXlRXK1G(AO_x{F;l zuk_i<+GjW6j+H?Hi~l6`ODMWA3gg!^OO~G94z}%)y=!VIr)ztwH-;Zjws-p4PVh4S z=i`rYLu%+}BUc(4F0--$S#(#~;F+%Rj}hMf+8x@~Ez&we%b0x-Z%r4NX%PQpZp2uO%IG z*r|`)L%ArD2}`zo`sA?{Xb_`I*Z|Pfv18-~C=a*wLbjTB6gp+QS2NcV#dN2hH7_E! zHjtTwz+Ewu_B&i$XTLP?P#?TR-*uEKZptq1?*-~qIA^l^6EAW8?GadBdzh8z9h;IU zzW?z(My+*?#?C0&HA|kO)XlE&N;GC+=OphOEmr2QqRWOCA@)=aZqMAm*N|qBvD(34 zI^ZSXr@*cH58n^?V9*HDA1c#{llf5bGQ(4&Bjwugx+M*?P_DI(zZ0XzzO;C!Ek*I3zKt!nen zi@7Hkq|2Z!l>YaG3{Iut2^uu@xuJrR&Y02j_Z8%oXCG1Ec0*M(=6!{%RV!PWqTtww zU8S-Kk=uVucm>d`x|8qB68%y1B?XYq_wTFB0z5ju9p`_Jy|UD}i!-(0FNsV&%YKR87G+d?X+EE( zN4TfF0H@pU!{3}z$Qfnh$|HR2lu<|*SJz71xo)t6sRR<}2xKw=Q7*nLw zb0Jvy9R@(vh6xG@na*dr)-_VQF@3N~qj-Uidu<1%d-#YYl=HtPo z%*N_tm;O-T7kC#3QQo2+egD0c+`qp{KY$kcB(1D<>>dMmP zq!ageZ34`b8^URg?yZSVlmME^y5t8oq9zc}Sb=UG2iZ^vm6Z;ajAGp%jz93+C(k78 z`8OO9Ejff^Uxk8E?u3?zDog;G|uLl z#YCZG%F;cy&AFP0NhcECxs%j(yp(7e2q{%u5(Aj(m1YXR|JHB377juhr=ZJ=0gBFc zpeV}^xO+jDG4%E>u<)#q=YueZ-XL?I%*ss0&9Vnog@hc+uEeh`RFzf*C+5EoyRrRV z`h*5>(OatZjX-3G6=u?4$>d&;e!P2tGUVrXz+gMc&vSBcw4}=33rSsuipUB`;pL9Y zN5l@K#Kevw?X|1rBD7j>-)s~9$Q{#wG}EiBYsBq3O6s1r5W${HrAFX*Nh;pH)dI5`n_!n1S>Ar1XH0zj>{<=LNs5gg z+S=OpeiP(L%;AKRx!8iZAwEb+QDyFWDL{mq%T!zfl0OSqNrZ349!QOfiVCIgpnv4) z0jUPt?}L&B6W13<1wf12XhHan!k+gRh&$#f(7tJ_VtuN&)v9!j6|`%98ty!3?>*A- z%VNx0O8yx738xpntVm(AxYP9 zEm+NtKUc7^913|~eTU=6YYY_7X*Uh2W&kZ}0%0K!())vWPdWcuwI$$I%aB0EP;c#+ zB7W-)2*Bi!H^#S_ib_=m0*f03AuaIV2SfD2Lo-(I#2HsR$}sh4D6e?ubl<8lUXWdu z4s6cKWpmCxy)iBX?eD49tph=K&nGZ(6r#Z$ng|$1qh-+3s6ksVj~fk0Yv1HDQtqDF zIRb!6f2CKxYvncIO)cTALWE;H6{F{#&o#ePCqv!UcXDznc1Iw+B~y}BSMwgp{-tN` zk=Gh7_)i9L#tOYHUK&h$hwOq$1CbTUT*FMe@vid96!v4MQmoi70WCHKYbzYdm15HC zZ3V!G6I$aukeM`RrUclLCKvld)TNH0p{YmN&BZP4^6J5Am^&p_JvohvS=!Q zTk)yGUO}g`1zIb1Oba2qkbW8>sUWXV2_Y$1#~hH?H-ZJ2ac4xy!6qqbg5(vSG;z7} z{-eanhlEPTCIt3WP(ESyEf)B0=QpcZV@xXZ_$L5;Nk&HlBOCjDt5o3U9%TbBQ**CJ+{CpjEw3n&Q4Z^p)q z5q<)_`<0s~y4B}ih^iP^M(Y}H zj#uNnnkx@+QF9a7BsshVygerK>$;`DKQB)A z;`7by=`@m1O*C0*Or@x>vm5^EvGIbb&#`Z7JOmrwJjV4ljrTn1-@$58{BaSa`Y1^B zmh;u$*;0|Lh-16;WAP#-J$hd23o#QM<>$rmzk~Fgu~o(~t4gv|uKIBXXd*CsTh2)X zhx6ij6e@gpd5(mzHi)5>?l7RC}HGXyP$@_E<58L*n zl@RDs?jEP>SULysdq4I+rQ%y(G&XKjd9h{D>kKF>!r^>rS^H5cQM|ZmS5Z{?TRT|I z;d^!_u^jzh^j`LL(>!TVci>uv{=K}|rLXdtB`uM1{ktFLr2UzgqyOzgbPV(U zz`TB}W^7XMPiKWA7n)CZ@WR6MhX--VUe+D1+em0X9r#3I!-jIyl@fyO(J=+D;aY_ zSc~=VXJlWU^G(7SoA`1t|2|3ev$FN>$oeiXSAW8M>GT1)f15LA-LiN7WWqa{Q*?34 zdQr!w8(aQqPS}@xcUbSZX*du2pvLc!LN*|PA@xfll=W~VoP4@)ZQl>Hm8k+tX zaG;A9??O^>3@FEpdpQZ~32mO8lTP|jLFd5`vxJPF33S?PC6Dit;MM`IT@7c*nLR<> z(Ew(G&$AlLm!8qmy=_LJ=O@2^rw`QC)un?KHRkTuXXE$S%Q-5=c?M`6xILLw{*P41 z8S1`I=SNDJ)%@r_0lGY~AJ1>moX{vBBCUB#_Vm}UJi?tKUtVS~+V4NaU0;g;@U}c7 z<+6wOcEm0pDf=s-WPWt4JOTSwYBhyHRW37%t$f>%f4|EN=aEI$mE&B`|AdX87zd zSpM{L$*yr7{iU8b;E?eF=gR_Oc(f(jdAOpucfG&bw^VX#CQZ<~?eyJt{8guR{?ZFb ztFeBbws!U#4g1tjd$#F9F2Ge+QsRnghgin7I}h=gTcIkzM1M`d7-YJ-uhcH^!#S(C zvD89AGR69;pMz0$6Ab`ev+lgUrsM(Mb&5q(c)rL{mAjM3q;PtZ@_ zdEZ5IS|kf2K&0R z1ze07-lHpcVdvCOf94L74#e*rF(rAufwURo^kHGPfZd*jLDVT4iC=GcM^Z9Jct%vu zh}2O7sz7&^7cur%=`06sw?W%^hhywz8Rbd-Xb{viBXB&V*T+IWLit=?r6pwMxq-c3 zRZT6V$CU6zZCpU3%GGiW|B~CYGX=)8o%VkW(BjW50FKFpV|%c9c5w2$wM(rxfuhYS z@g?=qqbyKeWN7=OfighfD_{FRpx)Si%%3MYar9^kZzo?D9V_Jz(r^Tjd6m^BlCqw^ zVE}K0P>TmD^!XHB2O{(U8%90?7j6Qt9Xhu1?1DuIeLJYTK)X%&vYJTaPfw&}knLSd z$@a!Oh7>C7=7jA;z-oveSkZNWEet6Wn?ZHPVr6BeIEW=82@!9HGPdB2u}e6(wuG7d z2&b2M(o06@4qoq{KGl%+lKs}qXWYc_b|L@?0j6C70a~z=^Ug7J*a)RcAVJNS?3Xsv zqckFsX!R*ka%n=VS;p-VMFdZg=kjr8od+fY*E9|% zRiD1(f_AF-@d^?-N3L>m|GDeY0FT@%Oc!uD9@8ynM0JX%@?`aFcd`&d-u>o)Q^EXe z$2miBiWj}U`i6#vw<=`VS@5?4Z2`Md^vj>C{PU!#B$(NzV1A;R$f3_}<~=61Bi2QL z!P8rzwje&^?QGVc(^|DTd0xj*X9hMPRKMj6A2V!NmJ8;MxpuhSqQd3xgu{ z&kq#rDEeG&JA4HFhfL4kRQ(?+i&Ye*AY2QoO81$IQ zZ6_~tt^DOa8y%@O1VJ@mL5JsKt4C|vmtJW9+wWVhrig5d0iGMVS0{0S?@I=V$14zg zoBLuy=ymzDe>Yt0@tfKJ!DfUePIl>+&7Tyj;`jpZoj&tlh|iLiy0J8K&;7iz1MJ6i;BZbN-v_Y-z~DPj!m4zW|It}4}u9u*)e3+P+_@R8lfv-A0b`dnS zAfdYqb3b-RM0e^FhnUe9=}_eK|1bZfoU!Y|iOtPdfRUa37Bv-JdRUO+I(7eQvu>ow z3P0eKUrB##8kNX%2p8v=*(5g5Uurygp04ATcPFVFez~A6cx$hdg;Sr_X{%`@za;Sn za8ID3uN$UdAJAwK%?w=f+juLlq+~q&2U@P>8Y*bZ;HzPT`;gG|{$|%~&MyS2NjYwO zz6p5Vo0 zmk>Yfq0^V%4n#75yE3}izK=dLKGA=zpQ=L_zlgAP-1T?HX#2h!*!r|6iAbbgVDpsz07n7~9k#8h$z^N*@)0Sv;NW04KnZ-SG(oZrr9TGRhtO>$d3j;9CgEGr zUu#}))G;(ds)H0>?ay2)^f?Ie?K)qH(e9;>$*}eG* zy2IJLIuiE>!>E2s6tfM!>R5G$tHdtSFZJFzbF=k9;8RaKjAUMl$Nu#Cp&QhLyB~!6 zm%oWuJIS0!Ske1*iZkTl`0ry;_l4SO{$05f|!^%L?l&Y z`y~WzhnU-5WiJq0<1N(+VrOItJ9>SF<+^?rFQ1#;lSj^HwbgV^3U(=~c^G?6bqG9x zHh7(T)~dh|mfQuxYJL6i z&r|dPh7GWhjR`-lmc7rgTd^==_Yi8i4o8;Cc!jUf6*OfGM6+PdY6T#yr>^>t3BW;N z5K!F=at^CoR|)spjDz`SW7&geH#0*bpYpO| zwWE>JL^UV4h|Wq&UxQ6AX+St_9m+5O3cejP=ou^IuZ7(9l(p-4X~o8p_^8=-R73;| zFnmIL`TpHC`6lIEWpG^*5ae^iwm0W_tN*QXB*7+oy7QwHLM@oV7wX4h1$8R19#|?gxix#zKSX`P}ck^N4%o#jT4J zVIem*j&>7spmOP|>9Hekuiw+g$=P1~1w%#70W|a0Z>`^&hJnF8y5QOQ@i1N5qEGh) zl3W|J2_t{QluhStOt8#~yvrHJoMe)%KY1aPyrW z<{iX$hge~cqDG5sqdIHAD&ebTJP6_+wxdUnUXvpDy2(F5e{}ZP3$9xmxslnoQW(F> z@!-jZZ7rj<+OJ(TL@cyeCNa14s_li!Wss%zF=4?GoG^9&Dy*v?YvDizv3_~=JoXJ@3W`{$p3 zLblp{vStyB=H9~Bf_JCz?hzFA#$#MxdC`YbYvKm26A3=i!gKCUkx7G%~FTJfx5(Ja6~`iQCq0J4-z?c^^_3lU)5_c9&CJCw3F@ zCc}Ip#GJS_>8%U)fn}yEh>7ZsFV*c7$Xe@qsX&<`c?SQKmui7M6-@n-wSRgRN-sB1 za5{>8h zYvf$PiwdvB1y`|^i@996M5~BuWX`*oZ0*2FlQGarb*c)z5t$(z?{{lS56T(q^a-)m z=pNx3+KSTkZNqF;`hOlY(iu@}AxF9JKZgo= zOu(;?oUf$iAADnSje1b^@un2h^0)S?E_PJzt<=K4+~}t7(N`5*{(&Hk7l;;wi>IIV zDDzFDvFYKIX z4%iRKEMdir#!Te-$w9+u$WAV9iQY!*=tO=A3QE2-^8Gg9DWEj}6oeLCK1-`KCyq~a z>$^^-{%lUPg(Gi^8MGIU14RX9qkwUKoI*q-c-{rTKG@3%j^%8N#9;FHGzP9ZS%dXQ(@{I^LK1P(t=Jymki;(^@9llIKNFdB>L+Mr0#rdO|438PY(#KeaUb0XR_oK$iQ1eae0h7oZk< z%_o?ejf}7RIb7KnkhQZvATE9<)xsA}ZwzWHh*smgU=m!epJCNT7-P_g*}ZnK+vNV* zzy9t)bE<5f(p&uXqbKys-l^|S|IVXf){Gh#w#{ZuTWdD*+KSYbNo*xDPnnLmW!E0CtK@_zx_<85u zk-rS^bMnufr4-yml5f(0PFuPD(~4hL$Z{iElJ@U9_|BhzAePsser(Pj4z}xR!3D6C z-4}5QX--_MA_LLzE9x1&yF_hCIQ4q_YuGE*QqRkg+?l_e4N{ zdCCXBoaoUn61LjG9(~j#h{X|1&8VVS`)0FJ`N+zo=5s8mhM*MzU7oaTiUiOF75-xQ zpTcVC7g;1>7^XzcRVCN-R4%;=YsF4}dfvj!GG-V{WC~^%uz*AIf;8C(1Qbp3B(bTfy)Pyib$1662L=tKYxxL20<1<%R!3sJ*K=-|VldNtkd&HsP4pH?t1O zBo;|ei}!EVKFq)sHI3+NM+^AY`02u-V_8HU4qPcjfHXnzGh1~?B$&`OzK$JbUbyq- zY*aj^wXm^yq20{Wu03fx1f19k(50UeCOzFs+yVI{g@17jVnyu}wA2m0wji`8O$Gg0 z%bl1)z|Y+2RPmohQUkAOQVAe%zCQj+Hw?Ac5U;JLDVKL;Qt~Lb z2y{^3rj{4MSm}*NXlOl0@bc{%=h3JxlJbz##Hw1PsSv2fjU-%qK=cRVnSw&`1<1k$ z83Px_t>bWf zYK>t0iS%N(>bP=^O_>}?*r8Yc#8nsOsh%DS!hqL1~NeAf71qI^=i)>jTxGfUGR4^tciAcwUa~YTG zuko*xgyL7`P!Covs%kzon;sokDp&VUytV~Srk<@ zLix-Et1t>Cc%WgVT<+|N|A^W)R9r_+6kN&bVTkyu_A)+0@cWX{jcHxB9~)oT_e&QO z5yyTE2j}4%Y@N4-QVVgokXmFMio2ZTtm!6Iq;N;WT7^st$l1-kPRNyjV59BOea9p(T> zazq9uuBGRynPc}I*bJXYLJqqk^MD7O&zNmw9yHX3l=zxtGXdiZ@dMr3mU6Tzbhk4A zv96j1F>G7e%U{sbuFemF60)tiIRYRgmD+&a_csgsofknI5pwE+9GHf$TVe=XD{0!d zD!v;$Jg#<6lpQ#<)s2DwA*}_F$%959v!tuq5Er=l5s{j_Mip^EbBSb7ZUSdJ1@w3n z(se_0As=MZBF$|%bOIs!Ve}#`vw45sgq)oRZ7~PBuQ(W4+j>5>ui~{{(<3_`OVNZX zN`sh2eI#5RaV0?M&MD|NsT}<1(fPZb;@&LyFN;1$n#xs6WTwmrg6>@Nck4QDT@R2P z!jSr*2GV7xLCMex4q*;Jh!JL9qUYfsLE?$~z;zaVD4w%ZF{3t$th9!*iWa1q-@^}3 z*^>fqpba1rGLH)fhD>~v8ggd~X-=d4ZRt*W8Fp;frm(a77Sfpumo+^IR3&}#KV1j9 z3Bs6_JKIK02Ds{p%xt|I*>Mx|zF=T*dN`kFvJ9%5d4YXSlF-JZ*uC!w&fthf5@0p| zNb^!zj!iqB0J{N*Pj9#f+YQ7YN2QR5J&lQG5p8MPmzwNOE?qiq>pL#Pc9tVKLUHC` zsz_x?jZo$6E({p<@8!JI3c5r(XZ-!ql2Krqm077|sOk2Z3B}E&1wPKL>W1^j1-*8& zhCTP8NxONPYd5x}-(j@|{}U)uU*rv`>wB3{0&;tkSx-SZV=H!&rycFsJy0$WmDp=- zfKlG>VAALzBES;NnJji-|W$Rmje~5sC}2P@=`DS_eKW7Z^m4TBRad3n0j4`^KT%`XdgE$K9S=Jq|NE?O#9Mi? zrlsTE;$I$EIGmsqov*mCx0L;;j7^2kBNX1Zo#_l*PITu(v%l!x+cMIFW6rrsL)3=ge|}?{gU&OuPv&;y3;0LfcI@YMRgWJW`M!I z3_V%GRn>auQH)O97eh+&Ts)>bTYX6AJ^ndj%>`;@{Ya?Tku7?B=9-)33O8QHN`Yqm z{kPs2&-nC7Epvb64!)DZI&(6o?~u_)HhMxq6RX~Cvd~=rd?f2Nm}daA65%)DH1ymw|aW7B3KUPdUz$=JA`P)w9!K_f?4l$21fQE^Ff z8|ge8{@GOnl0E>CQeucwATSXBi7{*Zor7IZg8h>D{pk8WjDbfE$biOK`d4}m>63$* z@nI>F=mvox$V&}KS_Cu>s<fE}X#s4s0L3@~G<0;saHOyok#6W1V9!BaKzFiLN#+ zDC8sIApVNLlBhH^G#e_Alcuh$tV}_k%Edfn0*s2|M6?Yy`zKC%m_}}+MT64 zOqK$NJ|B?oqIcVYNB@Czk#tf%MeMtTjqG)910zVEx&t5F`Znkl1!dL*UVp&8D?5YZ zMIpAHiABIeX4mj-1*{o+lQS?6*$+aup_Hg?nrw~=W&RBld65JTn~p#1UAF?rN<1Wq zYz%k_ddAuSE~S#a8!Ca+@gRjmV$d&rww&pB;p7|$d}+ua67YsRVX!o5B9nZdgAna# zcrWvB$ZY#M_V2|4u{abvhRW&wo44bcdw`16A0-Z=A29J{*kA2Sg2({-P@2rEn*Qo1TuqVBf#`v(&uD?xIg%L;UxFYh5q@H^#>)-<#45 z^w8vdAVTBe=AHz&)XDkFR@)mu6%?r}xIZU40Sr^6vET3$=lEAZ;v-QXux;& zqO4JhdlJTX2HrR2YbGNzcp_{d>#&5)dYq;qqhb(J#29qD%8~u%U?Qez;j^*u@!i7`li!6Sv?3 zA{of;UhdY_Y=)?1Bv8f;iRYbR=l<_J9&YstT2?vgo9Jxt@#%pe)z(rn;p8nuJqlqii)+$GFD>A$ zZd`?gQBj*f0^?$rcFC*oYYPTgyC@?>`$9%T#0iaITnUMc-%W>)=ngA%HW-eZkoo{P zc3hu-Z}35wS3UMi8YGH}V@CT-x4N~dz795W5~N9O_*2XND^qM&@f zYYf^h80^P~j6)rAYvpqTh)H5tUtgaCJgkw4iKkn)2>hs43KT7=kJTJFz|6SVm1lJc zihIzW)$=ifZxyLXk!S^^x@z3Nr6ZRcEbKTSoQe7kIXi3!Zvx18k~l$`#LV2!-I8H5 zO%}Iy8$$G4z|gG>t2=1`;Dji+t9ISNWOoHgDyphX9vnI2M4B=&F=6%o(NVy^=WvN5 zuViOIl;E36kAGK~J0KY81Q6MX#|9H@7k>kMAkOcLr8GcNGmdOx+$4|208WlYAm7lC z?^FyI@y`n4egqs4TN6XzbbHEZT{WRWeXX<^`aAE?nY(fW5E}J2(gWa z?KA-fuGou?1A3JX5IKkl3VUiB6|vh!LSPorheBVT=83;7EdYn1R(Y~_}Z&Jy*6HG(!cgyP?m-I7|{$; z^22ftMSG|t2?@M$v9Z0({)D>dHI=bK`LX$-F+_Jp8nyK75)l3HP^#L^Vp}55Vzz?D zXn{)sN)K*PTaxhv&WO_}31Vmh7XSDf90BO0LGFFR%}7NN_3P)4>fw*IvuTL~Ei3EI zBhDiQu<`K|0X+a&vtU{%hXhABRZ)>K9uzlkWdDTPj)lsBTVz;mhK#fV zANTP42~tn1iQMo;s}sL^w2Y*sg8ucqV5+9r)Uk5-4H;!+U&|apDBZ|*e`_5YdC>v> znvENX(u!{Za*L78A^Rm`9+*(RR0`^mtAcj41w1Nl&n;Iv1k9AbM74t-g#0jEr9L7H zI(mB#(ArQn$TLic@z%2f(pwM~bL$oXSkPjmYP)j0FBzheGvBBAsvr=2;4lryw#0Oy zxVa684g|L%N+&DuA`ANbwz2JedH<+KF9x`%LuaH-q&FvNH=$``zj(2Cc7ag!mVp^1 zC>}ss5G(9mYM-}EK-adnP=G)n9;JT##?@*2R~hQERW*wV_%aQIIIWWII)$%{8YyRo0;2VF5PUCT4@ zR|Om-W0Ym6-w00yCwQ{3ZI?XKs3EoGmwDvHN3uno%ErmK`1p!*{+w?k8t7A_PbtWc z?wp;7%IK-f28MH5iZkX4Zto8E5V{u9sStL};$=M?XWOJfPm#nim_-wVWIZ8iq=sgM z-Fb*>0FgN4nO-HMWMR4WKKJbP>l%(3+c>LzTo~+c#Kyq9^yQlgAgFb8OiXhS1udel z?g9X?v=sp8UUD@Ud!9)(qDfdMvylD!g7@O8jcxC?0AAq-z82CNcm?;TW%e(q+95Mz z1aHmiA)K(hynLCdCq#-gjzT+q(dEyXEBQr`^h4#vvLaphU2F(`Spt!8DtiyC5Y&}V zhxk48J6sUJ;0D!GUzr1(8f?3P+e<_Fj*C@2u$d(X0~tE-Nl-sE}!;TGL~vdmy} zNEr5G$Z=fr$H;_7R!ac5Kzy|h96FHY3|esQK||4Rvo>R!0P;dE!WXhRGYdijcYH75 z)Co21d|J%!2)o)jyUzWy%|3JdToTXLtu3tXPqz796A$rkXvt%#BUK~wK#nR3a#5!t zg9#jX)T1Cw_QyaBohANq&|~L5xq-X2cTw;i2!3Y-F2exDXDnSE6D5#2be#zzvTreFO1F?n3E`x~HRDk}*^oPRe$EzB!bsz?EytW<|XX_T>YmU9fRjEDp za&)0Prs~41_b2h;dskl6(E&UiM63x53Ra>c=`$FtGzpUtWA$^5==$5u`5>3Kn4X;K;|wGSXAr_Hm|6!Ev<=)tb*k+@*=h$$XP5io<5uD(yyBH65IGf z44fBo-8LeCPXK-DoDL_mj*$^Jl5Jx66lQ>9AeX?k8bak71@Dnj;VS+ZKs@hDn6x^- z2Qie8oUz^6-j-PeO`x#olCl9)`z8Xq4A={gspJlfc}on1*H`soag2nU+rj;7C_MfPUvJ)RHM=g9#Q?VorGUDG9Qd52ApT7mgnCZf`B=bG~Iq`Q&j# za{fZ(P@=G{)sC^`lLb`j!DNJ0Cs5L`S>l-2&0y{$0J;7%;?=K7j2g3`T%Hb67$K z)>QDottPr`FYTMKX%aKyy!w6MKpvR#7iS^73)TdE9ApaGB$;OZ4OlSjoKOq2b~4IydogbN{Nsw*l)={f%T9Z5v)7VxdoX7JayDr+WnsWNVqv z&%Hwp(n%j7a#`S;*f=;8ovK&$kpmtLFWar%GTy4x`o;BEBg&R=6VTN!9 zM4SlP_nu*x{QcugC>Cd!i9qbj(704q=a=JiQmI>zrQC&U@sWBkBIB!h$KyxGCu;;< z8nHsdmK>`6&2^XUg}bvD?#IDDr?#jL^kzarLhJz2?(HK{phQ;4i{Z3CPGeXlXuMZ> zyHJy9QTb@bXCNIjTb2d6+BSy~w$>&@xF+uZJ3!$wt3W+SHsRrf_Ke(9z}ZY`7`~On z#O4q49ghi2JpP@M!j)(x<_HQKt`}MP_h8Ot<1WvDgE^d@Mv&ox z!@#3zSw`QJ>cm(dASPIa<+YP7FVEnt>X#Xp-)%|hZ0=_?`EhY@%XJyEZ|%4e0k!x~ z;kjC(OaFw0M8YCydg`@s%Wib#!(3!Q=RF#_luuZZvv@Tsy;j=@>+Tg~On17C?h6tuv8QlPqP}7#NIm_uBrS$)CPx1V~5*ZAL3WHV%z9`lC=c#0g zB40M+)VJw9sstGP)b2^2fg7OBLS$2?pr=D=`}~5wr-0uFB+No6QvCN#M1$K+Pk|xn zP*6}1grQ;Z*fg%e*Q2&6nf}f|WL=%pkE`un*L03yzPwcwRqy|0YD%-W`KpX~cl$~g z1ba`tPfO_A3P7P!J@M+eC(eB_pd^h$;5bwToAZ;?;7tdh9+P$^rSa^in=9+VDGM-6 zkD>5J@57PWszk$KhR5YKyaR~9Z|aY&Aa~)P9fy^+ca3a!={@!wx9g2YYyIcs={)Tu z=mtuwJ?6SU?e!aL>2X+`+0+ThRbbu@N{es(6nH55ec5ZHbrKY{$wuDwTv#ca<6We2 zD&)TEqrl~3ALRtzetmjt(>=35c_Ep_^vvGlU$$%)q+2mA)SsZBuUD=8=-kW$Ap&l5hWV*4cDR&z=fA0N#kZBJrUewWc-)CUTT>Ek;XMiKtrC=?E%4rE6yUSleX*&-%IMARyDe$_v>W>Pffy(zCnmQ3%egSDP-j$sE$cR~U% zkRfaHjtB+z<4aZs2?~9So14dKYD-EncBKJc^7T6}ji{c*8ZF#%;9>pZt(@YaV*k7j zOM>c9){Bi(`IQGJP!A1_-gOYxL15~fC?Ve7_F zbEC(&+xlmYsp)E~>-^cLh_84&De2lNs%@!ZUlfP_<2iRO*;8wpdFq=$_=7eY3EIj| zmYzfmhi)*&0k2Oc*0=S(&ubiIO`23|@SN}9GRFSa{-dSyKL+_eJX2mu)E!W5edQW3 z2S*$EVdODu4B!>`4fEMkOY!;X%@ibbZPZqVsQ51GnyY%hKXScub=_emL!o1$^_xm- zzqd@c7Fzy#ljOFcY?w>NA$V3zV=mdwQv8TGyJqj2l^(2Y)TZ>{wMN+ z;{ZdA7x3d!F(?o6G*YEUh!qxL$JV+SV9rvsu~Yf|FVOvlYPi~#`>nV?jrg%M6(%vI z9I;|sAQJreP~0BpUSLQ0)e2#ikTv`*bK&uT0_rp^HE|Evf($R}!)y7!u#EFQERRq{ z_Zk}HHX&lks^`s>Wk7=I0}&2$km>dq5n^<)KivWGzO1Kqpx(rXm7P5+_f%@l0O5nD zdX1f9W^6|H9#KIqlPpXYLONtbHw@^F_k3}mB_N!&Aook)QF)vA1(kftcP;5=n?ZK7HE0pXPa8qU&C7#|3H7 zLv@U~v?oi=$3g!vDDx>hO?DKtsC4zyAWK6O#)#7Wzp7C3^Jbp?R)WqWl5yMi-`#)& ztunGjyzV0Qy|=4O3-gfJC#0UYgn6OTl;D#zFMg+_3KMGrNt5Do+9IGc~dub+O+!9|)vB#@ry9%{M2C<`XQ8_r+mAoYa9anDV{ z;?#|eo~f}Ap|)@c!jhoskXj#_SWSP7AWp8P5K~fb$rs^@j9Ixe1H;wyzPG11Tguw^ zA5x+(>J71SW@BUPE>lDLwpy^wug+c|LXqnoq*R@J5(8IQYnH@gb-b8ToU>7(xj(Z= zq{aTX%mwuuFIjhbL9g6V*6OUaPq*~+rdX-?9GHdns~}%^T$w=(0zt?PAJFvPjQKQP z5r4tr!LO^B+hwhmz4RSWaHeRVG%8HXCQG%raU&go^=~}T2Mg|j{&T<@c4GA9ey=iv znhlmga&ZCU1kS1B#S$VpDsn}M!^Zd8<}=LJdlf^i;Se$7{sgHL-AdrF1VzSyw7o)= zIk`o%yr~+bNfwTX4u3$uV$| zAiBQobm-7n2Dn$1Ax31Xt_8>&@Cedi4_}Zyj+F+L-BCSt4UJed4o2>zy?-zZLCGJd zJ`EPVDvI5*0fBpU$~J5QaX|RcMIC;XnaNO`F7nPuDD~F2_rU-6Z3D@_6gRYcA2#ax zWnRP0L8gzB2lI;@Xn5{zr!!26_;{&AG?ByFheqNU-V%gaY*`{H4i~pW3>2W0NH?E2 z5_wzAxlvFani3xnLN7zc^r`-;c&?u|K$##?Lkk)iWa3{l00Hn@CeP3(OrQSGxDIa0kp6ub+t7zB&Iv0Hp+e?zKZ z{#U@8c4@ixXl?NPYpmPnY*+6z2=-UaJ#rX`5h&lruodiP(H!5OR{OPS*qUDDlJap{ z`O%YbA&ohnxD-(>jhSxwvkmV;`hL$G{MowsDK}t{^256Bl`4*2+QDlReloVWUYHd0 zW7(ATnVD{Ce&){{N5PfsQTaEgC7rj{PE&+!%B(GrRDGV^+1}vuUW~c^k=0j1f{iWw zeLveB!FL>5HmaCAa(7~58Ev)&vhQ@sjUR}Nj+Sas!?Lw8-~0K#zcqON=adwCb)VSN zAgku5R>rZs{xW^H_y@y&OG^8&dGC14Nk1{3a$Nqf9XvR>yB?LH?ksk^NBxA;YbSR5 z<2!?c>~7Ee^FCPQZ!U1wQl*ai%<-8PYy+Cnf1m?VLjV-j{Q2|o)^gmro`oB*)x*$) zspCBzU;`*=pWD7~wt|b_7XHBKHS|IWGXP);rtd1BtjTcZi%})UN{-baiO~SqXbZND zb_*IEzXg0N__)%XB*O~Ye#hOt-PP%9D$i98r*QX7IA~`K-{O7sPi6j<4HLuL@12Yd zl(B5=L8=df&IdmcZb~PL5PWN&qvZT$^XV}CV&)3`dM90x1x_Yk$&?McaXy*W!otG# z?NTzC)`Z0X?v_EGhyTo z**zdps$!{kCL~f3xjr| z!MqX16ko7dK(;fT9S-ge^Xfi~x_S(Rjt}-h*!T3XQdYU<<(ja_*w~w;W+~GCY~0*$ zM*Q`RBjdMU=L2h~y(m4zLHG*s%R&uXFD>R%G;8+U7wg7}xhRyQlODFU6nx2e<7xPw znnkwwl|4Cm$+q~CiHL`T<=fvsPw`ZD-C*Zp8CpmZ-{Ezx*=|00#`mAn;O*d#OzoV3 z+jO1bo5e6pDvXqw0A^xQAdzU|1eGtjB zikQI`(ulgGJ5q*Kf3Hk2ivK@!eRWinakupVA_58uk`gKFkrlD*1=Ewv5KsLAhwxoV}A5{Ilf``g}JAat#vPR`4KNuXvOZ@9^Q{8P} zlYs<}@a^Rg>Xdy!1AD`ajk${h6I;q)iFVU_Pp4kzZ{>FKU(c10(c-PgOJN;1k047b_$jCMbbvxUvo#tENZB&w=*gfXCW|Dd|97~?#2Q~Yt8{A! zaW8shgz5I%F3?-KRP{t#lML7O?zx%p`0$#G?GD)Q1Rs;j_bXv{Gd?J`Hm5Ah?90|{ zU`8e+aSEGFdR{mvWXNf9kMzMH8WIKztTiSJ_R8aHH@)pg6JWsMrughAgSRN7!%0f- zsi#`GAm!y#sM^N~>syOA^Pf06Vxm3!HXYBoeQjv^JK3M$upL5O(gsXrJPRJ?n<}n( zeBR4HWGse#=j`p9bt|yaHPPHy){bPEl+g7rC?R%s*^+WZmyKQ;8T|E3l16YX)r(g> zL(RA3b?Gy^@8G{F6UJ-61G#Zs#Oy-*W}|NIsyd*%(e(UMc6Xkg^nYbe3imVJ^YXK z$kAeE6hf&*xkW0#qKR`R*O~dHx7g??Te%Z{$k3Oe5&v@^O~d@ivr%TUIK^o=z(X(! zRgPS#J&4=Nu7pu55jDll{x$?grq;xX z+Aq>X`_&T?5@3{QK_yU$igdf%d3Xf_Bn`n*B{AcRiiirV|^8k zi!j+!sFa0A0$F%p12NYEhfO~R^R;U-ds|pe{t??(i1dw&j&5oY-X8Id`|4b;RJNIshn04WZGh~so9t5&GvLtf^c z!=by)yg9ZO$Ec)xQpqI)*xRhSpRW>jC)h+mTWRj;ZMoyuzP;6rMLnBeZ@ck+>1KEA`Gk?1eXy{e{4KdBC*7#o{~8 zt#F@j*DQpONCfv9#<9hW`+F3!dxLmJXU|TVl-$rK7l#ahb}|Zz6snbq=`fvo0n4b%KB)DxCLv;mXz5`ZVGLh0v1gQ5=rBFECEW#B=i`dSh`Aua)}m=hu>~&DV&pbWE#(bG8XnmZ>#S^6m;ua+QBgaFXqAO!+ z@JmOgI4#QO*J4w%k=BlJj=qfT;S%}OJaLz`v83r*S(mH%axvKxW;7g|w!S?hSbz~K z`?mQw{FY*Kip~2NnmpC8;o?ZSwdwmay0X!0mpZ&TCyH=3AV>&}frM&CdC2zSMlZqG za|EJ9?H963rNX+i8&&u1_ zZRniY^HRCPPOMY1Pt-MOg zE!;)l2m0ofd?L)ZTO9E(!p@KJ=^5=QeLmnGAGQ`-o3_HmsMkVuD*6`R11x44l)=9+ zMG1qfXCBrfd!JI3PdOXE0cBQ>O>Tmyv)XL(HK-4_=!LeIs9QVrp*rzC z$;s;K4=Z|;+J#4J&i?-7fTpt2Vs04JVNQJPy>x~b!8&m4CS}F|i8(cw4v&=zrt;5( z7QFzku7k`LBM*9a(NXDr`qG?-Va8E3)d75NrTpOJ=+&peqs8&R?-Tle*!~@j@+KF? zkP%|LlJ<5r>$`LHgK}zIdyUs6IEmuHFP%jR_>L}2F9 z3*82vkFhljM$r|AGj@i3ASb{U<2SI$o2&5qE=Zwiuf;4yV1K#d9<`{db6tA+^$tf-b-;$)c?sl2|2Ks$#BlT~Qds17ULizB9 zS|+MvMo*y8Em8_mDjS`l_yzUYj$SCtMKRc0ovrCJ*|FYT7!#@rp(Kx#2fA7i& zz5Y9JHu0f-__QEv*iJ%2(7WP}#-})T$GL+vsvzGD^GqGfhYVp?Wv#DTsma1^K>2)? zjdm-c+daUqs9p>;3usQKSe75oyD3n0@=+7p2sA1W70O|I=1Jn`}YUz9rs~VYo(P%ec*cAg`Hxl)sK<-zf0~@gzlX0FI_d ziDo9YFlQEgn(Nx#cPz92`mHYI5YPWK%~(l{?v1+&tRxCyck&ERa0c8=w)f#3&~(GaN399_NeZ;FW~>zH$lB%C2GSBYGiT?Kn>S25={4q<-=_aPB?4mj_)J-v@lH zIf;ZXEKG}~k~7(!R4Fw3IJ(7T(8lVsC2^`N?ZjD;;s#FqZT8HxAk(N%O?8ZFm}s_! z6gBv)i{>GI`*FlEzR5BVO}?EJOP?^~oUG3uf_;-0f8(%=>5IrY6gyDCDL-Yp-8x2< zTQcnQ6nmV!i18v+Se(jp{VtD-DJ>O{m6Hp?4twu?7lk<|Au%zK6shh;awrs|ii$dd zNy6|m9he>d7_zfFbE8Q}ZnQ+apx=;q`RBCr=QLa7c<}l4)98+T!(4s;4=0dPaN-uqZVD1rv`0VWL!nd-o5Ur-*+jFW5 zXxVEV*&!^K*3jnZ*-dFg$xU2x&^Aof(RUy^L0Mg6!+L2+sy={T`XX$3ZJAe+Idb z;NdiI9+DInHJYY2`xDmJxTTX5dy0s{-gSvLQ)fxc-l7q?EL$fz_Yex%gu+A92F1+q zY(|BYH?gr-QKZkr?7sf>6p(M5W+}FwpRIfU;JIv%>9*at3t_-PYD`4LHE@Z^H|u0T z64The0wzuApbTx@UdWJ~4b}N1m(pC+rKNT!fH8ez5 zWxE}irSR_vJbP3Lu4KI*Z{nj|UOmP)T{|4o1*9?N^f2RLPL(mM&{Y}MvkKkV+frNd zA)hZFodKw2PB=E^D|oHA>UH4)*A=YrLO`m9C$p)qv%8fjN;g~ptM7}_KJ*tX*m8JZ zvm&RMPIX=LZrAozkY*zuu#iRhamWC_^V!gRQuAN!0Hs^8LgPf)DyYKO$KA0)ZKD=~t?;%mQ5w3G1+nW4(+5Hh(3h<~EvfCb8Ekn{QFul&*8j zoN>lyg<7YSPMgU*=jgXAu0$O+rv{sMmVKnSPPTL>pu-I24I$!tuk?@*HG%&r zmZv*m{UwY*Mmxx#;TVTA|9IV$3}5e?D3i z!48!BGkDQR1w8UKH3T!aXrbVuU5}mY3~#tGgnbS$c$BqFqL$5H20{NaOZMo zfWhf5e0+SPTeQ|yaiK_#jfBKYwy&>*z+B$e236XSVEiC|rIAkSn~-$zqA-9JS;EQk z7T2{ERhqu4kJKOgpEJU*_E{Hg3pM*BpvtRLlOqtbpynB2S=@5QCYa-A?d5>>bK(mY z?13(Gp7s;{21C|joAvHNPOan20;0{v4zGN1G0nNYP4V&*3;ctn{VxI4sb8;aT~bA?W4e>ZL(7xy8WOD)vi>U$0I=Fe2Y^rPZ42^n7MeoSn!XNRPj-dK&XQSw8Y*srfq=gF=t~qKp$tz{Cqj^BBV?0Zg zd4G*aSJXmJ=zi-(y}4)(kzOMmYm6>A4)}N3rjr%~zd7Pq4{$eNoN-_TKs;A}ww6;h zT9oUOcfYKL=s@3e&M_O7bz~AcT6cKbDe2(xMl7JTAVCZ&L>>8KNcke)B(`+lq8aVf z!tWfE)-%AbGSl|gzpwnJWAw36>2tWroe6MlOATy22xLn!LV{%*^f1G!-hqT7R&Wa@?QS%89fbWD0< zgN1C>{RkOEwWiki`leoR*v8)FQWIj`>#Qp%oPeGD?Y-;5r`j4x-=+W9+%j%wy?$R} zSa{2@_-#8*V79Z!Ih1XGF*prRexh(1Hu8NKtRLXf+%Jc0z6a3K8@fz0AS zc7q|6bY2dFq}!1{#zhGj-iR_i%$e`n9lkASzxXh{b=E?8`;Oa%@3K_MGIAC3&Xw>$ zyCnS@@)aCEe+8!-%bYGr(AiFks+57_L-1oim$W0rY{5Zlr?KI_=PvQBru0* zKBo^PLz}rQo#}E(V!YczgLh4Nu(|p&^gRXHHo1Hjni(*#_p3?51@S!Ze7(n(xnnKz z1ZSAQo0>VcHJ!JM-%wb9-&*XWRn%-gp!IS%`2*kera7;JY#h`U7i+N9o&x`2PUx2* z1N(6F(X`m%5t`;80e2`8nesv2PiiU-$}2s;g)6g!KKKm>T0e@@j7V=m$@~_zi6$e= z_?mwZt0e2`Mx20=u5Y4kVrsk#xtmQ}aC3ism{BA z)=E*US?JQKD>b5~r@ga%PQx?q`bJjWN-%NQl1n>|?F|cp(Jnp-2|Oe>UHf)1^7^d@ zQ;8BIcYq3===6+mKu2T^o4DvO&g@)loS$|uAZTED|J{6J1lol=FD~%)*c@OHbc5|^ z>CQb)aKC>E_uT`X57HeQm9Fa>jS&&lh@>4`eEX{-0V>`mWm+xYYW)IIa>J~ef~WF1 z!WA`?;~zGERKSZ-;SN|?>vt4Ia&=yf(oQpHVcz*R*)buB%CrJ`8%Nj6g|c__2Qmw{Qxcg->#OMhrXJGr$SQ^&)Wf zV3`o*#k|_HPlt<1)XPk^-KKp%&o^SczQ)ESaVFQ(@l@^ZG?iLY;%1r9f2KQSTTcD& zzKPTD3utKw1<-zGHoUS?Yfq37ghFMPkrwD}<`M&9> zvw%%*F`KSO75aFz!pn93?2$6~LYPp5a7Z!Dy#0<-ldVaVbk^%t&S-_+)Y zLoya-0iOyeY+?>aqHSeYpAL*E^hd)vAeZ(mcruIBK_Y|tm1Zz5s%zz&8n;mCGGoc0 zwOG-bq-@k`^y@;|N!4;1sP%=o(8c^6F5*CoGFukYQw0webqU_Np^4zN&<>`(qW=zM z^whc|g>YvMRlETFacN=`FrE~Q80>9@U-v^KczDEw^`u)Q2{|;d)lSv&us?&T8 z@(38^-Y^7bA97>mcZD^7d73?sWh=0a)$V#hsbar4sN$^^?vjajez~1F_-PYlvHZh% zsM_97(Oh9~#AaIRZVTpnZ4=rR(LWykt^9A;1NbJi7KPk-ZO&YPRM%x}B|f|^T%P*Y z6t^dIjYsyZ)DZ~|cnLWLd$YJQ9zWnxqbRoint0RRe$E@2khr34symqc&eMMU$(k#a z^)>qwV5c13$Qr{7615`s0aVR(4xt@UThaEe$6Kq0H+FR~cT3GK5l+O3g)w5yHcOk1MFqH+342#X^>p@(V7IJpi79^PR@V#-v zpq#JOQWJVU>OC}lC|pQ5@M4{%zSl&&a)H< z#}PT~99AoH6bo24{9Okgvyf-jMHB8kWHL2)s_W?6nMpAFY<^stlg$bg?{bH{i(bJ92E14za@K4VFmx9?-mm{$bUMZ}O zzHcyxgKYzr#S}8L%AM6fZ8BfA1QMSVkXzh%+C~jQe|CQD-Q(SOv;tzAOpdb&1qhF) zWTv*cm|?@-eZ$n{OAdM|mP`_}h=*xCgUT)-EjV(y;Vas<;)a|f=gjz`OnXN~k`FMV z6<8qiLU?MtMtK!e65hCryDOk#fT9+Q6>@Ch9+2*ZnOa9WV-ie(;w*CmnS4J{lzC*h zIr{gjbwHhvEBQ;USQIcB2lHqyq=`8DvCnT$O@n8^JTyU?jqr@$#_y3XH>4Vs?l`Px zddqA@Z4X)B1S)-Ez=8Q%`B9yw!IKr=2WMg+6>lIZEluVs)50;6yk2U2ytssfgtnCz zN;!DYDzJTD@#NWeW%JV$y*H;fl_@z3JRv*#2lBHhai+*tdB@KOk1hRaan|z=^?Q5= zX*ry8E(pMFPdhb+J%th{Zt+?dWgn~2|3Up^n}==lCHmL0udMBG;W`6TVz2ldy3vZi zv(MQ@;bhx%5Nzg_jt_tv9B|ijqz`UR-p26>I~&8lD4Omr#H$<3DXkO^lpxM1nN?`V zvq$$}#x@_9x2T{X^tR%NCbNcg9Vh*+yXxJyqHTZ;Ip&vR>gYM6^`Cl%U-Y=;a|B_` zJU|BKBA(UdWQaX4Y-928@Pr}0(NEjIq-37z6GcrZvKDCl!U`m0M%U;?aW-~_lzaGJ z5gEoHFP*bF`DBm4`% z=Ur#f&$K~~SI8AWiJlj~^ic|NQgGZq*KY(-=J#ReZ>H;_@b&A;1fAad!{ZyGs4hB= zjEDI`3lT5?GYAJn#;Lnc^YnVt8_(f6x!LysJBNQ?U!q3&jPr$uH$`q52wR65a)>VE zxpw=)47%2#lGR83<*QfIAQlSoWpI-b-NP{Ht$G~n*m-KfrYQQs2Ev_!>TqR4^-w}e z>TPN&s#MK2Mb_)LJscE%zh>p`*`)wBJ56nJ-_5~l5yGhR>z9+7dm_I z_WBBnmi?0O*0VAoXmmc0*8o% z(<6=n&dvVO@%>SK$u}K&fmcq3Z)r)S1-H$My{t0#uCEItzvPmuDeBxZ7*$(r7ZuSH zbq$4ts5SQ}_?}E1@+ZHAglLo%dfR?JL0AdvrB|BCC^{r0;aRg{M8GL!gDe45-#2IC zaa>~Rg5;V7^_d$|Ldh;rGN)s&vT)pON-<7rnQ8mzYx;ejj$SleEkbul?3RG|&$Ek{ zVT!{6Ln;}Q2+MI$#s9u}>a?uP@Fypn0-IpFrFD3vh@iBiJkG-bpBqm}J6gFNaIpmM8M`!B2^g`$_||Z>wturWAv(IkR#^qGx5aNr`L@yz!dJoqM*c{sP*_y z#5&>VV=}%^WBn|R5^q@cz7juXk0ZHywRw5^i$71k$sa?6`BjbN;_>Xnyt|Bm=4~{? zAi`dy{e+nT{IMl~PM5*DB7r0B60N8XR=`C)Hn!q>isw|~OVcU*bRjeYOjl14AV&c5 zr}PqBq<}{vaSP8Xy6MP_BusjMs?m&dBBOqS9{!Kty$=)ep-GfU*e-Z*KV)-?;q~Sq7K0| zw*?l+*l=)g5b0Iqb;Fmbsi)!z?LSe;K1eBRo?R)c;0cM-8+ng#H}I#H*i`{_*vuB! z1jY#05Z5GFMQmTnA3t1jUh9$P9jsQ5%d`g5_h^ErMDcnK?X5w?G1F*Mk6Gks)!z}Nb=EsJaMRp^IOUF&qw|<@h{XlfYUie zKoiyXmWlbAxXO^!odsxMS2S)P(+HPMc@FMRZ90WZZ_=`R#%ayJh=h_EfBWW#Z~?oi z*QZXOmV*tw(XHn-hw5ifzdD#C_3oN1SW6Wo;|3hCawWs>K7I@ZkHW!XHWn6H$dIKI zwEuy36TeH>Ci-`k?Ed(zbc=m>j%ML=oZ)kE{8_1jW|KR|W1(DIK57-Rg#5+FfTsHHvdvWS?UsXil&BO8^t;Eg& zUGS}ZX4IU}aS?KB*kT$;BfOPPkc#ZO^Q?a%n?km&)L~!0W~YJI1eUVPtKJgDtBudX zG+vlXGp%kObfWH7)u=CxsjZ|kq0Lh&(*;CvO!b(4B?l4`XiX8MEV|~fS_tCWG zCE9m&H8fS&pL!{Wp(S;kPw%#Dn+%uKh0taUH6w9$@*q zP^D;V`^{>oudqc873*A_raQK0pIX1g*@zC>;2~s)MDAVA68;%qHxp5~z~+aQPeA~1 z^x#eBgOpIG;g@irY^%*NM4kM*MT|aQ=*8dGH{VLhnG+ylFO(0ds`!8^1Y97mBG{oW z@G9kD!)%b5wTkLfrH4pf!#P>+koZb~i&k zMeeXcD=dvtI7TYjjvVSF;waZ^lQTUdgUqjC%C@Z8BnaXiKwsc8y&;g^EG_n{*a6a`*+a#~uEN4nMv;F3v_hO`p%o+u>%Y7aRSr|k$z zF-4ul@~$b(Cz{a_S69Xk7C6=QeZKu@#(jNZR1;Y;Lw%gRP0{6jxM1MedamDj2NuV} zU=~`@T7jNRYy{WftB+F^Ls{g*FrK1^8;=9pX^yhFo~K$xL?%*AzhB~4G}`+ZjH#v} zyV(1y_wl-RS))YNW%Xi=Xd-^Zopg~u0Us_ro77IGv_4?m3 zf!sq*PEOmz#)ngm#HRZXhRU0Y@3ag^DvOEcnjnSp@(;0nIx|N05T$T>+uvUMg|TFp zXr=-M;j{Pf%r~9kVMk4nj_3}fFn!V7wn|+UOcpbQuVG{k2yk&4g$-*mz z3_iq$SKmc>(_8(@22{tZH=nkdukPZhzj(At<~;n?O#Pmj{Hx)g(s8YC zf8zE~^QIrO@X6MbR*mZ>qh{cay|rJ@O)?Ddr0KSCg$q*gYXkS9l3i+o@y}F{Ornv8 zz_EBjeBrM=w?K5dfP?CtkwB9Yh0P}nYg>7gyGJt_a_nOz`b5P%JulFc9gIIzw8nPu zXz`fy7nF4_(+^rEpL+dTUS6vJ8!>XQVdRTF-p-t~cRY=A>wx&VAO1wUoP%@AjJx}= zHSw8%u!QvvM2o_*ql1T+~|MD0&*) zy)XJTzloiF8hjC}mDD(?bj0zt_K99QeT+U-BXoth|Gn=vW!&=wzZXf1g{+pNFS3LO z%>NC8i71LTwMlFL+{vLOeiQI+WeA^uD%8tz{CetlvF^BuOo_@EN<&8S~f&I2u-h91Bh{P?MH7yZ~)mg)PIi`{L_H zpl*FZ&)41!{x(s8v(XX?TZJDez*nCWVhS9?US-;^0^MBDFiT)Cy=XTh;%9pL^#fTL zR=HhMdz0!oZayj|WSIxFr1PB1OukX`c&Rw4=fOM8_e)WR#K?<$>sg-irpMv_{->%q z_vD{VeZ;-GqHsLf=1v`U;F|A=0i>%Xz{4Ppx*z$+kaOAgOyM&E!(Y~?x@Ze6ck#V@m!~%Oc=nvcu%p%-pb#J>F8kT7Di@x=^)^xEMH~noI5eFoSa#tXlAb zv2h@&=#F?DY$br@;V(aqEUf}5B4Xkcm;)k~GD=D`)n7w-bExU!n60J8(|lE&7JW#P zvqg~C)^5(_M9DI}Jf9FSUYwLIB*f~Ob&8Eyc+vPy=2+*a{!UNfMubkG^&Eym;SO@{ z>~RXU$Ko#juUVa@5liFR5?l<@BdoaYtW4&ccV8tWB^|&)gOyBM6u**zbkU!P+m>N> zL2WD)aWqhKG(Bfg?<-)vIn_YXHA_BJUBH0-IiB-e2YIFE&GoKS-OuQ7Ou-=0H8)HTtcp=MjTnTu!d|E;?26uiASuc?&I8}?es1FYZ`^F&q-^vs*|q2y@kOOD zyRR#~*w`{P;^DNPi{R)||MR;}V{^ceo}j7YW{inXneXC_jgT9Xo9iyN0e5b=JqgmW zfh{iEdfeClK^UUc;>*J&O1>sZQXlh&bLwV>7`%22>BV7DZsAmyxm0!qzAxulsU*;v zCrkUbGM;x(OS!nLV`e$&fdOqjBD-2IQ7;(usITfp_sSQe)Q{2k5-rrsh=g}+w{iYwNeJ8`)6O>+L;sa05 zl7vQh8E?vjrZ%W*U&9wxmjMb@=rd3A*x#PSk6{@^>`g6wOxsnB4b-$(9#u&XQr`B* z+){@5flU?27HjqACiRSXcN_O&is62*s!;&oBGL`L@Qmx;{2=v!JTR6d4 zZQ7fgk-41xa@Z;4`*FwC@90WB3)fKpifv-SNQ0*(hb>-Gt}As@^9(&Yl(N%{1c786|p2d?383Gg|O5X&)Bij={x)3rYRh$Q%chcLUN8kRp&`Igk@wE_J%=Hs`Wo z?zd!>u^o?!dW%RHdu=@QD-!n2NG+g!|JaL(vOr5NpMMy4?j^SMhpzhHo;kG_#k7r- z2l8<4yT+fK`;XwIe4?7N`Zod%`CsV;?kD81CO@x`Em;T z<*z_61Y%OX#90o#?|Xm$jSaLmpSJC-zQII$*Y(yIlHpwHQCT5p+w8sH|FcN&@P^in zW?CUdg7HDy*SCyafbvH(Hw6w>4SoR1e^ezUhjw$kOyBJpaYD`9OeH?=WO#zFZ;64Z z)}}d#CI4wV?WmjNn@&jaE1$k6Bl9jbb%seK-s1aYn~uR!NMes%h4l-Z(`Ebfdzo>U zB-A)F<2663Uf>I*l>5FlT`XSC>>ezCSk4W&|8r=%^pbmkAKxvB#C`u_{GQ2})9KkhkP6ya)?peE^O?fQ6gxJGLmo&a+*i6 zu|^A0#3Tf%L@odAn}^j@hHrc0dx>j?{5Thx;j5zMB1crhU9|a`r+sv?XBC#??49cQ z-7tToe}cmD>~P}S42|1W(1}&I8D#{KnGv5qna!a1FZ<*qz?8fsPqW6W%If#Gd){}^ zn9tRV@7ZFJflE>S#sgt6IyPGAW+Q z6$#hl!)?cLum{`iL{!#VW%em&Y3hdAFV#&+sAY?}|M3=1POgM)CYUOFd76#`l)U$d z5ZVQ`f9=A(Uu4k3$gGuc1%VV-PbfAW;~%ZHo(_j=9yW7dI!>qW`mMl*2sltuZaJq$ zaLQqe)2U1(hn%5Kz!jaSI1_KF?6;*5qpfE;NR1dHfh%TnN5?M%UteE&u#d~qDNzgn zU3hEa13zXSo(@9;$EuH)=s6!nKWfB|*VJ&YsJm@1amRTdm1WyI9PF$jeXr|$dg_ML zt=BL{WXY;BPL=-~^L$$%I`tA8Iz0KcZNS~Omw$qj+cB3mm0Ld1e=Ay0X5{Q-aQq*r z1r&m+OrIGzp+t??3&Vhgy$O3E&EBxsMRW#5^>=+!qUduS^@HM%8e4Kg-+`k>I?(XgNT!o%ygx21^ zg;BP?InQ`ohb@Sc*8f$s5RgTa`;GM=ElW7Ozi79&IZgi_<^>a+x|~X5@XbOkL3MaF zAD%U&(YFDN7C+{GN90Xu5dCC`X<+%Y4D(@vg9LyObR!vPu+PI^PNc1NZ6H}<0TRym zG42oU-~TPDq^S4{X}IWfQa_bh4c?&tl&OAMfK=PIk8}BcUS*AWT>TbJ_{QfudHuYv zT`p}V`p?g0Nlk@}ziTiF3VNjjh2Z=D2Z~;L&oN9syC7*<@hMF-iTV27A?p(0ES%NE zrO~D*RiKmr2YpU~&;kaCnPP|wo&`(6JkDVgQvjyAD?_N<2}MsvmPmC0-4AE@<>RC6 z8pN}4G9Mfzz+CpShm9Pn{p#yG#|O;v912e>ddV+PUUSx+7wPYuH9m9Z^Z%%gAyZYB z*3d2Ggc_xDJ>S9MreqiVCL6$HPm3m}yIh5GNmR*w&>KKKT!u&NT33OivL#cF9n0ZbzvPjhLH2nQEUNxGH&m>?~t-l zbafrAJ_?Iy+nQx#y{hRYnw_c?b44vXDrc!`C(aMU z1nw^x8=d|586%p+B2=~Kp0-7w!Ep)Y6Q)ww3Bxx%-+UB6DwKzqPfkWgP-?83+_rQq z;DY4jWPz$N$XCgr+IWm(1H)UFSFl1XykX+>b;k5r+{~Hy&~Zq@6ME+bRr_y(FGCM8 zc?KmTGQa~jM4t9Fk%Jc*8O}ehzIL}?pI&For|)QMi`xuKTY0i-ievbQjNc@KLC$oN z9iV0*eoqEfA?N=t;ySG){h50nytTkxTx{6?HzxF=6|Vs$!P>+t()o-T9K6#_NEj{R zK8sz;&%c2fkU3@&Qe51E2|N-GxO)0z7hX-+zr3$9Yeuwv@@te$PZB#Vtw%bMje)A& z+Uxo|MLtQnV`FdKw#2kZ9@N+c-n(J(#NRtMs)I^4f>zA^VMYf-X#S{}%wP^((E3$EtXG5RWZNRy$)LV{}SU1rk?b(z5>3ZalY6m-sW zSJH9}D!1Uf)xPd?Ff1fW;q4g*^Ovt&;+laK?S}5QLnuX#e>c8(OGdxMCT7Vjiz7yj zqVQRBf}3b25hBE`I)vrx3(sK>(X`~Uq24onbqB^}n|qU*S7IVCQCBo&cV&0eBjt)} z<1fcmRh?VisnjDG9%}S$j?p+5_W22}Il+8hkm$qvUa4_P84rA)^(W{!nQ(VgbmpW9 zg)%+J9P5`PR9G^#9Jmk@dj$@~k2MaH7l7tE_h+=Lk zk3%tOW#sYJpV89UWUfJF;UKErO$txKxJ%+yPmlc<=t zS0q6%;bfY$6TWwvY|LT;%;e%8m7Bp1ts}crKFaVuf8ELKT?>`XrB|+sjOy{|EY+5B z^eRk1e~R!}%?p%s^8or`d7Uq+9o!jMAP(EkO~f&BQ*CZW+vj}ydj`7O?MF}i>S^R` z$glJ^=6y3YrpCo(PF#2*BBZ+-ACbwC;jgUE&jrDcT08 zdz1Rg&+|hUL4K)HIWCJD1#S$n)weOd$eV~ef<)4=K8%RZe4|#hPG#V6MQnetGdUwe z4(ZKkgq^bVSn~Ll{fPC!fqritLUjbHJ`wd$@#vgxTi&yvS> z{8d9Q%1mxe&+bX{h5?k<6^hOH%G0EsD7{n>@Sg9M3Sit{{oV8KubpWxpH9-KI!%bO zWjgGLdKE=qEbTaZl395L{8eUgj{{0^QT* z5du#UL}Z{;G^;#3`KF`bOj3;|?5k!CyVe=*SM~O?T=%KXPdW8gT7IO7CSo7)x>z*~y)skzoZ$P@NKoA|sGi;V5? zt$)Yc(f22IL(cKj=?pE9=(q-ptFmC2F<+?<)XxZT;%i`Cv+ynXspbeBk{<~$^zQPe z>cm}0j<2wWAcOJ;@Ak~C+Y@ljZ=0ER#ot55QDCeP{Huvp(Y0Cf>mUW=Z)VU{nv`%? z8hg(jJ2yVNVvw>lY{_t_4(kTiB~{YT{5@7n*o|(HDZBZ7==Du`vOuOs4-<9kccP5%Yg(@^;pNn zNxit10_o|#wd45&1RvAOq-Kh{WgU3|bD8N92earC8WZK6;bY?ugaa7fuWsBrc-yHV zzC3k7mEukeLY5^}&u0a8H+X|BI=eok&6>j&uucU@ zKf#cym3K`!2Uki-wuBIDzz}|rGcVrV!vlS*%g`^ev*A-BVnfyotNQ^gM-t0gkMYI- ztRi)m!b=Bdlo|#B>K(CYFIAi>*!=Ka(JEUWI(priy+S4H@{Cc0EOm^VGFTQm@i^in zD*1mn)agINIsW6mPQMjJJZr2FyfQ(s?~csJ#CxeOtJV-a^LugvMGJk@>BaHP89+}JWlgDGHSRt=v`WCxPLgXnY1@ZK=ZH0iJ0sq z!Q?BnPLHz_zsE1dd|CTK0Wopqmy-g=;L3QYb@tvBvi^muI+wFK7W=kOpQ85oHWslamh6mPMnSA9g$w*PDXsEwU6YNZpc?$YE4ipWNCQDUo&cBzLj`J$;4?-NJ3fE=cD)ROuQn5jkZ>q) zqeBwGK}piy`*_P067syLe(ohHg)N0g|9PP3E>BvruD|*R9ylm|j*OGd1Jp_;$_>2Hg*+HUJxUR$n!5UWM-)9dITKKD zy#tT10&tSDAfgzOug5Ubb9Eb9Veu+GV+c}w{au%+dZ+$vSDDUYpzte|fJl<1wku+g zYe>J_+^MZXFY{cBnSzb93|*O#*o(Ljz)+x(Ro%DyuyG+E0V`BC?F2$M9&s02zxu5+ z9P!8)@j1>0P?{;66Gv;|HJFvbNbsrL9Uebo1!=-bSQ;W6g>nrV)g6xG1)Rg9DNkhC zwSEg;_u0VRc*iq}GGWEWHoqi`BsqXk_?08+nRVB_E#;3|tO!SH*s%sFOR}fgd&S(G zPoESSp(o~gO^Eua_NAa5P`EXaa&Ll(;2s?l#cyMWQ}HlhIS1ZGh7Q(9{4Ys7(%XP( zOrTtA=HH@Dy39B>{m)kq%UNypyD!hrI++_$i3}+mydg;j2bXBefjpo@!;rmQd9%m( zLu{X_*x4x6cyC+^>lyx)!&LtKEo|mHEThEnD3%>T67rt`GASMgq7_DMn69s+vZu?L z_pg`T>1Wc`b8e+G1NouYq=TN)gXT=pxn`0F=f*AqC^nPmlu5MN%#V3V!FO z0kD+6a2^3mpCdRqlUB(@ds7rC{X*z|r!@w48l0CDkH3$;^siK$ zH?~a^eoh8D(uVzC!pODA>&|6% zFQ|6{85@iaO;I8iJbVK*7y6UfueU4oCQw8yE_UcN#2!J_dWW!s6`gEl-Ikxd||Oa~2g_XDbN&6@87e{o z|F5<0j%#Y`wv7exfPw|YgA}o$AYcKcNKr%-q!W4(5kaa*k2DLSpkM_>dJz(O@1P>0 zAe{uIOED0tKxm=7xq|21^WFEpci+9gxBpTIJIUH>%`)bgW6D01-YlO|wYJooOnK!u zL=8%>oB4GzajO1<3`>XOt47slJ3h#q;E%zJXxu;K))e3M(QHV*Sk#JEB#+Btv#Vv( z{$+s9lniZK_Uc4eB3_`cF}KjpWmkX zED3{qjbWv;-^HQZ>bpHN^B+&7MJD>8s8Gj!As~rL-0##bgK{T{ThF1K+F5=HI-Z5O zb41+(c&f`6l6;iTog<&gcjImJIxm30nLsE!OYt}<+n{7fcL>H?emj$`J7iC z=?Ww;+t7YYz2S}dn!c-^{DFgI6LXSw6_z(yukYFO6ZkLV*B~(XoUQ=)dzC^lXC@qX z3S}s@N~_R0Q9(jmBuFuE&atuo^J)~w@54UA$4zv;xBIR_m*;a#%ymG+|Li;I3q@0) zeN5_Oy71abhgR36&Sns@C#;x71$h_v$IPM~o-059?qGCIKL)XSfCkps3?=p*MA|-? zf%fW;#-;-P(!R+o2?fHTao)CyUuPShFX~^yLY|WdVSvnjXa#*paeXTEBp8YSEckWr zY!~I+*S2Tw>5vtx)J4}1w_u3s1G8UE)&1||{imFM`UH#>sE-|V(&Pi**1Po)pE>Gl zgw_OyqeE(2;+qKm%R2$`fzZqdh zm^6V`*n=}@a1tma%?{{bx)|Cy1Y=g$V59<}-!M}3&y61O?;h-axCe&^Nap=TfMim~ z`5+6fnQ|*~ISmYf2$R*9Jb}mesWCMG)j~r`w20r)pRsq>&zhT1T_A*>gT?gg+1Lol z?NdbNWZ{z7t1i2cfaX>DfJvxLE9e&Z^5tl7-&XqBpKXY`okkO%_)-_v;QQI_+e%O^ z?xK!ICYYn#U1yr?510>#A$WZ1fSm0^HamlalmtPw;{4FtMe~m$Oqa6&9Cx}1Tgv8M z{>VUR$GUf`FNPR43nC$lL&_RnjK6c1+RMK#^FBgQac7Y;nrEC1Bks9xa6UedsJn;{ zha`zrZXDx#U68q{vAkXCpq!w#+i6*n!y6pUtH&x-+17Pplzpm>0$I$jCcf`=KlHYp)z;2jvrXK=;=b3I0c|-$ZEPs6PFP4%L z7uR(t9zN6+X8}lgRD{6N?Hqm+!Su!MDDO{vUb;T6p1dEdV*d0o2sq?Mj9>ovdNIuH zdi^YLa0H%ohaP#`mm|y#K|nQeer0ty185W%pgL3%JXCIR;Fv%ZZL9oUsBuH)eP@00laDi!Lm6`Nlr$_WW2DSh5G{kfdd}6aGx(^!I%Q!r`+m~ayoVa zdIK=(Q_@g9IrK{()zb|?Apb9OA|?W{?|Hn({+RTHEFm!G6qXUyFHxF+KWx9WZY!{x zPk%-M$sCFc?COtfnB;>TXI~ZV)HnN)h3bZb<9BLL1J7|>kwFym?#)Tr4CWYHBx9K# zxYFRevL6H#76(wpatfqGaX1^FFLN7)sFhkHgHZFKz|%YZyw7WJWba|d)0XL#Fx?UIf>cLE3DUJ z3?#jqJ>8jEsCTbScWY0;Q^PhgZYuOP#ng-JJz#ug)1^iQyg+@}2ku&Mtrusng{j3S#$964NeK}OB82en zO61VY(}h@IRAf{9(~?b-8&LFQ7rzRGU(D$*DeEC_gw|VL@0q8NVvRIP3Ja^D2|y-) z$bho0y88C=RQ9}Gnm?u=C2`b}mXr3BHdL$3)w0J+s{n)2h`i~YD>TaPI5nds^P*;H zCyr0ScCU$>M}bQ9$(OrozEjPH!Slb|+vI)HWjA+=Z|Oa^A@f_c>=-$Hzf5Ld1+)So z8-_|tPb1L_Xa!fA^U{sYfm+PNo&3;r4NYh-dV4Q}?1<01>pgo!wph2G#t!BP{>XWj zxB94iB9BpN`{{M2!iV3c9e5Ox9-!rOLJxvz9TA&CR{$Em0xt*HN&_>L5Ncy}$b<$U z5VC5%Tduc3SOl^*f;Y9+rL|ANQeb>Izb6mB40wmohT?;wv{Fo=`h_=a*WWk=@lfV07qXV~ zDb~o^l(>}Ev;+(eG`r6`1Z=r z-NMFKzO3K?HB7d#5L%^k5Q2%u{S%c*Rol1(AQO1@YK9_j$j~E@ zu&I}uy>Yt^on%%C4HwQw)u5kHkHO`f{&a5LgU$Q7)-Pe@FBE><@+u}k3mYKY%} z-0Jpp2iH|Qtv$g{VlWr4tE}o`i0k)XIRM6ZMnhm)9Wh*u*3M*74e*CRk024!eVse! zl0szK6mSu6B+oDlz1!Lvq(tFn5@V=xf6`lcuKpA<5P=v@YegZ6)(^K^&L@0Hs?FZx z{RqFm8Hzjdk$0}ND}VSrj5|EY@0_^T_Dc~@b5KIx)nFB?48r7#2vh1)@cX~G2`TX{ zDI12LxN#Sq^1G&y8?we#1_iv^xb@bK#zRw|9(KK9lZQ5sTRHWNevZu7pYH>OoF)Y! z9%COnU!ZG*0+j~KkS=t<*mE&6!gw(lZah{3nX{(M9gB_HSnZ!LrGoo~wB?Gx7k31Q$)it3=6;8Hq)z zoD1&?6k>rtkQDNqeq46TjNWelI+(cVslA6@VV&Hxu~FVzCZjMb5efx7)o&VyUXzW={Tt`keeVg`}=_OX(T8yW}{=op4gT7{ghoJ5F zBLC?QcD1ULn2GJ%b_i!};HSjuu#;7O%vutII2}xm9%L*iguq+zb$Yrk&;n589gK$< zSyt_X^z6>NO+%p_>ph4| z={-UBa9Wu{IRFe~U#JNl1@H2<`K)BgxC$KE_#MzuZRg55?9+cBSAn&A>-uP&(tnto z?%Qyd5``o%@7jP{{_@__7T{v2|Ge;?bnsa;mivNUL=EsU=-!GD0)8T7;&Mu`F0Vqn zs|%#8hPZVfcC6tq9Q*ywN3O-?3p^jvG8q^#^Vv_t@y%|dTV!-NgUIu2jhMt zRJVX@ONE>i;|REu^xKiAqGP5Ack;R%KaisMM-H5W*+C#pyZ*;Hm|=dh-DH#AT!jqAvl3<`$ zebmB!pWI-0RO*{{|7WX)hkScAptNOe)TrU@u!YHqthzp2V_n@^e$KP8@tq`Ptdj*; z-I%mz#}DfR0N%WRu$cd&j_E#sX`e=7lfV;FY~=UO-hKl}84E4^A!Yo!L*Yj9xw+y( z;MOrEl(01Mi8W3!Z*y?}#`E1D6!Bn!)!o$>ez^Ii4?&9qBCp zO3IEO=+y| zpq*9F3K@lHw%yzJJ8>Al4bbwOEdQq^OP;8^bzKH(%^iG-zBy)g706d#?(?rS?`{RgY z8D+oBi{AOGkVIh*ayN_DK@hc4^?m(40;?4IWU9z$G{PkJC9Ml@bbv!~pU390AM8gL zR~OgX;DcJ&8y^5;ji`VATxB&LKG~JFwdMo;KZ5Ni-Wshp(>({N1K@h=;28(PGxn1r zZP=_zd8-6+-)uqOkAl5~y{$9C0)oFPWHoxCG`@A&Ro|wEA`h)T59vGcN|A_$NM7c( z>*@W_vGm*Jcf(VWP{I~U#sI}pgQj=4DVgI>o&CejJ%QlRA&G(0VE{NJ?dEa47sW2! zIx%UVK=p<1im38A@_Wc$p#d}0uW|D@~J;zN`Ul+pWf`+c||}l^h2Z|RMYlxU*MQ9o6smS zE~`gEPgvAd+U(Evb_C8z zZfaZ$aQ1*O0yJ9>@S1JU+95Y;1$9fSkP-haX+fN!AXnb}t9AVYE0T9CJZYCb57l^$ z%9711oOXn_MsnzQKOzv;+K6(2T9&L0ZRoy{+`IUo0wb?sCk;YXhB7avo=fJhG*K+7 z@Wx@mde|FUY8gu!tOq4zzw2_RogIFXf8C*odvX#md1+<90ml(0?7@xp4xGSVhO_Mn z${kq+6!R?ogwfC4(d&m(tE4WwDOL@i2Ae1cxFElqj2^d)X`21`)HC(I`+E@ti%eiI z`E2{!i)i=7`iKw$t!-DWv3zOkujwUYXamiwL$P3rpAXFd82V}^OA4b@`C2(BL9`8w z>0QakaH~l}dk5#P9-~Pub6kw z82Oi-r%}*1s>))k3&`|utlI_*%F7%kc&;({Duznw*_Qpo*|d*J;nnJ(iHL|JD``&>%AF*+c1U{j}zq-Ov&sw=91*`G!RCgtg4MC^l__EVl8b_ap^F zkti$iQ}cF&$zm|tGE=f1%8%Zg1LWIBVM0MzTX24El1sOU*+t*(w;wsq*WlAeQCuTltR^pm&c#j6<|P>je>5De%0p z+h7ea9w#BofYxoan&IS-dT4y&bt{4*#CHpzSR0hZn3JxQNyG~N!vY$C1++!%QW|0O;Aq>ijTq4znbHH2R)^P7j2yWr#ab z55#mV1OLEG4J!nR>KC#Qmsg=ZA+SM6{+sIqrJqQlq3{m)y6V2GU}rFmC;r-qUpqP0fDh*$kBhzoR5M~^(TR>B()XV#3HG1Veuj>);bVsW z%>?RwL7iJ6OLaZNPve7md9=Ab;F-`KKhJC}FIIg}!X`bk+PT*1+b9L=N7hGY!%tl+ zO`tEp1i&w+*;i~=EppcT$)mjR>+ZAYya8kXnSJ=v-rpd6V~=>! zA}H4gc@yWD+0 zJS_7VMu_udML^vC`l)eonO5mI$fsa0AO39kpLXU!Q9KQ(d_7yhK$9~=n4*~rgzP8h z4!1AdRdGg}{WJu2t~IDjzF4wHnG(raE&;PuCRIUhD3e8I2FAzeA)`50sJsfFBHrGH%75P*{|7hAX~rU5=?Dq3HRjb;$@c^68C66iTTgiukD zMp|XNFk#FGF;ZB)5x}{)zxQau)Tm~?4E(v*4mdWv(HV;nlBA=cP$%RE?ts~{(s8{| zm8NkbaZD;-m=vAzr1(8Dh$W%YXl-PXTgyOCAcp4?RHPu9Wi47c0(~48p@$X7q(4AsWgpn#Vy1*3WnlsybYY4V79i zeIHFD;}>l)GqVws!4iXq_y~LTm(+JV!hq#a$M~@}o)s6b#cOK^` zB54V9iS7cQtB8&jx?&8_i(}DtKv$zRO2>B5(;NB^IDu%Bd2@&S9({Da(Pb_A*D<31 zms2S7SIPQvlo4Xyo!lq2btcdP{i2Tkhos*Nv|g&~_?v0eh97;7nI%RRRRho^9}jO& z){nrVNETh_?&#dx_b#pd0Q2}9nB_|Ozdvvj%D)V6{Qhkngqq1BYscoVzmGAUK?{Q} zPV_|GksYRAj}_5li~W0PQsG{|f8}rn(^b<2c=MHGyD3~Jt?FlH*o+X7oWg~6Ib7Be z@6L}iia5b!jGB9Y>;$m{z{^3H^+3y8)#=ZrUP+9ibdwD)MQfhc2^|+R$zSZmYk61a zm4GC)^{-!JLSh*LSQl4l6VNN5=TN?I4Ty46(9*<;0&w~j&b}^}6$jGO)M^;kLyn44 z9#KLeA-ZB<32YC8(z0Xu!ucrApMMrn2q_5`)xr5yOHn|*+Z5RdtR(L)pXJ4iSFa`@ z?Ksq%AW}McM@L8JLCGMQTx%8WedsPDYv{@{tz?8GsDm?q2YTlL-bAXnL+7`ciTGe6y}~6# zcQi2gli2Y1!#>oZ4wt<1KEHHD|4IVxo78y4st+ja4|%F5faXT6aLJ{B2B7Ytp^&K1 zXu55(l2s0AXHfhWfPVh$?-mw`qlfgvfKI5as~g%<3!H)d@87>itXJ_N{p4tZ4+@^) zYLwX6DE5d`Oj#{j*53UNw5|Tw6nTMt$)T$l>$?K|Y8@Z0!VHZz6Zc4#IPBq6L$73k zA*A0kmhq2f6N#>J+csf?GF+aqbDh`@Uf!MVUxKOIYi&nO?Cwb}fnkm)>p)uD!@ORu%8 z1Ot$QiSG(D*C{(|vy&!Dr-RcIqE36>%;D|vZGxDVepw$~KXfhAqMC&pBM975SJMXE zYswLm5~7keUPBtXsjq!~)s&#@u=>;PpWD7`3k$ET9qqKFBvjOhbZ-9{YK*Q!$yok1 znRGdN%)EAg-5EcJwKjmt%F2;a)4E!l@fI$4rZO_&XY>BJz8U*|EGRH2kSibfUJIjR z42?ITR$Ms@Cyve$Re|JG9C=y#6~2?bv2L$`K#jwh4{JRIrd@YFT=aJVDWmzgzz3VZ zWaJ462((){iui?AH|zu>(gXI_SAoIh8J;M-bQQq@`uoTuA^q<#{U~Rx&}@Ql?*x-Pe^nRHc{NjFOuDz?|HuqJ;Wnzj+|(Yfz|aWYh>IgI#CG_y6jTij z_g5CVv6U}Ak(8L@lY~mlSbk#H)T!YcgA+*nru+v?n9K(fkOZ#dGIji( zt%zvt7uD6<+^>O@Kk5^R@?G(So$s2mHF|j$%JR_VcRrrcMTkTh{1g}%NN;{&%{?J& z_#J`%aCtDXH_Df-6Gd4j{X(|3H2}E65N7qoQwO%YXQb$N%YF*w@BJbhT|Vb5uI>w-W-dSt(*!;Tg?l z<(S=}>*f(sNe%X*K8rAC83s>HUo8Itl2Xm`;cf>Hh#g1fZJV|HKq*c4;vG!Mw((`~AbT-GA8^45k2Z zwzaSf#EWj^|9*w&KVJc1`|lqYGW_kD-U*jy-(PIf za%1D93L1`rCfU(1*^s{i#ee-oe{C=bWzZm}wd=pR82{t1vay()ImrOKAS&wjv+~n@ zgtAKdf6V#MSNzvc>fe00^T^C#}U@BMp)tQkO+{P%kl zGO#yV9|L8o=T?H?eSN=v{mSM`wk5}jb@4d9N#Q5AH20JoERUu}?k`_-W=p$UupIG= zKjK!8EJxa)xM#;{8JXzNr4=yAzCVV^d2;=owAqEvleaq*{xWu$fBUHK`rijqcp*Fe z$$(fdW{LeD8|&YEoqulRzfReI^O4HKY23ZyIrOVtaB<1aweQe9AHpL9 zno6&el3;zj-oYRx`gkMLGTd;5*uMSy6CHcXk#xNBnKQSOk{B_Qe4B!*uKb!g0B5NT z)UiyVTTmK>NmoBV;nRYtRv_9Pc4r%lJT6R!uLMW3|F#Ut%@33?S} zq?F*q-io?4F1Y|bDJO+pIZ-3$9p?xKj2AVG~6805CgOS@<`ygE? zwTD>bY_RL^G+c!(DafyT=r&68JGVB58eGBP{TrY-p$d2r`ucT)t{$1^8HHWnyWm@!Snv z6g>WJYO!sUOV%^>Ok7c#0?mnDQ{Z_qdQ|psqS>l5(3dzhJ#9Hy^D^7x*A8|F8uucpNcMDkyHwHL_kdFkQ0LK49K5bgoF zhm&1^7gxli9LaUJn9Yj_S?_j-4v5@3W2VKRW#@abYs$(J@SQ_J0CA|{@%eaT_byl9 zundBlMXO^8_T@ziuez0hpx|J;Z3b~w2ZRUpge#VAj*Gx%zISRVP4f+|$w=s4YP%2q zGE8%N8wWY4pVY>U8$}m8lQZ>E&)FF7hw$`@6_XE~~@PNwFdr znO>9uSvOzQic#fj0zp8=x)QXmg^|AyJxL|CA+DUSEW{g0ZIDl!-28HIaT?e*q#rl2 zc`hJZS);`3eIUoGAr#{7n{ohaT%#GGIh$&8w7H7av<=pIL&qVhyN&$0Xb!iu+YrV8 zVup-q9H|yj^m=^u@bWsAT%z9AVP?b!zP)#EBN`z%I?FA26A>HzX^?QrwQ~8|BHhX- zV;u#xA0O^ags@plN9O?az`g`M&a=?zaHL{+%+aVf6pYR~7nj@OVw0 zc^Dk*)-ME4dMEO-;RfdBv18Mdlao>zdT8wPVQAdwgvasD9zNlym0Sz>F?aeSr`zK- zQ)zuoQ&ZC|Xd^N4r={2&1T4LdUp z;#%3Qxn1#Di+Xmty1J2VIi{hnURf6$-UhEi2G)!2fa@*^%{#;F`QZkRqx!uwQoDi_ zL5`5#Rk-8tw^K(Y-FI6Z%BZ@_PQTSD0&3+za1^z6u6V%!yH1FG{a}&+M)o1Z4t_Ww z;h$HFI(>c5wVdzyewXXUVD>S~I+qHcWtYThBt{&xOQ^YnxFX;cCF%Tj1y}Rp2vT$# zhc#GYqkp_en4xwOOpY2H>WeHqr#-rJ;@6YmJbtsUg;A@(rKg0Deh*u1F=OQvy};Pw z%b@{6}d$S5E61_(p)0;x8dfO>z+pY*P=RMWC>q-;j-T}{&bM_}|n z^e6ukFZ~<0_1}ti{!J+Mf4g<>N#F`pT|PFnuCN6pC1(2VD6M#wFoZNLo}g~8bA+5> zamfvSXb@rqM**hCzl5Ax5)^ZDcz|T;i#!cxsr|XS>~-d&L=PB>TVK)ueVwDTAY0+^ zv}k?z5V(m&&xHGnoxHE=u6g#}W}G2YK;Yxt`STPSRwmf0E_`r!7S}?Q1cH0GWhwE>YY&zfBf~Ox)qWmCapTBK{iEhWwEVdc1EZ7zQE8M zhvJK11TCSRDZ&iSApLy%Jz$B>DEi5{Ou}KiJj$7_<4V@AbXa*I!B)M=fiKTxWVj38 zg9A&IE1#~H*3#1YAPL9qA^oe=FMx=-gHdnIl`M&fFm^%R&!8EEI_ggVzOX3(i)8#0 z5)&^%M?Rn+Lj|KLOf`LdSCVKT!&yU)_7w)4=LIMpQDOuS95^TCGPD-k^mdHT535(ybI*HZ-R{1rmjt51z~1B2JkoW zf<1Td-lb->7MWyTJ`+j9(*_sF(P7A|ENo(681Cpwg4vGHxH||3XqO(Ib}>@jwSs-u zt{Q|e4S&x`y65jdT!fcQGsyW_TCGW*hJjdN&|t0X;Zd00$93$o9};`NIW_QQQ&*VH z3S_{~Amn)1WNoM6Xp!d#Yi@6r(qKEQRKU~m-NG5H%i=7)`kK7FUuBTgUM!%tm2lL9AR(wwEw$zNatPTBQ8D{c!PBVm^`pR?Q zsnX0ooT^ETdfx70h{FyXJg68w4cf2ET-qU9sq=E)i@7b!tqnQQZ{6hPi|9FY1 z>xu%!7^6FP?##5o7_wdQP1YS15Rrz0n%(*63>XilALTu35cK|X<9=YM`{wfa!jlgL zH=R^H4};_#+)c!wr<^U;Q z1qM>sw`opm^i=#TT85^-u}m`ve9=%0HyXZ!Yxi#Jj<@wiDXp?LJ+Pjday+eMfXYH) zxjUSQ%d>EElIJ5(sO=p0B2))T-KW~Rn!+?$jS26ClAh&yx01fTzL|a<6CYU=nIQ?j z{E0+a_(o;+$+eL(;%GQ$M?~7(Ds(Ee576*<@_<)s!vk>o-j9&Es2{<1#HDLbixXXd z%`1I6)MmwQ_bG=6pRbbB(D)>M+BdZz4F#r|^d7wRQliA`7aez&#qM_gR90ebH^IsTn)REo%Lw z+za{zc0K=cwO|5dJZu&~P!*ryJ~eQ$!;QU$;P;u6v@+qlnsV}$=KH5i&N`dZf2IIm z+X&=d**jK1z(ha3A&lv3Ytei^@2a&VsCAbgm6J1L^d(LA9Ti=Myuc2Q_`+c5GHq^T z4938Rf?AIO1f(;T&~M?gIn((jOj=82!ndUw#22JYC*2u-@q$ezqj>!gSSWo!g03>p zgR<+jBZ7i0VUaSobOYXzGVQ^Ty3q`IIY1OFG(>LYTYmInHw+ELjl&<=xnR;h9B2RL z4gGG5Rm_&O*)b_D-LRAxBY&?vgUa)+-N-T<26IWq6{I-BoDnH+Y2qLhe|_G+$C(*A zPQ&-TRSU2v)siwRdCTeJpVaw`i6RYJ7T)xjhB-9TbzQq1;+rpogQy0*iZ~$KS>i z`pkt8P}=1KdWi&$>e{<5G#*L{c2eNYX5 z3%Z`bIZtzgHDY7`a1OME{Iy9nhU|-nLEBc~hUSllOt+z#y>Es_pqyv-!!KidM+3=G zx6~$1`7F)jQj$gp<+-{gZX=|+K0-J)$QAq96vnbFPQlQ531bqNY47i#WH6e0b`j>$ z7jT;~*off+74uB6F_0d91yQMPTV1WTSrBGvKYPUXvrz}gX35mK&%E=z%Vy^!Nnedqj)IETt7d3K4BBfS*lsLAtc0j+@-&zkU=9;bWmD`amvnSqj4RrV6;YU6jV*};EOj;z zTjmgnCU{Y+!GDV5ZVM56sP7qhQ|IQ5^6j!Z(#yNcy-QY>vZ{GnU;w?gw>>t*Zm`R} zC(mAw+|nLIMe4)j+QvpWT(Rke=0EdRjU3Ao<$NLjp^SE=jPE2)>ryxCF6X3Ov9O?4 z!$Er>Z48nf#jB3?go+6TR+$7jJ#SmAx?u`f?-80hHN_RJX%gj(X;1$p>aCSZ-j%&V z7pcW=)yu8M^&g++$4e&cWL}I=vb>>;FkJpEFM5MbMO6^TNCVO0tPG>= z9?00B1VPYr&XN6u6`3ktY-xL&ZslzXg8?cR$yUx;D-qgdb9)H*kmzum?rIW^8}cf( ze!V(K2qu42ZLV0JHR8US{LX+OuQp7iZQWxr(ex0pW12w>1jP(W+^#L!5QPujcx{+D zVz#;%78PJPufY(KS9y&+7hkn8(%;G@B$vx=jw=Y}RT#$9VPSX=F{qg6zSJJ{ zG7V(Te05@<817CL-pQ}_0wgk;+`I?h-QJMj|9H=-^m=cirI>=WAiI~*oL*2DzA=w) zt`=Mql^kY@!^@7tkb`~vY7tM6W)}$4^_^#YTL06#y!Pw09HbK$@JNi;%D}*YDqG*$ z_mI@l{|FZ$!93HY=uZJazzt`hJ(2WNir-apu8g`s+)b*^WwN<7GxTFdisJvLYjn!7 zvY*cKLIi6QuDo(`h~d2Ea_P;NxF1R~QN*C56fXh?t~JQX=C(p!vbjk?Qg(LpY~Rxq z=z`Aht;aJQL0#dMj#P<^Bd1Okysx&U^e4RypUa}63ejUUOcN40h2X{3^s~Az>yVf_xm7_gVMBHM+mQ6x*TCaDy2JD`QpW_5}z@Yy*89M+KUP9-otr_ zSZdblopJV;?@~W6k_H(z-|EP(8lGU5y!*t}2co*(1$am=8}3rY@yCdq$>KA!t*y_X zEc>x_zl#J+3m0Hc=trBKwGLF^a)Lk$oL0CfC8q;w+#!}nP<9mSzV4!*FW8Z z6!XVY&Kgn6u=A&H^vW-Q*-eB|N_v|J1Z(zr$nx6Vpa=A-t(Kmcr9wtq2=%&X)Je!fCRQS05oA520g_F#EGB?B#u`m-tUZp zKKP_+ClXUtB!%Xq3L)x9L@P&>puleXVX}N38RtzqHE4B*;GsJqt|2##vSu_v*k=x` zvlky~2;Tz9>eSR!0$wdrDskczyKIV$BlV;P!+Xjt1K)b@`5ys1o>4|bFUEj>?4RCy zVs^T*@i^u}ib(3&Pxmv+slHGY6^=H-E~j?yNIzRFQn94P7@3#y*jAT2{}4&p2>V(; zhms{eVl>{7Z$FahSXUc~7vzGuZn4Mbf?mQ0Tn-{KZxA@AJr|ZIRI4IVb^tU6J)xr7 zV^07)VY=}}_3Hu~%JRUt^A+z)BnOce4H8WR(ys22$O%aeGxff=vx3179yFRyL%u&F zE#GPkl4BLaH3Q)0ewrHZ8D-NTb_80E3y4V*f^MLX9~PiJ`Zww1z&9lsHJ zP8FeHo?|GlQMR=-Hi(sEtf965g)7e1JMo8Xk0q`TDehK=kglhohW3@SH$`9qvMrLQ z%*a>?bpG3QNV$*OjeH?2R*|ZTFBsWV7&cO8eX81JG`C547(YHbfWiQPV7BhvpIXZHoBG>nBetYfDTZWN>o zn2vaxPD{p&-?ANau5(vy)$Or-9P7if48nVDL5oD6M=2dUxwum0^v-0b$dPq?hl_EI zr+$)+on;%a2MhBOpAa@8E1*QT0*qK{(pe5MsKEc+6rF@h&M_DAnmmSL11M$bl=sFb zKWi=PvQx!f#B8+R6Kh2(It3nIwKVP%4_hm^K9v z-*?>QHdDV#{_pFPw|!aMcZV`;Z<3lWoF`~JlJZTLNFi-Y&l)@0>J`BdK$shaWr>lC zKJA(9Kr>Gy=wOM-QXv%!b&e5H>VkQhMs-nr5D3g4QC;SN?cM3E2BV?z6OgUdF|6OC zER=cOo;(T2w^Ug@pn;A??9O#$Kis-c#h66gC4DollaFt>FZ_zpsh|HXHeW_EoTJUl5{flyFFVEV^O5m)*k7j3J2S#UX} zyN=}ZPT*)HsIalWnZGgU|Ul2z<`jhB?oG`jb)C|oM%}u0iJ0POevAisZOy;M5@vXp}I)!0XC&Olh zS(U(;Z8*@pn4=&xuXoG57U=@Yt>$EB4_L|3UV}ALa?~SJ0KaJp;UkjbvKmyWKiA)MhDtL-l1W0T!oL6Ct2$a){B#`^Kz>U+qqBVafDEh}DyCS;I(ZX)Ru zfE--H86X&f(5jt=tC)l1BLs3^nbH*$nv#-IhdtQMH(fpyFGCS+XFVE<#mAh% z#FV%Q+w$l|J_nUQN&|5$=;6aGnh$V6aFNv>v)`?RV60UBNQyI_$Lh?PGj7wj0Btrd z7uiD@PKwU=<`r9o$>iQ0=_TV*VLWe;U%&x$0Xa=cw>uy*uBK#?kt4B9lQ&W`g| ztMRw0Ae0sYSo3yzRm~wrKo{wT*52Q`X8@dD;)6v=V@N%CcB zkG+th?1g=p%5#d~uLu*g2bz?4iDemS6S>LX(-Zkx8}Myrz9tnAr3Iw8@6_jD(s}v8 z=8P`2Go`38nh4XuEUSaLix-V$LWrHj<(JQ%#rX!~JB(;-OJ8Jh|$eUFC5G7keExm zM>&}~ev%sZ5dq%a=kjtjmY);3dc3Ap)zy=*oPda!Zq}kcjXG`tLjX5mK$F3Rf|HkvYluA8&D)~E(-@9`~3dujoG+Y-3zHlfoxBM+clDtWZ;;eAgq5Hr<40n;r|g|eb5$G0uj z@acipIE8d=qnfvG-!8Q1jnv#$0iJTfq%Yd1_|y_A^28qLF885YgZH1t@i?ax9ufg( zO#KGQblqJZ0QfO}0Whf}-LTjNpYEm`w3kQn<~g8Yr72AE$l!(;=34JW*sN25*E#;P z)Q^gxYuCRcJN(}ecKTobf1(b|tU9)_*kCZ1_uh;2C2GI7xF~q{82t^H)~`_{|5!(7 zWBA=e0R#YzXAzPBP&nV(cX+Z5sy%RB*o;;KgE_Dfs!ee1_<#HJy=!-w4A}7BdY9B2 YXA%qaJmd@I(0g4}R6Cb`_VS(o1xfr80{{R3 diff --git a/quadratureshap_n8_speedup_vs_nodes.png b/quadratureshap_n8_speedup_vs_nodes.png deleted file mode 100644 index b7c94b7de26f3b30a55c1cc6f73406b857c1f5c6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 71642 zcmdSBcT`ht_bnPlKtaF)qDWJ$AfT@z(gIQh3!TuL1*rj~69`2_tbhej5kirc06`%1 zZb3x|H332usUd_G0)!;ziN4==erKHj?zm&zaWjU)h+${%XRl|ix#pZ}KQ=Ma7vK}) zgFqkxR}FN{AQ0{h2!!jwP9E?RE5qb&@Rw?Uo^^n^&)tCFn|>}3!SjqeZ0>roI9&<>5Q~nK!C5ms-hzNzn@U>@pDydwhQb4N7?0TVB-&g9L93~Y;zQQ zAr9FFfn3$SY!UK)ewg>oeiz2tcvwmfeeV9wqdSipn46mH*dbyXZ8gvBwX?sIcwgXS zR@Tu278Y8jA2ZFq2p*NqI3;~_dv|J7H$%H&0x`n=-FCNZVE(1o%gJC#{ckUCoOt#9 zPH9-X2L08qm5p%D1#wTuW`K+O^E!7KBJl6qN4vKF|LV}+2r)uXx6`LjFZ|>3 zLzo~OW$)gT`#e%=`?2##UIeMM;jj>))TM1}rFIUD$tr#JNN5e*+23d6@4G|B5|>Fn zjRP}X*%S=6p>(n>h4l6L;i8bGUqSsh;|7*mrFYHdH>`tqwjKdyYB%$4-! z-0dN}LruWP6Nc~lGw*D!&aSPId`JnJteN}IitH+FUmVpd)Q;t|tPO&D4pn_j3t#?D zXh4Qx?5q90h3%&s#usN6xmJBHR-0Q2F_y}S$>*CG^~v&?tZpFbFq=R zMoOg5mgN(XL-$$-Y+TbOTi!2Ew&~Z|L-q*7@CsG5rz+c3`%TUh?0gKXlF-x}$lxc* z#%;SGkVGkPbH~_)4z&dDMrKeLwVq!3;9l6;{JlsE*2XmIqDSwkTKY&Be5@%tXz7b+ z8S9sBq^{;_r_RE(OoIAy;;xKJuYp@chu}0dgMDXYWIhEg47p=0i~MdZC7^pd2KlOnzjDh2?K|H>TXV_hb>&jABOnt z*CICsw!c%_e=#azrCoU#uWF*t3B6^05SMQTsEurIR}`|M$iBL}NC@^aJ#^}A(B^EO zT^}0eke|1>F_EBI@#U!~>4}Kk{LAYKAFKVRYfAXP2{G0v{vr+M@BZ{=xhIY;nwFziItPbqauF4&@a6C=^=|vUMV;) zDy&MTvBBjnoQeJZ>ZDy;vb+#tV4Iff&0{SM!jIz)xPZWU1`%sILIo!p1m_dBe>`~L z)Hk#h)-gG7Wql%n^i;9IEqLf&96uND?t{)QKVE-qKr$qk$$LL4-}$yDa=j2uz543J zz`0iO{eHIG$2GSWW9pZG$R)oM8$BXb;WN5`?YGRSNrrd4S6S=Qm#W6Ug6dRfHaXlh zIg1qRJE{-Ay0$RvSvee(8j1X~d4I)R-nmVDxh9ypAcV3jcN>@0*NMe@=e>RBQ90!I zw4Sk0%NoJaMiTE6rg zTubgIHB1KG?wCa^dX|LqbWM_l`K@$~1!{z?1;}4LszK?s^Y)R*hdo9(5wK39u!^~c zCX+>Hr?{z{P)8)Kk40K7s}vv66KRQFWt&vSz3eHUOPke5;DHF^Tus$>*GuAjzkB;? zz^1{WMhtVQ3<;mGIaPD1bf`dre6b8ghz?D*_@R5XXiV7DTYrD$a~~mOY$S>u1vKT8Ka=Z+Z|lBcl5aS zlc06T2Z9Il{p`53J?w~n9+HkEl);Su#weRbguU?W)RXT4YrIgQMyVAPzqF?-+(H8C z)qj?L2{~0R;9=+0oBx6wLxuOK41ByDr^f%mrtI!QnPY{QjdB8u$*66%li6EWEO^(J z5>v;hH~*!{Ts0>TRhPlAj>AQY4o-;KoCW`bhI2jCjMOL>!f1U~ zOh)PZwWVgPaY7Hvh?FrhU0zKX+xs+zR~rpO)@K{xtb|SPo~c1 zp#)Xh8`&(fpmbQztl9}--11UxHAn=0TMhiI4imO(SKwl>Sw6MJd2`5ZM9c&iFx$P5 zg-;pLug8Z@B3(n7sQPHw(USoanbWBON$?P;_=ItAV(xzRA3X z)#Ej!G4p8RrzLdjs11yv2rGuMPi@zgg}1%;DyAM@GI7tL|(0G6Hp#X#IzPW)(@s6 zCr?SHzI^%1uFCgjvU^=&D~-3fLkus1rkzAO!@s9q5RcPz!fW54~Hla>}ys z4YjCpWSK@C9gQUY{{E`$r+#H9-E&ZRG`z>EE@bJ<%HeajbbB4+rbNC^coGWV&feRM z%qQu5cOP6Bu4SNHo9}Zsu(zf)kE9&wT%(YXDfWI>L&Qgu&z?jS`;u;DB+J`v<{_ukg2StEAxpQMK+!om z*b%{`5lh-OK}Op&Qki`1CFN8T;6D2K@%~Q=c#yopIcY>uNl9tG^n%aGP-PHhuw(_K zWK=lLqmNGwbyV-kx?|{MQprjeOJOJ+z$! z$;09s({XJ24%jm)5#HOKMm5=66Kb?xbN?YM6D(eXM;|&CsVQe!IK$Xl!RtV$aYRQl zY5|;FhaVe6KN8evx^XtK+7+LEre~HGN}A3|aeU_@2`ZI?E-DX--2ClFVGJ$5K4sdX z(Qvyp{z%>iIPiH5yb{rVzhBKXDsn{Aa8Z??)xb|nd(m@zvDO!qE*6QFmW7`J`!bz! zJN1J5uZpL2JGVOcg1Z7T9aD-^&e;6&?990QjiMI9E^rZ`s}EXk(#Rv;x`T5uq#BFx3y|!#AX;huB4v5Pd#+1Oj<<5~0O)>CWj$ z#wHodWYY(?>!NLO9>oZ2k=}R26sL%{14gOp zfk?dD{%%s1(95=p>5cX4(i0sJbqXJku!MDk?!UbRf zM+Cm*|FCQ~Zp;564%5Zhp0>s6aOp%We?Pg_t~}Zg!hOuUc4I1am=e8=p*+~!joD|} zH{qd}IR}NI-8B$v^8*vIym`i`1s2xhIPB&{x6US*oQHGU3mN^Iz#P64uxQUPzJ=k! z=|x!tZ4iZ+JAFMMS|rj|Ga6NihcbR@K9}9fa{2U7fQ9lWIoYBn!B=+HpUm>O4Rb~F z8Ii`pTFnCx<;x}Bk*=ZEx-skj0sj!)XTby+(9mkK;39O+xE@ZrsK(w}aS+La#$+bz zqpaX7S$tY>HN(`4fv%qU-OEh+XeBC=HGmWEf?k!6)`qju?|Xt~U}he+0u5+cH`v9w z2zXpgBPiP5q}|1jgp0V4BbG(BlPoNse!UTey{bYOfOxp9qS#xlu$eEP{P7yDx4%BS z=NfDuJRDg3j~(_p={+I=qivq_SVYdUJk`7AG#;N#-2*2&CY7O2;_CD7nJ-M6HxH0a;^?r-A0i=wW0a(Qc`o1Kfj+Zimge(G?m3KSCF( zr)!WAzBn{G7bc*HuIa5fV1Uo0ld`9~BX0B!N3V+2^KFeq>*26wfn{!Cot{0V=v$7| z$Fd7Fn@(3*MT-J79}Mn4ilnEhO=yjti@radHtZi-NiHj<<{1=pJ7D!(ueh_b@de)e zxgJv{0Vh7IOuwIiYJ$M4wu?jWLup1fkeOJz2{pDuq1*LBWv}XuQ8~2Zn&U)grp|DX zm;uS?)TUrh#5LEO`I|Pb(X+wQ^_I`G_lBH^`r5VF5p`+y!>b{B)!N?8XEXCBDO8$j z;DOf-6Qyb~H%5y9Ze3%iXGNo6+(nctq)O=MriOD@mY#Q5@8+&82N^TOXgSulF`}N) zIF*BcN{CH1#_EmMZWfZ&R4dDeHYIoIqt2yW&SaL%DDgZYQ1?I}pJDbBK9EP*8!OWc zxhobo%YvyR_IR0+cl6RWg*LxPBN3nKx0WKR!r7PcNEe7Td&9kov zSoH~e4VJq%ygYt=%KF8A>%Fp#-*-n3Q zQRQYHr4M8E%^k-KBoo>-huahz-@CkPl?*%6vjV{KfO5pvy1JDbcZb{okxq2D`maYa zTS*P*I{N%jb!onXd(;#`|F0?8Z+@Gr)o4x#9)k{COy!t#ZN5vRTgy+C)dMNZ9C>W zDm)L$K@c5QY(HQHS{}N8{*NV+cWt{SYOE}}Zt{#>g;)PI7@U;Zaq7|M>;&jsBx|-O z&$YS{An%2S1M;@(dEg^9x$dhQ=D4bb9vk827MRxbWY~$4?Y1d?lRskZgOK`UsG)hj zNv1g7m+$0at=voGZC@99#^spOG)t0G+1>W7HkI)8#l)W3^RqAm%s#|OU8qg&q4Rg2 zu~le}cj#)LK=3kq9Kx5rG}NsxeiCA?%na8C_q)?r^CR_)jJ4aD9lk06Br_8l0fxA> zrnUnDS<;0Fj3v^(h&ruDnrcqHxxIWLA$Sx?)Jy;ggKDF5|8(3Vfvzsp0{_rQ}VRG9^;EWZjq7G^CaeUxdzxO zr8~klWf~%wn+MBBGH(IJe_=Rbhzy# zV^>GY!9`b6onz#pt zo(u{VVgS09^;BWGvTG-9YlQy9)?mkq?Wq9Jor%hgMzWC=yxYs$(T2$j<|VT@@=n&s z8;!7_{yk?2+XAZaImsux@jivy-Ei;tra{;&8)LYn#%Hu1XnPovvh@?r(3L%UzYSXz znB|C*;%CB)dpsfla#);9t`PO9j@+0Ob#fHH0YQRmWJrP5f_Mfs>BbV=2GLT#=O`GZ zlWeq&_M3KLsi~E_L(#F9rJYyz#HdwB@{GQ`LS4l+v9q#6T0yFC4P~-k5qF0gRlO~N z_`;wUAwATTq$)6S_}Ephy4dWV2BpNrnety_&0`m;-WsV83ch>z{OT&8dhloVFgb}P z*tXbcm<=LMKK-GPlw>LMN`8fM^`U1bMY{qvalxi)`~qe<(TaV3S^<0!L0m6m_xINv z?QXvGsf)39-I2T7*&Ly$;g~9rFvtkWizN#qaAG_4iv)Cw#Olm(8iN;gxjxSdvweFB zm-C2EiRjkzH(PeF*9zCf#de=l3mys0`_jhY4xqdPrgB_9(m%&&k_8`cx88l{6tvpl z-dQr_tvqgovPyl`L=*Skqc4ztVD6qIRzA*X6l|;K>>%6?lk5DZ*LPxZ4=d!clsF=^ z$x~q8P=w6l`5dHe{$;i7@z48>SE+F^B+n48R(Dheoa9-w*F)7fpSbtB$3W)V>49Bd z=W;5XC|5_Z<7?-l85(eGG`#N@4v!bW9bB@- zWB*Nc~)@gh6Bz28GdfjIC9n-G6rBfw|&Outg^?4NtYFGPk86cJtEHMI<0y z(fqV?gz`Cmbb6i2_t}lWFh&Lu^^&G+0RQtKPL zM}k1BpFUBsze>Q3v_#kdz>s(;DzRXTDsx*xT$KCS zyG<4;6?U69q`4GPh)F4Ne7ZStmnb?&KAnat_Zr#u!bA|z7Kx*KH}u+a?;UPgTCqbTV%0}Dd;ZHZkXXz@m1}iB~B+o6C+&H)SoyY^*>1~+^ zx<2Diw-w7_ROCv~S8m!ko+|%Fc6P1K+17ZWk_%b|{ zENiLs=50%agE0EkYqPWY(F|e7D@XJ2iRs2<3r0p0-gU9UA9Nj)p`64X=gFX(Yes;Kbnis{nEm#qX9mtX1vFP!c zvb=3KU^eLUfvYzBg}Fkzp9emS71auhE^Ij`P%K+xnNyy-vVG>gY#b_l`KlqohG1%C$o2#?t5Rj?P$ z{MuYTZj!MYIbmJ(bo-%LJG6&|+WoqRc*)pJgdnU7(Ohj6nrogRVtwk;3fkn;_x9;C ziz|oP;nU7j`|F&ZO+t^o^Q-l0CPVk}VV0NeyZW&+KNE+35;1N<;e{v9Itv!^2?~01 zMG7i80M1IAhqCG2`&L1DIN%ifZ?$LElC*OsT_?Xt$;GtCH$?PhWb&U1oyS~!>@~EN(Yz}ua(YHQ?YxaD;v;h+{VMx9v3Zu;m zyCDgE9$NCJ%8>v93Mu@y`r=Zx#<%%?(js#X*|A*{)yyVdF0Hd1}u(qTU&02cX|7~B!hUA@{ zUUi%iYIhSnHJfIMk-j9>Xa2%WFOvtcyrVMsQBKUROF~m0Kk}`>Y>F>#_qdnk+u>=N zG3Shpb9L0z86?yl(27PG?@~X`aOP=!LJ0S(73y-m8h!Ue5w~EF=Al!{3}vL~%^rXS2 zz1&Vq#hYnYGq@5DU-a^gNkWjK5vF0$LEfRwrKgTrB;iCKpi7lX%rvTAl4=Ql?}4Tf zieTxy9VCWaE)YB_e&cu;2=35KJdS4>EX(xJHdCYz#<<+-x%l^kI|r&knP&0^t%~zy z?5AORaH*gR4f$9jCYWmCjm7r(KyhRm9Kd zKRILH*w@78151ycNINQF2%@<;?~ML(YcbvN)Mq}Tw_aauSE1$Z!jUHd8I5kccE4LT z8-FXhd7}>KC@lI6i$M*JyfYoJlnhIvJ`Nu&X^G-+a8lOjv>R1xCsaTPQ-z=cYPO_! z+n4$9N0oJ|ICZRFnULwDw>^*t(HdVqQBL+r9#I~@Zq$}LM(}LU;8F^mJ%@c_w`*SN zeVPG}OIsW0=tqIpAQ5B>gClPG1-WkQc-YE!n~UVJV*vyEJ`cbCZpv9Mj#h1jUuznn z@~+fR)CI3)x zzMgjI36D%~RO3j-W5=lXcB+pGsDf1ge8S=HRZx(J&VNj2LcWMt=?19c zc%FrBOr|i|k?Z4eI!4pX%wlIXd+s41c1W#lY?rI7yH;Ux;gM05FRcDj>P7fm1i`&C zIvc}l4zn%iEqP2J?-aC<+4ZcRuDv2auklAH`d#M|2pvbG8$?2MWt#AhZ7=Sy+=sww z5>6PspV_`saWxcZD_b$Dv94VX5R1z`BH{C-6)xSGaPY8eiLsS6{gizrsJH@G17IZ9?@d zH+GHBvFZKr^iZnPA|kZ7bE5rTIUE1J#;w;E8Z$fc1CMSl zbh2eMI`_8!CHpwHLml=ZAKixWGWEGdO&~b)`8?cx=oexwG;+}ia7wJjGjDjTmRBhx z2p+HpiqjEdA4ejS3SW$#oboT=GS1u%r#bdTKO5@YB%EIGSlZ*YwZ5cKP7?$* zbeOa(rWM9G!aWjw8LN+KKI~m}ghYKEqq1i#HUNE)FqWFRmM3O!h$WIlXPG|=5Z3U! z+5%cLOWx*6^}!~`YS#{Xz;Z4sIpRqs0i8t5?xdL6ZP>CGb@*&$V{T+KT*;z+5Of4L zdSb&QUMyGeQph;T{f6K+`|FIVj0D$p?Ch*KP^|F`-HTfo1-Jc1CFXei8n+}e9t7=5 z3BD-CFqA3oshPJe@sq?Vi$f~NNBPwv=e!h(_>oYYv6Qm6eX%V|cjXVzeQdfLBEGyD z7bWW@yif>eTyAZF!xsaZAF(7Fk^36jcNa_MObC>3pT)-y=4WB`IZZ}M7tl}o{0c<#9NPe%7p=Hu zfX8l|18ktI4Yw1~F{}uq+AZb0`N^~|SrukQ{@r7&`F!x=zII|JBXCTsuL2!lel|BT z6Nujn5e~fRHE3u9kWueNR?TVpAoo)4kc^A7Wd8Prh5BjmO@_*y3X{% zDXHbi8UO5(%gv9_!#Jcnsjr!b61@HNdZ!=xe5P;c;#jkSG@5Q<{ClvQHR+3#ux^NCq>WyM~+wZ>DCo$g^( zK%yPqtXgd0A9gjPrVvoj6~5_Ji9JI+N3a)!+U>Z=vt`8|{`IbdjbRM)`bNiEh&-ku z(-)Sq*|5@puj4wAx!ohNd=+RMuZZtp;Jge#C^Z5s~R6{b64{Sd@P^}6W z37xKvSy8RM!4RgbtTv=0*9I1}-X&WxctOid-95;jt;e^$<>C9qSe>0{hzz*$J{d*^ zn|xFUA>arpra&uQ6WtoAuvE`TObkl}7uMt2q{cY@u2BcrV632!s4|Z}*-<@Fhu_z) zUN!w^@0uCf;`C@!G#8RrQ2g6H3KA$(4Y$5LaZ$s9LPD&(-73gt+f{l?_(1`cze^4E zNl0@@*?;PW6MbQrSqYSt3%~0)`n%f1v-72CDe2Rc%F(s|OSCi4T?yPRp?ZO|(OYP9 zzHJqxfii6P#_|Ak>ccCe{iiiPT#M|y!a-1aM zR8}BIb{ySrtiIPzGPAd$ZDdE>r6ey`@Z)*3ubVp3R(3e~ik+;2&<nGN7p5%z=rQK-JjB>Y_a2772+jkGkn@?1G9up2j9Dm zz8_xZ#;fH~TTBRrHWLJwW7O&fAwYNFyry#)U=Q!_s`D#*7G2!1IrnJFsr1g((Qhwg z{Fws+l|F3?`{vXX#Qp@5G4M$3xdRBO1_rfpG#E%>3{VcHYMW1)`-IYkgu^s((ak;l;yj zgd_7@&ZLP}CPXh<2ecyLK=t(R`Fu1gevNEW(w%c%R5XN}Ct!ei4$UP1plBJ^C{1-i zNiNa5D=XILTjG(1`Y?Kp`>nlNVSpq5T?_JrLw^xPT3){0Dob~l6KEO3fjwze07#I^ z!3#rPQrHaF{*s&CVX8}?_YbR=8@g-EAuXyxC^#ZVPM9WD`Z)Ew>>4W0-m9hT)!!M( zQF)QoocLf*kcn8S6o@Kt`Y0ne1L)9zna<|eLg2LtroO630;CsXbB#hCX`SB@w|B6( z9EtN9=!TOAjXa6Pl!{M42m3AC-{q>iVr*<2Os%cB31?79m2?`Jh^GV7&eSC- zB6$+piinhhp?`BE=Hbi#AO(rQZsl~5!pIWQ3Zs4k>cP+iPIGasU=TaY?7Ez2Fq@H7 zxqC3-4v;yatl4t;xNNw6Go}JF5wF%^1vHAvdiTa)(5cjF1pu7V;MR4yvI$87?6{~O z$HwBSDqRqxKCZio%!6+dr1E3_Fu7DRwlRq6>UgTZ zuM`S^;sX7M#17&X&an>vQSb)vD5#P`f9pdeivA$a6k+E6Q86!wfuO1btspPmPm?Tc zlJ-wJ(Y$h-u)T+AMo&wuQ`vF?PniWn*i`LF3}yPT;oPU80`rsf6|gq;Lf*jXGiK5( z{Q$D(2+ab9cQ$21X>kHdO(GCTbId=8eWwrdf{!e70r%GL79g-tG}#*o44asf?oDC> zA9g@4T?p-4$m9rEfG~>wsT*T(5k#9*iXix{M~!TO(A|(nkFMB60y72$NESGt$kdV| z*2j63|51aCb@miHs9|83C}dak|FC+RY)gY&I?5w$Q)pM&;gFY?*BZw8_-H#M zG4axb#J6YvE{$EMH{%P%k4 z{cE}KKWmpLiCi^JJ?|P1Y+pZscf=bMtdjRYkRvJ7Eoy_Jb{`Zeyh82#KXMK=$e zqz8m{bYAtKj?{w!WC>KE6p2f|(j!l~jvb@81)?7t!A^thUnj~5YselwU;*?m;s)+I zZ(x?ha0*j`fNJ3Rxj=Zv4>%Li3r?wnx-~-e}nv!93Ge3Jf`-o50Uz#UgFyQJWA?d@elRuK4p3)4POtKnc z)CA0$0TMm|27G2>%HH#XcI7^!&@7_(I?E#+QI~uqdGN}fOT0j*X&my@0JVt%vb;Y@ zrv!w9o%$ZFlZR;cY%0Ce;3IV^BTu&!@f03lJgRh;}P*DZf}!Po_q3o>nO<_P)*B z3iFoI%;h%4+G!?WkaPs|gIgtntIjg(RyxYgSqhFP(_xu&!qYX2@f>_R6r=Xsl)MV z%9)w(go54AX_jI z3sL>PBMz_X_aZs@Nc1#1DSnQA+%DKZA!`FRl$;Gu>m7^gO&lTQ_tA~`(W_wyu zU2c0+Vqa(SI7yqp6V_#$X(I* zO{myP=FfBu(osNNsMOL3b`!?l$fpjg-tbQ%_qc$aVbuN;^k zC~H-YWQ7OY)P;QX0qf8J#EQ=WhobM~;^UwvH|s)B63F<(U~r5;P^S_Pu6y(Np0cY6 zC}@KTs?DDr1!g~gCjHjie_X;j*f)Ryvc-_PB7ntb5^PHEJg&upUuBUji|uElPNKZ|mC%tTn9Lk_n4#Pn%a8I7@uk=At0m2WUD>^j{MTUV6XBZC5<%r}~dyrrj zI93CfYHBmDQ~;e*`cgyzN(o3Y7!F@)w@G#gu*=kr9a7y*XpD>uZcOg$0YEjFR2RCU z4K;ps!Z18Z&PuSh5EwS@t+)-8x}>j^nNWQ)z@IUO$ZXDCL2}6Q@9zWNDe13_p#zQS zz;|+H6y)S7kW^AR_+_oHz`S)hFHl10t(vJ!W9Ec48UJ4rhYrx5 zZRdoYu=rdn@HBS;7kptqa8-11Jl(_}m-|qgcBUd#KWut8kG$S*bm>SNd`^Obd%3s; za?FD5iV)D#puNq#+jV;n9C_2>XXW{{DRjEr9=OQ;iRe{3FS$nNx*O<;@lv zRG)MHQTLD3mVE*}rmJYOM}`V*ujS2``jLl$&v`KeMdC*Sv5V@}y8L6|T;{?nV9B`q zPN8cRR8;`tei5)lpTQm==89?o7q=>3c>SggwF_8?!%W)H(ge&qLR7eUo>5@mcMwh$ z7**CnEj5e9`ikKVs7KK3M;A?ajkj}oV4IXQ_iCLyGUyEn=ipZeWL(~{uHPYeuJ^S0 z`kC{t9nY@q$-B&|i{1k}Br%kGS)_SD;|kpYbdPTr%=@rY$H9C;nXUs?Kbw#k5!F2nsjG!YN^PB^)UI+a=*IMNQ%L%OF zo8yE9ggKS!bEVD;Y%%p2h%MJ$H1`j%mr957o~&}t#$)+Sz`= z`4P!7+&kOC*A%#)X}^12hT)NMVAGs$L^3yNK@-oFuehjiytacT;h?D~CI+CO$SwMF z3AKxNlzdYZ9PBmcI7TJ;xE9NV!TV=yNbvb?k!*L`Im_B-z%%E{ zxWrwBWUK{Z?I@~ufEjiTV1U}PWNg(hK6W{0)_^}FY;|UQQ#1@e!SSSnNb-40a5zhQ zM5s~c7?{*8Cv;|2VBaMubSn#&qU#Z=paeN!-r_acL-N>U(h69(s-UBwOvhg*?fHYg z!EMof-ZI@@cH*6)3<4P!lk-8Og2H|vRAsEbq^mah+I*ff>%l3F(Z74I-R%Bwwbg$no_hhH;9c^WQ_8us=DG?Ui5_uTj@{9H zb>KuE3MdeL3+2ZdxcoD;`a-94mmf0h*z7ugsNxeJSK?xHLxc zGas8;Lsn2OYH_aT<;=P`1P!>@BxLFlQWhTku*FR!vZBKUef)i-vX1%pXv;OZ7#MNd!J3+~R)%dA8AlLN_0 zazjMO6I-XjVv8;bq0`X5Z!*P(%Q8s*fCsKX@390dNac$!pVrbCUJEZH=kX$`@Ydqp zayIJibA6{=pU0!!a<1vJuXEeX8l(hif!-2N?@@Vab2L1HZ#y;L^XT(cen?a1GwNN) zW4H09;tH1zt03vy*qo%z(I(ff*B_Cua&kpJmOdxh4c9_T@8H9t?#J`F>Hfy{*?G*c3L`iuE{ znSd7?)~iSM245wr!hkW6^V0ggGh;771N*b2%bZ2DJFzc}E>V&bI8Lyg%6fIYQJOVYG;Q5Z5}mAX_t=q;JL3 zD=Sj0AQQRijjqP(jbmkH9ja{(RM1z0UHf{H#cA=otN|Ab=ZyL=vz0ET#tZ-1P90Vv z1kUaTolffUxAn4()?19NaXtkAkEl6^Zt<1VLZ9WT?K11DtzoYl;} zpPVGlX;Ya_F^BHnxpU_oo1?;koR0&(Pdm_|55wKv-3fJ}{#F}9`KB_Q2|gr9dmHQM ze@|oPXw;Phk;VPueYjHRFS!q#+U7{}4!sGWmb?RgPi!?KUS;ad4fuuL~b5J8WUVj`gdWMbcEdQe7tQn6K*VfbpOGAH&(i%^{^5|92@!NlWt@aL96U-Hbjx>I zNedpCMOhY|QwQdUM{caS6>$TKGLNA6n}WG&!0%grFxVUETco9Gaq8bW*<<{+zS2(- zNd!@CPz1TI?RJYZx%8X*CbyujvrTJP9d8p&o;6N*Y~GXsr8Vb{VM0L>k_ z9{x!{xt{}yd8931xT(CS2m@Z8iN6u9^$%yMImt_pU-Ze3cU|G}YG`e<;z_`q^(R?5 z&T5ZwWCba8V6M;~zvo>Kn2CkW!m^IiBRUVO_&i}+yk?bj5=3 zQ$^mhYU4PDS2Cn2_EY56Qd~WtZx$|ce-*Pod-g`hb5M3_$Y2cPVVzTSu=5wCzh{(K z2ojAMzJpd|FdjovUtK4w^%>78*#oclpHRhvfze0cmf*X)yXW6|7T>Ep{W|?3+_pb7 zgzEqT@C@YQ>7Q@+`)yh|hJHytW3w;E>hFOepM|zv&_*LeY-%U(PY8NYW|rO=m(~Kl z!Cr9&Y{&tQZU_f}$A3Hkw4@vskVio5wYydeAVvBC<#QJ#lS&ZgF7HseDPXX&+Jto2 z%@^?O7oD3Rf2b*-0c~uG{TsyA(ot6`5J6Caf}6iZI-!#rp5T|DxU<(U8{o=?X~6pZ z6gbF{FqGdoPkepu5ETyEDm##-3PB7Mw^W1q-5^l+NkB}10|x2s6riD$BTp3_kjlB9 z60~AI6Aq>>s-pm1G;%NW@027&>jFrOg{@&}cY^H)`=5NsxiS=_)1x`E@?ksfuE_2- zAUJr~gv@ zj9q0hqj3`o^3ITB#D)u?rRT+KId;8p>$PyO#H?LO_=X>;+CMJ|G}9i0 zLideXP{ro^?nU;0ma_ccRZMv-0|6C%P;vNF$gho!(*~UR&5o{9QX0NsO1xy$4K zsFbo&v<+9U(K`k1dWBvSYnJrlgfjyxk{KD-pVGHWtDLfkD5ZZVU>Qqmq?eKK<|l>~ z{=pBEP9@)M?LFnwDgkT0mUK#-`sLq}mZ-HeZ4(JkfqC^0`?{e5+%{xZAsmeYT%`v>SD4%+orJPlKjL_fZij&>$CSEH1I%u~0 z3urOz*qlPb0cLi<(!CGVk2BTsFm z3u?l_jO4-}#sD;+kNMohUjnFcXQDMBSW^>B zptjZKJ^M>to@#*U_@M5qCv7&CCyRjZDCqNE{eCda=mDx2{-1hBjN?;%OO-9 z4@M{h1GT|e&WJ4`Q-`xKMeS};Gk!_nF0)=oY#^b_XWXF&ZK^{*Rcx@ zpwc;z<2kJa?!`_Qa0%YNIvg*aj?oHWh)G~5a)JkHp?RkZ4@LwVIP5T(=YLngjBA+y z?OILf)9nLVAQKC0s2gk0UehQfqULjL2kxP{BSpj|Ky=nzJl1^o=bQ1C(AeKu?&a@o zojB~9ol$HLz&Z)+b0V@9CD)Q<2LSdTX4Cw~>H;G_2rGbI{5y~or&-tx+cq!;!~@`9 z8h;aDBAnP!aeptwK;1%XA~65}rA;}XdKUi09H8B5EqB%edMvSrW9loesRHSp0B8g+ zxep1@O*am$>jgTs3M3;)ln2vdbs31ZMK#i-gyKwGVWX8msHdvqBruyAqXKEFZ<7(J{eW>0qisY1=mpxAl>1HqD} zQCr^RZu_Ywm)-U(-&J+jw7_J36fCr;#Hp?L>0ss-c*epS@RVBvEv|2e3(DeRU)Jh!lrq=ae0LH+7&qgNp?SYoP`Z zdBt$uQ)p+)vGA5%C)XK9l|G{*4+SOWSZn}fsy!xRGD~c3E=6tQvR5j7ecJphZnTq} z2xtK1wuyzh6(hQJ>~D@fV5e*g?eer@JdNh!tzlzcst#JH`hUfzW}bw=)z)WFQuXs^5BB z+3{n|R6=~eY7N%}u2|XdVzXm%Vjx*DJHT+~^SQde86q)qXU4s6W_noJP%y#?2l*(O zU#-u7fIqUY^cP1%GfyVoiBh1 zRuG#a%YVj>!{enQx4HmRH5LIRf*yxGQgd0MVabY4K#mXPC{N6A;A+u>H~d|RZ;?AX zFa0jSeG;*KQ=I0I%5~#Vrxo#kkrK&SN&ZvK+f45$b=_`7!0~Zqf#vETcFD zAZxW`N0#GH6mrexTl3C7UR!1cS0rI9#xY^?PFW;7BDiQx+QV?)_Z+Zd=WGu6WElKC z+hZreQJytPc5tYG)1w21xnYrE=*o_36wr6pbfd1h_4p$!LdE$3He0ol z3~NS2C}e9h=lU?UsMZ826uI^91xerA1NL<}(2O{KmiQhH1ZAO8fsi&CAz+{H_(SZ& zB6y(5bGOCj7*n7svhkqy#DDj<0P2!z@WOmV1Zd0r?2Vga4qmx^2nzUjF99r;M3tez-4RPs}P-m3Y|JcY_*M%oeLBK7)+pJrmN{0h?9?Ge0Yw-k_}pfg0yBa8r? z>OT_o^+4Ye*&No20aNvU@)w0`=4|^qIK?;BQZo{?6K`#=KM;Fna!X^~ z`SHHv^FQRU0h*(MM2BKM1CU}U9J)UfUM?QkYr4kC${XJOCC$b4YqoMu4Rx@5lV;!% zf^$eE@GP4eS{ChKA=o<$n;g=+Ex$tS9r1XI{l2ahvdV>-2_VjS0$IwhzN&tjIQ9sA zmO9$lcvyVj-=%#lCqKdu`TPltl({@TAn&(r6*O~RVAxXOr$yV&*gC+mmw^tV=IjPw z;(lHXPX>h6Fn6|x-V3Q1A;Clce2%~LYv*AwXvF~smW9^i*lY18>REt?teu+%99jQ+ z6@UAKg?wJHlc zx~vaKfl~mY@juU2X}o^}exmN0TZ`-8?;T^y03_yqyiH{0$)uYZe*!==B&K+~aEUr?MD!!z@W1N}K-CE@trkL-zph<(d>_G~xN*jnR%arfSVSoiPS z_=QkJi{|3 zG@T&-e>%xXLB2*Pz>rx)0gevfeigfq%$|k2f5_^Z7MT;`jS5FJx>NIyJM3 zh;P|dI7GHW*Mm!W8s1#zih1bAQEqCc9oO;AM31wEg)8M&!Bs|3OfQ?7Y z0!d|1VBkw(7P-{MKH*35k5;?g{Qbdz{qw_aA4h{6seve4H6zn*O892i5t;qJ|6H^h z=W*z62{PL>UD84)(U?-_};<{Pxmi~VKm_J4G=;q@p^Ap&A{{uy6 zEz6qO@`UpLo_*Hull(7CzvhPx)`M?&8tz(CbC*N#yeifB8w%A_LQ3R`n+*$;#aAej zm(cU9Ar=%lj=kwf!7n|SyKn(ajO~l{F19dFpaIm4O*sRA`;I7k2tj6E^SvmNMR**# zLGMT*_X!HTRc0Z-pG68~ZNTz(2QLQ-NITa6+v!4Wj*m5)jQfCQt^VT?T?!CAcx)ovBezZ@v4y zF9Ipx8!$BoMWq||70;XUTDDw=H5V zBQ!MbqFP;|dD7Qxviub7K5H1~>D?2s=x>|2=l_LG+>@Ub6)S8DqoXN-AF+Kl*gHEr z=khR9D1D6!DGRRVvi@HnCQ<$$@vHm&@cb7G?#%y>4E&yG@_{~71k$t>Q6zGU>+LId zyS?#bIx4ZEATeDtU}7W))X>y*jZi|~s2`Yna@XD?CWk?#%SAK{P)o#o43VmyEHHc* zxZ9?gO7ipx55Q~PmFoH*2Qo=Cfa6{~zCvyB6GBXr$YrJkY(!;yp}M1^BVLw5Svo_3 zyIADnDZyF>KqLh)cohn(+nX;|ULM5Sox%QLVFT_2D1`jLwtIlI2W8a@(_#(L7sw$h zl|X(eA0asuf{+0d%R(DkIfBdk-Zyg|jzZ4toWr(FfDpRjBtkmp1#Tdl6VS!O z-z2SRK4G?jj24{p3nhL~m&AG|YI4g?eOxAX!W5jjOzPA))>JLid4H5fq_4u2u;0Jh z_VtoAtK;5wB%?3js;M>`=BP`V%Sfen_XFzBk9Il2VYHQ^`1XZk8q$h9A7^71C7@mej@W~9bC(*uj$S6kA#xwAou#a*Q0kX=7Y@YdE3018P$Qw75TM|qw#|@lq zhv+l>lq2sc!a5=La4v>j5k*9_=|Kj_k3Jw z9DM7`wcn7p=ccmS?u;hqIONgq0c)LmM`X+ywkR5sr27}`KBa9sveM6 zYviYIb~z=1B|QhNTA5?_RSv#Qo_<~}u2uPow9cSds3bOi;CVblgWFJQe@K7h^RtI} z%*>thf~0zL%XeEh1siG6_{^JP=c#eq5GE_==!v{epj4f{XAfI}L{Z;5m5XaE6;n&F zT$@WIs|V8n%FqPg9s{;2_ebe@Eb?qRm>yuv0pLkDE)}SLxvS6K^kEkIHjSlh2@FD^ zu1TJp4Yk40R@62niKlD~y|`BY-tTPQFj6V)u)l!VPMX)BQ+nzWKS`lEC2u#sQxTqX3=xq#J84BQkqajw+`+qf8J?ESk3`9aLN|FDev`adBEoqzl0X;ej3qlcJYO%(m<8sAc8BD}A2X%}#Z(?$GrUZf}&OVf$dQxuiWVrEq~{xi{ChM#yjI0Ct+(AKkbn47(not z%K{fPQ;o6vMYHErGVT4dX_;@WW8}3h%3y^`}a1(W}@Y`DbjILXb!Y=Rw#;4F+`AatMiV$X)qa z*4ZXX2{=w{B*ahzZtH8ju0g1Zjb9R$Z8$#P$pai9S6?>n0n5hhU2mO0a(64JT3h3d`MG7=Q&IDlTasIzaWZt@ z-r`iagxVAfbfuOnUKnLfT$He`5vwFOptSMH(PjEz1+UX8A^rtUx}X}axku|&w>tiC z_eW{PV*cawG#DkVsrIV6d(ZUX`2~6zp%CdB-5&v%v2aWVs-v350_!jN0fT6MJ;?Du z5uSf>9YWe;#^0{Sm%a}(s0|O|83By|5jyAnS#(qr!~vzX3H#7$_kn+@-s%c0W%{gI zl3w?;rsc*8F*!?~2Qc(wxn*7%dzXHiFV>GnD~qk6?Kep3O~U^ftIKh;t{#=xA01_H zP`uE;?)F7d$=u!Rv@Ul#{vu|m3R~0ifiU>QjU;M3kI1nEz?{gadAC(71J|gyVnWh} zpT!LOcB0;8t;bjDp3YM-8im@z42*2`pb77ZGi`UredzWb5$RjZL?hV7L0BwIQ@xsc zIqf2SW2PA&_1Ah;iW|(gvSf~QuIA1htWcDw;QSp9U*D`|{Dv=MlWN3SLg2jXlDEp_ zjiew>-b|&*$41!EtOT>D1iLra4lf+6@O#O=p;B8@I*_0s)2W?I355$QjmJTibC?X` zc2ICn`x5YCH?P;{&r>`po9-;qxooy8oMGMtt;=j-+|6;L10U}1(ABp{C`ElW!P;H! z>Rlavm@c^|SKyNu{inf?d+KaADx2gJ6oBis)!j9DR=8le zXB6d{0*_@`cV&i(y{A;LYDoV9a)#x-Wv=Mp+>fM*pB`42!bYe=jXS1EJHE%Np=PN) ze`i|UTU#MQ?09MH`&?_07Z$~;-9;sq!%3r4;0=6P)2p9hrF-0kdigw%v>eaWRnlCP6qe`N(VD9i>@C!4W<-ChRz^a?RkPY<%0nBd*t#C}rPyXS3^a z-&o!*F}}2CnIOS*e^wArJ9~5QU)bt`Vs`mPSNqQ#Aq!Q3$Aq>}$2sEjom+(d-l7 zo2MP`&joX(HvNEfjX+t7aj8Snj?BAP(>JCyDZRPa$KX%=b!cVgSupIyHdJRwlH2{G zot5;kAa<(7-{;6BV;#Edlj}-&u<>m+^*zxa#`3HDg-2{tcUia(dw6BkMdWme+qEXI z@`z|O3tS-i8HA8$2rSFKXV;aGI;m(mnyj(S&)C8tuG|^d;BT>vT9rv`_bR7ah~vbm z1$2%>(G2vR*36CJEF#Cb*opx*F4&V9`f?BegBDI$r?d?OlbmB`5;oFC{N&ms?K^7- zk7sLRs;NpwVwxh?`_48K<>H&j2x^gmwg@L8rE*1Z|M9_=uHKuGq4Vhpn;Cx%5zkb{ zYnC&A;=QC~NgKjKRi5spY8iLRCtAw+aCL_Ha4K%9s204_$swS#l0$IY-&aC)EWXca zI+Q=?)TRq@*G&4lzZm<*KXC~qDOpZ*?+ULUhW3XdgDonlQ^l zEkl6jb)TjR3<5IG4y~muRq-or8`ShJYfs}nO*@>o56GQ;ROh^#g4#+KeVByGv?-L+ zPCfzOt4(UK@$JihxJZ|s)gevvLVhySN*B=t)v@T({bwgOD4f`G#%NVWMz*7y;nhRk zn^a@yzERY*DCxE`vio6~sQNS^s^(mNe8k z5!>!?QOylr9ceoXyYuJr#Pb|4wuQ{^Z_TTuc>W?9yWg*W>dEG~3KUA`Rf8)|W zcLCllr%vONBz|?mvJktdc=YJ=Yhu|&0xn(KP{%pUex>In!ZX>PZteHZRW#v9MH@Rs zV(7KBC)}*khzy*RpcA9Ljx85pHQ#-8W9Tyw+U9=ch(yerW17dbPJ@tpc5t4le8aEd zeSqAID0#rKSAk%Qg!DCxO}f3?P!=~amx360t@_N&^A^w-8|o65BlFC~nujC(}UoS>XI8t1M>Jt7g-6?`3GOF&4DqSj&PT;J;+-p zONzw1BKIfUY0kf};w|6ZTz7^N$Uy@VAhdiAJX}!5R9Bqq0eQX8xZvF5uv9&twTu)9 z|NcNg!7?zhc@FomeNp5wdr$uPi2vV))V)_yPr@KA^5i3d{vnY4?fmBsKq#H-5wO`d z#wU@HT4+;Q0uNqtLO)JT6hjoA8W|dDBood@wXHilMexku@DfOH|ALq1#d($}P!tU2 z(D19`YZ%)b(4V@@Ev5rD%G_^lI>Imn6xWm}1%W=io{?$Joc-P-DXzO6x-=XYq!Oy< zj|b*2(;I4Xol>Bxb4>JC1ao8H^wGKRIm%51v{O$i4jv(wykJL|LR(!sh&(&NV|r?v z-m%#$O8v}S2J*&pVi`U?4Hg3dxw~P#`F>XPVbM3%BpCt3+aV@1(-q2^7U~ zJyQRJZzJ11ki0CBayvvrS@W7aR98R%DbY1AK&bla5B&PbzVFi1;4NW3Cqnnt0(M&3 z`(|#9aivm#`&OZYGW=WFg!iZdv)*oW@bhDy715uE5AJYfonk`eYm z*W<2Ff!%HaEJ-cq)Owe`Le>Oeiv4;Pa|gwF%C+_`U)ybx)__{r<+mmA8F8wX`r1xODHWpyDE)68cTj``1_T5P zS=Oi;rO5n=jsIj#2kG6ZFCirU+U7!^LCdkZOK_BziY&7Z49R$Oq^Ih+j>N4${EQgB zl9>bG%;?F8&-VN%f*sCOp3SF(io?92A4*0=tpWBUdz3V4FX9_eukSI*eR)0b^_iCt zjoA)pmj#~Kbd;IXmS_e(pJ$j3}ZVXWcz+x?KX64IvoK~6T}KPTwS6&%t02MfMT5# zahVSMFs(VEa}O=jnODF(h9Rf65MvS=D(IT02XRew=S!nZBU(vpnoeTTL<-P_2lgg? z`&>9VAM{lv_F))7hhg)Yj<&THgk}j~N6*Xxk!|6~|MP`41H~{I#i?URXlTg;28tj9 zX*OKFoXdJ7f|4C}_VPTz+P=f6jc0lNUR=^0}?nehQr?X!T> zz})cD#O>(?~u(c}eCr?7(7-2x*Ijx-G{~;OG-Acft z%-C$8%eBykgjyF@fN$;s0!G){)gGzX#A`uL&|ywxY)7H;N?1K{MrSRin_UB9&449- zXXYh%ztx4AD84KZ;zl5qBP5IDhG}p{G!ZtsD^yOV#i&GStu0FqwXf3bX(L^CFSlf3s@XV!XFD0P;ll^2CU=ST_c@NdARiGO$w zHfu4}a`%8Zayt$G)3NT0bC|kwnk|et?IJ@|&Cuuqrh#8e6nm zfJg9~MKRN3;iGPh!&6&d5u?&5cnMJXwfFmK*DxH}{p%Jxg?gqXXC-CnBBwN@FOOEu-itux2kWmgvoEh9JJ9!{ zTj{pn4iy&_UC|2b=D7Z62k@9k2Sz4XzV6rN2IHwW~IN=do-Rk<)&Hrwy|-G#~)BfPANB$+rJNC}9H=E?7o;{I$W2xg zqP!Hw!4P5?=_J+Teh&UkB%o{y{5tqxTU!bSOVuGtS)n32e>%19eEVl`uJ0lF+&=E>Eq3yZ>S)X|Tu+KF#Um6b+ zwrN7jIN^jXTMh0+TTE)m!%nuOe`VSTrAy+`?{|2&{$33T+otH+q~V}z;V;qh(ilmu zJ#jfp1}am3;-_oU6qyFw{MR~MtL>!@?=~}-FUj#bnb22 z=ar#mFB-|mBp9-3S)l1MmVJEg-R%nf>y?h4C)qz_CFy9GNO5fy7WOnunN1}GdgT$5 z$wMr+_Mb=rzCiiEU_bn;NdG@*-yeL?nnD72{RBZK`J!%Nzb9j09_PzSZ?-z|8sI6+adOw4S%AYGYR6dTB?b%d0lD@KRUwwdO~w=k6y`n`E$2O~%hJZG+|5qypuVF)o%bp84_G}+7B&%B=RI88u#q$Fw6Blu(~06oKH8x>Ol zL6nwYIEJf^+1%m2RG&%KQ5BHXR~^)vGla$VF8=2R`1TIPg7p?m(_dI;zzZlJo#+|3_; zT|m0-L^+a6SFPVlQ7BeFtAgx|-_8tHdef+opzgB-M<%tO`KM2)=v;OF{wcjDKUJNtXDUf8 zX+fh*WaoHTR}>4c=Ts&_mkTjk>u#6{m40|GTO=pip)?r47hK6r9uLN>JSXTdvuS=gXzXjjc1pF+A zmJ28@MMOo-Pdh;U(Ty=R7s{_gyDvs&H_p6J#1Q(jqt|&7xX~_8$Bv|h4Lq46)(G0v zvE09x^b_{5!`D}|D4w{8Tsj`9$>}JRR#^{6koXp7w9X|pLZ*~S%&5tv<5+)4SdwNo zy!g@qU%P5-pZ!(pDb_OL>=B;)vh@gDy99sy6JJba;ZdCr?Q9QUlrDG&WUUTg`}=W7 z&txq_=g*;|C04P8DkYpY#Dvuvjx*y`23cuBG}3(?r@ z;{_usYfxX(zzF)nap*Vg^30dwbd+89tq4@o*bNroUPJZ$bb!s9JoNyH&G1J6Qh^FF zXff3ONz8(?mB>2_hpr&sQr-*XIm~=5 zxg(Vnq{K{;Z}nqgxG<2_4|`&%ayQA_!Ov4Ee4j-n2&Ww3I0MkEWJJdj zS?+|`KWI`c+~@2uRf?A8ClFOnoQRA>)|Ubqrn40c{d(Tz(KURn3G_39@2UaoRX>QH zBNG`dSo1O!`);<9b~!z*0cZ@U)jvea6@lS>oWOk6G>py7+!u~8Kzzw+u`9h2qeCO8 zzpp^TeztxRmC#=H_I*Xe2dvGv5d^}K? zT6fw+lk3wyL~t`GnmYj{j+Z@aM=XUjj>WDU{dW@~%)gZEY8Y4Kq(7 z(r4!%9N){w-|?ZS$VSR-=E{9FLdul*S@Uix39gX=C%I$TylLI>FKQ9+Mr+V-NuKwF z*KVfI*S(Awby}%VW7QV2-QYgXy8j1qv;JwKIsfn2E-5ep%IcHY>RI*WC)><8*_S^c zUL5Nk)H>NdMLhOAa;lU`ca&xwMrsk#hF#c#d9nvyK?JyYZjX!`ty_E-m6jx?1H4jFxK^H$K1un8bQh&ZJ z1%}Z%6-q2|izag5=!QOLfHeZm-V!=2g7fjj|AY{LUSJ8Q1gj~6h>f;=zqCxT@9ER0 zM&r7o*yPp0=F^9+(hVGs!B*d(aFnI@t>{p~36P#y)+d1Cr*%%C6*Tf*#xhCq+C zkE)IyrS)}4G?|J^o{v}2JqD`K&EE|1#o~Ia4QTsaxTC!)7-jFijXgH3r5~N}n8lMa-+ay?Cu$uxxurQMZH@WEk(zAgdQE{WZ~j8E5HUp zF{Ad)k_&Y8#;oX`gtkA8eJnHYkFa6S=N2nqbokss!+SO7)k1mhFpW?xTikOL$MyI3kCMoAg#RJ@qCs?BT!%_ zzXk-Gyt}m5+&?p(k5kRJ9vHq4lrKlOz{s~TP#+Urp4~Vk$Q-^@g5^$Cc%6kAr7*Y( zM%>IlJM;FT7_R2grl(-(h`nC^apOgP=hVatujorcbf*I9$KC{vdv1yD#g-FPCG?(F z;jeupj3(77MFv_KE$I6b$P-B`eIQk>;#> z;NDg9T+FrVe)-=t8cxo0>oj+N3Ud#K=cw3eXKFPJIhj9%zx3e0QZ|wWB(1vi0U717 zihAv4MMqJvobiqJI6IupeD?GV}P<26%s=EoNpT_BE^jtLI) zy;@Q>snR4?+|<$_&W;XXOV>g8@NLS9)8fP$v$XJIW;B^Cb|s~Stb^H{8GW0qz*5)KCQORh35O zwaHK8MAXiOzo)-Qd0u&Fr|Pt^b?v}I&g0R?^jb2Uy;j3@smFnKkLdq;jMFUp>%u5h z_iGbP0Lgau+*$0FHIp^jZ2cFLCVj2SXSRnA_)T8yzhU&Gvysd#TCC;frw#$0@Z(Ce z^F%?v4aCz3sT|_h?9dX)@?6C}wuDiwY$ad5nTQZ`5*T-mm6@@gP1*Kw?6XXo{zX4{ zjxc;DWZNGJVNw;IzE|Z=k3lok@|X9iivYO}6R$%0ST5Wr)Q`pwKT%s7QI>@k1E$9c z@*FoWlTI*CSVjw{|94cH5yXdyE%AA=)-(xo>QQCp+;LYI)!KNUHJwa}Ov|wUsC&=8SRJ*kNyFh~w93EUFG#gYgb8C# zS90;Ma*os%pfyZwi{vS(K1pY zsvpnSCWW7n)K)Vo=-=ecQY(q#bcc zWtg`!%#+2sEJhBEGh6B8(@mxYh zebXeAFrDcKH~=yXdsfXaM2q~w<0`*=uF6##ZMNm=?QQ8oQ5MKA?FHm>{5WV503k~{ zm3kTYO8`>DV>vE45$Cn2KGk&Dpb;b6TJ&~HxXXwHcUA6t_AuHc-_d3qFJ1$R+#knu zd{`*dU)nYRJl8i}oB4PA#}%*m6XAdVAI3SWJIcRdZHkN9QAO9`9y==etTySBoBaUw ztDgcT4{RY~H4(b{QDESil`B^&>u#9)J6JTW2C~d6P<;yBk9iX# z<=vV82(%jp@ea+AODGZlt%!>21I3w`1lf10PM~;9Aydt@a+1B#1Z)8MhJiNwrAN2h4^Mq>6u;n$F$cTgOr)2=bUCgQ>sU8>gUf z);M;|j~Ku>5D(&d?CxhV4gM++w>qorj>t?AT=bcj;C6E5jBTGxk!L&Tc*Z?({U-2o z%}_5V15?PZuW){hNiT`3haf%@soiXFPXfsD;^)Y6v*^gQtMJ?`AkD(L#(7PiO9pYu zF1I8DB-m%{Havccpv3NHEuZ~W&)3UGDcOK3iL2T4EtfHzPlG*j?3CjLASQTJ{*9N~ zzLEBnAwvp=0Rc3my=*7WP$7wBfToU@!^Xajxy+09>F=-Q!q;LVMk4C~skpEYWdIkd zL$YQCST!9;(EjpN;LDtQ4c5$lMz9~;Eg5=zN*A%Yy*4J#8xY4P8GgkZnDiq*e(i$B=kFJKmNGEv?k0_LK(95#7z-lm z*}rn#S-3ZyT26dZ@Tt3zL?I|P9=V6;J?(du68xRHkULVtqj6R7a=th ziYZaRZ`nlr=4+LVnE%;_eZi1w458`>0)O^at!kjg{T>$~+-uXa!w{?L|0tv*{iRgU z;aorl=q9pR+`p!2S+V(+duO^||8uMN!TL<(&$V!lWW{F*oGXhHEbToeWi zv%@a1Gt@cRj?`z&T7AV_#3~=YT!KO_ zosf8R&Cr|bm7nWJ3!SA*f^(Qd2auct#Ha){=JnCvd_R0wptR@6xFkIr-M#*Q>vE#$ zys8&Xo!_t5R5Outb67p)Z?pUVRS4_J*nMM9(=$RH3AuiA-@p;ay&77q;95O5)Tlbp zF$t#7rgoxM1`zO^czMFJ+0P(E3*a3~dq>GLLghP-7BE35Ml)my zmyA?g?Bcm)upv9#1Qw>}{3r6;m1XZ?HDHsPn&v@|m?-rj^LMz1@xpbmVzDIjou%uK zy|zcw4=|WNIOBDwz!SiUN!o0i&aZ@D?>(clf5Aya?V_B*0yEPR!K`7O@D*=+i)OD1sGEA6NTOGvwtllh_ckvlE+i_|E?K`?g|tt1rsx{a3}8_{7-t9>rbAWkzSveD)hM><+OBITXC z%(7IU^kEOluCjC%9S^a^_*$`mI}lKnaqlr%x$_IL0|dF$@)Kbc&Jbz5tdw% zG}2F;%U`1IWs~KKW1i8 ziK$+mA{$j^X|N>-WIt^IY2m@CTq*qdsGRYw&C$ffP8W4w&-Lxhe-TIjtZlnM#IXd* zpMh{z58c=qG)vAK-;Y^LJ+1eQ$W64vLES8-g}Z}>Isyk;=AXF}H|^9;_qn2ZzO z`xmDymkmuT!6v?x?WTBUGb4}8mH_q!404pK|^?fOG z0$UAeT2`aEsLzs#YK@HQ(@XiWN|dc-IGEr)?5*7IAq9FVdabGhQanOT(-eq$p7C%{u+ckQ9#|2&f7Y~2-JZ|rk;4>pY z?Z`OsT}ZsQxA3#Fk)}{!Y5((y|%MoPa2a4>kZEskg3zyQhczwJGXV7FyX6Z-SkD>L}9db*1zPlLmXM zdp@yWTyGP7>=dnIxv=V{@@|s%5I5npT}ny1{=$#AZ<;yNzJP03;_)aHUWQ;(IA9Vo z!tdhvXO560CDzqF7OZ7ou&X`wrIK>aF5hA{L+%Y*L`F=S*lv|dd|Nnyr}wG@{wt?KkM!thW@(U93Zwe8%XEz30B1 zGH7*o~2_JPo0Ur@^DP;)vTORdf94VjZOr_$TeD1)Z;o>H{z z_7$H+a`3f>#D{4@hs>_WcGC(5Q_;YmXkFz@?9wkKAdgiJ;w6$TOyBrItmt~Gdt*WY zb?4OUp)DZdpG8EA#6Z0G{1-A@p@#%9adGI8I>c^e193|c$!Sl=c!w*(kNni1_y>#;)5R9np&I z({X>?_DJwe#|Cl{F5VrS45?f#!A0wg4Y?m{RVjI(U^*rv@o+(}Z$wJ7w^KqHqsw8s zO~9bd!`OL1Q)7fg*bmq>cwFe zz~|n#1f@3j2~T0IWE2XhGVe>KPwTyL?vr$G55N42ero3zP=RxHQ0x~(wy7-oZbSQE zRvNCh-fTfc`bsmWh%YgfiSm_SZxt0!&}pXh^IPE_YXWXTQsenz(ZqWhC6rTlGllyr z(+_f2Js?ff+H2qppaGLrzP(bt2ff}>J3k{q-!e&ErSi@S#Sq?==Hsu9Fr}I-=AD!s zhMCg$jyu^bm9wAjA zoTXX6ka2mA2{JF8#_~=MF4Q@vtbk^|bLMdSu$Nj`Tskl2zx%NJ4@+6}4R$#8C>1v| z=Q!`ro8T5PHZ;sMkPIgJGM|ODqH%KPhgsBD+YH|xsmck`oUR^OtcZujq|8~>e8upe zBXxNe=ppl4DDAgq{z|&|?3HPjZ;43(Meq|!gevif@o!(@zW8pp>vEcm`2!-{kb$G4 z>w>UKeMO)0A_eih@4_y)nv2K09BEgCT}Q@SV9ui6lBd~L8jazaS__3;r(a6uB4(4 z7s>{xA<3;=un!s6*j~jr+4;2ATaBgqTe(G(UniEG^l3U^{(nOOVgD~Hg^&S48!%IU5~Ok2>O_vid# z+?s#@hSWoEsyePWZE9#~zjq=ev2Ai-^%luDic#h7fc@V!+G&BE>|pxYcS-3EFAbz- zlrZbkG&aXM_avN)X?>;G#vFp~^mE{?wH`Ub{s3kEiWX5o63~5pM~Mx0oUmY-riB2l z`j5^<4P>rf8G-7p_A1JZ8BV(q!iD8}m;ya|dii`rnX@kXj>OOJi-*>mMP~QHW$J5{ zuPscwTSgyopVq@f7LxOb+2nz1_^Z4LvPv;ewK#uRr%pWD_$=*8tOrxIZgi(%g2J#X zr+Yl3j&u~~hJzE6Hx4Z_8IG+QBw9?;4)+kZR&GskwxL@uV;;t3IMPhfDtH#d@fKku zC0J&~@$%@5hwcnFxI>bEB43+0X{ov%1u}KJ^3PnSs)K_@zmmkQMV>sFOuBnWE2?kO ziws7ne%VaW*@DrS0gY}^u4Yr4wq~UrNfqx8ROSftL47H%Lr+B;I_JxdJlQ_)EkTzT zt?v12#V602Y_*zrVQoB2PhtZURoO1P40jr#X0n&QLX(fDs(c-KqScIQd{#roND+;`xcGLq4%1_KUwl1bV(WDP>!lZuI)ALHb?=RVOn$ zEpdJRY}Ik#J4^Ij6TV}Atg^C+$}l?YlV`$vBJS;L@0UMb^l6?Z*C!cQ^<~?ni$jgq z^*rSQll%uBLflo7K9-PoCANCU+S5#>Fa&8v_>YXUv|`z~!6reP!Xn@2Y-e`$g^aWN zuiUYrUdbGdO73cJhiik8-l4^`ieLF z2EC>>b$~BB@kX^<91dlUeh=NVD#hc*eW>f4=sXs*y|350j?`JTXOR(S*LEeb+Rb!9 zm(b!CxNY!yoo-{GUcbF<3(xYXyKG<9t`mo@Ex96fty;RROPovfVdY2rJ&XG1T6VaDGq0t&j9T5hzILF@yYF5VCedP2MjW#QlLTD*3a><#-}it{>9jmK z*%%Vyqrde(@ltLt1hne;D?)Z`?WFYG8JmZ9Vm&h@ug#|LpQL)c9GE#eH+v5H4cjC_ z3p5KAa0GHJ?>5KLr!a5w1l|1x0K@N{+aAb(uyyXA940%x@#ar^kU7u1mg+aSP*(%i z>M{ZzI06cxHrm7q$bd#r!)4e1Mq~P>>v9(LZF-+}69<$QjYF^XO(<( zs(@Z+e!KC0ZS8&0chaT%#3`@S&nyZXL^aGU1x(Bmt;iYILr|nZI?T;fM_+s9rSzGp z{Y8E$ajUiblHW+W{a^L?Xa7hwT@dyZJ|l^1#}OgKdT76ookF>DNp}9LyFo$cfLNqK ze8D{$JNpsJF%{+^kbAz#wId_t`NzLIGuuJqlK>#D$#w8V4$>H1@pfcl+@of5UwcN{ zQ}M`=`7T<(83`hX_;}bpWbT`vhB053YqcTi@NUDR#c=Qp1;&t+sXj-^UrgsNgHrn4 z@YL(q2MLRZ|9sJ!o>}8W3iX{**HtSt-27bT?A9nxw~@O~wfnjKvxr-DRkTR;(%b8P z6KtNsMjn6KyzJ4bcVCy@7AX`y)W)#%cH_YCjL+HP9SlRDD6DvH1e<@SCE74#qne%? zy|bi8^1_$IXz4#*wq?5=KJ;6xb?mYPL^)exS+Du~XJlq(%84H?J_3nFU`hRQ`Al|7 zm=c)W&587im{&pbD1s~m`Fjr4(WbxH+EGrouAo{&a)o<1SbA*z1}%`78%n`D&)9E4 z+#?_aPnlo6V)imADfO#ef(Pd5;(Owy4p|^SL00k+CZ`sp+CzLIii}1hA_qtZJd&{*otlk$Gl* z)6Y+IUKyWwT)txG+}agP=XRmE;W)HW3}ZdG&4~7h4rlLwAM$t|<{9}Pfd|RIuURtoM;fp)q;|yXxg@~B| z9G++Zfau%a93=S%w@3wA>G*;+&Lh{sxCcJ3^@L7NPtf&^%=~cGIJJ*WkW0_dUV-(K zkW@!}wA~2m`1q#edkjM`g!dX^p)gucYX~+$ME+pfHGb9O{{9LbP1406j^S}CQ8(Yb zeLDlE5YComHDC{ZK(^{ZPQM=@h0TJB_mH5Uulk0EnW6-zrfQuC_NJW$eR5N=k8E<@P& zSnQ@bVTNN+z#ItuL?r6<7~NqA)Ic|#G7cT4P0Jv*6!iu((FC`@$ru56$4<+ajc1IgcTbZo|ou$DN=n!6LP-^ zfcnY&L*N>GN~@~Pndi{?kOd0En z2NBS5Tdi!V!<+_oTbL9#r)LRYdwXLQ&p`j$^-&IH1Xkd;dBFgKxj>N&aqaXEmMZ*P z^U$qIX^3)*T`mA%2w^@tVFxal=P4zI1P?p->a;t{^kP0Ca+%z(dh5eJ_*Iy8v-uPA zr0pu<(WP5MBoi&8RNf5da_eB3d?b}C=J43AIcn4F04`=N2o9$Sw08QTTmrF@&vxiG zBhZahBi|tcKqt4OP4GLV*=G=qqICiTnFCm#&Ao=4Of1Al5Bs0p>au@G}JsvHoqs2v5d6M zCVwGK&I$diNp=r&oU@qoesjt>f2(_`sXOWQ8A-%k%xyU|!F$D`X@4gA=}Wsex4k9( zB0@PfKOH_sU=HC+JQ$^2rOExusTPtq_GgtWCzzxcfdGUa-Z{rc`+HIfPohm-=>7E= z4cj^bEGWmEb~sM=a2OVQ@n28`s~w51nAD$Bi3 z`yFE!o{|rXWny?eMJ~1K33atiM>s2!4T-Gmj@lO$=A&NH-IWu1Eh*mHIo@~DSoEN4 zuFAS#%wxchu7;Q)!?7K11zTLL;A%$irn^;t0n?B88&vPegbnyjMJg- zKSJ=X`*WYc3yKTk?_^72s5!v-10{plM>3f#$E2S%Gk1HXt|s&(;^IqsA&I*#kK_pF zGdVFW6Tnu#=AH*g$2l0-`Fr@?nd{;#I!>*3^MC%lsIQ|FoP-sff9cY^QI}^#tkEcA z`dLWdH0X&j13zL7z9;ODNOnqMf62D4>mOgqByI2w{_-+K$tT*-VJywB-;P(2|YT0N3> zaY%P6ksj4?{DLjP^|=A!aS!9cSLh@Jf1|r0&eY0m92k#hgQf9IMMocR%(1zPn zwL|21k4HsxR;Ub1J&|Tr+831;Cwi&rTIVijGReF(#;hOS=og}_FP{bdDbZPx<^s9+ zjRN749JRZTS%hen&&CA-Vkth-&v(t5Sg2%gfU_EK;D6_*Zp_DIa*CFLo z+G~2bNYb*exTCkuy>l=&WuLd{VlipskMv=G=3{XPLnXR#muS+V9a#CEwe{$-C))|>RlNl~n+m%tJo3M$o{?hkD^#}8UDx>shZH20D5~ki4 zs^;nbS!l}Cv2d~OkW?y||O*Om^jXUk^nY*yNw zIc>a2E59lR?jB+l0|MX3te_AupslDcf~!RLbCtC(Jsy6u&a%B?jHr5u)`B>NtcpL* z#KEE%Wy&ep9v0B@;N>RqV8&xo)?u<~@$(}cxLWuw4mfX)v%DeGyR$FsrKWGPWZwq1 zz4m<@?(v{d3^-_I%d6+peqkVij?{uMS`a@DpzX0zaM(5Wrcn=}F~_$QD<%&8}+xKd=Z>hd?h{WS#GN-r6!m&02s8C-6!KqoV#PcYUf2v4<~tKnRPTbm4yrJNNR)U zds*R-W{f}ymhbI}Rm##y@;=*tIBZocbhDCosqg%0Gt7JdEV7fBSCk;i7j>}VIeQWD88O=(cFtN6Tt|vUNu!S_L zc{3la9ckyD#8G{{X|7|~g0Go$dwyPXDk_ccc`GAW>3nvcpsGRy(5c7{qY5@y=Q6% z?4`OAN`vZ$#^NkW^X-a13~4$?mLA`@^!5Gqf4$X|a}A~)Ihw5Zc6Vxd_U^RE;i)mc z1@gv{n=Vzl=(2`XxEHS4N}Fuh)ov<5lnG7RXtiFDZ6ggTIep)UaCKd=*CdnoM064$D;~57)+1H6Jm@4$vf${s(h!9aVL^wu??dKtZrT6j8C3 zNZBYTVPFfWOj5!?BqklwC5nj&CQJ~dyIXkiRY3)$gh?uh#H1U6eLYkC)>_~G&e-dW zvBw#Qe_mf=*6;T`&wXF_70G*Bk&tIit(6s5>I$dVOg=m6E*^Q!TCZ}+Vdi$*i&w?U z?Eb!9Xzx0WnR^xQPhn%iW0*YP7hY;4cSvAqM3S~Zb6qOCuo-tnhy8L@Y7o60*uR)1zk10MNXXLZXlYc~vGI#wcIr?96 zv1h6tI_cHT&ze2@El}PyE1S;^O-x5u}#;<#zgxSkK(Xgtg=k! z#cxx2HWp7$OkIe~9Uiyd9c^T@ygEJX(Xr|FZ6c4I#Ke!QJ&K+vjyBPBy@%0pqgigd zO^CM|vF?*=wtIc3SJHK4V{_GOS8ffPt=6rFM=x_<-|<4sirOFjc79KI%&Y;@T(H@2 zWSFGBEZzR%Z17#pf&q-XdxG>lqh3|C-*Os^HqjshbTH%NCtNBhuR3*OU!Mtz6;e!u zz>AEHiBZl}$!U=|X{8AlVwIq~G#N}s-1X;ZzIjpK8a7oYGq zTp5aDx9^}*REzN}V}7S8n^%qALbeBImYS4_fFslvBIZj5>%b4JX|k2))n{B<`PBl} z>1|@1Yvk~?;(r+KC5JR=&Zi(pq*3KCMr@E*`lp%ur!o@DEOK~XPQG+ zs()njy33mr&>%LMtIGQvCTSnGfbGwP{RHkC0(IC2qzgL+EiEWdk3hu>MXclRaR6- zD{fNOYrA|hb@SeCt#rq9-O;g{v~yF+jMSYUPgQJ{*toIa)cr*gTedWQ{Wd-BR^fc? zO<4zb1b!kZ`Pcmii(*cVuQNTBo^*SK+w-dEBM>EsN|1T@2(ksXsLzTmeny-phKEmc z+$j`PF&GPVzaS78xyitIPLOBFo>Vz(PvVrSjb%~Qr!_lYgv4#*x2?;Vu#5V->-lIz zp(aNe_qkMg?i+TU6CheP7&D{XDhF(Za;r~rMcN+ClHYdl*3x_J1GyD#c21dM>0G=* ztzKuSJe_L2YwzqTT+w`Edu;yn!b)ik&G@4hJ}ps|r!AZE-Dk|IzRBL!^UPqgkBzS5 ziddt%Iy-73DRCd;Wu-o|X9%3nw_H8+NzS-u+YgW8Hi<2(Rn2@>x!v+8o#Q>rtLHJ16Z8t-VR;&Cy+2>#P66BDUwUf?R zC8An)psRA^@bhY^sLHBI)o?Fy5voW`+R{#&&fy(=8ziVfAT2&hFG!!Z;M zcGr&XIUI~AJ$1|ImBHL50;1vv=3o01p{f{1*Qu%j3&7rQ5`(vl_JzE93RK>6uM%`h zkFkeG`TR{(f%}|$FJ&Cc2N5DTk8@jvCmVIIX?ExKv7Cu->`KG6wo{Asv-=$125uJ* zdTtNEf42Ce07>J>_+vYa=ew{?Ynm0ufnAy?rr1GTD|EXDjEgP@8iKs+8gwDZO9&d zRhw5iFIqkGIj?+&!E*V{Io?Mp9^HS?$RYy{+s(Z7-){HuJrr2qR%xFae^^;L4ZQj> z3v3SCmV%(&#+k>aI-$+h5i@Va(#h~)m6mCjvY*BsPs|-p8%ZXfmX)^WDkYbM5~gyT zvrJLz^nGPojfYQXcuXF%GdTw&REKAEKen5FDT6X?ars+z)t`;iE}5otLzB1K8c5Wa zzIuLX`l!HtGsdSW^GplZv=4D|;y=|D$wPYFUwiT3AaT=>!vvbBHfyZ+v-5G*<+)}r ze|+5al;`JzazR!0=i6rrjb79RHhCPXFqQup-*(BdRC!YJYgZlg%dB8?j4`d)7eyKM zLmb7T3#C^fbn&a-T>4nV{HYmm(?8LyKn54A_aVRKcnfMF%)i#5N#_ zPYtf3#Bbg3{c58}{`r@OrFVyD-S(AQMc$w4&l|aH6L=VVYAMO;!9iHNAEs1=-wn7! zH)Z`;BdD5>R{Tv;oMMU!t!87#J=s*$8_3lsID@tpf_hZYsNb5g4Y z1mo+MlpB&Wk#cT-B3Bvm7}Ei>>A=vAMtt1hvaJp7%zou)x;8Q$PZycaJ-`*7#_nrp zD&3PbbUbKxhJb#?xAV0{Z{K9LnT$v8?77ih$ik1e1b<=wf%7BNU7onl1FA1g9yJ3j zjZmr3ch&PnqfRqU-(R%G%_Q5u>qQ+Og(G;|#t-4i;z{Xm#Uvz4B|nemzhynrL)kYg z6*nPe=>CHTvEZ+rJrO@5EYvqoly_I^3dNXX>WtgFw(*uX->z7&Q9DrQLjBB2mZC(p zlYi3P`N@+f{*NCoRi{3B)d0*65zL8)t9gB;@u4i}BIm>P{+S_@Pq)mb&7MekjAXX$ zjM3eg#KU@oqAQsCpiws^o(o`heV1WE7bgKzCU6^1t?S}Eu$=W_kRxVp|4dhvaEtV^ zj*gij|LOsq?eZcaC4@ynDY}Y>fzqinZo`SE4~>Qgq1~-lh!RRz@cS*8mPbU9`m8nd zG9$o$VYy-Nv5%G|Rd|ebtZ|yh=Lme5#G6-_U`&M5d-eH6%+^W7>@!q0!0icP^yELU z&&K!HtGm8j?UBWS@Oj&s(;TsFpZ0e>sgb+Sua((YfE`zS$l%IT z{gUzlwOMlX} zms+spc^w~p-6ie#8@#f?nN+c$3(KC!Gm&tI{Tp+dLjQUv4&K-Y6sU5a17);2M ztliIgYd7AEb33n;14@0Zes#4u*m|6N2#!er@$LF%tD0DO5Crg9_$;1i+d?VOs*ydwu#UNbdXbd*3 z;)NFMV152RWt-|h{}uRRM>2m zWDkE;szY-gnX1rRD|h;Ra$f6)Mc=-`*^ztWk+@Y;9_C!-i{`K_n@;)VaQPNcoIrjd zCX)!EaEhC{yLjUNnvHmykkd?Cp>JOeeC$Hw4Ntzww<7t_+5zDa2a zym-FUg4a|dY02&R2}^kA)4wjGgk1SEPP@fXhPb-~BH@o}$BOx4p3%a&8)8^uVAaE$ z4Os(;P9O_`=t?wrr=Sb2CCO3iSFAexZsWVT@~Lt=B)8R&9dWTt>Ix+c>Tm`CaxyR>dAs3(PEonwkE(y}_q@3Nyi*B#jYX9XHQ}!(rd_khhuRAsRvN-AWNK#Ga$}-3IP|8yS|GU#N1P!^bs_f6#A0>l(zBu< zY-JLA4MK$`A3T5;ts}(G<*1PIAP2UEKLcOiM9KtETwCu2jaF}0AXkPDGvNmWyjRb6 za~-{^ixc|elWlLI*fS-jFc^?$%a;W|#|T>eexK)TdT2|~861s-?zX@gb~HW^nEz{B;DfNb2Q$Au1Q*oC0RI;J-}wcX%}7KqHsDryT}( znU*qNU?*dWfVULlT#=az((?$hT!f#~4Kvq2Q{DBU9B%d3u8G@2osJy zPmrs;zJJNj@j=+JxGCip|9)v6_ZKJO@kBv_5Q>> zreO;ggS%qMs#U9!WVK1%fb*ud2a$8B(RcLVa&j+*b%NkE(r0s&K^=jcP5($pL{GA2>NH5h z$aIysz7QLr5g6{WdQ5W-I~&Z=_kWrM5oXeP@1MKtoX$NiSVWT|qt3G4w&Daa2q11R z-7w0qX6__8qtOU$eCLQ+J$2`ah*{i!J}<*P9{xBHYLua<(dh5iZ6yO+Kyz}fdrZpV z*Eny~21waWTXF+=e5HTANsKu%X&)YM=@~)JN6LpD{xtkZ@%=qb;rX~mR6=n#dw+lZ zbY7{@RCkqa*;;8Cxil9-he4uN>jqB}$~z=OY4_2-4(0Sv(;Od(daia#1@OH><#Td}Vl zHdz@X9wvgj$o&K<0ZHMphq)hC4yvAqOWz0)_9x|C5CrL2JZpMMrQebq^&CM325=ne zYuah`^+0xCArBLM;y+0FXIEVJ0aH_Dq~DN1JxCKem@Ko>B_LNad}a^(4|~vj zHPPh9ATAwQnCD2tsf&pv(!Q$ADv09>8K9G;jL})<=2J$kWY#5ZS}n2j_V)@VkkUav z(?!S62SpequT*$C<(KPLj)j(n=<=dYl%Yg0dp4@D;`s+mQg0`Q5W|?vdLlzXS!`DE z30s+@4*PWt#PAX`A(%s_kt8S#%mcczqMS0YA;5MyETPn_Qkopex&lcndts`PKbzEjcO~T%^vfdfE2pD&}g{Z6e}AZ5wv)52v+y305Csx_Dj@l;x0sgdghDE>sef5z0Tmx5Fbg)QINp@nr5WlCT=WI zPuU6dn)*uhiyUT&W?f;fA)6=Rtu(L8G2-c&;LLr^IQHJ^yGr=&8SsU@^Splb<3YJr=_xv{pp9Ses=L(Pv(~czd>36tM>>fDTu#ROy5L@|oL9S=6*d=PW^5e|~=9z6vewns> zgI?aj)$gX4lQvh{hrS58>op;NnQjZAJqBk3>oY5R_USI3;oa(~4!Dl>G~~D13%rYx zs7XUxYvkWp$pc zTc%E{8rs6~yqi#LKs-oN^}~&<>d!LEq!;yT8|=ZfIa4{EIF_>vS%#2l7A>9` zPu351X_h{&0s{Y>@2R{3isG*|dfQ+DoS`mI7YN#b_3E@sG6jsXm$VnF^!$Fjrk#1* z=3@%$_Yoq=NYI)rfB9O6Ywb{7In%`Olo}UlL5|9@%BY9CP|gi=Jqi|c^LE9S1iuuD zb=2{ZeWymw)%6byr#0S9?lk+d;L*ym zMn!KS(Jg8}V)M%2dzOnrF5@ugBkUNWysVfK)vv-Jf~8ru>=R@|_{xUTjsPf#;;^{} ztp6QhwA1A5mdkwxyP<~Cog?GWQ!f#_@j$G!+L72|2v(J3#tinRw-6Z{=NBTXENr}L za%j=-3gHx4g|3h#tH65ED2T0I8+|r_i*1P+YhBpN&xaV~-x~W-Ca$cCE zMM>m_>>nPamSsSNsZk(&Snh7ssHUW(*Z#w?hkzpK3dACE!g zqO7R1h_$$&a@5pc=-`;EK5V+Yxsb354aI-(?JzhSqpImfc!vmq>@FO{NudLm2;I_l zq6a&AjzoFMo1B{FHwemKydba^fq!M*zbR&#gC+? z>)i{T+6i<6hOYOV|`W39w28u5P2UoIDqbOcx-BSw|M!BeuJx_E`f3yAq z&1T`;yj$CkCES?i?jlQGU69xd%Gi9*Xe`0TZhJO~lY3r_yt=^{2yV5s^oH+FiV7%Z zNpqA9p+W<#0LFGBRF(78N|Rbrh~Tni6AKm274wul4NIwyMXSc)$HfT?O8oLXy1iYlUr*2M;_{D$Vlj*I(%K!;QJ zlWj%^*LzgI_QJUm##3vTu#a6e>Zu+V2r}R15!+(sA0PNt7H{Ni8~BwIK#g%7bz4)`G@7zqp!l_rGmuX)Okhd zl5bO3`wYE|wsZU4AWq>*x+5YUL*KkLQYg0s*%gL)V+sC_*jcjWEAiWkN8Ym1gA`ZW z_BP7JY>{I{4Os?|s3D5SXGF-8r()xNxXgw8%aGT50SlwUzyH-ML*TcQ2mU7F7by$B zKsms{TE^s`B4WQIknrz+HACk9XLqUgPc!8ID-V66=!XR8+7!b_o7R5B$9=olT5^lb z*Agh&DGTISFWA=?JIa!P#~0?RZ;{p6zNJ=DpByz5`UTdTP&`m6XLu$1Q+>L)i4J5g z4S8!{rn%tflzRg0kcBVQ`$u8Jn3Yw}Q`8rs7@c{uf@-be(KR*0cmCWW_|6Ob-~Yt6 zkx9adAo&K8VM`){b|B5}>({UKhG>4uYX`_H#FuLE`X!^>Md0Hu(IlOc^NwHNr_j|_ zdrSvmf6_u<_A`LkI6!!P!=d;0&lTq$q|h!RGbs_?0~cyH6L%gf=AUj;RN%)f+C>yi zZ*p1SKmLebx%$8475{e8f6Q8;WMyS__QcMMA0D6Sjn{||1dLxgGl_PU!r;$eI3bI)SHca}Kmzb*O|dRKa@iX)nr&;(AU_lA+h@p6qS_r`900pyZh?TR<*U22^dH1>)2 zcBya3C7vJBdueZ^7Srwu-Nn{axUt~*{EaPb7DC=gUtbc>+zi+x9OXCLD zA51A`>6Hb)Z-@o|!TMhGGWpAWVlVE0?_SG{+_r5Mx`a549;`cG4pew>+0`Vve!0g| zDC>_rM^u}@=+PoCbk77y8hmb!0n`Y@ss~!PEE5n=Ji-E{b-|kjA)P`YQsAO>fSSd@ z<^KH@;zKoGETHbB>Y8AF)$fDh-+E)%tftB)mf&L`vmPNHEWIlSt)x#F7e>PpG9cC$7VAe26s1equ(KGeZlU=FsT_d@^u_#%4 zG9%ji!Sd+ZchnI?M{?#_bT;c_*xD6ht%CK!m#b2lKw)c~`IZw2EZ4(?=<$^_V8Np;q%f8r80ldrIf&7b{- zC1MhSZ1#>=gkG8gDnKKe1ixS4t?H6>*zzcWSl8Ou_*Qh^-Ora;;LSyvV_gFYND5*I zEQy7^b2l6gNOmsNp2WKQIoVFXmj?B+eK$a7o|g*|Xmzy4CJj3v()o1+*ao}x_N10) zL_6p?DX@R|1bok8Co%cdH2@*pebgCz%0eTKw?MYCc!C5D6LW(eE2PJi+-2`1=DJd6 z2)g(J#?~XGupw5XUg{j(glg?vPdrq>lQtujlk+ES^H9+%HRlPV$Y+4tkdW}Il^C|z zGVNg+W%NwNGvABB&bR9|bx7=h8^)pa@;ce}u{{3l4ZqQc(HsJt32JZpjXtDr%A1}r zopJkg-D{FqCJ|Wz@w6eX5v4D7*IgA(NamE_Ep*h`o3+j+wgWG9@cgzu;LJI+I49E_1x|@yK_8Ph z>56l$8i_v|_hLHr8k*o8;Lrrs5)&waZI?4Kw4&@>l;Yx(;kh~oDyhGz`hLPPTQ5*z zht&o85e%dIDCds2*aYq!iO>AZLdE;@U zD*qHJljk|)HX1)VF%D={;*66UiH5L#rnf3;i3K+7cQw+iwW}EUeSc#4cO$_u7YMN& z7%iYxz{V(jlRjfJwm=OC^Yr~z$|cpy1HDWCfP2yCnGwOW@{n&cVWERIFPlWd0I(l2 z8w8H2lnVXRT{ z8oLJ25OfX#>1>#m^7ZO&dOao3v%XTgdis{1-ze##YtyP=kKHA1Oj7GHP7sm)Gs$>& zgZom^ykQ4*_hwbtBy22nlk+>UCM8N}CbQOb1`C5XRGzA>>TW8jQx)$R3m@I#`cyGjZ^uWUqm=KgOP%8g zPu?XA+MeA3b-l}b(=J!&l%3!8+4F4tvz;m^RrY`S*09oq zo(qe#?-_cT4dEj4r&8pq(zU%s0xy5PZf>B+d#9auIzmyp(*C3Aaz};|zeF^Lk5@;1 z#ET=!vr=-PEsBf1&MB)_!dLEOI#qQ{Hr!6^!?dJC{P0z$r^K*xp*~AGf}&lKa{KW@ zK|#TR5fb6z6DD*?+4TzbWY;egLG{Pmk$p|zkBYZWEtfWXXBDVA`W{7*6<$uVjh5_~wynxraD%GRxAD z^RPPoUb=28@fs(@Fj(*ij%wbcFA;+$;-t=k)Y-=6AskGZ-%D1Ba$A_GHa1-`*PX?UReC5BjWyl=?njNUgR`tWcLPKVXeo=CZ{Aof>eJB__osO}e~ z6(FheJu1!HS@bP8t%mewBp#qwAvSSQrPO}cG@B60D7H!4QXQu31weYl!Y@l5J@CAK zmEB-T3Mxhyj{;8aQ@CRU@g*@RGOV>Ps9dYfI0aATb^vGL&c~_Nhn!}Xq;aK0r3GOx zzFoJQ5%8^3R_B~vOiF{ObryM0j8?^d`g1@sb-u;dzhS?)tqk-WTIJP(gwqNnYeJy zeQJuaM`_dt(X4Q-5gZiR)yp6!OQk}=^YwROM)4U{8DF!&3ob+QM%_)cCmoRs^Dkv; zyM3Q-$TW9PJ$4ygMK^&ntFod}sNx*dJJ8T<)6u!(bxKjzqH&Ml-(tS5WQ)l^n|gNX zW&_1{j@v~-UQrXG?8-eZgB$vc2TFg_vLeke`BGAKc(0H8ScK%@&?is?&c}AG$k-?|jtK~w z6ETv?^j_$6ocHKb^WIjkZhh8ur0^?0S81<*smaxFvr4z`Ejv4H(l7ap=5x(R?V^C7 zJk#hm8~&kFCB5Ox>KV6+;$nG?R9l16v#IA*Uq@M{Lei(cB$4+hpT+BgD@Z;6hU9Vj z$|VMlQj~6UU!oWmV-nOhy|l~_xbvG_mWTo*IFz!gAFjdyqRw8}=EIgZxng0BfEU|g zch~!C2C6u;WzK5V4=e|uahle`J2;Vvo|Y%$Vps{>+Pdp)N>ho{^yy9ubHpB;vWoxZ z{HYjuh-3S?8=S9qLz1BDUfbGwVW(&yXGuk~cfs~|O~kd2;F?4&4A08tmxx$B8N-{4 z&*U+e7({%>R!y{QKC#?sm^qjtm(8+7b+ao;yRbVc>IyN}MXk4(u4foyRMaa~{;Dj4 z1SR})J~g`WFw$`@`F9|YLz$EVJgou`3D1UjpUsyWy6cJpNUEFK*OiG85)~KnZ(HaJ%@0YU#s=55G&(BWN9V!Y`oYn_^UwfO@BW{0zjg;_0{TL=50D7sox0 zlG$=K8NgeVhtj8rAQ<2I!Y9*uH$tAkGWeQag|e~`bFLP#7t|w@&}?f#;x;w<5wW!# z1yT0u&BB`HWb}4{=PoTgW|fjb#-Oo{G!dPfN)=gDFGU68~bf2zw8R0zZ0@j@w><3q~Us`c@PC!(P zH@JRrS;Z?1R25eVr?5I(ibxiDFFEqBxt9~)eX~&uBo}bJj(RwvtuLCOZkQnUj7%Ri zI-Wjx(h)@HBGcez?PIRtKL_9yMR3*)&Ed`J@0K5@mT@O-K)^+Mz4guP6Vx{*1y*Pg zv-936?UjpV*SUDPGO)Wd0$ZI@tn_n?TYhK>an_EW?C#8 z^YwolJL$obvc(B2XFq3IfotL*d_Nnly?LWf$%iNWcEKnorAsr+{?W$TyfDYom=2;o ziEdUa2#GSGM#gX(E&J_+T9Si3KFU-lxL~mI#y$+4oW!FK4G=wSkh&{<$K&9r>lRrZ z|IpYGo1k+eYU=82jjhfYEID?GJ)B>EwWFiMoKldJ#xnemlG0Bw@a(!7%hUn`O7PrG z|HV1Ha(dLu896J&Ive_vs20k3A392`*ks40^FXA8NZ(LQpmnjBz$XdZNlfJ#w zj(SR6 zpNg|vRAPUbZoYwGzy5^J1TXkohrrU(nwU_@KAw4V{@~FVTU%Qt zMvdj1M!Jm&WY85KW4|<*v-PGxojo`G8t?Lp!~mQ}-8-lY2u`b4XEQBg`pZh%28+JG z4|jY?e3^QwypSV5Du++lw-5W|{MwmXopi6yd#}PpQL5FIJ5oT=G~p<<{P#kWokPF( z(zPq#@us!9Clkj%?;B;45)K;Q>e6FXx56o6d7ldVOr+B&ghlqNQzrPMA9=x@KWGd) zxr5D0*P(L4JKg#2dEp7(bKxU=q}tnSWH}a9FvOn&r`pfdZ(`5uku1rw=Z6vH2<-Xy zENwwMU7sXidrc@|v%086{_kSVzJ>T2lg2f!o%mSju1`#RpnctUUpt9cgUbBOW>QFH z(PC*KVzyJl8%=8affZ!?3r1zE1Yqslja|Q-rq8sn$o+K|s5&iBTze?*gL`57%+!)F z$+pR+Z;wfeaB9}l-S#nr4L7VuY*2U}>`*l@7`EEC&kKN*Gd#A^!Q(?k8&L({N8(uQ zpo6nKY0r%)*QyFlfmH1@Cw9bAH~3U>({}tY5H;yPnSAlYPFIQaroISkEiA=rb=nbi zno|L(G*%31`*Rz(>@%Vik~p<0?O7)Khv3d4=Dp+6`E|y*LraMh1w=SNweL)EGD(66jhquu7Ua+)M#>Tog@DPW!zx`IN zsvg;mqd|9-$*&KeE&=@Bi!}j~Kfz7kk#IJnW{>{YZ>_>Y{rNTGvd5bZ0qGxx)Q-idNq*H4bFERosdC(&HIyOmeCYb7 z9u}jQJoQ86|IemQ17~5wLYRVpPUIQ9Cu;zsJN!D#MI*5;H|QUH8rEAU)PK|e=9d9r z*cl$WHKMu1K6p3M-D81%s^j232GAD9HQM=ZQ{5Q&zPHLFX8ve=E)heq!Ag{4#5Nhk zfTkO^Q>@1)^tF0%k1qoMgFn%YPh{8{M#9a3)9TG z-((Vhof5PAg1=lEOy)W24;$lbSCWU^1R&%4ZgB2g@fHS2pji#}t0tHRCV$)p>};oB zT)0Y55Afq>@q|}9;;OCvwDGWQ$xO;NPSI;L-8vT1uDbyBS2@`>FbH|m1`;{xj(%ZimCC*p9) zZ%k*g^*S9pz+JX)A~qeR*RNmC139=o^Y+$l+Zx=evrLx*kSKK^M$OExlIfzuy-iKD zCUU}Ry#@4>$!G`0t%EtbzavtazxIItZGSHgoYTI{-nM3RhKl{ju1yMI6B6~nwPAps z-0t*451&mMtbv_@1;%189l^1j9aUyP4me_yal!_i_sm(m>p?goO~IcGZ{*m2o9H6+ zzc`4SpMj!5EReMP?NHc}O_~s-;*JjE^h_g`)WgHNa5(ZGnX65r@apWlznsPTNJEhb zq10r(O_?>ZB%Gj_8oVLUccbg5F1cff_Ga4f$H>pJQ6`7r83X7rlR%=Z^O(g>qB+t) zb05ta+T$nNDY`Ruf~yH>Vm+B?u-+1=&Vp-;>x6C1qOUkNKaDI!|C$WGg%&Q=3$zhE zGRDiJ7imxlL-FI*DY$D8wLl7Pz|0#BKs4)c^LL}PPa?^mb2mJ14pOf$o*-<58~Vet zB&Ci_?#*`-cL;U(<4csNoH${}l)!;|Zo>=P``P-gj|NG?tTT-N8r*VP?;nGP@;#HC z=^%N!G(N%%#k8C`tfA}W2PB|0Vm?5TQY5s)M@NvhZ%BH%9<&XLwjZH zx9jViY_yxnv&&edrbhyE#6^eGPBfo)5tNb9PX+Y0_M z1o@s(X|ojlsXi9tnrTqLCV1I7elAC*_C;dF&Cy#IFdjU$?6P9PAo0Bdb?OM6X0Yg7Vt)|@h!dnNwGwM6%d)PxtA3Vt=FD@Yr%_SVVB-E(hvt7-^&X*m<$^eEod{X&s``S`4ZA$5rFr7E z3Z_=~6{=dS!p>VNSEKr-*@qWCJ%q|T44YT9n zElu=uTaQnNhiSG%p3)PHUtaz7p-Mo)C{2;Vo%t`5AJUe zJiPkVPhi`b>Zv`58k`wK(#e^}x$xl;JQdZHAf`&48`*4E9rnu7#LbQDwzn6Vz1KE%OjxdRPlmjcN^^5WU@Q>u3GFO zugUFX)+Xh__sa65_c}36YHurHgQf9z&W%mb$bGo>4bN5f=meW>Fcz(mKfn}83H03l z`X$?s!!Zxio`}5}pJA1l|C;`7T&`=OIxAU z5Ip=Ir{>q5wLK-`F;qKCI8Zc%0a&U|HPvv&T=M5E+h&R{N0?$ZPQ5s^rT1D>C5K#) zjGJKAlDw~(L>jLneWrPW9Mwa-yVSJq&x9aduiU&Xq~`tzG-A^~s$jld@6Iqy4yenBRH$=( zknd14p_|?{Z5b(003XOFZPL^}6H8fY>UVfz^AG4^+D4AY8t%0j0m(n|fbi}(_IT-1 zd*{lgaUgyr()o23SE9a*m{%CMt&DH;-uHK5U!7BgmrMrD+sjZc{B`?F?mE0r!;XzH zcX#wT3SLc1JOh6IYVl-ti{&!bq1h{?%{2Y(xVx=l**WhuQxk~XQcEnZ>H#np3qBdY z8g^NQfaar)9F2AJ3t-CES*?Wk_aH0_hB9s72o!Ja1zyxuXg$n7zp8{#xlljT)D&M< z5{+Q)aIN(-dG|-m+5qz>Pvno44m~lhPuZ1RG`8!QuU|6MZbY+buU+%1Iz?}!fXNnV zt$DT=dYlFiH*U46ulov~*9l$`65f=)gx}<5@ZKgY+wW0_tL*oB_iub$ z>8C4JcAbc!>{OU?#VqQaM($6HJF*Jch^CoD#qrn@HUNo{?>fTd$m8Bg-5A^cQp^%= z7^R^2*mY@Xc0Pz@#`LK@sx~^(k`BBPp2blybb*9{qQMWFI1l zKAAfU`s?kkOEb9^5@nT|Q#Pm(5j~{_!Ol%i=6q_S18Ptj7gf7!M{s*+P^xn0+Z8|R zOyljSty!0zT~a)>BBD(3w$sPsn1v5Bf7QTq4p45NGIck(Dz}Zf)7h?i(XkVywgR6b zji1D=Q4@G!QrmT!Zz{jZq31+$WROH^y1c%*(Xg5MUlVC5*6(LhLg&0vJmuMcVrN5Z z+z!iUbsOv9$u|=3J|(W-75VxkZ8Mo6?^ES);odFqrPGT4DIbjL6s7=vo+YCOhI#f! z`Ja9J1(!lz_xJE!^@ydyip0{ue2U<>H9dQ&)!ebAkg{kjjA^cTS%#9g$+qe(cP5o7 zj&ap{IE^qP^(wjDf*ijub0!1?Sy9!h*Vh5y4yr_p8FRF7=kD-x+HNW&msh=h(X5n+ zbGmzpPh)3VV&F4QzCE20FS~OFKb34H`7=KX&z^|((DqCS%uizw0swmOJ4!b=r%(4@ zSh23GfQK3!?KKV3gx%y0p%izJLDy=$k7Y*ti_3nY8|ZP(-u zn_cqhp^`zqyR18%GW*8hM%Vp`eVmf#1rIc4eavQNz0FS(=yIrw%Qg0Y>VNjgk-7f& z9~xZ0-oIqkXz55zn#LAe_+&w&d8{O%HFS2gn*95xm9b{$udOnI0QOd)5Zn3(JIgK- z37az&PeNEz51_=0*`+TO_NnefHgn)wUc!3QG)DD{N!^m$>|z1QQ34Nim6Gddm-RWb zs(Z6^M@?l<#E);!_kX$hx3{h0^g=Aj)P{`d>+!mma?Yzf4HMOjduVojZcA?Eju)?L zT%I;XhgL&`$@*%YFGoKa715qtXWm`^;9}c@#WCl`LtQrQryU53kNZ9jUut;;@R7FX4v7Hx-ED1T2|?0CiJZ4CpjK53r| zw>9gY*cYJ09kiqDjGoX=^W%gTO1>1dE$Nk)CkNa7p9Xet3dg9ee&5udNhj~6&6wqW zP#i@_3A2_DW{ewDWrQ678}CPXE_d>s)O|96!4Z3f%TCle<{(pGppZ^ zyt??MCwDylO7+lFl|fS(?M6t|U#9&QU@97841Y>V?&(`Tg-@MTF_)-f>yYA@-g}fc zdqc{rir;^;GWU08p~p{hrC*kfz3uxwf|?+^P0?0jr8&HpanjhREgxp38C zCH;lDFxJq?v3?fYm{CGi+OXkooxkcH#VCDak5T%IJ>oLwz-xs}7&k2WW16G2o^y)t zp|(l5)q!N8p5N4Lk%|9z)Xo2h8u@z#{9mDF{|Bt^>*ZP!18b$E?(qIPF2OMlX8TpQ zfW9rDrYN`;?&0f+MRmh47Qi_oNrarN73uOnMolM<~8?!K0h{)#h!Qv1My#9ivLe^bSrrw zM>a~~)Mq*uirp>5yPC3~>pvi!&j05_tIqnp&ry8;gOB3RGG}r772gk`2_k9*LZD70 z4$;t)5g=s*$OQi=Cf2IoW3*F?;hc5u+DfO0Nxe8k7p}k*Mf&20r3|n=tm0RK({K3I zMLsA#LxYW?n)Sl2T8m|@Odu1KN{~3pI|D-^-Wby85wg+4-gFk2MG@VWH~xKjcFWb} z-SFc~1_GiD9NrbpGLcD>xjE_Mplb%%d=^~Q_u{*ufl0iRGku||V;2w*?XZz$_i!&+wZP?j&npD0cM9gDHk0#*-YhjGM zrf(k3@9Di9+bIth3iZV!AcZ54(EF-$ogH+LAUT-}vTK_Aj9blLS^*nRzarUs=KzcH zbOrTPDDVC_W%CmV$db!1@Klj_s!YfjS=QU&;xY*uMf9K`9`WQT<7N&fcblQ_^jv0b za`^J`X((M2W$ikz>rYC!Y~rezb&gjkW{DQ}D?HU2Pb^r`wLr4C_rw>T>nMMDZ6>H*(ddPp z{|d`&G!FbV4{;GGJ`QT$24i&JNEo$O{@J86*xch$nE7z;cjroWB| z(!0d9!ghf(8^vSIm|h-KXG{n;Qwea7(#s55jU-}VqH6iV?8>mNIvSOlG@>h`9R&f%XJ31+LWgdRKz0Scz* zxEe4;vj;?}1IqdrEFzI} zKCq)gPy#8&A<)pczkkL>_b0}MGRo}@8kmepu&r`vq9_{Jj_HDfoY5y?;LmPplZ1W+p5v3radaQ&Z^DQ~$z>Be zEV@vbQi|_lW2nOnb1Hz)S1Dee>Af(iWhi8Uypj=ZyApJHlG@v2|1rML5jQ~3C`KJx zkeS?Ud4ipHfc0LAy>8N%d(63FyLInmwXCOP!L7eRgz` zxLJC2K#rqi%)}O}JEn&t&v2FiKCgtCBLqZlKyOzKlK;({HrZ}TPWo{+GYLG11uO7=iz zVK77g0Oj*D=T#jcK_y6V(w znphojr(JNT%)CtKxwE^+G9YP@qRP|l?@6|)!{$y1BR2L_?VU}GAi#_>CgM97i(R3? zA41|pmmY;8D0}34@42KK{;%SdH#;7MUSKCgxs+VP!#=ZJ-NFu5N#$bTd9(Jq0X z00g2lG`LP~Mv{tyoa4X+NXOqn*SCH6BQWitV7ClHD}SU^Jd znsy_#Mmpd?NkG5@OQ%ldS`CgioD%y#CBrg{WvGEgP@P885ws=hB}_hyNRTizs@LRp z)qotzgsEf-rX*R0mm1<>_5rO*w7@GPd(>@H?1_=Qp4BHy<8e;J3H*f_kG1Y6Y)%$L zmq0XaAH(^8qOI`gB`6TjoY99n<(%^i$ zG!gJsk4V6$q<0@NW;Q2~)!>iRkeUxfD8gAT-c4~e*|Oc>HN_MuY>e+D=e~DKuJq6Z z5k$NjA99p7uirH@{n_U8)*<3vlboxNZqd}^)YFyg0&<@Y2;B{nT0wIOwnEA@?w6b6 zg9t&BPkziG{SNW<53a_pK`-jVb{nx2B*FU^^)XR#g>wKgwa;vVg+#Jhg&tH=J$Vp% zRP4mII+`p+GH9e}WFU+4c0}dnF^sxgpgfpvf;PAXC5=*=;|AsDj}7x+nm;L~7IC*N ztt&2Ds&%mcd(|K!%x`SM)N2aFBC5WNOOE?A3D!tQoycYpkJ@CMECk~0pDm04i=H|i zH%Ka6Pz$aO=qFR5C+c0_W5cD0@q`6V+STnZ;BgxdO`cYoF&LzinRpB`bRFyEX~X^9 zw*qle)5a+)oqoQi5gC%vGYiay^Q38Y7oMn;%0PwS{w&SW&3NU<-S|XDoytnVn9sx3 zFcuI!HG*BJJ>wQJqA3}vG*v3;bE~l*N7-oI5KyD%96RHP5%0%bWjl90<=xEf`p`5@ z`Go}`c}a0bY4n;CO@83r+G)Fykr@!Sd2)levri0!l=@<(mtjZ@?qrn1+e3`mf=<&x zR=nse$>Am~Oh(W<1dD;5J!MXPlS>CBENfI_uf889Jcs;gG0az^n$5I{ZXTlA*C$%` zoQuP<+-Fo`H6m*Co1JC)a$_BbW#_K%OXWX$R$svy-22Q9Pz^q~(DvhqzJCK}pU0Xx zbOIG~FB8Uyzl38Qw>qY$kBlg;aF5BD#hJ;j|RFu*Zbc#xJ(5^?h& zlFMeKG70r=bdOzr+KIkdTo(FHrQ=OA)9DsiGClHg`Knxp_LX-jq>t|FGd;~Uk3TIC zfsWI_&>b-R@M%=>Y50fZI9}aUsU29B9NhF(&fQyPwhqaUR;vVU(RS#i|BdQ6=_4sf zC+4qv0ToFe$>(DXKP4i(hOX1`FTG}$)ipIH*(vUeH}y-KL>^L0Ho^z%rRRmZ{#bEE zb&edW`BAO4br!@u-U3ssJdCQ1o)Aiq?cSfDA=k@~vbnkbF!OB4l1 zhQB$Y0{LGyS0qB{?A z7(CVYv~eT;9Oij}a?aniyv6h~UOaowc8RItQRCryGmn0JYCOoEsQD9^6)p%KwFFyykEgZ(HSQ z6}y^a{t_GeC_0_JPWUL=vwq^VGR)M?pTYgv(+(ziW?p_lWsSOA654jX{-!u)$MCt_ z?xd14L0=6})7rBixf(JKdTp~?9{BXu-3`&MA%W?g3uC5h0|}Wz7kZt>kYF%g;$|#1 z50B=e$FDE#)QJvE2}BE;F%|BX8vI>APGwl6lxg?M?SD0QUQtb_X&6V57KAuT1RZ5W zMXF#yFan7{OahL6M0x-!3Byu27&^r=Q6 zM5_z}aoJO49(hO=jvJNFGY|hEK?#dbx}DWn{|flIG5tG~kZALQ)Kfwqg5n!1~!O&+To!^GY$>?r9YO&`6+@99z1I@pnL(}lCXI>K>!ey3P?!u^4Nf2AsnJ@+S4-`ob*1!Cu&?UiPwW42 zxWLK{n}|J3at#Y_6Lb+5j_oS&9>1kdq&kb2WLFjIkMsxRX|8XhXX2d;phBE#yfF!# z)MaJf+tge*>zMa!KM@V8%z2|fg!$eyBgr=Jk*EBm)5RBH!Vdpudb&SIBU1)>~-zVolO82igClx&mmpU*BhLqbXgLeuZO! zInD=>l;6;{2E`T#Xj$;S_H_}CLK2s*K));Mr%MAGjQ50sjjjs_I}oycwx(`%!BW=F z$mw*OYkaYTL)X`)&=k>#v9`dNrgb(=$QsP+SG%fzx1aSDa!>YHBpqnO)Ou0)4l;cq_HzaiUo?d*4=P^t>*YoYz zS^>e4y7XrFOGj6Ki5GFGAfy6B{vXqnD|wt$X$UrkKPMz zGY)GIz!NR=7@};xxExO*XFC3TU#5jAG3MFNK2(1@ZA}ZBE&yFG7W_&+aL4TpC?1HX zgfoPXfBA0Nms-E~PQ`grS^jxT%H!h%XlS$_13#J!&POhomeii zPuWM9=lQwhp5Dl16kLRodOF(;0RVFkcE~)0Sw+ra+&eI4N=C0fIz&kjuO+!u7&j^N zkUlV5m?YJ4N`K5pbckCErmuoObg$wz3>bvV4ZU6%=?}F{fxOS68!muD*XEX;kwS@p zw8gB50PenRA1ED+HLEKIR$q#Hm4xc=*WbsdnBy}7X%ZvMU4JJ=yw}wJ#YUuGic~^Y z5OR19!bM=!5*d}?PE%+cJNc-lc|T>Jf?UZyrMNQtnC(29uqhB2Zs2|TLdEvJJAc1I zl?8{;{|JSxG+vu=Jq9^geh!KZ2&#;)Wytu=PaG?X*6uVo#=NyP;Aw;lQM6P}z;->S zDPinGul&~ESsZ<*mbO99NzkaJu9bTpv;<2^Al$SjU^{J#d+{3%goSKhYTMv_L2;-c z>Eg%eF4;7df{n&Z-RCK){ZSwE-pqH?niEZ}kC&{)>Ws7<{ ziy^P=$gtQA_jz0rv}hYmVu$fP>;I?9Koa@}%(zo7xs!El^=@@c3{KkT`BW^CH7HU? zYrV%8#lPEJA&^>WC+VvoSw1#>LZkkfJP~HCogB`t`T~uP_Bbp#87}hDzK9?HNmUj%g5$0JSpoKl6bhNG6ng@dV}-- zBs-m}Y4E@(jWRktTDw+2aTl$G9gqGTBOASnPpl}TwFli^1m`DD1gGGldJ;Bj1=P$i zmtPsy|4m~8%5hygwS9i&5uYt{)%`zWa?sW~rKzySvO1k8GI(@ypnMXsf8i}~)KxX1 z)0?D!hh5D;H1O??s0+`cb@)5MI7ND{fLy_?ZWn8nRE(f-Xj+!{Vg_jtF0@LR5Hvw2 z*e8`mO9ky2i?4XmnBJy$jvs8!3x>2pTN1W+H%9Ji(hde_Y55w@IYa;>rSufhIU%9% zcZqdb%XW>GEaB*?u}YEgf?XfUD9T0Gwxkq;q`>XKBm6yxzD8bf4dANcBt8%ftYv^? zZ!Ckh%y6!jepV5kv{I){*T*jYfQ$JV1a8o6scrl|1<1WHG0&;?>W_oI`Cy0xy@kW9 z;@P)0T=wf{NUk53J5d%eF50xxNdo4(g7p#ms~XK@%vDN@4*uqHVpzpCTlyc^jQf7W zB{Qblm>rHSH;rE`?0usgY)MUNRQE-QTB&LZ+-sm4RSm)o7R?UvMb=U2x|>z=HIl*x zoI%xT8x*X(9xwrmiF-i)TGbmenV%qyP|s$Vp0;dRSL&d4vk#?bJI*Nz-;y(V;c}-NCZaB*I@niJ!(j!JEefHi+~O6g)_1`c$L(_Ky$pf6h`9Z zz^;&rpwgLG;dbeaI1m{^crcH(=6mP(BY7tqRToW(9r;~JK5?(7Yh~H5fFkhcuL*z9 zu>u7l62;@NMOfGL(HW9f@irg9tuqXm^s!=IRht;ZpB7CMJM*3J-O4pc0rdM+MCDh# z@9m>Gp+E!~r-sgtCE{YQp7;n;?SWLG~AhS1D`qShF5YUwO#H;fU3#`l6y^(5+P z9}tpzVn27yg9>o8yPQW(jxX05x~`zWDBxa=H-d*DEpXLwL&#!5XvmZep$Q;II?nQ= z9fSNL>zra<4Lb#48j-m1w;^GT-;RtV)lrBLp1Q1rW9kiYgk4OoC!!FN9^KwQokM~6 zn#f7P6MI4;kUK7mVc=Yl*PG4(&ANSPTS_9Ibu_8;$t z1w~mwrR5fGWFbhm%hJ3ftT#b{h#*nKF zN@bvqrokyQ%Q8;-Y*_?Q&vwnvo8$kMV=O5{aSc3s9k|j;wodQ*K(#&v!ERnW@Q>%lPi)!0Vs2i8n z6M&5BEh%xC!z=d>gQp_GWX$B`q7Ryax2bsx8<1=H1@_-SWi9G|Eu#Omoc;d+pZ&)` w9X|j6U=5)5*NMGEyUuU_{%a6iKb|*bQ$X530Aj&>z5oCK From 52d3086c039669b2c991efd2eeafaf6b979b2870 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Sat, 11 Apr 2026 07:09:21 -0700 Subject: [PATCH 12/13] Remove quadratureshap summary artifact --- quadratureshap_n8_summary.json | 92 ---------------------------------- 1 file changed, 92 deletions(-) delete mode 100644 quadratureshap_n8_summary.json diff --git a/quadratureshap_n8_summary.json b/quadratureshap_n8_summary.json deleted file mode 100644 index c7ec09f2ef9b..000000000000 --- a/quadratureshap_n8_summary.json +++ /dev/null @@ -1,92 +0,0 @@ -[ - { - "dataset": "breast_cancer", - "family": "real", - "depth": 30, - "points": 8, - "mean_nodes": 37.37, - "speedup": 0.4245682082044919, - "max_abs_diff": 4.76837158203125e-07 - }, - { - "dataset": "diabetes", - "family": "real", - "depth": 30, - "points": 8, - "mean_nodes": 191.01333333333332, - "speedup": 1.5975975642047238, - "max_abs_diff": 1.52587890625e-05 - }, - { - "dataset": "digits", - "family": "real", - "depth": 30, - "points": 8, - "mean_nodes": 46.516, - "speedup": 1.5759131851301158, - "max_abs_diff": 2.5033950805664062e-06 - }, - { - "dataset": "easy_linear", - "family": "synthetic", - "depth": 4, - "points": 8, - "mean_nodes": 30.61, - "speedup": 0.645007986369304, - "max_abs_diff": 1.9073486328125e-06 - }, - { - "dataset": "easy_linear", - "family": "synthetic", - "depth": 8, - "points": 8, - "mean_nodes": 370.47, - "speedup": 1.1889959234572578, - "max_abs_diff": 2.86102294921875e-06 - }, - { - "dataset": "easy_linear", - "family": "synthetic", - "depth": 16, - "points": 8, - "mean_nodes": 1467.43, - "speedup": 2.3738298777126716, - "max_abs_diff": 5.7220458984375e-06 - }, - { - "dataset": "easy_linear", - "family": "synthetic", - "depth": 30, - "points": 8, - "mean_nodes": 1928.48, - "speedup": 3.459982033859825, - "max_abs_diff": 0.0033998489379882812 - }, - { - "dataset": "random_labels", - "family": "synthetic", - "depth": 4, - "points": 8, - "mean_nodes": 30.54, - "speedup": 0.7424122662547481, - "max_abs_diff": 5.960464477539063e-08 - }, - { - "dataset": "random_labels", - "family": "synthetic", - "depth": 8, - "points": 8, - "mean_nodes": 321.22, - "speedup": 1.8998626925981703, - "max_abs_diff": 1.7881393432617188e-07 - }, - { - "dataset": "random_labels", - "family": "synthetic", - "depth": 16, - "points": 8, - "mean_nodes": 4273.45, - "speedup": 4.594703258820682, - "max_abs_diff": 2.8759241104125977e-06 - } -] From 108402fd3656588514ff5322f680ddcee9a47b34 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Wed, 22 Apr 2026 02:05:14 -0700 Subject: [PATCH 13/13] Add SHAP efficiency error experiment --- .../README.md | 49 +++ .../benchmark_fashion_mnist_efficiency.py | 307 ++++++++++++++++++ src/predictor/cpu_predictor.cc | 3 +- src/predictor/interpretability/shap.cc | 193 ++++++----- 4 files changed, 474 insertions(+), 78 deletions(-) create mode 100644 experiments/2026-04-21-fashion-mnist-efficiency-sweep/README.md create mode 100644 experiments/2026-04-21-fashion-mnist-efficiency-sweep/benchmark_fashion_mnist_efficiency.py diff --git a/experiments/2026-04-21-fashion-mnist-efficiency-sweep/README.md b/experiments/2026-04-21-fashion-mnist-efficiency-sweep/README.md new file mode 100644 index 000000000000..682db19388c9 --- /dev/null +++ b/experiments/2026-04-21-fashion-mnist-efficiency-sweep/README.md @@ -0,0 +1,49 @@ +## Purpose + +Sweep requested depth for a `fashion_mnist` lossguide model and compare efficiency error for: + +- CPU TreeSHAP +- CPU QuadratureSHAP with `4` points +- CPU QuadratureSHAP with `6` points +- CPU QuadratureSHAP with `8` points +- CPU QuadratureSHAP with `16` points + +The experiment records `mean`, `p99`, and `max` efficiency error, where efficiency is checked +against the raw margin: + +`sum(phi) == predict(output_margin=True)` + +The generated result directories are local experiment outputs and are not intended to be tracked. + +## Commands + +Original `max_leaves=128` run: + +```bash +PYTHONPATH=/home/nfs/rorym/xgboost-wt/shapley-value-algorithms/python-package \ +LD_LIBRARY_PATH=/home/nfs/rorym/xgboost-wt/shapley-value-algorithms/lib:${LD_LIBRARY_PATH} \ +/home/nfs/rorym/anaconda3/bin/conda run -n xgboost python \ + /home/nfs/rorym/xgboost-wt/shapley-value-algorithms/experiments/2026-04-21-fashion-mnist-efficiency-sweep/benchmark_fashion_mnist_efficiency.py \ + --out-dir /home/nfs/rorym/xgboost-wt/shapley-value-algorithms/experiments/2026-04-21-fashion-mnist-efficiency-sweep/results \ + --points 4 8 16 +``` + +Follow-up `max_leaves=1024` run: + +```bash +PYTHONPATH=/home/nfs/rorym/xgboost-wt/shapley-value-algorithms/python-package \ +LD_LIBRARY_PATH=/home/nfs/rorym/xgboost-wt/shapley-value-algorithms/lib:${LD_LIBRARY_PATH} \ +/home/nfs/rorym/anaconda3/bin/conda run -n xgboost python \ + /home/nfs/rorym/xgboost-wt/shapley-value-algorithms/experiments/2026-04-21-fashion-mnist-efficiency-sweep/benchmark_fashion_mnist_efficiency.py \ + --out-dir /home/nfs/rorym/xgboost-wt/shapley-value-algorithms/experiments/2026-04-21-fashion-mnist-efficiency-sweep/results-maxleaves1024 \ + --max-leaves 1024 --depths 4 8 12 16 24 32 48 64 --points 4 6 8 16 +``` + +## Generated Outputs + +- `results.json` +- `results.csv` +- `summary.md` +- `efficiency_mean.png` +- `efficiency_p99.png` +- `efficiency_max.png` diff --git a/experiments/2026-04-21-fashion-mnist-efficiency-sweep/benchmark_fashion_mnist_efficiency.py b/experiments/2026-04-21-fashion-mnist-efficiency-sweep/benchmark_fashion_mnist_efficiency.py new file mode 100644 index 000000000000..2106d4298f75 --- /dev/null +++ b/experiments/2026-04-21-fashion-mnist-efficiency-sweep/benchmark_fashion_mnist_efficiency.py @@ -0,0 +1,307 @@ +"""Run Fashion-MNIST SHAP efficiency-error sweeps.""" + +from __future__ import annotations + +# pylint: disable=missing-function-docstring,too-many-locals +import argparse +import csv +import json +import re +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import xgboost as xgb +from sklearn import datasets + +DEFAULT_DEPTHS = [4, 8, 12, 16, 24, 32, 48, 64] +DEFAULT_POINTS = [4, 6, 8, 16] +DEFAULT_SEED = 20260421 +DEFAULT_TEST_ROWS = 512 +DEFAULT_THREADS = 35 +DEFAULT_ROUNDS = 100 +DEFAULT_MAX_LEAVES = 128 + + +def fetch_fashion_mnist() -> tuple[object, np.ndarray]: + x, y = datasets.fetch_openml("Fashion-MNIST", return_X_y=True) + return x, y.astype(np.int64) + + +def tree_stats(model: xgb.Booster) -> dict[str, float]: + dump = model.get_dump(dump_format="json", with_stats=True) + + def walk(node: dict, depth: int = 0) -> tuple[int, int, int]: + children = node.get("children", []) + if not children: + return depth, 1, 1 + max_depth = depth + node_count = 1 + leaf_count = 0 + for child in children: + child_depth, child_nodes, child_leaves = walk(child, depth + 1) + max_depth = max(max_depth, child_depth) + node_count += child_nodes + leaf_count += child_leaves + return max_depth, node_count, leaf_count + + max_depths: list[int] = [] + node_counts: list[int] = [] + leaf_counts: list[int] = [] + for tree_json in dump: + tree_json = re.sub(r"\bnan\b", "0", tree_json) + tree_json = re.sub(r"\binf\b", "0", tree_json) + tree = json.loads(tree_json) + max_depth, nodes, leaves = walk(tree) + max_depths.append(max_depth) + node_counts.append(nodes) + leaf_counts.append(leaves) + + return { + "num_trees": len(dump), + "mean_max_depth": float(np.mean(max_depths)), + "max_max_depth": float(np.max(max_depths)), + "mean_nodes": float(np.mean(node_counts)), + "mean_leaves": float(np.mean(leaf_counts)), + } + + +def train_model( + x_train: object, y_train: np.ndarray, depth: int, seed: int, max_leaves: int +) -> xgb.Booster: + dtrain = xgb.QuantileDMatrix(x_train, y_train, enable_categorical=True) + params: dict[str, object] = { + "objective": "multi:softmax", + "num_class": 10, + "tree_method": "hist", + "device": "cpu", + "grow_policy": "lossguide", + "max_leaves": max_leaves, + "max_depth": depth, + "eta": 0.01, + "seed": seed, + "nthread": DEFAULT_THREADS, + } + return xgb.train(params, dtrain, num_boost_round=DEFAULT_ROUNDS, verbose_eval=False) + + +def sample_rows( + x: object, y: np.ndarray, rows: int, seed: int +) -> tuple[object, np.ndarray]: + rs = np.random.RandomState(seed) + row_idx = rs.choice(len(y), size=rows, replace=False) + if hasattr(x, "iloc"): + return x.iloc[row_idx, :], y[row_idx] + return x[row_idx, :], y[row_idx] + + +def efficiency_metrics(pred: np.ndarray, margin: np.ndarray) -> dict[str, float]: + err = np.abs(np.sum(pred, axis=pred.ndim - 1) - margin).reshape(-1) + return { + "mean_efficiency_err": float(np.mean(err)), + "p99_efficiency_err": float(np.quantile(err, 0.99)), + "max_efficiency_err": float(np.max(err)), + } + + +def predict_contribs( + booster: xgb.Booster, + dtest: xgb.DMatrix, + algorithm: str, + quadrature_points: int | None, +) -> np.ndarray: + params: dict[str, object] = {"device": "cpu", "shap_algorithm": algorithm} + if algorithm == "quadratureshap": + assert quadrature_points is not None + params["quadratureshap_points"] = quadrature_points + booster = booster.copy() + booster.set_param(params) + return np.asarray(booster.predict(dtest, pred_contribs=True)) + + +def write_csv(path: Path, rows: list[dict[str, object]]) -> None: + fieldnames: list[str] = [] + seen: set[str] = set() + for row in rows: + for key in row.keys(): + if key not in seen: + seen.add(key) + fieldnames.append(key) + with path.open("w", newline="") as fd: + writer = csv.DictWriter(fd, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def make_plot(rows: list[dict[str, object]], metric: str, out_path: Path) -> None: + plt.figure(figsize=(7, 4.5)) + series = {} + for row in rows: + series.setdefault(row["algorithm_label"], []).append(row) + for label, vals in series.items(): + vals = sorted(vals, key=lambda r: r["requested_depth"]) + xs = [r["requested_depth"] for r in vals] + ys = [r[metric] for r in vals] + plt.plot(xs, ys, marker="o", linewidth=2, label=label) + plt.yscale("log") + plt.xlabel("Requested max_depth") + plt.ylabel(metric.replace("_", " ")) + plt.title(f"Fashion-MNIST efficiency sweep: {metric}") + plt.grid(True, which="both", alpha=0.25) + plt.legend() + plt.tight_layout() + plt.savefig(out_path, dpi=160) + plt.close() + + +def write_summary(path: Path, rows: list[dict[str, object]]) -> None: + header = ( + "| algorithm | requested_depth | mean_max_depth | max_max_depth | " + "mean_efficiency_err | p99_efficiency_err | max_efficiency_err |" + ) + lines = [ + "## Fashion-MNIST Efficiency Sweep", + "", + header, + "| --- | --- | --- | --- | --- | --- | --- |", + ] + for row in rows: + lines.append( + f"| {row['algorithm_label']} | {row['requested_depth']} | " + f"{row['mean_max_depth']:.3f} | {row['max_max_depth']:.0f} | " + f"{row['mean_efficiency_err']:.6e} | {row['p99_efficiency_err']:.6e} | " + f"{row['max_efficiency_err']:.6e} |" + ) + path.write_text("\n".join(lines) + "\n") + + +def write_outputs( + out_dir: Path, metadata: dict[str, object], rows: list[dict[str, object]] +) -> None: + (out_dir / "results.json").write_text( + json.dumps({"metadata": metadata, "rows": rows}, indent=2) + ) + write_csv(out_dir / "results.csv", rows) + write_summary(out_dir / "summary.md", rows) + make_plot(rows, "mean_efficiency_err", out_dir / "efficiency_mean.png") + make_plot(rows, "p99_efficiency_err", out_dir / "efficiency_p99.png") + make_plot(rows, "max_efficiency_err", out_dir / "efficiency_max.png") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--depths", type=int, nargs="+", default=DEFAULT_DEPTHS) + parser.add_argument("--points", type=int, nargs="+", default=DEFAULT_POINTS) + parser.add_argument("--test-rows", type=int, default=DEFAULT_TEST_ROWS) + parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + parser.add_argument("--max-leaves", type=int, default=DEFAULT_MAX_LEAVES) + parser.add_argument("--reuse-json", type=Path, default=None) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + args.out_dir.mkdir(parents=True, exist_ok=True) + if args.reuse_json is not None: + payload = json.loads(args.reuse_json.read_text()) + metadata = payload["metadata"] + rows = payload["rows"] + completed_depths = {row["requested_depth"] for row in rows} + pending_depths = [ + depth for depth in args.depths if depth not in completed_depths + ] + if pending_depths: + x, y = fetch_fashion_mnist() + x_test, y_test = sample_rows(x, y, args.test_rows, args.seed) + dtest = xgb.DMatrix(x_test, y_test, enable_categorical=True) + for depth in pending_depths: + print(f"Training requested depth {depth}") + booster = train_model(x, y, depth, args.seed, args.max_leaves) + stats = tree_stats(booster) + margin = np.asarray(booster.predict(dtest, output_margin=True)) + + treeshap = predict_contribs(booster, dtest, "treeshap", None) + rows.append( + { + "algorithm": "treeshap", + "algorithm_label": "TreeSHAP", + "requested_depth": depth, + **stats, + **efficiency_metrics(treeshap, margin), + } + ) + + for points in args.points: + contribs = predict_contribs( + booster, dtest, "quadratureshap", points + ) + rows.append( + { + "algorithm": "quadratureshap", + "algorithm_label": f"QuadratureSHAP-{points}", + "requested_depth": depth, + "quadrature_points": points, + **stats, + **efficiency_metrics(contribs, margin), + } + ) + metadata["depths"] = sorted( + set(metadata.get("depths", [])) | set(args.depths) + ) + metadata["points"] = sorted( + set(metadata.get("points", [])) | set(args.points) + ) + metadata["max_leaves"] = args.max_leaves + write_outputs(args.out_dir, metadata, rows) + else: + x, y = fetch_fashion_mnist() + x_test, y_test = sample_rows(x, y, args.test_rows, args.seed) + dtest = xgb.DMatrix(x_test, y_test, enable_categorical=True) + + rows = [] + metadata = { + "seed": args.seed, + "test_rows": args.test_rows, + "rounds": DEFAULT_ROUNDS, + "max_leaves": args.max_leaves, + "depths": args.depths, + "points": args.points, + "threads": DEFAULT_THREADS, + } + for depth in args.depths: + print(f"Training requested depth {depth}") + booster = train_model(x, y, depth, args.seed, args.max_leaves) + stats = tree_stats(booster) + margin = np.asarray(booster.predict(dtest, output_margin=True)) + + treeshap = predict_contribs(booster, dtest, "treeshap", None) + rows.append( + { + "algorithm": "treeshap", + "algorithm_label": "TreeSHAP", + "requested_depth": depth, + **stats, + **efficiency_metrics(treeshap, margin), + } + ) + + for points in args.points: + contribs = predict_contribs(booster, dtest, "quadratureshap", points) + rows.append( + { + "algorithm": "quadratureshap", + "algorithm_label": f"QuadratureSHAP-{points}", + "requested_depth": depth, + "quadrature_points": points, + **stats, + **efficiency_metrics(contribs, margin), + } + ) + write_outputs(args.out_dir, metadata, rows) + + write_outputs(args.out_dir, metadata, rows) + + +if __name__ == "__main__": + main() diff --git a/src/predictor/cpu_predictor.cc b/src/predictor/cpu_predictor.cc index bef04f358177..4d8418eae09d 100644 --- a/src/predictor/cpu_predictor.cc +++ b/src/predictor/cpu_predictor.cc @@ -761,7 +761,8 @@ class CPUPredictor : public Predictor { } catch (std::out_of_range const &) { LOG(FATAL) << "quadratureshap_points out of range: " << kv.second; } - CHECK_EQ(points, 8) << "CPU QuadratureSHAP currently uses a fixed quadrature size of 8."; + CHECK(points == 4 || points == 6 || points == 8 || points == 16) + << "CPU QuadratureSHAP currently supports quadrature sizes of 4, 6, 8, or 16."; quadrature_shap_points_ = points; } } diff --git a/src/predictor/interpretability/shap.cc b/src/predictor/interpretability/shap.cc index a50ed19de038..999d15c7f134 100644 --- a/src/predictor/interpretability/shap.cc +++ b/src/predictor/interpretability/shap.cc @@ -238,24 +238,26 @@ void CalculateContributions(tree::ScalarTreeView const &tree, RegTree::FVec cons condition, condition_feature, 1.0f); } -// Keep the CPU quadrature recurrence on the same fixed 8-point rule as the GPU path so the hot -// loops stay small and the compiler can fully unroll the basis update and extraction work. +// The CPU additive path supports a few fixed quadrature sizes so experiments can sweep point +// counts while keeping compile-time-unrolled hot loops. constexpr std::size_t kQuadratureShapPoints = 8; constexpr double kQuadratureShapBuildQeps = 1e-15; constexpr float kQuadratureShapUnseen = -999.0f; +template struct QuadratureRule { - std::array nodes{}; - std::array weights{}; + std::array nodes{}; + std::array weights{}; }; -using QuadratureBuffer = std::array; - -QuadratureRule const &GetQuadratureRule() { - static QuadratureRule const rule = [] { - auto const rule_d = - detail::MakeEndpointQuadrature(kQuadratureShapBuildQeps); - QuadratureRule out; - for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { +template +using QuadratureBuffer = std::array; + +template +QuadratureRule const &GetQuadratureRule() { + static QuadratureRule const rule = [] { + auto const rule_d = detail::MakeEndpointQuadrature(kQuadratureShapBuildQeps); + QuadratureRule out; + for (std::size_t i = 0; i < Points; ++i) { out.nodes[i] = static_cast(rule_d.nodes[i]); out.weights[i] = static_cast(rule_d.weights[i]); } @@ -264,24 +266,26 @@ QuadratureRule const &GetQuadratureRule() { return rule; } -void AddInPlace(QuadratureBuffer *lhs, QuadratureBuffer const &rhs) { - for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { +template +void AddInPlace(QuadratureBuffer *lhs, QuadratureBuffer const &rhs) { + for (std::size_t i = 0; i < Points; ++i) { (*lhs)[i] += rhs[i]; } } -float ExtractQuadratureDelta(QuadratureRule const &rule, QuadratureBuffer const &h_vals, - float p_enter, float p_exit) { +template +float ExtractQuadratureDelta(QuadratureRule const &rule, + QuadratureBuffer const &h_vals, float p_enter, float p_exit) { float acc = 0.0f; if (p_enter != 1.0f) { auto const alpha_enter = p_enter - 1.0f; - for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + for (std::size_t i = 0; i < Points; ++i) { acc += alpha_enter * h_vals[i] / (1.0f + alpha_enter * rule.nodes[i]); } } if (p_exit != 1.0f) { auto const alpha_exit = p_exit - 1.0f; - for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + for (std::size_t i = 0; i < Points; ++i) { acc -= alpha_exit * h_vals[i] / (1.0f + alpha_exit * rule.nodes[i]); } } @@ -297,8 +301,10 @@ constexpr bool kQuadratureInteractionUseLatestLiveIndex = false; // H(t) = H_without_j(t) * (1 + (q_j - 1) t) // after the zero-fraction terms cancel. The conditioned on/off difference is therefore the // precomputed return-edge kernel divided by that partner factor and multiplied by (q_j - 1). -float ExtractQuadratureInteractionDelta(QuadratureRule const &rule, QuadratureBuffer const &h_vals, - float p_enter, float p_exit, float q_partner) { +template +float ExtractQuadratureInteractionDelta(QuadratureRule const &rule, + QuadratureBuffer const &h_vals, float p_enter, + float p_exit, float q_partner) { if (q_partner == 1.0f) { return 0.0f; } @@ -310,7 +316,7 @@ float ExtractQuadratureInteractionDelta(QuadratureRule const &rule, QuadratureBu auto const alpha_exit = p_exit - 1.0f; float acc = 0.0f; - for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + for (std::size_t i = 0; i < Points; ++i) { float edge_delta = 0.0f; if (has_enter) { edge_delta += alpha_enter / (1.0f + alpha_enter * rule.nodes[i]); @@ -323,25 +329,28 @@ float ExtractQuadratureInteractionDelta(QuadratureRule const &rule, QuadratureBu return acc; } -float ExtractQuadratureInteractionDelta(QuadratureRule const &rule, - QuadratureBuffer const &edge_kernel, float q_partner) { +template +float ExtractQuadratureInteractionDelta(QuadratureRule const &rule, + QuadratureBuffer const &edge_kernel, + float q_partner) { if (q_partner == 1.0f) { return 0.0f; } auto const alpha_partner = q_partner - 1.0f; float acc = 0.0f; - for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + for (std::size_t i = 0; i < Points; ++i) { acc += alpha_partner * edge_kernel[i] / (1.0f + alpha_partner * rule.nodes[i]); } return acc; } -void WriteWeightedLeafReturn(tree::ScalarTreeView const &tree, QuadratureRule const &rule, - bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, - QuadratureBuffer *out_h) { +template +void WriteWeightedLeafReturn(tree::ScalarTreeView const &tree, QuadratureRule const &rule, + bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, + QuadratureBuffer *out_h) { auto const leaf_scale = w_prod * tree.LeafValue(nidx); - for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + for (std::size_t i = 0; i < Points; ++i) { (*out_h)[i] = c_vals[i] * leaf_scale * rule.weights[i]; } } @@ -462,6 +471,7 @@ struct LiveQuadraturePathState { // Current additive SHAP formulation. It consumes the weighted subtree return and writes one // feature contribution per return edge. +template struct AdditiveContributionFormulation { EmptyQuadraturePathState path_state; ContributionVectorView phi; @@ -474,25 +484,27 @@ struct AdditiveContributionFormulation { } void PopPathSplit(bst_feature_t split_index) const { path_state.Pop(split_index); } - void HandleLeaf(tree::ScalarTreeView const &tree, QuadratureRule const &rule, bst_node_t nidx, - QuadratureBuffer const &c_vals, float w_prod, QuadratureBuffer *out_h) const { - WriteWeightedLeafReturn(tree, rule, nidx, c_vals, w_prod, out_h); + void HandleLeaf(tree::ScalarTreeView const &tree, QuadratureRule const &rule, + bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, + QuadratureBuffer *out_h) const { + WriteWeightedLeafReturn(tree, rule, nidx, c_vals, w_prod, out_h); } - void HandleReturn(QuadratureRule const &rule, bst_feature_t split_index, - QuadratureBuffer const &h_vals, float p_enter, float p_exit) const { - phi[split_index] += ExtractQuadratureDelta(rule, h_vals, p_enter, p_exit); + void HandleReturn(QuadratureRule const &rule, bst_feature_t split_index, + QuadratureBuffer const &h_vals, float p_enter, float p_exit) const { + phi[split_index] += ExtractQuadratureDelta(rule, h_vals, p_enter, p_exit); } }; // First path-local interaction formulation built on top of the quadrature traversal. It keeps the // traversal and weighted subtree return shared with additive SHAP, and only changes how return // edges are written into the dense interaction sink. +template struct InteractionContributionFormulation { struct EdgeEffect { bst_feature_t split_index; float diagonal_delta; - QuadratureBuffer edge_kernel; + QuadratureBuffer edge_kernel; }; LiveQuadraturePathState path_state; @@ -517,15 +529,16 @@ struct InteractionContributionFormulation { // Traversal still needs a weighted subtree return, so the interaction path shares the additive // leaf behavior and changes only the return-edge algebra. - void HandleLeaf(tree::ScalarTreeView const &tree, QuadratureRule const &rule, bst_node_t nidx, - QuadratureBuffer const &c_vals, float w_prod, QuadratureBuffer *out_h) const { - WriteWeightedLeafReturn(tree, rule, nidx, c_vals, w_prod, out_h); + void HandleLeaf(tree::ScalarTreeView const &tree, QuadratureRule const &rule, + bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, + QuadratureBuffer *out_h) const { + WriteWeightedLeafReturn(tree, rule, nidx, c_vals, w_prod, out_h); } - [[nodiscard]] auto MakeEdgeEffect(QuadratureRule const &rule, bst_feature_t split_index, - QuadratureBuffer const &h_vals, float p_enter, + [[nodiscard]] auto MakeEdgeEffect(QuadratureRule const &rule, bst_feature_t split_index, + QuadratureBuffer const &h_vals, float p_enter, float p_exit) const { - QuadratureBuffer edge_kernel{}; + QuadratureBuffer edge_kernel{}; float diagonal_delta = 0.0f; if constexpr (kQuadratureInteractionUseEdgeKernel) { @@ -534,7 +547,7 @@ struct InteractionContributionFormulation { auto const alpha_enter = p_enter - 1.0f; auto const alpha_exit = p_exit - 1.0f; - for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + for (std::size_t i = 0; i < Points; ++i) { float edge_delta = 0.0f; if (has_enter) { edge_delta += alpha_enter / (1.0f + alpha_enter * rule.nodes[i]); @@ -546,7 +559,7 @@ struct InteractionContributionFormulation { diagonal_delta += edge_kernel[i]; } } else { - diagonal_delta = ExtractQuadratureDelta(rule, h_vals, p_enter, p_exit); + diagonal_delta = ExtractQuadratureDelta(rule, h_vals, p_enter, p_exit); } return EdgeEffect{split_index, diagonal_delta, edge_kernel}; @@ -579,8 +592,8 @@ struct InteractionContributionFormulation { phi_interactions(i, j) += scale * pair_delta; } - void HandleReturn(QuadratureRule const &rule, bst_feature_t split_index, - QuadratureBuffer const &h_vals, float p_enter, float p_exit) const { + void HandleReturn(QuadratureRule const &rule, bst_feature_t split_index, + QuadratureBuffer const &h_vals, float p_enter, float p_exit) const { auto path = path_state.View(); auto const edge = this->MakeEdgeEffect(rule, split_index, h_vals, p_enter, p_exit); this->AccumulateDiagonal(edge); @@ -588,10 +601,11 @@ struct InteractionContributionFormulation { this->ForEachPartner(path, [&](QuadraturePathElement const &partner) { float pair_delta = 0.0f; if constexpr (kQuadratureInteractionUseEdgeKernel) { - pair_delta = ExtractQuadratureInteractionDelta(rule, edge.edge_kernel, partner.p_child); - } else { pair_delta = - ExtractQuadratureInteractionDelta(rule, h_vals, p_enter, p_exit, partner.p_child); + ExtractQuadratureInteractionDelta(rule, edge.edge_kernel, partner.p_child); + } else { + pair_delta = ExtractQuadratureInteractionDelta(rule, h_vals, p_enter, p_exit, + partner.p_child); } this->AccumulatePair(edge, partner, pair_delta); }); @@ -600,11 +614,11 @@ struct InteractionContributionFormulation { // Tree-walk engine for quadrature formulations. It owns feature evaluation, child descent, and // the live path-probability state, then hands leaf/return events to the selected formulation. -template +template struct QuadratureShapTreeRunner { tree::ScalarTreeView const &tree; RegTree::FVec const &feat; - QuadratureRule const &rule; + QuadratureRule const &rule; std::vector *path_prob; ContributionFormulation formulation; @@ -623,7 +637,8 @@ struct QuadratureShapTreeRunner { } void VisitChild(bst_node_t split_node, bst_node_t child_node, float child_weight, bool satisfies, - QuadratureBuffer const &c_vals, float w_prod, QuadratureBuffer *out_h) { + QuadratureBuffer const &c_vals, float w_prod, + QuadratureBuffer *out_h) { auto split_index = tree.SplitIndex(split_node); auto p_old = (*path_prob)[split_index]; @@ -642,14 +657,14 @@ struct QuadratureShapTreeRunner { auto c_child = c_vals; auto alpha_e = p_e - 1.0f; - for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + for (std::size_t i = 0; i < Points; ++i) { c_child[i] *= 1.0f + alpha_e * rule.nodes[i]; } if (p_old != kQuadratureShapUnseen) { auto alpha_old = p_old - 1.0f; if (alpha_old != 0.0f) { - for (std::size_t i = 0; i < kQuadratureShapPoints; ++i) { + for (std::size_t i = 0; i < Points; ++i) { c_child[i] /= 1.0f + alpha_old * rule.nodes[i]; } } @@ -663,8 +678,8 @@ struct QuadratureShapTreeRunner { (*path_prob)[split_index] = p_old; } - void RunNode(bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, - QuadratureBuffer *out_h) { + void RunNode(bst_node_t nidx, QuadratureBuffer const &c_vals, float w_prod, + QuadratureBuffer *out_h) { if (tree.IsLeaf(nidx)) { formulation.HandleLeaf(tree, rule, nidx, c_vals, w_prod, out_h); return; @@ -676,11 +691,11 @@ struct QuadratureShapTreeRunner { auto right_weight = this->ChildWeight(nidx, right); auto goes_left = this->EvaluateGoesLeft(nidx); - QuadratureBuffer right_h{}; + QuadratureBuffer right_h{}; this->VisitChild(nidx, left, left_weight, goes_left, c_vals, w_prod, out_h); this->VisitChild(nidx, right, right_weight, !goes_left, c_vals, w_prod, &right_h); - AddInPlace(out_h, right_h); + AddInPlace(out_h, right_h); } void Run() { @@ -689,9 +704,9 @@ struct QuadratureShapTreeRunner { return; } - QuadratureBuffer c_init{}; + QuadratureBuffer c_init{}; c_init.fill(1.0f); - QuadratureBuffer h_vals{}; + QuadratureBuffer h_vals{}; this->RunNode(RegTree::kRoot, c_init, 1.0f, &h_vals); } }; @@ -835,21 +850,19 @@ void ShapValues(Context const *ctx, DMatrix *p_fmat, HostDeviceVector *ou LaunchShap(ctx, p_fmat, model, process_view); } -void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, - HostDeviceVector *out_contribs, gbm::GBTreeModel const &model, - bst_tree_t tree_end, std::vector const *tree_weights, - std::size_t quadrature_points) { +template +void QuadratureShapValuesImpl(Context const *ctx, DMatrix *p_fmat, + HostDeviceVector *out_contribs, gbm::GBTreeModel const &model, + bst_tree_t tree_end, std::vector const *tree_weights) { + static_assert(Points == 4 || Points == 6 || Points == 8 || Points == 16); + CHECK(!model.learner_model_param->IsVectorLeaf()) << "Predict contribution" << MTNotImplemented(); CHECK(!p_fmat->Info().IsColumnSplit()) << "Predict contribution support for column-wise data split is not yet implemented."; - CHECK_EQ(quadrature_points, kQuadratureShapPoints) - << "CPU QuadratureSHAP currently uses a fixed quadrature size of " << kQuadratureShapPoints - << "."; MetaInfo const &info = p_fmat->Info(); tree_end = predictor::GetTreeLimit(model.trees, tree_end); CHECK_GE(tree_end, 0); ValidateTreeWeights(tree_weights, tree_end); - auto const n_trees = static_cast(tree_end); auto const n_threads = ctx->Threads(); auto const n_groups = model.learner_model_param->num_output_group; auto const n_features = model.learner_model_param->num_feature; @@ -858,7 +871,7 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, contribs.resize(info.num_row_ * ncolumns * model.learner_model_param->num_output_group); std::fill(contribs.begin(), contribs.end(), 0.0f); CHECK_NE(n_groups, 0); - auto const &rule = GetQuadratureRule(); + auto const &rule = GetQuadratureRule(); auto const base_score = model.learner_model_param->BaseScore(DeviceOrd::CPU()); auto model_data = MakeQuadratureShapModelData(model, tree_end, tree_weights); std::vector feats_tloc(n_threads); @@ -885,8 +898,9 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, float *p_contribs = &contribs[(row_idx * n_groups + gid) * ncolumns]; for (auto j : model_data.trees_by_group[gid]) { std::fill(this_tree_contribs.begin(), this_tree_contribs.end(), 0.0f); - auto formulation = AdditiveContributionFormulation{{this_tree_contribs.data(), ncolumns}}; - auto runner = QuadratureShapTreeRunner{ + auto formulation = + AdditiveContributionFormulation{{this_tree_contribs.data(), ncolumns}}; + auto runner = QuadratureShapTreeRunner>{ model_data.trees[j], feats, rule, &path_prob, formulation}; runner.Run(); auto const weight = model_data.weights[j]; @@ -909,6 +923,28 @@ void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, LaunchShap(ctx, p_fmat, model, process_view); } +void QuadratureShapValues(Context const *ctx, DMatrix *p_fmat, + HostDeviceVector *out_contribs, gbm::GBTreeModel const &model, + bst_tree_t tree_end, std::vector const *tree_weights, + std::size_t quadrature_points) { + switch (quadrature_points) { + case 4: + QuadratureShapValuesImpl<4>(ctx, p_fmat, out_contribs, model, tree_end, tree_weights); + return; + case 6: + QuadratureShapValuesImpl<6>(ctx, p_fmat, out_contribs, model, tree_end, tree_weights); + return; + case 8: + QuadratureShapValuesImpl<8>(ctx, p_fmat, out_contribs, model, tree_end, tree_weights); + return; + case 16: + QuadratureShapValuesImpl<16>(ctx, p_fmat, out_contribs, model, tree_end, tree_weights); + return; + default: + LOG(FATAL) << "CPU QuadratureSHAP currently supports quadrature sizes of 4, 6, 8, or 16."; + } +} + void QuadratureShapInteractionValues(Context const *ctx, DMatrix *p_fmat, HostDeviceVector *out_contribs, gbm::GBTreeModel const &model, bst_tree_t tree_end, @@ -938,7 +974,7 @@ void QuadratureShapInteractionValues(Context const *ctx, DMatrix *p_fmat, contribs.resize(info.num_row_ * row_chunk); std::fill(contribs.begin(), contribs.end(), 0.0f); - auto const &rule = GetQuadratureRule(); + auto const &rule = GetQuadratureRule(); auto const base_score = model.learner_model_param->BaseScore(DeviceOrd::CPU()); auto model_data = MakeQuadratureShapModelData(model, tree_end, tree_weights); std::vector feats_tloc(n_threads); @@ -973,12 +1009,15 @@ void QuadratureShapInteractionValues(Context const *ctx, DMatrix *p_fmat, std::fill(diag.begin(), diag.end(), 0.0f); for (auto j : model_data.trees_by_group[gid]) { - auto formulation = InteractionContributionFormulation{{&path, &latest_live}, - {diag.data(), ncolumns}, - {matrix.data, matrix.ncolumns}, - model_data.weights[j]}; - auto runner = QuadratureShapTreeRunner{ - model_data.trees[j], feats, rule, &path_prob, formulation}; + auto formulation = InteractionContributionFormulation{ + {&path, &latest_live}, + {diag.data(), ncolumns}, + {matrix.data, matrix.ncolumns}, + model_data.weights[j]}; + auto runner = + QuadratureShapTreeRunner>{ + model_data.trees[j], feats, rule, &path_prob, formulation}; runner.Run(); }