diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 3861409ea69e3..76110e26c0fe9 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -50,6 +50,7 @@ int cxl_get_poison_by_endpoint(struct cxl_port *port); struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa); u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, u64 dpa); +void kill_regions(struct cxl_root_decoder *cxlrd); #else static inline u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, @@ -79,6 +80,7 @@ static inline int cxl_region_init(void) static inline void cxl_region_exit(void) { } +static inline void kill_regions(struct cxl_root_decoder *cxlrd) { }; #define CXL_REGION_ATTR(x) NULL #define CXL_REGION_TYPE(x) NULL #define SET_CXL_REGION_ATTR(x) diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index 603cc746aa497..64d96b3310e73 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -565,7 +565,6 @@ int cxl_dpa_free(struct cxl_endpoint_decoder *cxled) devm_cxl_dpa_release(cxled); return 0; } -EXPORT_SYMBOL_NS_GPL(cxl_dpa_free, "CXL"); int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled, enum cxl_partition_mode mode) @@ -597,64 +596,6 @@ int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled, return 0; } -static int find_free_decoder(struct device *dev, const void *data) -{ - struct cxl_endpoint_decoder *cxled; - struct cxl_port *port; - - if (!is_endpoint_decoder(dev)) - return 0; - - cxled = to_cxl_endpoint_decoder(dev); - port = cxled_to_port(cxled); - - return cxled->cxld.id == (port->hdm_end + 1); -} - -static struct cxl_endpoint_decoder * -cxl_find_free_decoder(struct cxl_memdev *cxlmd) -{ - struct cxl_port *endpoint = cxlmd->endpoint; - struct device *dev; - - guard(rwsem_read)(&cxl_rwsem.dpa); - dev = device_find_child(&endpoint->dev, NULL, find_free_decoder); - if (!dev) - return NULL; - - return to_cxl_endpoint_decoder(dev); -} - -struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_memdev *cxlmd, - enum cxl_partition_mode mode, - resource_size_t alloc) -{ - struct cxl_endpoint_decoder *cxled; - int rc; - - if (!IS_ALIGNED(alloc, SZ_256M)) - return ERR_PTR(-EINVAL); - - cxled = cxl_find_free_decoder(cxlmd); - if (!cxled) - return ERR_PTR(-ENODEV); - - rc = cxl_dpa_set_part(cxled, mode); - if (rc) - goto err_put; - - rc = cxl_dpa_alloc(cxled, alloc); - if (rc) - goto err_put; - - return cxled; - -err_put: - put_device(&cxled->cxld.dev); - return ERR_PTR(rc); -} -EXPORT_SYMBOL_NS_GPL(cxl_request_dpa, "CXL"); - static int __cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size) { struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); @@ -738,44 +679,6 @@ int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size) return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); } -static int find_committed_endpoint_decoder(struct device *dev, const void *data) -{ - struct cxl_endpoint_decoder *cxled; - struct cxl_port *port; - - if (!is_endpoint_decoder(dev)) - return 0; - - cxled = to_cxl_endpoint_decoder(dev); - port = cxled_to_port(cxled); - - return cxled->cxld.id == port->hdm_end; -} - -struct cxl_endpoint_decoder *cxl_get_committed_decoder(struct cxl_memdev *cxlmd, - struct cxl_region **cxlr) -{ - struct cxl_port *endpoint = cxlmd->endpoint; - struct cxl_endpoint_decoder *cxled; - struct device *cxled_dev; - - if (!endpoint) - return NULL; - - guard(rwsem_read)(&cxl_rwsem.dpa); - cxled_dev = device_find_child(&endpoint->dev, NULL, - find_committed_endpoint_decoder); - if (!cxled_dev) - return NULL; - - cxled = to_cxl_endpoint_decoder(cxled_dev); - *cxlr = cxled->cxld.region; - - put_device(cxled_dev); - return cxled; -} -EXPORT_SYMBOL_NS_GPL(cxl_get_committed_decoder, "CXL"); - static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl) { u16 eig; diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index bcf77e604dc3d..97b1e61ad0187 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -893,7 +893,7 @@ int cxl_enumerate_cmds(struct cxl_memdev_state *mds) } EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, "CXL"); -void cxl_event_trace_record(const struct cxl_memdev *cxlmd, +void cxl_event_trace_record(struct cxl_memdev *cxlmd, enum cxl_event_log_type type, enum cxl_event_type event_type, const uuid_t *uuid, union cxl_event *evt) @@ -920,6 +920,7 @@ void cxl_event_trace_record(const struct cxl_memdev *cxlmd, * translations. Take topology mutation locks and lookup * { HPA, REGION } from { DPA, MEMDEV } in the event record. */ + guard(device)(&cxlmd->dev); guard(rwsem_read)(&cxl_rwsem.region); guard(rwsem_read)(&cxl_rwsem.dpa); @@ -968,7 +969,7 @@ void cxl_event_trace_record(const struct cxl_memdev *cxlmd, } EXPORT_SYMBOL_NS_GPL(cxl_event_trace_record, "CXL"); -static void __cxl_event_trace_record(const struct cxl_memdev *cxlmd, +static void __cxl_event_trace_record(struct cxl_memdev *cxlmd, enum cxl_event_log_type type, struct cxl_event_record_raw *record) { diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index 759b43364ed7a..c673b81e13062 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -26,9 +26,11 @@ static DEFINE_IDA(cxl_memdev_ida); static void cxl_memdev_release(struct device *dev) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct device *parent = dev->parent; ida_free(&cxl_memdev_ida, cxlmd->id); kfree(cxlmd); + put_device(parent); } static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, @@ -570,23 +572,23 @@ void cxl_memdev_update_perf(struct cxl_memdev *cxlmd) } EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL"); -static const struct device_type cxl_memdev_type = { +static const struct device_type cxl_class_memdev_type = { .name = "cxl_memdev", .release = cxl_memdev_release, .devnode = cxl_memdev_devnode, .groups = cxl_memdev_attribute_groups, }; -static const struct device_type cxl_accel_memdev_type = { - .name = "cxl_accel_memdev", +static const struct device_type cxl_memdev_type = { + .name = "cxl_memdev", .release = cxl_memdev_release, .devnode = cxl_memdev_devnode, }; bool is_cxl_memdev(const struct device *dev) { - return (dev->type == &cxl_memdev_type || - dev->type == &cxl_accel_memdev_type); + return (dev->type == &cxl_class_memdev_type || + dev->type == &cxl_memdev_type); } EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL"); @@ -778,13 +780,13 @@ static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, dev = &cxlmd->dev; device_initialize(dev); lockdep_set_class(&dev->mutex, &cxl_memdev_key); - dev->parent = cxlds->dev; + dev->parent = get_device(cxlds->dev); dev->bus = &cxl_bus_type; dev->devt = MKDEV(cxl_mem_major, cxlmd->id); if (cxlds->type == CXL_DEVTYPE_DEVMEM) - dev->type = &cxl_accel_memdev_type; - else dev->type = &cxl_memdev_type; + else + dev->type = &cxl_class_memdev_type; device_set_pm_not_required(dev); INIT_WORK(&cxlmd->detach_work, detach_memdev); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index f4f7dad473beb..e31054b9076ad 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -459,6 +459,8 @@ static void cxl_root_decoder_release(struct device *dev) if (atomic_read(&cxlrd->region_id) >= 0) memregion_free(atomic_read(&cxlrd->region_id)); + mutex_destroy(&cxlrd->regions_lock); + xa_destroy(&cxlrd->regions); __cxl_decoder_release(&cxlrd->cxlsd.cxld); kfree(cxlrd); } @@ -2017,7 +2019,8 @@ struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port, return ERR_PTR(rc); } - mutex_init(&cxlrd->range_lock); + mutex_init(&cxlrd->regions_lock); + xa_init(&cxlrd->regions); cxld = &cxlsd->cxld; cxld->dev.type = &cxl_decoder_root_type; @@ -2193,6 +2196,8 @@ static void cxld_unregister(void *dev) if (is_endpoint_decoder(dev)) cxl_decoder_detach(NULL, to_cxl_endpoint_decoder(dev), -1, DETACH_INVALIDATE); + if (is_root_decoder(dev)) + kill_regions(to_cxl_root_decoder(dev)); device_unregister(dev); } diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index e820fb63b95aa..198fe19833c58 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -710,141 +710,6 @@ static int free_hpa(struct cxl_region *cxlr) return 0; } -struct cxlrd_max_context { - struct device * const *host_bridges; - int interleave_ways; - unsigned long flags; - resource_size_t max_hpa; - struct cxl_root_decoder *cxlrd; -}; - -static int find_max_hpa(struct device *dev, void *data) -{ - struct cxlrd_max_context *ctx = data; - struct cxl_switch_decoder *cxlsd; - struct cxl_root_decoder *cxlrd; - struct resource *res, *prev; - struct cxl_decoder *cxld; - resource_size_t free = 0; - resource_size_t max; - int found = 0; - - if (!is_root_decoder(dev)) - return 0; - - cxlrd = to_cxl_root_decoder(dev); - cxlsd = &cxlrd->cxlsd; - cxld = &cxlsd->cxld; - - if ((cxld->flags & ctx->flags) != ctx->flags) { - dev_dbg(dev, "flags not matching: %08lx vs %08lx\n", - cxld->flags, ctx->flags); - return 0; - } - - for (int i = 0; i < ctx->interleave_ways; i++) { - for (int j = 0; j < ctx->interleave_ways; j++) { - if (ctx->host_bridges[i] == cxlsd->target[j]->dport_dev) { - found++; - break; - } - } - } - - if (found != ctx->interleave_ways) { - dev_dbg(dev, - "Not enough host bridges. Found %d for %d interleave ways requested\n", - found, ctx->interleave_ways); - return 0; - } - - lockdep_assert_held_read(&cxl_rwsem.region); - res = cxlrd->res->child; - - if (!res) - max = resource_size(cxlrd->res); - else - max = 0; - - for (prev = NULL; res; prev = res, res = res->sibling) { - if (!prev && res->start == cxlrd->res->start && - res->end == cxlrd->res->end) { - max = resource_size(cxlrd->res); - break; - } - if (prev && !resource_size(prev)) - continue; - - if (!prev && res->start > cxlrd->res->start) { - free = res->start - cxlrd->res->start; - max = max(free, max); - } - if (prev && res->start > prev->end + 1) { - free = res->start - prev->end + 1; - max = max(free, max); - } - } - - if (prev && prev->end + 1 < cxlrd->res->end + 1) { - free = cxlrd->res->end + 1 - prev->end + 1; - max = max(free, max); - } - - dev_dbg(&cxlrd->cxlsd.cxld.dev, "found %pa bytes of free space\n", &max); - if (max > ctx->max_hpa) { - if (ctx->cxlrd) - put_device(&ctx->cxlrd->cxlsd.cxld.dev); - get_device(&cxlrd->cxlsd.cxld.dev); - ctx->cxlrd = cxlrd; - ctx->max_hpa = max; - } - return 0; -} - -struct cxl_root_decoder *cxl_get_hpa_freespace(struct cxl_memdev *cxlmd, - int interleave_ways, - unsigned long flags, - resource_size_t *max_avail_contig) -{ - struct cxlrd_max_context ctx = { - .flags = flags, - .interleave_ways = interleave_ways, - }; - struct cxl_port *root_port; - struct cxl_port *endpoint; - - endpoint = cxlmd->endpoint; - if (!endpoint) { - dev_dbg(&cxlmd->dev, "endpoint not linked to memdev\n"); - return ERR_PTR(-ENXIO); - } - - ctx.host_bridges = &endpoint->host_bridge; - - struct cxl_root *root __free(put_cxl_root) = find_cxl_root(endpoint); - if (!root) { - dev_dbg(&endpoint->dev, "endpoint is not related to a root port\n"); - return ERR_PTR(-ENXIO); - } - - root_port = &root->port; - scoped_guard(rwsem_read, &cxl_rwsem.region) - device_for_each_child(&root_port->dev, &ctx, find_max_hpa); - - if (!ctx.cxlrd) - return ERR_PTR(-ENOMEM); - - *max_avail_contig = ctx.max_hpa; - return ctx.cxlrd; -} -EXPORT_SYMBOL_NS_GPL(cxl_get_hpa_freespace, "CXL"); - -void cxl_put_root_decoder(struct cxl_root_decoder *cxlrd) -{ - put_device(&cxlrd->cxlsd.cxld.dev); -} -EXPORT_SYMBOL_NS_GPL(cxl_put_root_decoder, "CXL"); - static ssize_t size_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { @@ -1236,6 +1101,19 @@ static int cxl_rr_assign_decoder(struct cxl_port *port, struct cxl_region *cxlr, static void cxl_region_setup_flags(struct cxl_region *cxlr, struct cxl_decoder *cxld) { + if (is_endpoint_decoder(&cxld->dev)) { + struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(&cxld->dev); + struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); + + /* + * When a region's memdevs specify an @attach method the attach + * provider is responsible for dispositioning the region for + * both probe and userspace management + */ + if (cxlmd->attach) + set_bit(CXL_REGION_F_LOCK, &cxlr->flags); + } + if (cxld->flags & CXL_DECODER_F_LOCK) { set_bit(CXL_REGION_F_LOCK, &cxlr->flags); clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); @@ -2598,12 +2476,13 @@ static struct cxl_region *to_cxl_region(struct device *dev) return container_of(dev, struct cxl_region, dev); } -static void unregister_region(void *_cxlr) +static void unregister_region(struct cxl_region *cxlr) { - struct cxl_region *cxlr = _cxlr; + struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); struct cxl_region_params *p = &cxlr->params; int i; + xa_erase(&cxlrd->regions, cxlr->id); device_del(&cxlr->dev); /* @@ -2620,62 +2499,16 @@ static void unregister_region(void *_cxlr) put_device(&cxlr->dev); } -static void cxl_endpoint_region_autoremove(void *_cxlr); - -static void cxl_region_release_action(struct cxl_region *cxlr) +static void endpoint_unregister_region(void *_cxlr) { - struct cxl_port *port = cxlrd_to_port(cxlr->cxlrd); - - if (cxlr->type != CXL_DECODER_DEVMEM) { - devm_release_action(port->uport_dev, unregister_region, cxlr); - return; - } - - if (cxlr->params.nr_targets) { - struct cxl_endpoint_decoder *cxled = cxlr->params.targets[0]; - struct cxl_port *endpoint = cxled_to_port(cxled); - - guard(device)(&endpoint->dev); - if (cxlr->detach) { - void (*detach)(void *data) = cxlr->detach; - void *detach_data = cxlr->detach_data; - - cxlr->detach = NULL; - cxlr->detach_data = NULL; - devm_release_action(&endpoint->dev, detach, detach_data); - devm_release_action(&endpoint->dev, - cxl_endpoint_region_autoremove, - cxlr); - } else { - unregister_region(cxlr); - } - return; - } - - unregister_region(cxlr); -} - -void cxl_unregister_region(struct cxl_region *cxlr) -{ - cxl_region_release_action(cxlr); -} -EXPORT_SYMBOL_NS_GPL(cxl_unregister_region, "CXL"); - -int cxl_get_region_range(struct cxl_region *region, struct range *range) -{ - if (WARN_ON_ONCE(!region)) - return -ENODEV; - - if (!region->params.res) - return -ENOSPC; - - range->start = region->params.res->start; - range->end = region->params.res->end; + struct cxl_region *cxlr = _cxlr; + struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); - return 0; + guard(mutex)(&cxlrd->regions_lock); + if (xa_load(&cxlrd->regions, cxlr->id)) + unregister_region(cxlr); + put_device(&cxlr->dev); } -EXPORT_SYMBOL_NS_GPL(cxl_get_region_range, "CXL"); - static struct lock_class_key cxl_region_key; static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id) @@ -2790,6 +2623,19 @@ static int cxl_region_calculate_adistance(struct notifier_block *nb, return NOTIFY_STOP; } +/* unwind all remaining regions */ +void kill_regions(struct cxl_root_decoder *cxlrd) +{ + unsigned long index; + struct cxl_region *cxlr; + + guard(mutex)(&cxlrd->regions_lock); + /* no more region creation */ + cxlrd->dead = true; + xa_for_each(&cxlrd->regions, index, cxlr) + unregister_region(cxlr); +} + /** * devm_cxl_add_region - Adds a region to a decoder * @cxlrd: root decoder @@ -2828,21 +2674,15 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, if (rc) goto err; - /* - * For accelerators/type2, region release linked to endpoint device. - * See handling of cxl_endpoint_region_autoremove() below by - * cxl_memdev_attach_region(). - */ - if (type == CXL_DECODER_HOSTONLYMEM) { - rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); - if (rc) - return ERR_PTR(rc); + rc = xa_insert(&cxlrd->regions, cxlr->id, cxlr, GFP_KERNEL); + if (rc) { + unregister_region(cxlr); + return ERR_PTR(rc); } dev_dbg(port->uport_dev, "%s: created %s\n", dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev)); return cxlr; - err: put_device(dev); return ERR_PTR(rc); @@ -2871,6 +2711,9 @@ static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd, { int rc; + if (cxlrd->dead) + return ERR_PTR(-ENXIO); + switch (mode) { case CXL_PARTMODE_RAM: case CXL_PARTMODE_PMEM: @@ -2903,6 +2746,10 @@ static ssize_t create_region_store(struct device *dev, const char *buf, if (rc != 1) return -EINVAL; + ACQUIRE(mutex_intr, regions_lock)(&cxlrd->regions_lock); + if ((rc = ACQUIRE_ERR(mutex_intr, ®ions_lock))) + return rc; + cxlr = __create_region(cxlrd, mode, id, CXL_DECODER_HOSTONLYMEM); if (IS_ERR(cxlr)) return PTR_ERR(cxlr); @@ -2942,32 +2789,27 @@ static ssize_t region_show(struct device *dev, struct device_attribute *attr, } DEVICE_ATTR_RO(region); -static struct cxl_region * -cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name) -{ - struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; - struct device *region_dev; - - region_dev = device_find_child_by_name(&cxld->dev, name); - if (!region_dev) - return ERR_PTR(-ENODEV); - - return to_cxl_region(region_dev); -} - static ssize_t delete_region_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); struct cxl_region *cxlr; + int rc, id; - cxlr = cxl_find_region_by_name(cxlrd, buf); - if (IS_ERR(cxlr)) - return PTR_ERR(cxlr); + ACQUIRE(mutex_intr, regions_lock)(&cxlrd->regions_lock); + if ((rc = ACQUIRE_ERR(mutex_intr, ®ions_lock))) + return rc; - cxl_region_release_action(cxlr); - put_device(&cxlr->dev); + rc = sscanf(buf, "region%d\n", &id); + if (rc != 1) + return -EINVAL; + + cxlr = xa_load(&cxlrd->regions, id); + if (!cxlr || !sysfs_streq(buf, dev_name(&cxlr->dev))) + return -ENODEV; + + unregister_region(cxlr); return len; } @@ -3166,13 +3008,15 @@ static int __cxl_dpa_to_region(struct device *dev, void *arg) struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa) { struct cxl_dpa_to_region_context ctx; - struct cxl_port *port; + struct cxl_port *port = cxlmd->endpoint; + + if (!cxlmd->dev.driver) + return NULL; ctx = (struct cxl_dpa_to_region_context) { .dpa = dpa, }; - port = cxlmd->endpoint; - if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port)) + if (cxl_num_decoders_committed(port)) device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region); return ctx.cxlr; @@ -4141,107 +3985,12 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd, rc = __construct_region(cxlr, ctx); if (rc) { - cxl_region_release_action(cxlr); - return ERR_PTR(rc); - } - - return cxlr; -} - -DEFINE_FREE(cxl_region_release, struct cxl_region *, - if (!IS_ERR_OR_NULL(_T)) cxl_region_release_action(_T)) - -static struct cxl_region * -__construct_new_region(struct cxl_root_decoder *cxlrd, - struct cxl_endpoint_decoder **cxled, int ways) -{ - struct cxl_memdev *cxlmd = cxled_to_memdev(cxled[0]); - struct cxl_dev_state *cxlds = cxlmd->cxlds; - struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; - struct cxl_region_params *p; - resource_size_t size = 0; - int rc, i, part = READ_ONCE(cxled[0]->part); - - if (part < 0 || part >= cxlds->nr_partitions) { - dev_err(cxlmd->dev.parent, - "%s:%s: invalid partition index %d (max %u)\n", - dev_name(&cxlmd->dev), dev_name(&cxled[0]->cxld.dev), - part, cxlds->nr_partitions); - return ERR_PTR(-ENXIO); - } - - struct cxl_region *cxlr __free(cxl_region_release) = - __create_region(cxlrd, cxlds->part[part].mode, - atomic_read(&cxlrd->region_id), - cxled[0]->cxld.target_type); - if (IS_ERR(cxlr)) - return cxlr; - - guard(rwsem_write)(&cxl_rwsem.region); - - p = &cxlr->params; - if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { - dev_err(cxlmd->dev.parent, - "%s:%s: %s unexpected region state\n", - dev_name(&cxlmd->dev), dev_name(&cxled[0]->cxld.dev), - __func__); - return ERR_PTR(-EBUSY); - } - - if (ways < 1) - return ERR_PTR(-EINVAL); - - p->interleave_ways = ways; - p->interleave_granularity = cxld->interleave_granularity; - - scoped_guard(rwsem_read, &cxl_rwsem.dpa) { - for (i = 0; i < ways; i++) { - if (!cxled[i]->dpa_res) - return ERR_PTR(-EINVAL); - size += resource_size(cxled[i]->dpa_res); - } - - rc = alloc_hpa(cxlr, size); - if (rc) - return ERR_PTR(rc); - - for (i = 0; i < ways; i++) { - rc = cxl_region_attach(cxlr, cxled[i], 0); - if (rc) - return ERR_PTR(rc); - } - } - - rc = cxl_region_decode_commit(cxlr); - if (rc) + unregister_region(cxlr); return ERR_PTR(rc); - - p->state = CXL_CONFIG_COMMIT; - - return no_free_ptr(cxlr); -} - -struct cxl_region *cxl_create_region(struct cxl_root_decoder *cxlrd, - struct cxl_endpoint_decoder **cxled, - int ways) -{ - struct cxl_region *cxlr; - - mutex_lock(&cxlrd->range_lock); - cxlr = __construct_new_region(cxlrd, cxled, ways); - mutex_unlock(&cxlrd->range_lock); - if (IS_ERR(cxlr)) - return cxlr; - - if (device_attach(&cxlr->dev) <= 0) { - dev_err(&cxlr->dev, "failed to create region\n"); - cxl_region_release_action(cxlr); - return ERR_PTR(-ENODEV); } return cxlr; } -EXPORT_SYMBOL_NS_GPL(cxl_create_region, "CXL"); static struct cxl_region * cxl_find_region_by_range(struct cxl_root_decoder *cxlrd, @@ -4282,12 +4031,11 @@ int cxl_add_to_region(struct cxl_endpoint_decoder *cxled) * for the HPA range, one does the construction and the others * add to that. */ - mutex_lock(&cxlrd->range_lock); + guard(mutex)(&cxlrd->regions_lock); struct cxl_region *cxlr __free(put_cxl_region) = cxl_find_region_by_range(cxlrd, &ctx.hpa_range); if (!cxlr) cxlr = construct_region(cxlrd, &ctx); - mutex_unlock(&cxlrd->range_lock); rc = PTR_ERR_OR_ZERO(cxlr); if (rc) @@ -4563,119 +4311,94 @@ static int first_mapped_decoder(struct device *dev, const void *data) } /* - * As this is running in endpoint port remove context it does not race cxl_root - * destruction since port topologies are always removed depth first. + * Runs in cxl_mem_probe context after successful endpoint probe, assumes the + * simple case of single mapped decoder per memdev. */ -static void cxl_endpoint_region_autoremove(void *_cxlr) -{ - unregister_region(_cxlr); -} - -/** - * cxl_memdev_attach_region - bind region to accelerator memdev - * - * @cxlmd: a pointer to cxl_memdev to use - * @attach: a pointer to region attach struct with callbacks for - * safely working with a region range by the caller - * - * Returns 0 or error. - */ -int cxl_memdev_attach_region(struct cxl_memdev *cxlmd, - struct cxl_attach_region *attach) +int cxl_memdev_attach_region(struct cxl_memdev *cxlmd) { + struct cxl_attach_region *attach = + container_of(cxlmd->attach, typeof(*attach), attach); struct cxl_port *endpoint = cxlmd->endpoint; struct cxl_endpoint_decoder *cxled; struct cxl_region *cxlr; int rc; - if (IS_ERR(endpoint)) - return PTR_ERR(endpoint); - if (!endpoint) + /* hold endpoint lock to setup autoremove of the region */ + guard(device)(&endpoint->dev); + if (!endpoint->dev.driver) return -ENXIO; { - /* hold endpoint lock to setup autoremove of the region */ - guard(device)(&endpoint->dev); - if (!endpoint->dev.driver) - return -ENXIO; + guard(rwsem_read)(&cxl_rwsem.region); + guard(rwsem_read)(&cxl_rwsem.dpa); - { - guard(rwsem_read)(&cxl_rwsem.region); - guard(rwsem_read)(&cxl_rwsem.dpa); - - /* - * TODO auto-instantiate a region, for now assume this will - * find an auto-region. - */ - struct device *dev __free(put_device) = - device_find_child(&endpoint->dev, NULL, - first_mapped_decoder); - - if (!dev) { - dev_dbg(cxlmd->cxlds->dev, - "no region found for memdev %s\n", - dev_name(&cxlmd->dev)); - return -ENXIO; - } + /* + * TODO auto-instantiate a region, for now assume this will find an + * auto-region + */ + struct device *dev __free(put_device) = + device_find_child(&endpoint->dev, NULL, + first_mapped_decoder); + + if (!dev) { + dev_dbg(cxlmd->cxlds->dev, + "no region found for memdev %s\n", + dev_name(&cxlmd->dev)); + return -ENXIO; + } - cxled = to_cxl_endpoint_decoder(dev); - cxlr = cxled->cxld.region; + cxled = to_cxl_endpoint_decoder(dev); + cxlr = cxled->cxld.region; - if (cxlr->params.state < CXL_CONFIG_COMMIT) { - dev_dbg(cxlmd->cxlds->dev, - "region %s not committed for memdev %s\n", - dev_name(&cxlr->dev), dev_name(&cxlmd->dev)); - return -ENXIO; - } + if (cxlr->params.state < CXL_CONFIG_COMMIT) { + dev_dbg(cxlmd->cxlds->dev, + "region %s not committed for memdev %s\n", + dev_name(&cxlr->dev), dev_name(&cxlmd->dev)); + return -ENXIO; + } - if (cxlr->params.nr_targets > 1) { - dev_dbg(cxlmd->cxlds->dev, - "Only attach to local non-interleaved region\n"); - return -ENXIO; - } + if (cxlr->params.nr_targets > 1) { + dev_dbg(cxlmd->cxlds->dev, + "Only attach to local non-interleaved region\n"); + return -ENXIO; + } - attach->region = (struct range) { + /* Only teardown regions that pass validation, ignore the rest */ + get_device(&cxlr->dev); + rc = devm_add_action(&endpoint->dev, + endpoint_unregister_region, cxlr); + if (!rc) { + attach->hpa_range = (struct range) { .start = cxlr->params.res->start, .end = cxlr->params.res->end, }; + return 0; + } + } - /* - * With endpoint locked leave the caller to safely work - * with the region range. - */ - rc = attach->attach(attach->data); - if (rc) - return rc; - - /* Only teardown regions that pass validation, ignore the rest */ - rc = devm_add_action(&endpoint->dev, - cxl_endpoint_region_autoremove, cxlr); - if (rc) { - attach->detach(attach->data); - goto err_unregister; - } + endpoint_unregister_region(cxlr); + return rc; +} +EXPORT_SYMBOL_FOR_MODULES(cxl_memdev_attach_region, "cxl_mem"); - /* Link type2 driver callback for stopping use of the region range. */ - rc = devm_add_action_or_reset(&endpoint->dev, - attach->detach, attach->data); - if (rc) { - devm_remove_action(&endpoint->dev, - cxl_endpoint_region_autoremove, - cxlr); - goto err_unregister; - } +/* + * The presence of an attach method indicates that the region is designated for + * a purpose outside of CXL core memory expansion defaults. + */ +static bool cxl_region_has_memdev_attach(struct cxl_region *cxlr) +{ + struct cxl_region_params *p = &cxlr->params; - cxlr->detach = attach->detach; - cxlr->detach_data = attach->data; + for (int i = 0; i < p->nr_targets; i++) { + struct cxl_endpoint_decoder *cxled = p->targets[i]; + struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); - return 0; - } -err_unregister: - unregister_region(cxlr); - return rc; + if (cxlmd->attach) + return true; } + + return false; } -EXPORT_SYMBOL_NS_GPL(cxl_memdev_attach_region, "CXL"); static int cxl_region_probe(struct device *dev) { @@ -4715,6 +4438,9 @@ static int cxl_region_probe(struct device *dev) if (rc) return rc; + if (cxl_region_has_memdev_attach(cxlr)) + return 0; + switch (cxlr->mode) { case CXL_PARTMODE_PMEM: rc = devm_cxl_region_edac_register(cxlr); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index caee223c0e611..74d3e89c7c210 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -294,7 +294,9 @@ struct cxl_rd_ops { * @cache_size: extended linear cache size if exists, otherwise zero. * @region_id: region id for next region provisioning event * @platform_data: platform specific configuration data - * @range_lock: sync region autodiscovery by address range + * @regions_lock: sync region discovery, construction, and deletion + * @regions: regions to remove at root decoder destruct time + * @dead: root decoder dead to region creation * @qos_class: QoS performance class cookie * @ops: CXL root decoder operations * @cxlsd: base cxl switch decoder @@ -304,7 +306,9 @@ struct cxl_root_decoder { resource_size_t cache_size; atomic_t region_id; void *platform_data; - struct mutex range_lock; + struct mutex regions_lock; + struct xarray regions; + bool dead; int qos_class; struct cxl_rd_ops ops; struct cxl_switch_decoder cxlsd; @@ -390,8 +394,6 @@ struct cxl_region_params { * @hpa_range: Address range occupied by the region * @mode: Operational mode of the mapped capacity * @type: Endpoint decoder target type - * @detach: accelerator detach callback for device-memory regions - * @detach_data: accelerator detach callback data * @cxl_nvb: nvdimm bridge for coordinating @cxlr_pmem setup / shutdown * @cxlr_pmem: (for pmem regions) cached copy of the nvdimm bridge * @flags: Region state flags @@ -407,8 +409,6 @@ struct cxl_region { struct range hpa_range; enum cxl_partition_mode mode; enum cxl_decoder_type type; - void (*detach)(void *data); - void *detach_data; struct cxl_nvdimm_bridge *cxl_nvb; struct cxl_pmem_region *cxlr_pmem; unsigned long flags; diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index c98db6f18aa29..09ad9b20a97bc 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -97,6 +97,34 @@ static inline bool is_cxl_endpoint(struct cxl_port *port) return is_cxl_memdev(port->uport_dev); } +struct cxl_memdev_attach { + int (*probe)(struct cxl_memdev *cxlmd); +}; + +/** + * struct cxl_attach_region - coordinate mapping a region at memdev registration + * @attach: common core attachment descriptor + * @hpa_range: physical address range of the region + * + * For the common simple case of a CXL device with private (non-general purpose + * / "accelerator") memory, enumerate firmware instantiated region, or + * instantiate a region for the device's capacity. Destroy the region on detach. + */ +struct cxl_attach_region { + struct cxl_memdev_attach attach; + struct range hpa_range; +}; + +#ifdef CONFIG_CXL_REGION +int cxl_memdev_attach_region(struct cxl_memdev *cxlmd); +#else +static inline int cxl_memdev_attach_region(struct cxl_memdev *cxlmd) +{ + return -EOPNOTSUPP; +} +#endif + +struct cxl_memdev *devm_cxl_add_classdev(struct cxl_dev_state *cxlds); struct cxl_memdev *__devm_cxl_add_memdev(struct cxl_dev_state *cxlds, const struct cxl_memdev_attach *attach); int devm_cxl_sanitize_setup_notifier(struct device *host, @@ -776,7 +804,7 @@ void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, unsigned long *cmds); void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status); -void cxl_event_trace_record(const struct cxl_memdev *cxlmd, +void cxl_event_trace_record(struct cxl_memdev *cxlmd, enum cxl_event_log_type type, enum cxl_event_type event_type, const uuid_t *uuid, union cxl_event *evt); diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index ff858318091f1..798e5c369cfc2 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -48,6 +48,11 @@ static int cxl_mem_dpa_show(struct seq_file *file, void *data) static int cxl_debugfs_poison_inject(void *data, u64 dpa) { struct cxl_memdev *cxlmd = data; + int rc; + + ACQUIRE(device_intr, devlock)(&cxlmd->dev); + if ((rc = ACQUIRE_ERR(device_intr, &devlock))) + return rc; return cxl_inject_poison(cxlmd, dpa); } @@ -58,6 +63,11 @@ DEFINE_DEBUGFS_ATTRIBUTE(cxl_poison_inject_fops, NULL, static int cxl_debugfs_poison_clear(void *data, u64 dpa) { struct cxl_memdev *cxlmd = data; + int rc; + + ACQUIRE(device_intr, devlock)(&cxlmd->dev); + if ((rc = ACQUIRE_ERR(device_intr, &devlock))) + return rc; return cxl_clear_poison(cxlmd, dpa); } @@ -183,27 +193,59 @@ static int cxl_mem_probe(struct device *dev) } /** - * devm_cxl_add_memdev - Add a CXL memory device + * devm_cxl_add_classdev - Add a CXL memory class-code device + * @cxlds: CXL device state to associate with the memdev + * + * Upon return the device will have had a chance to attach to the + * cxl_mem driver, but may fail to attach if the CXL topology is not ready + * (hardware CXL link down, or software platform CXL root not attached). + * + * The parent of the resulting device and the devm context for allocations is + * @cxlds->dev. + */ +struct cxl_memdev *devm_cxl_add_classdev(struct cxl_dev_state *cxlds) +{ + return __devm_cxl_add_memdev(cxlds, NULL); +} +EXPORT_SYMBOL_NS_GPL(devm_cxl_add_classdev, "CXL"); + +/** + * devm_cxl_probe_mem - Add a CXL memory device and probe its region * @cxlds: CXL device state to associate with the memdev - * @attach: Caller depends on CXL topology attachment + * @hpa_range: CXL.mem physical address range result * * Upon return the device will have had a chance to attach to the * cxl_mem driver, but may fail to attach if the CXL topology is not ready * (hardware CXL link down, or software platform CXL root not attached). * - * When @attach is NULL it indicates the caller wants the memdev to remain - * registered even if it does not immediately attach to the CXL hierarchy. When - * @attach is provided a cxl_mem_probe() failure leads to failure of this routine. + * Failure to probe the memdev and/or setup a region for the memdev + * results in this function failing. * * The parent of the resulting device and the devm context for allocations is * @cxlds->dev. */ -struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds, - const struct cxl_memdev_attach *attach) +struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds, + struct range *hpa_range) { - return __devm_cxl_add_memdev(cxlds, attach); + struct cxl_attach_region *attach = + devm_kmalloc(cxlds->dev, sizeof(*attach), GFP_KERNEL); + struct cxl_memdev *cxlmd; + + if (!attach) + return ERR_PTR(-ENOMEM); + + *attach = (struct cxl_attach_region) { + .attach = { + .probe = cxl_memdev_attach_region, + }, + .hpa_range = { 0, -1 }, + }; + + cxlmd = __devm_cxl_add_memdev(cxlds, &attach->attach); + *hpa_range = attach->hpa_range; + return cxlmd; } -EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, "CXL"); +EXPORT_SYMBOL_NS_GPL(devm_cxl_probe_mem, "CXL"); static ssize_t trigger_poison_list_store(struct device *dev, struct device_attribute *attr, diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 4bfcbd260381e..2feb79fb8c3c8 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -879,7 +879,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (rc) dev_dbg(&pdev->dev, "No CXL Features discovered\n"); - cxlmd = devm_cxl_add_memdev(cxlds, NULL); + cxlmd = devm_cxl_add_classdev(cxlds); if (IS_ERR(cxlmd)) return PTR_ERR(cxlmd); @@ -965,7 +965,6 @@ static void cxl_reset_done(struct pci_dev *pdev) { struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); struct cxl_memdev *cxlmd = cxlds->cxlmd; - struct cxl_port *endpoint; struct device *dev = &pdev->dev; /* @@ -975,11 +974,11 @@ static void cxl_reset_done(struct pci_dev *pdev) * that no longer exists. */ guard(device)(&cxlmd->dev); - endpoint = cxlmd->endpoint; - if (!endpoint || IS_ERR(endpoint)) + if (!cxlmd->dev.driver) return; - if (cxl_endpoint_decoder_reset_detected(endpoint)) { + if (cxlmd->endpoint && + cxl_endpoint_decoder_reset_detected(cxlmd->endpoint)) { dev_crit(dev, "SBR happened without memory regions removal.\n"); dev_crit(dev, "System may be unstable if regions hosted system memory.\n"); add_taint(TAINT_USER, LOCKDEP_STILL_OK); diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 90ccbe3103860..943409033ed41 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -962,6 +962,7 @@ static void efx_pci_remove(struct pci_dev *pci_dev) efx = pci_get_drvdata(pci_dev); if (!efx) return; + probe_data = container_of(efx, struct efx_probe_data, efx); /* Mark the NIC as fini, then stop the interface */ rtnl_lock(); @@ -979,12 +980,11 @@ static void efx_pci_remove(struct pci_dev *pci_dev) efx_mtd_remove(efx); + efx_cxl_fini(probe_data); efx_pci_remove_main(efx); efx_fini_io(efx); - probe_data = container_of(efx, struct efx_probe_data, efx); - pci_dbg(efx->pci_dev, "shutdown successful\n"); efx_fini_devlink_and_unlock(efx); @@ -1240,6 +1240,7 @@ static int efx_pci_probe(struct pci_dev *pci_dev, return 0; fail3: + efx_cxl_fini(probe_data); efx_fini_io(efx); fail2: efx_fini_struct(efx); diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c index 52b2cded76daf..9aaac53ba9888 100644 --- a/drivers/net/ethernet/sfc/efx_cxl.c +++ b/drivers/net/ethernet/sfc/efx_cxl.c @@ -15,19 +15,16 @@ #define EFX_CTPIO_BUFFER_SIZE SZ_256M -/* Called with cxl endpoint device locked for precluding potential related - * cxl region removal triggered from user space, allowing safely mapping of - * such cxl region by the sfc driver. - */ -static int efx_cxl_map_region(void *data) { +static int efx_cxl_map_region(void *data) +{ struct efx_probe_data *probe_data = data; struct efx_nic *efx = &probe_data->efx; struct pci_dev *pci_dev = efx->pci_dev; struct efx_cxl *cxl = probe_data->cxl; - struct range *cxl_pio_range = &cxl->attach_region.region; + struct range *cxl_pio_range = &cxl->region; cxl->ctpio_cxl = ioremap(cxl_pio_range->start, - cxl_pio_range->end - cxl_pio_range->start + 1); + range_len(cxl_pio_range)); if (!cxl->ctpio_cxl) { pci_err(pci_dev, "CXL ioremap region (%pra) failed\n", cxl_pio_range); @@ -37,13 +34,21 @@ static int efx_cxl_map_region(void *data) { return 0; } -/* Called at driver exit or when user space triggers cxl region removal. */ -static void efx_cxl_unmap_region(void *data) { +static void efx_cxl_unmap_region(void *data) +{ struct efx_probe_data *probe_data = data; efx_ef10_disable_piobufs(&probe_data->efx); probe_data->cxl_pio_initialised = false; iounmap(probe_data->cxl->ctpio_cxl); + probe_data->cxl->ctpio_cxl = NULL; +} + +void efx_cxl_fini(struct efx_probe_data *probe_data) +{ + if (probe_data->cxl_pio_initialised) + devm_release_action(&probe_data->efx.pci_dev->dev, + efx_cxl_unmap_region, probe_data); } int efx_cxl_init(struct efx_probe_data *probe_data) @@ -107,22 +112,20 @@ int efx_cxl_init(struct efx_probe_data *probe_data) return -ENODEV; } - cxl->cxlmd = devm_cxl_add_memdev(&cxl->cxlds, NULL); + cxl->cxlmd = devm_cxl_probe_mem(&cxl->cxlds, &cxl->region); if (IS_ERR(cxl->cxlmd)) { pci_err(pci_dev, "CXL accel memdev creation failed\n"); return PTR_ERR(cxl->cxlmd); } - cxl->attach_region.attach = efx_cxl_map_region; - cxl->attach_region.detach = efx_cxl_unmap_region; - cxl->attach_region.data = probe_data; probe_data->cxl = cxl; - rc = cxl_memdev_attach_region(cxl->cxlmd, &cxl->attach_region); + rc = efx_cxl_map_region(probe_data); if (rc) return rc; - return 0; + return devm_add_action_or_reset(&pci_dev->dev, + efx_cxl_unmap_region, probe_data); } MODULE_IMPORT_NS("CXL"); diff --git a/drivers/net/ethernet/sfc/efx_cxl.h b/drivers/net/ethernet/sfc/efx_cxl.h index 1c294cd1df56c..6833e01be00f3 100644 --- a/drivers/net/ethernet/sfc/efx_cxl.h +++ b/drivers/net/ethernet/sfc/efx_cxl.h @@ -20,12 +20,14 @@ struct efx_probe_data; struct efx_cxl { struct cxl_dev_state cxlds; struct cxl_memdev *cxlmd; - struct cxl_attach_region attach_region; + struct range region; void __iomem *ctpio_cxl; }; int efx_cxl_init(struct efx_probe_data *probe_data); +void efx_cxl_fini(struct efx_probe_data *probe_data); #else static inline int efx_cxl_init(struct efx_probe_data *probe_data) { return 0; } +static inline void efx_cxl_fini(struct efx_probe_data *probe_data) { } #endif #endif diff --git a/drivers/vfio/pci/cxl/Kconfig b/drivers/vfio/pci/cxl/Kconfig index fad53300fecfb..16e7bb377af44 100644 --- a/drivers/vfio/pci/cxl/Kconfig +++ b/drivers/vfio/pci/cxl/Kconfig @@ -1,6 +1,6 @@ config VFIO_CXL_CORE bool "VFIO CXL core" - depends on VFIO_PCI_CORE && CXL_BUS && CXL_MEM + depends on VFIO_PCI_CORE && CXL_BUS && CXL_MEM && CXL_REGION help Extends vfio-pci-core with CXL.mem passthrough for vendor-specific CXL devices (CXL_DEVTYPE_DEVMEM) that implement HDM-D or HDM-DB diff --git a/drivers/vfio/pci/cxl/vfio_cxl_core.c b/drivers/vfio/pci/cxl/vfio_cxl_core.c index 2d6b804d8537b..1e61b49f1cf72 100644 --- a/drivers/vfio/pci/cxl/vfio_cxl_core.c +++ b/drivers/vfio/pci/cxl/vfio_cxl_core.c @@ -212,21 +212,13 @@ int vfio_cxl_get_info(struct vfio_pci_core_device *vdev, return vfio_info_add_capability(caps, &cxl_cap.header, sizeof(cxl_cap)); } -/* - * Scope-based cleanup wrappers for the CXL resource APIs - */ -DEFINE_FREE(cxl_put_root_decoder, struct cxl_root_decoder *, if (!IS_ERR_OR_NULL(_T)) cxl_put_root_decoder(_T)) -DEFINE_FREE(cxl_dpa_free, struct cxl_endpoint_decoder *, if (!IS_ERR_OR_NULL(_T)) cxl_dpa_free(_T)) -DEFINE_FREE(cxl_unregister_region, struct cxl_region *, if (!IS_ERR_OR_NULL(_T)) cxl_unregister_region(_T)) - /* * vfio_cxl_create_device_state - Allocate and validate CXL device state * * Returns a pointer to the allocated vfio_pci_cxl_state on success, or - * ERR_PTR on failure. The allocation uses devm; the caller must call - * devm_kfree(&pdev->dev, cxl) on any subsequent setup failure to release - * the resource before device unbind. Using devm_kfree() to undo a devm - * allocation early is explicitly supported by the devres API. + * ERR_PTR on failure. Before devm_cxl_probe_mem() publishes a memdev, the + * caller may use devm_kfree() to unwind setup failures. After publication, + * cxl must remain allocated until PCI devres teardown. * * The caller assigns vdev->cxl only after all setup steps succeed, preventing * partially-initialised state from being visible through vdev->cxl on any @@ -253,11 +245,13 @@ vfio_cxl_create_device_state(struct pci_dev *pdev, u16 dvsec) if (!cxl) return ERR_PTR(-ENOMEM); - pci_read_config_dword(pdev, dvsec + PCI_DVSEC_HEADER1, &hdr1); + if (pci_read_config_dword(pdev, dvsec + PCI_DVSEC_HEADER1, &hdr1)) + goto err_free; cxl->dvsec_len = PCI_DVSEC_HEADER1_LEN(hdr1); - pci_read_config_word(pdev, dvsec + CXL_DVSEC_CAPABILITY_OFFSET, - &cap_word); + if (pci_read_config_word(pdev, dvsec + CXL_DVSEC_CAPABILITY_OFFSET, + &cap_word)) + goto err_free; /* * Only handle vendor devices (class != 0x0502) with Mem_Capable set. @@ -275,6 +269,10 @@ vfio_cxl_create_device_state(struct pci_dev *pdev, u16 dvsec) cxl->comp_reg_region_idx = -1; return cxl; + +err_free: + devm_kfree(&pdev->dev, cxl); + return ERR_PTR(-EIO); } static int vfio_cxl_setup_regs(struct vfio_pci_core_device *vdev, @@ -332,6 +330,12 @@ static int vfio_cxl_setup_regs(struct vfio_pci_core_device *vdev, ret = -ENODEV; goto failed_unmap; } + if (count != 1) { + pci_err(pdev, "vfio-cxl: hdm_count=%u, only 1 supported\n", + count); + ret = -EOPNOTSUPP; + goto failed_unmap; + } cxl->hdm_count = count; /* @@ -370,128 +374,23 @@ static int vfio_cxl_setup_regs(struct vfio_pci_core_device *vdev, return ret; } -int vfio_cxl_create_cxl_region(struct vfio_pci_cxl_state *cxl, - resource_size_t size) -{ - resource_size_t max_size; - - struct cxl_root_decoder *cxlrd __free(cxl_put_root_decoder) = - cxl_get_hpa_freespace(cxl->cxlmd, 1, - CXL_DECODER_F_RAM | CXL_DECODER_F_TYPE2, - &max_size); - if (IS_ERR(cxlrd)) - return PTR_ERR(cxlrd); - - /* Insufficient HPA space; cxlrd freed automatically by __free() */ - if (max_size < size) - return -ENOSPC; - - struct cxl_endpoint_decoder *cxled __free(cxl_dpa_free) = - cxl_request_dpa(cxl->cxlmd, CXL_PARTMODE_RAM, size); - if (IS_ERR(cxled)) - return PTR_ERR(cxled); - - struct cxl_region *region __free(cxl_unregister_region) = - cxl_create_region(cxlrd, &cxled, 1); - if (IS_ERR(region)) - return PTR_ERR(region); - - /* All operations succeeded; transfer ownership to cxl state */ - cxl->cxlrd = no_free_ptr(cxlrd); - cxl->cxled = no_free_ptr(cxled); - cxl->region = no_free_ptr(region); - - return 0; -} - -void vfio_cxl_destroy_cxl_region(struct vfio_pci_cxl_state *cxl) -{ - if (!cxl->region) - return; - - /* - * Precommitted regions are obtained via cxl_get_committed_decoder() as - * a borrowed reference owned by the cxl core; do not unregister or - * free the decoder objects from here. Only vfio_cxl_create_cxl_region() - * owns the region and decoders. - */ - if (!cxl->precommitted) { - cxl_unregister_region(cxl->region); - cxl_dpa_free(cxl->cxled); - cxl_put_root_decoder(cxl->cxlrd); - } - - cxl->region = NULL; - cxl->cxled = NULL; - cxl->cxlrd = NULL; -} - -static int vfio_cxl_create_region_helper(struct vfio_pci_core_device *vdev, - struct vfio_pci_cxl_state *cxl, - resource_size_t capacity) -{ - struct pci_dev *pdev = vdev->pdev; - struct range range; - int ret; - - if (cxl->precommitted) { - struct cxl_endpoint_decoder *cxled; - /* - * cxl_get_committed_decoder() does not write *region on every - * failure path (e.g. when cxlmd->endpoint is NULL or no decoder - * is committed). Initialise to NULL so the !cxl->region check - * below catches it regardless of stack-init mode. - */ - struct cxl_region *region = NULL; - - cxled = cxl_get_committed_decoder(cxl->cxlmd, ®ion); - if (IS_ERR(cxled)) - return PTR_ERR(cxled); - cxl->cxled = cxled; - cxl->region = region; - } else { - ret = vfio_cxl_create_cxl_region(cxl, capacity); - if (ret) - return ret; - } - - if (!cxl->region) { - pci_err(pdev, "Failed to create CXL region\n"); - ret = -ENODEV; - goto failed; - } - - ret = cxl_get_region_range(cxl->region, &range); - if (ret) - goto failed; - - cxl->region_hpa = range.start; - cxl->region_size = range_len(&range); - - pci_dbg(pdev, "CXL region: HPA 0x%llx size %lu MB\n", - cxl->region_hpa, cxl->region_size >> 20); - - return 0; - -failed: - vfio_cxl_destroy_cxl_region(cxl); - - return ret; -} - static int vfio_cxl_create_memdev(struct vfio_pci_cxl_state *cxl, resource_size_t capacity) { + struct range hpa_range; int ret; ret = cxl_set_capacity(&cxl->cxlds, capacity); if (ret) return ret; - cxl->cxlmd = devm_cxl_add_memdev(&cxl->cxlds, NULL); + cxl->cxlmd = devm_cxl_probe_mem(&cxl->cxlds, &hpa_range); if (IS_ERR(cxl->cxlmd)) return PTR_ERR(cxl->cxlmd); + cxl->region_hpa = hpa_range.start; + cxl->region_size = range_len(&hpa_range); + return 0; } @@ -511,11 +410,11 @@ static void vfio_cxl_dev_state_free(struct pci_dev *pdev, * CXL.mem device * @vdev: VFIO PCI device * - * Called from vfio_pci_core_register_device(). Detects CXL DVSEC capability - * and initializes CXL features. On failure vdev->cxl remains NULL and the - * device operates as a standard PCI device. + * Called from vfio_pci_core_register_device(). Non-CXL devices return success + * without enabling CXL features. Failures after confirming a CXL.mem device + * abort VFIO binding with vdev->cxl left NULL. */ -void vfio_pci_cxl_detect_and_init(struct vfio_pci_core_device *vdev) +int vfio_pci_cxl_detect_and_init(struct vfio_pci_core_device *vdev) { struct pci_dev *pdev = vdev->pdev; struct vfio_pci_cxl_state *cxl; @@ -525,16 +424,16 @@ void vfio_pci_cxl_detect_and_init(struct vfio_pci_core_device *vdev) /* Honor the user opt-out decision */ if (vdev->disable_cxl) - return; + return 0; if (!pcie_is_cxl(pdev)) - return; + return 0; dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL, PCI_DVSEC_CXL_DEVICE); if (!dvsec) - return; + return 0; /* * CXL DVSEC found: any failure from here is a hard probe error on @@ -543,11 +442,13 @@ void vfio_pci_cxl_detect_and_init(struct vfio_pci_core_device *vdev) */ cxl = vfio_cxl_create_device_state(pdev, dvsec); if (IS_ERR(cxl)) { - if (PTR_ERR(cxl) != -ENODEV) - pci_warn(pdev, - "vfio-cxl: CXL device state allocation failed: %ld\n", - PTR_ERR(cxl)); - return; + ret = PTR_ERR(cxl); + if (ret == -ENODEV) + return 0; + pci_warn(pdev, + "vfio-cxl: CXL device state allocation failed: %d\n", + ret); + return ret; } /* @@ -569,12 +470,13 @@ void vfio_pci_cxl_detect_and_init(struct vfio_pci_core_device *vdev) goto free_cxl; } - cxl->cxlds.media_ready = !cxl_await_range_active(&cxl->cxlds); - if (!cxl->cxlds.media_ready) { - pci_warn(pdev, "CXL media not ready\n"); + ret = cxl_await_range_active(&cxl->cxlds); + if (ret) { + pci_warn(pdev, "CXL media not ready: %d\n", ret); pci_disable_device(pdev); goto regs_failed; } + cxl->cxlds.media_ready = true; /* * Take the single authoritative HDM decoder snapshot now that @@ -594,11 +496,11 @@ void vfio_pci_cxl_detect_and_init(struct vfio_pci_core_device *vdev) * firmware pre-committed decoders */ pci_info(pdev, "Uncommitted region size must be configured via sysfs before bind\n"); + ret = -ENODEV; goto regs_failed; } cxl->precommitted = true; - cxl->dpa_size = capacity; pci_dbg(pdev, "Device capacity: %llu MB\n", capacity >> 20); @@ -608,27 +510,25 @@ void vfio_pci_cxl_detect_and_init(struct vfio_pci_core_device *vdev) goto regs_failed; } - ret = vfio_cxl_create_region_helper(vdev, cxl, capacity); - if (ret) - goto regs_failed; + pci_dbg(pdev, "CXL region: HPA 0x%llx size %lu MB\n", + cxl->region_hpa, cxl->region_size >> 20); /* - * Register probing succeeded. Assign vdev->cxl now so that - * all subsequent helpers can access state via vdev->cxl. - * All failure paths below clear vdev->cxl before calling - * vfio_cxl_dev_state_free(). cxl->vdev is the back-pointer used - * by vm_fault and other helpers that only have the cxl state in hand. + * devm_cxl_probe_mem() published a memdev that references cxl->cxlds, + * so cxl must remain allocated until PCI devres teardown. Publish it + * immediately; cxl->vdev is used by helpers that only have CXL state. */ cxl->vdev = vdev; vdev->cxl = cxl; - return; + return 0; regs_failed: vfio_cxl_clean_virt_regs(cxl); free_cxl: vfio_cxl_dev_state_free(pdev, cxl); + return ret; } void vfio_pci_cxl_cleanup(struct vfio_pci_core_device *vdev) @@ -639,7 +539,7 @@ void vfio_pci_cxl_cleanup(struct vfio_pci_core_device *vdev) return; vfio_cxl_clean_virt_regs(cxl); - vfio_cxl_destroy_cxl_region(cxl); + vdev->cxl = NULL; } static vm_fault_t vfio_cxl_region_vm_fault(struct vm_fault *vmf) @@ -1054,7 +954,7 @@ int vfio_cxl_register_cxl_region(struct vfio_pci_core_device *vdev) if (!cxl) return -ENODEV; - if (!cxl->region || cxl->region_vaddr) + if (!cxl->region_size || cxl->region_vaddr) return -ENODEV; /* @@ -1126,9 +1026,7 @@ EXPORT_SYMBOL_GPL(vfio_cxl_register_cxl_region); * @vdev: VFIO PCI device * * Marks the DPA region inactive and resets dpa_region_idx. - * Does NOT touch CXL subsystem state (cxl->region, cxl->cxled, cxl->cxlrd). - * The caller must call vfio_cxl_destroy_cxl_region() separately to release - * those objects. + * CXL core owns the memdev and autoregion lifetime through devres. */ void vfio_cxl_unregister_cxl_region(struct vfio_pci_core_device *vdev) { diff --git a/drivers/vfio/pci/cxl/vfio_cxl_priv.h b/drivers/vfio/pci/cxl/vfio_cxl_priv.h index ac8ea3893c7af..2873005b88c91 100644 --- a/drivers/vfio/pci/cxl/vfio_cxl_priv.h +++ b/drivers/vfio/pci/cxl/vfio_cxl_priv.h @@ -25,9 +25,6 @@ struct vfio_pci_cxl_state { struct cxl_dev_state cxlds; struct vfio_pci_core_device *vdev; struct cxl_memdev *cxlmd; - struct cxl_root_decoder *cxlrd; - struct cxl_endpoint_decoder *cxled; - struct cxl_region *region; resource_size_t region_hpa; size_t region_size; void *region_vaddr; @@ -36,7 +33,6 @@ struct vfio_pci_cxl_state { resource_size_t comp_reg_offset; size_t comp_reg_size; __le32 *comp_reg_virt; - size_t dpa_size; void __iomem *hdm_iobase; int dpa_region_idx; int comp_reg_region_idx; @@ -135,10 +131,6 @@ void vfio_cxl_reinit_comp_regs(struct vfio_pci_cxl_state *cxl); resource_size_t vfio_cxl_read_committed_decoder_size(struct vfio_pci_core_device *vdev, struct vfio_pci_cxl_state *cxl); -int vfio_cxl_create_cxl_region(struct vfio_pci_cxl_state *cxl, - resource_size_t size); -void vfio_cxl_destroy_cxl_region(struct vfio_pci_cxl_state *cxl); - __le32 *hdm_reg_ptr(struct vfio_pci_cxl_state *cxl, u32 hdm_off); #endif /* __LINUX_VFIO_CXL_PRIV_H */ diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 25320f5f7c9e7..f7e7a084b6315 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -2233,7 +2233,9 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) if (ret) goto out_vf; - vfio_pci_cxl_detect_and_init(vdev); + ret = vfio_pci_cxl_detect_and_init(vdev); + if (ret) + goto out_vga; vfio_pci_probe_power_state(vdev); @@ -2260,11 +2262,9 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) out_power: /* - * vfio_pci_cxl_detect_and_init() may have assigned vdev->cxl and - * allocated comp_reg_virt[] / hdm_iobase / region state above. The - * normal teardown via vfio_pci_core_unregister_device() will not run - * if registration failed, so release the CXL state here. No-op when - * vdev->cxl is NULL (non-CXL device or detect skipped). + * Registration failed after CXL setup. Release VFIO-owned mappings and + * clear vdev->cxl; PCI devres unwinds the memdev and autoregion when + * driver probe returns failure. */ vfio_pci_cxl_cleanup(vdev); @@ -2272,6 +2272,8 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) pm_runtime_get_noresume(dev); pm_runtime_forbid(dev); +out_vga: + vfio_pci_vga_uninit(vdev); out_vf: vfio_pci_vf_uninit(vdev); return ret; diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h index f3d7f66552393..a590ab255573b 100644 --- a/drivers/vfio/pci/vfio_pci_priv.h +++ b/drivers/vfio/pci/vfio_pci_priv.h @@ -147,7 +147,7 @@ static inline void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, #if IS_ENABLED(CONFIG_VFIO_CXL_CORE) -void vfio_pci_cxl_detect_and_init(struct vfio_pci_core_device *vdev); +int vfio_pci_cxl_detect_and_init(struct vfio_pci_core_device *vdev); void vfio_pci_cxl_cleanup(struct vfio_pci_core_device *vdev); bool vfio_cxl_reset_capable(struct vfio_pci_core_device *vdev); void vfio_cxl_prepare_reset(struct vfio_pci_core_device *vdev); @@ -168,8 +168,8 @@ bool vfio_cxl_mmap_overlaps_comp_regs(struct vfio_pci_core_device *vdev, #else -static inline void -vfio_pci_cxl_detect_and_init(struct vfio_pci_core_device *vdev) { } +static inline int +vfio_pci_cxl_detect_and_init(struct vfio_pci_core_device *vdev) { return 0; } static inline void vfio_pci_cxl_cleanup(struct vfio_pci_core_device *vdev) { } static inline bool diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h index 0c41508828382..c6d48d9c8fb85 100644 --- a/include/cxl/cxl.h +++ b/include/cxl/cxl.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -177,36 +176,6 @@ struct cxl_dpa_partition { #define CXL_NR_PARTITIONS_MAX 2 -/* - * cxl_decoder flags that define the type of memory / devices this decoder - * supports as well as configuration lock status. - */ -#define CXL_DECODER_F_RAM BIT(0) -#define CXL_DECODER_F_PMEM BIT(1) -#define CXL_DECODER_F_TYPE2 BIT(2) -#define CXL_DECODER_F_TYPE3 BIT(3) -#define CXL_DECODER_F_LOCK BIT(4) -#define CXL_DECODER_F_ENABLE BIT(5) -#define CXL_DECODER_F_MASK GENMASK(5, 0) - -struct cxl_memdev_attach { - int (*probe)(struct cxl_memdev *cxlmd); -}; - -/** - * struct cxl_attach_region - accelerator region handling - * @attach: invoked at cxl_memdev_attach_region() with endpoint device locked. - * @detach: invoked at endpoint release. - * @data: pointer referencing accelerator data for attach and detach calls. - * @region: initialised with autodiscovered region values linked to memdev. - */ -struct cxl_attach_region { - int (*attach)(void *); - void (*detach)(void *); - void *data; - struct range region; -}; - /** * struct cxl_dev_state - The driver device state * @@ -283,27 +252,6 @@ struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev, }) int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity); -struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds, - const struct cxl_memdev_attach *attach); -struct cxl_region; -struct cxl_endpoint_decoder *cxl_get_committed_decoder(struct cxl_memdev *cxlmd, - struct cxl_region **cxlr); -int cxl_get_region_range(struct cxl_region *region, struct range *range); -void cxl_unregister_region(struct cxl_region *cxlr); -struct cxl_port; -struct cxl_root_decoder *cxl_get_hpa_freespace(struct cxl_memdev *cxlmd, - int interleave_ways, - unsigned long flags, - resource_size_t *max); -void cxl_put_root_decoder(struct cxl_root_decoder *cxlrd); -struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_memdev *cxlmd, - enum cxl_partition_mode mode, - resource_size_t alloc); -int cxl_dpa_free(struct cxl_endpoint_decoder *cxled); -struct cxl_region *cxl_create_region(struct cxl_root_decoder *cxlrd, - struct cxl_endpoint_decoder **cxled, - int ways); -int cxl_memdev_attach_region(struct cxl_memdev *cxlmd, struct cxl_attach_region *attach); #ifdef CONFIG_CXL_REGION bool cxl_region_contains_soft_reserve(struct resource *res); @@ -327,4 +275,6 @@ int cxl_get_hdm_info(struct cxl_dev_state *cxlds, u8 *count, #endif /* CONFIG_CXL_BUS */ +struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds, + struct range *range); #endif /* __CXL_CXL_H__ */ diff --git a/include/linux/device.h b/include/linux/device.h index ac972e7bead46..04579ab2ca840 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1010,6 +1010,7 @@ static inline void device_unlock(struct device *dev) } DEFINE_GUARD(device, struct device *, device_lock(_T), device_unlock(_T)) +DEFINE_GUARD_COND(device, _intr, device_lock_interruptible(_T), _RET == 0) static inline void device_lock_assert(struct device *dev) { diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index 79f42f4474d47..b90ac0fd2779d 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -1766,7 +1766,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) cxl_mock_add_event_logs(&mdata->mes); - cxlmd = devm_cxl_add_memdev(cxlds, NULL); + cxlmd = devm_cxl_add_classdev(cxlds); if (IS_ERR(cxlmd)) return PTR_ERR(cxlmd);