Skip to content

Commit

Permalink
pci: Consistently use pci_vf_* for suballocated VF memory resources
Browse files Browse the repository at this point in the history 
Some of the bus resource methods were passing these up to the parent
which triggered rman mismatch assertions in INVARIANTS kernels.

Reported by:	kp
Reviewed by:	imp
Tested by:	kp (earlier version)
Differential Revision:	https://reviews.freebsd.org/D45406
  • Loading branch information
@bsdjhb
bsdjhb committed Jun 4, 2024
1 parent 9805612 commit 871b33a
Show file tree
Hide file tree
Showing 3 changed files with 284 additions and 4 deletions.
118 changes: 114 additions & 4 deletions sys/dev/pci/pci.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,10 +164,18 @@ static device_method_t pci_methods[] = {
DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
DEVMETHOD(bus_delete_resource, pci_delete_resource),
DEVMETHOD(bus_alloc_resource, pci_alloc_resource),
#ifdef PCI_IOV
DEVMETHOD(bus_adjust_resource, pci_adjust_resource),
#else
DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource),
#endif
DEVMETHOD(bus_release_resource, pci_release_resource),
DEVMETHOD(bus_activate_resource, pci_activate_resource),
DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource),
#ifdef PCI_IOV
DEVMETHOD(bus_map_resource, pci_map_resource),
DEVMETHOD(bus_unmap_resource, pci_unmap_resource),
#endif
DEVMETHOD(bus_child_deleted, pci_child_deleted),
DEVMETHOD(bus_child_detached, pci_child_detached),
DEVMETHOD(bus_child_pnpinfo, pci_child_pnpinfo_method),
Expand Down Expand Up @@ -5687,14 +5695,30 @@ pci_activate_resource(device_t dev, device_t child, struct resource *r)
struct pci_devinfo *dinfo;
int error, rid, type;

error = bus_generic_activate_resource(dev, child, r);
dinfo = device_get_ivars(child);
#ifdef PCI_IOV
if (dinfo->cfg.flags & PCICFG_VF) {
switch (rman_get_type(r)) {
/* VFs can't have I/O BARs. */
case SYS_RES_IOPORT:
error = EINVAL;
break;
case SYS_RES_MEMORY:
error = pci_vf_activate_mem_resource(dev, child, r);
break;
default:
error = bus_generic_activate_resource(dev, child, r);
break;
}
} else
#endif
error = bus_generic_activate_resource(dev, child, r);
if (error)
return (error);

/* Enable decoding in the command register when activating BARs. */
if (device_get_parent(child) == dev) {
/* Device ROMs need their decoding explicitly enabled. */
dinfo = device_get_ivars(child);
rid = rman_get_rid(r);
type = rman_get_type(r);
if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
Expand All @@ -5716,13 +5740,29 @@ pci_deactivate_resource(device_t dev, device_t child, struct resource *r)
struct pci_devinfo *dinfo;
int error, rid, type;

error = bus_generic_deactivate_resource(dev, child, r);
dinfo = device_get_ivars(child);
#ifdef PCI_IOV
if (dinfo->cfg.flags & PCICFG_VF) {
switch (rman_get_type(r)) {
/* VFs can't have I/O BARs. */
case SYS_RES_IOPORT:
error = EINVAL;
break;
case SYS_RES_MEMORY:
error = pci_vf_deactivate_mem_resource(dev, child, r);
break;
default:
error = bus_generic_deactivate_resource(dev, child, r);
break;
}
} else
#endif
error = bus_generic_deactivate_resource(dev, child, r);
if (error)
return (error);

/* Disable decoding for device ROMs. */
if (device_get_parent(child) == dev) {
dinfo = device_get_ivars(child);
rid = rman_get_rid(r);
type = rman_get_type(r);
if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
Expand All @@ -5732,6 +5772,76 @@ pci_deactivate_resource(device_t dev, device_t child, struct resource *r)
return (0);
}

#ifdef PCI_IOV
int
pci_adjust_resource(device_t dev, device_t child, struct resource *r,
rman_res_t start, rman_res_t end)
{
struct pci_devinfo *dinfo;

dinfo = device_get_ivars(child);
if (dinfo->cfg.flags & PCICFG_VF) {
switch (rman_get_type(r)) {
/* VFs can't have I/O BARs. */
case SYS_RES_IOPORT:
return (EINVAL);
case SYS_RES_MEMORY:
return (pci_vf_adjust_mem_resource(dev, child, r,
start, end));
}

/* Fall through for other types of resource allocations. */
}

return (bus_generic_adjust_resource(dev, child, r, start, end));
}

int
pci_map_resource(device_t dev, device_t child, struct resource *r,
struct resource_map_request *argsp, struct resource_map *map)
{
struct pci_devinfo *dinfo;

dinfo = device_get_ivars(child);
if (dinfo->cfg.flags & PCICFG_VF) {
switch (rman_get_type(r)) {
/* VFs can't have I/O BARs. */
case SYS_RES_IOPORT:
return (EINVAL);
case SYS_RES_MEMORY:
return (pci_vf_map_mem_resource(dev, child, r, argsp,
map));
}

/* Fall through for other types of resource allocations. */
}

return (bus_generic_map_resource(dev, child, r, argsp, map));
}

int
pci_unmap_resource(device_t dev, device_t child, struct resource *r,
struct resource_map *map)
{
struct pci_devinfo *dinfo;

dinfo = device_get_ivars(child);
if (dinfo->cfg.flags & PCICFG_VF) {
switch (rman_get_type(r)) {
/* VFs can't have I/O BARs. */
case SYS_RES_IOPORT:
return (EINVAL);
case SYS_RES_MEMORY:
return (pci_vf_unmap_mem_resource(dev, child, r, map));
}

/* Fall through for other types of resource allocations. */
}

return (bus_generic_unmap_resource(dev, child, r, map));
}
#endif

void
pci_child_deleted(device_t dev, device_t child)
{
Expand Down
151 changes: 151 additions & 0 deletions sys/dev/pci/pci_iov.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1070,6 +1070,12 @@ pci_vf_release_mem_resource(device_t dev, device_t child, struct resource *r)

dinfo = device_get_ivars(child);

KASSERT(rman_get_type(r) == SYS_RES_MEMORY,
("%s: invalid resource %p", __func__, r));
KASSERT(rman_is_region_manager(r, &dinfo->cfg.iov->rman),
("%s: rman %p doesn't match for resource %p", __func__,
&dinfo->cfg.iov->rman, r));

if (rman_get_flags(r) & RF_ACTIVE) {
error = bus_deactivate_resource(child, r);
if (error != 0)
Expand All @@ -1086,3 +1092,148 @@ pci_vf_release_mem_resource(device_t dev, device_t child, struct resource *r)

return (rman_release_resource(r));
}

int
pci_vf_activate_mem_resource(device_t dev, device_t child, struct resource *r)
{
#ifdef INVARIANTS
struct pci_devinfo *dinfo = device_get_ivars(child);
#endif
struct resource_map map;
int error;

KASSERT(rman_get_type(r) == SYS_RES_MEMORY,
("%s: invalid resource %p", __func__, r));
KASSERT(rman_is_region_manager(r, &dinfo->cfg.iov->rman),
("%s: rman %p doesn't match for resource %p", __func__,
&dinfo->cfg.iov->rman, r));

error = rman_activate_resource(r);
if (error != 0)
return (error);

if ((rman_get_flags(r) & RF_UNMAPPED) == 0) {
error = BUS_MAP_RESOURCE(dev, child, r, NULL, &map);
if (error != 0) {
rman_deactivate_resource(r);
return (error);
}

rman_set_mapping(r, &map);
}
return (0);
}

int
pci_vf_deactivate_mem_resource(device_t dev, device_t child, struct resource *r)
{
#ifdef INVARIANTS
struct pci_devinfo *dinfo = device_get_ivars(child);
#endif
struct resource_map map;
int error;

KASSERT(rman_get_type(r) == SYS_RES_MEMORY,
("%s: invalid resource %p", __func__, r));
KASSERT(rman_is_region_manager(r, &dinfo->cfg.iov->rman),
("%s: rman %p doesn't match for resource %p", __func__,
&dinfo->cfg.iov->rman, r));

error = rman_deactivate_resource(r);
if (error != 0)
return (error);

if ((rman_get_flags(r) & RF_UNMAPPED) == 0) {
rman_get_mapping(r, &map);
BUS_UNMAP_RESOURCE(dev, child, r, &map);
}
return (0);
}

int
pci_vf_adjust_mem_resource(device_t dev, device_t child, struct resource *r,
rman_res_t start, rman_res_t end)
{
#ifdef INVARIANTS
struct pci_devinfo *dinfo = device_get_ivars(child);
#endif

KASSERT(rman_get_type(r) == SYS_RES_MEMORY,
("%s: invalid resource %p", __func__, r));
KASSERT(rman_is_region_manager(r, &dinfo->cfg.iov->rman),
("%s: rman %p doesn't match for resource %p", __func__,
&dinfo->cfg.iov->rman, r));

return (rman_adjust_resource(r, start, end));
}

static struct resource *
pci_vf_find_parent_resource(struct pcicfg_iov *iov, struct resource *r)
{
struct resource *pres;

for (u_int i = 0; i <= PCIR_MAX_BAR_0; i++) {
pres = iov->iov_bar[i].res;
if (pres != NULL) {
if (rman_get_start(pres) <= rman_get_start(r) &&
rman_get_end(pres) >= rman_get_end(r))
return (pres);
}
}
return (NULL);
}

int
pci_vf_map_mem_resource(device_t dev, device_t child, struct resource *r,
struct resource_map_request *argsp, struct resource_map *map)
{
struct pci_devinfo *dinfo = device_get_ivars(child);
struct pcicfg_iov *iov = dinfo->cfg.iov;
struct resource_map_request args;
struct resource *pres;
rman_res_t length, start;
int error;

KASSERT(rman_get_type(r) == SYS_RES_MEMORY,
("%s: invalid resource %p", __func__, r));
KASSERT(rman_is_region_manager(r, &iov->rman),
("%s: rman %p doesn't match for resource %p", __func__,
&dinfo->cfg.iov->rman, r));

/* Resources must be active to be mapped. */
if (!(rman_get_flags(r) & RF_ACTIVE))
return (ENXIO);

resource_init_map_request(&args);
error = resource_validate_map_request(r, argsp, &args, &start, &length);
if (error)
return (error);

pres = pci_vf_find_parent_resource(dinfo->cfg.iov, r);
if (pres == NULL)
return (ENOENT);

args.offset = start - rman_get_start(pres);
args.length = length;
return (bus_map_resource(iov->iov_pf, pres, &args, map));
}

int
pci_vf_unmap_mem_resource(device_t dev, device_t child, struct resource *r,
struct resource_map *map)
{
struct pci_devinfo *dinfo = device_get_ivars(child);
struct pcicfg_iov *iov = dinfo->cfg.iov;
struct resource *pres;

KASSERT(rman_get_type(r) == SYS_RES_MEMORY,
("%s: invalid resource %p", __func__, r));
KASSERT(rman_is_region_manager(r, &iov->rman),
("%s: rman %p doesn't match for resource %p", __func__,
&dinfo->cfg.iov->rman, r));

pres = pci_vf_find_parent_resource(iov, r);
if (pres == NULL)
return (ENOENT);
return (bus_unmap_resource(iov->iov_pf, pres, map));
}
19 changes: 19 additions & 0 deletions sys/dev/pci/pci_private.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,16 @@ bus_get_dma_tag_t pci_get_dma_tag;
bus_get_resource_list_t pci_get_resource_list;
bus_delete_resource_t pci_delete_resource;
bus_alloc_resource_t pci_alloc_resource;
#ifdef PCI_IOV
bus_adjust_resource_t pci_adjust_resource;
#endif
bus_release_resource_t pci_release_resource;
bus_activate_resource_t pci_activate_resource;
bus_deactivate_resource_t pci_deactivate_resource;
#ifdef PCI_IOV
bus_map_resource_t pci_map_resource;
bus_unmap_resource_t pci_unmap_resource;
#endif
bus_child_deleted_t pci_child_deleted;
bus_child_detached_t pci_child_detached;
bus_child_pnpinfo_t pci_child_pnpinfo_method;
Expand Down Expand Up @@ -158,4 +165,16 @@ struct resource *pci_vf_alloc_mem_resource(device_t dev, device_t child,
rman_res_t count, u_int flags);
int pci_vf_release_mem_resource(device_t dev, device_t child,
struct resource *r);
int pci_vf_activate_mem_resource(device_t dev, device_t child,
struct resource *r);
int pci_vf_deactivate_mem_resource(device_t dev, device_t child,
struct resource *r);
int pci_vf_adjust_mem_resource(device_t dev, device_t child,
struct resource *r, rman_res_t start, rman_res_t end);
int pci_vf_map_mem_resource(device_t dev, device_t child,
struct resource *r, struct resource_map_request *argsp,
struct resource_map *map);
int pci_vf_unmap_mem_resource(device_t dev, device_t child,
struct resource *r, struct resource_map *map);

#endif /* _PCI_PRIVATE_H_ */

0 comments on commit 871b33a

Please sign in to comment.