| Message ID | 20210113121222.733517-3-jiri@resnulli.us |
|---|---|
| State | New |
| Headers | show |
| Series | introduce line card support for modular switch | expand |
On Wed, Jan 13, 2021 at 01:12:14PM +0100, Jiri Pirko wrote: > From: Jiri Pirko <jiri@nvidia.com> > > In order to be able to configure all needed stuff on a port/netdevice > of a line card without the line card being present, introduce line card > provisioning. Basically provisioning will create a placeholder for > instances (ports/netdevices) for a line card type. > > Allow the user to query the supported line card types over line card > get command. Then implement two netlink commands to allow user to > provision/unprovision the line card with selected line card type. > > On the driver API side, add provision/unprovision ops and supported > types array to be advertised. Upon provision op call, the driver should > take care of creating the instances for the particular line card type. > Introduce provision_set/clear() functions to be called by the driver > once the provisioning/unprovisioning is done on its side. > > Signed-off-by: Jiri Pirko <jiri@nvidia.com> > --- > include/net/devlink.h | 31 +++++++- > include/uapi/linux/devlink.h | 17 +++++ > net/core/devlink.c | 141 ++++++++++++++++++++++++++++++++++- > 3 files changed, 185 insertions(+), 4 deletions(-) > > diff --git a/include/net/devlink.h b/include/net/devlink.h > index 67c2547d5ef9..854abd53e9ea 100644 > --- a/include/net/devlink.h > +++ b/include/net/devlink.h > @@ -139,10 +139,33 @@ struct devlink_port { > struct mutex reporters_lock; /* Protects reporter_list */ > }; > > +struct devlink_linecard_ops; > + > struct devlink_linecard { > struct list_head list; > struct devlink *devlink; > unsigned int index; > + const struct devlink_linecard_ops *ops; > + void *priv; > + enum devlink_linecard_state state; > + const char *provisioned_type; > +}; > + > +/** > + * struct devlink_linecard_ops - Linecard operations > + * @supported_types: array of supported types of linecards > + * @supported_types_count: number of elements in the array above > + * @provision: callback to provision the linecard slot with certain > + * type of linecard > + * @unprovision: callback to unprovision the linecard slot > + */ > +struct devlink_linecard_ops { > + const char **supported_types; > + unsigned int supported_types_count; > + int (*provision)(struct devlink_linecard *linecard, void *priv, > + u32 type_index, struct netlink_ext_ack *extack); > + int (*unprovision)(struct devlink_linecard *linecard, void *priv, > + struct netlink_ext_ack *extack); > }; > > struct devlink_sb_pool_info { > @@ -1414,9 +1437,13 @@ void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 contro > u16 pf, bool external); > void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller, > u16 pf, u16 vf, bool external); > -struct devlink_linecard *devlink_linecard_create(struct devlink *devlink, > - unsigned int linecard_index); > +struct devlink_linecard * > +devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index, > + const struct devlink_linecard_ops *ops, void *priv); > void devlink_linecard_destroy(struct devlink_linecard *linecard); > +void devlink_linecard_provision_set(struct devlink_linecard *linecard, > + u32 type_index); > +void devlink_linecard_provision_clear(struct devlink_linecard *linecard); > int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, > u32 size, u16 ingress_pools_count, > u16 egress_pools_count, u16 ingress_tc_count, > diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h > index e5ed0522591f..4111ddcc000b 100644 > --- a/include/uapi/linux/devlink.h > +++ b/include/uapi/linux/devlink.h > @@ -131,6 +131,9 @@ enum devlink_command { > DEVLINK_CMD_LINECARD_NEW, > DEVLINK_CMD_LINECARD_DEL, > > + DEVLINK_CMD_LINECARD_PROVISION, > + DEVLINK_CMD_LINECARD_UNPROVISION, I do not really see the point in these two commands. Better extend DEVLINK_CMD_LINECARD_SET to carry these attributes. > + > /* add new commands above here */ > __DEVLINK_CMD_MAX, > DEVLINK_CMD_MAX = __DEVLINK_CMD_MAX - 1 > @@ -329,6 +332,17 @@ enum devlink_reload_limit { > > #define DEVLINK_RELOAD_LIMITS_VALID_MASK (_BITUL(__DEVLINK_RELOAD_LIMIT_MAX) - 1) > > +enum devlink_linecard_state { > + DEVLINK_LINECARD_STATE_UNSPEC, > + DEVLINK_LINECARD_STATE_UNPROVISIONED, > + DEVLINK_LINECARD_STATE_UNPROVISIONING, > + DEVLINK_LINECARD_STATE_PROVISIONING, Can you explain why these two states are necessary? Any reason the provision operation can't be synchronous? This is somewhat explained in patch #8, but it should really be explained here. Changelog says: "To avoid deadlock and to mimic actual HW flow, use workqueue to add/del ports during provisioning as the port add/del calls devlink_port_register/unregister() which take devlink mutex." The deadlock is not really a reason to have these states. 'DEVLINK_CMD_PORT_SPLIT' also calls devlink_port_register() / devlink_port_unregister() and the deadlock is solved by: 'internal_flags = DEVLINK_NL_FLAG_NO_LOCK' A hardware flow the requires it is something else... > + DEVLINK_LINECARD_STATE_PROVISIONED, > + > + __DEVLINK_LINECARD_STATE_MAX, > + DEVLINK_LINECARD_STATE_MAX = __DEVLINK_LINECARD_STATE_MAX - 1 > +}; > + > enum devlink_attr { > /* don't change the order or add anything between, this is ABI! */ > DEVLINK_ATTR_UNSPEC, > @@ -535,6 +549,9 @@ enum devlink_attr { > DEVLINK_ATTR_RELOAD_ACTION_STATS, /* nested */ > > DEVLINK_ATTR_LINECARD_INDEX, /* u32 */ > + DEVLINK_ATTR_LINECARD_STATE, /* u8 */ > + DEVLINK_ATTR_LINECARD_TYPE, /* string */ > + DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES, /* nested */ > > /* add new attributes above here, update the policy in devlink.c */ > > diff --git a/net/core/devlink.c b/net/core/devlink.c > index 564e921133cf..434eecc310c3 100644 > --- a/net/core/devlink.c > +++ b/net/core/devlink.c > @@ -1192,7 +1192,9 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, > u32 seq, int flags, > struct netlink_ext_ack *extack) > { > + struct nlattr *attr; > void *hdr; > + int i; > > hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd); > if (!hdr) > @@ -1202,6 +1204,22 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, > goto nla_put_failure; > if (nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX, linecard->index)) > goto nla_put_failure; > + if (nla_put_u8(msg, DEVLINK_ATTR_LINECARD_STATE, linecard->state)) > + goto nla_put_failure; > + if (linecard->state >= DEVLINK_LINECARD_STATE_PROVISIONED && This assumes that every state added after provisioned should report the type. Better to check for the specific states > + nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, > + linecard->provisioned_type)) > + goto nla_put_failure; > + > + attr = nla_nest_start(msg, DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES); > + if (!attr) > + return -EMSGSIZE; > + for (i = 0; i < linecard->ops->supported_types_count; i++) { > + if (nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, > + linecard->ops->supported_types[i])) > + goto nla_put_failure; > + } > + nla_nest_end(msg, attr); > > genlmsg_end(msg, hdr); > return 0; > @@ -1300,6 +1318,68 @@ static int devlink_nl_cmd_linecard_get_dumpit(struct sk_buff *msg, > return msg->len; > } > > +static int devlink_nl_cmd_linecard_provision_doit(struct sk_buff *skb, > + struct genl_info *info) > +{ > + struct devlink_linecard *linecard = info->user_ptr[1]; > + const char *type; > + int i; > + > + if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) { > + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being provisioned"); > + return -EBUSY; > + } > + if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) { > + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being unprovisioned"); > + return -EBUSY; > + } > + if (linecard->state != DEVLINK_LINECARD_STATE_UNPROVISIONED) { > + NL_SET_ERR_MSG_MOD(info->extack, "Linecard already provisioned"); > + return -EBUSY; > + } > + > + if (!info->attrs[DEVLINK_ATTR_LINECARD_TYPE]) { > + NL_SET_ERR_MSG_MOD(info->extack, "Provision type not provided"); > + return -EINVAL; > + } > + > + type = nla_data(info->attrs[DEVLINK_ATTR_LINECARD_TYPE]); > + for (i = 0; i < linecard->ops->supported_types_count; i++) { > + if (!strcmp(linecard->ops->supported_types[i], type)) { > + linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING; > + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); > + return linecard->ops->provision(linecard, > + linecard->priv, i, > + info->extack); So if this fails user space will see 'provisioning' although nothing is being provisioned... Better to set the state and notify if this call did not fail > + } > + } > + NL_SET_ERR_MSG_MOD(info->extack, "Unsupported provision type provided"); > + return -EINVAL; > +} > + > +static int devlink_nl_cmd_linecard_unprovision_doit(struct sk_buff *skb, > + struct genl_info *info) > +{ > + struct devlink_linecard *linecard = info->user_ptr[1]; > + > + if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) { > + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being provisioned"); > + return -EBUSY; > + } > + if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) { > + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being unprovisioned"); > + return -EBUSY; > + } > + if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONED) { > + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is not provisioned"); > + return -EOPNOTSUPP; > + } > + linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONING; > + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); > + return linecard->ops->unprovision(linecard, linecard->priv, > + info->extack); > +} > + > static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink, > struct devlink_sb *devlink_sb, > enum devlink_command cmd, u32 portid, > @@ -7759,6 +7839,7 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { > DEVLINK_RELOAD_ACTION_MAX), > [DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(DEVLINK_RELOAD_LIMITS_VALID_MASK), > [DEVLINK_ATTR_LINECARD_INDEX] = { .type = NLA_U32 }, > + [DEVLINK_ATTR_LINECARD_TYPE] = { .type = NLA_NUL_STRING }, > }; > > static const struct genl_small_ops devlink_nl_ops[] = { > @@ -7806,6 +7887,20 @@ static const struct genl_small_ops devlink_nl_ops[] = { > .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD, > /* can be retrieved by unprivileged users */ > }, > + { > + .cmd = DEVLINK_CMD_LINECARD_PROVISION, > + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, > + .doit = devlink_nl_cmd_linecard_provision_doit, > + .flags = GENL_ADMIN_PERM, > + .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD, > + }, > + { > + .cmd = DEVLINK_CMD_LINECARD_UNPROVISION, > + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, > + .doit = devlink_nl_cmd_linecard_unprovision_doit, > + .flags = GENL_ADMIN_PERM, > + .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD, > + }, > { > .cmd = DEVLINK_CMD_SB_GET, > .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, > @@ -8613,11 +8708,17 @@ static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port, > * Create devlink linecard instance with provided linecard index. > * Caller can use any indexing, even hw-related one. > */ > -struct devlink_linecard *devlink_linecard_create(struct devlink *devlink, > - unsigned int linecard_index) > +struct devlink_linecard * > +devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index, > + const struct devlink_linecard_ops *ops, void *priv) > { > struct devlink_linecard *linecard; > > + if (WARN_ON(!ops || !ops->supported_types || > + !ops->supported_types_count || > + !ops->provision || !ops->unprovision)) > + return ERR_PTR(-EINVAL); > + > mutex_lock(&devlink->lock); > if (devlink_linecard_index_exists(devlink, linecard_index)) { > mutex_unlock(&devlink->lock); > @@ -8630,6 +8731,9 @@ struct devlink_linecard *devlink_linecard_create(struct devlink *devlink, > > linecard->devlink = devlink; > linecard->index = linecard_index; > + linecard->ops = ops; > + linecard->priv = priv; > + linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED; > list_add_tail(&linecard->list, &devlink->linecard_list); > mutex_unlock(&devlink->lock); > devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); > @@ -8653,6 +8757,39 @@ void devlink_linecard_destroy(struct devlink_linecard *linecard) > } > EXPORT_SYMBOL_GPL(devlink_linecard_create); > > +/** > + * devlink_linecard_provision_set - Set provisioning on linecard 'Set linecard as provisioned' maybe? > + * > + * @devlink_linecard: devlink linecard > + * @type_index: index of the linecard type (in array of types in ops) > + */ > +void devlink_linecard_provision_set(struct devlink_linecard *linecard, > + u32 type_index) > +{ > + WARN_ON(type_index >= linecard->ops->supported_types_count); Wouldn't this explode below when you use the index to access the array? Maybe better to just warn and return > + mutex_lock(&linecard->devlink->lock); > + linecard->state = DEVLINK_LINECARD_STATE_PROVISIONED; > + linecard->provisioned_type = linecard->ops->supported_types[type_index]; > + mutex_unlock(&linecard->devlink->lock); > + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); > +} > +EXPORT_SYMBOL_GPL(devlink_linecard_provision_set); > + > +/** > + * devlink_linecard_provision_clear - Clear provisioning on linecard 'Set linecard as unprovisioned' maybe? > + * > + * @devlink_linecard: devlink linecard > + */ > +void devlink_linecard_provision_clear(struct devlink_linecard *linecard) > +{ > + mutex_lock(&linecard->devlink->lock); > + linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED; > + linecard->provisioned_type = NULL; > + mutex_unlock(&linecard->devlink->lock); > + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); > +} > +EXPORT_SYMBOL_GPL(devlink_linecard_provision_clear); > + > int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, > u32 size, u16 ingress_pools_count, > u16 egress_pools_count, u16 ingress_tc_count, > -- > 2.26.2 >
Fri, Jan 15, 2021 at 05:03:19PM CET, idosch@idosch.org wrote: >On Wed, Jan 13, 2021 at 01:12:14PM +0100, Jiri Pirko wrote: >> From: Jiri Pirko <jiri@nvidia.com> >> >> In order to be able to configure all needed stuff on a port/netdevice >> of a line card without the line card being present, introduce line card >> provisioning. Basically provisioning will create a placeholder for >> instances (ports/netdevices) for a line card type. >> >> Allow the user to query the supported line card types over line card >> get command. Then implement two netlink commands to allow user to >> provision/unprovision the line card with selected line card type. >> >> On the driver API side, add provision/unprovision ops and supported >> types array to be advertised. Upon provision op call, the driver should >> take care of creating the instances for the particular line card type. >> Introduce provision_set/clear() functions to be called by the driver >> once the provisioning/unprovisioning is done on its side. >> >> Signed-off-by: Jiri Pirko <jiri@nvidia.com> >> --- >> include/net/devlink.h | 31 +++++++- >> include/uapi/linux/devlink.h | 17 +++++ >> net/core/devlink.c | 141 ++++++++++++++++++++++++++++++++++- >> 3 files changed, 185 insertions(+), 4 deletions(-) >> >> diff --git a/include/net/devlink.h b/include/net/devlink.h >> index 67c2547d5ef9..854abd53e9ea 100644 >> --- a/include/net/devlink.h >> +++ b/include/net/devlink.h >> @@ -139,10 +139,33 @@ struct devlink_port { >> struct mutex reporters_lock; /* Protects reporter_list */ >> }; >> >> +struct devlink_linecard_ops; >> + >> struct devlink_linecard { >> struct list_head list; >> struct devlink *devlink; >> unsigned int index; >> + const struct devlink_linecard_ops *ops; >> + void *priv; >> + enum devlink_linecard_state state; >> + const char *provisioned_type; >> +}; >> + >> +/** >> + * struct devlink_linecard_ops - Linecard operations >> + * @supported_types: array of supported types of linecards >> + * @supported_types_count: number of elements in the array above >> + * @provision: callback to provision the linecard slot with certain >> + * type of linecard >> + * @unprovision: callback to unprovision the linecard slot >> + */ >> +struct devlink_linecard_ops { >> + const char **supported_types; >> + unsigned int supported_types_count; >> + int (*provision)(struct devlink_linecard *linecard, void *priv, >> + u32 type_index, struct netlink_ext_ack *extack); >> + int (*unprovision)(struct devlink_linecard *linecard, void *priv, >> + struct netlink_ext_ack *extack); >> }; >> >> struct devlink_sb_pool_info { >> @@ -1414,9 +1437,13 @@ void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 contro >> u16 pf, bool external); >> void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller, >> u16 pf, u16 vf, bool external); >> -struct devlink_linecard *devlink_linecard_create(struct devlink *devlink, >> - unsigned int linecard_index); >> +struct devlink_linecard * >> +devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index, >> + const struct devlink_linecard_ops *ops, void *priv); >> void devlink_linecard_destroy(struct devlink_linecard *linecard); >> +void devlink_linecard_provision_set(struct devlink_linecard *linecard, >> + u32 type_index); >> +void devlink_linecard_provision_clear(struct devlink_linecard *linecard); >> int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, >> u32 size, u16 ingress_pools_count, >> u16 egress_pools_count, u16 ingress_tc_count, >> diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h >> index e5ed0522591f..4111ddcc000b 100644 >> --- a/include/uapi/linux/devlink.h >> +++ b/include/uapi/linux/devlink.h >> @@ -131,6 +131,9 @@ enum devlink_command { >> DEVLINK_CMD_LINECARD_NEW, >> DEVLINK_CMD_LINECARD_DEL, >> >> + DEVLINK_CMD_LINECARD_PROVISION, >> + DEVLINK_CMD_LINECARD_UNPROVISION, > >I do not really see the point in these two commands. Better extend >DEVLINK_CMD_LINECARD_SET to carry these attributes. Yeah, I was thinking about that. Not sure it is correct though. This is single purpose command. It really does not change "an attribute" as the "_SET" commands are usually doing. Consider extension of "_SET" by other attributes. Then it looks wrong. > >> + >> /* add new commands above here */ >> __DEVLINK_CMD_MAX, >> DEVLINK_CMD_MAX = __DEVLINK_CMD_MAX - 1 >> @@ -329,6 +332,17 @@ enum devlink_reload_limit { >> >> #define DEVLINK_RELOAD_LIMITS_VALID_MASK (_BITUL(__DEVLINK_RELOAD_LIMIT_MAX) - 1) >> >> +enum devlink_linecard_state { >> + DEVLINK_LINECARD_STATE_UNSPEC, >> + DEVLINK_LINECARD_STATE_UNPROVISIONED, >> + DEVLINK_LINECARD_STATE_UNPROVISIONING, >> + DEVLINK_LINECARD_STATE_PROVISIONING, > >Can you explain why these two states are necessary? Any reason the >provision operation can't be synchronous? This is somewhat explained in >patch #8, but it should really be explained here. Changelog says: > >"To avoid deadlock and to mimic actual HW flow, use workqueue >to add/del ports during provisioning as the port add/del calls >devlink_port_register/unregister() which take devlink mutex." > >The deadlock is not really a reason to have these states. It is, need to avoid recursice locking >'DEVLINK_CMD_PORT_SPLIT' also calls devlink_port_register() / >devlink_port_unregister() and the deadlock is solved by: > >'internal_flags = DEVLINK_NL_FLAG_NO_LOCK' Yeah, however, there, the port_index is passed down to the driver, not the actual object pointer. That's why it can be done like that. > >A hardware flow the requires it is something else... Hardware flow in case of Spectrum is async too. > >> + DEVLINK_LINECARD_STATE_PROVISIONED, >> + >> + __DEVLINK_LINECARD_STATE_MAX, >> + DEVLINK_LINECARD_STATE_MAX = __DEVLINK_LINECARD_STATE_MAX - 1 >> +}; >> + >> enum devlink_attr { >> /* don't change the order or add anything between, this is ABI! */ >> DEVLINK_ATTR_UNSPEC, >> @@ -535,6 +549,9 @@ enum devlink_attr { >> DEVLINK_ATTR_RELOAD_ACTION_STATS, /* nested */ >> >> DEVLINK_ATTR_LINECARD_INDEX, /* u32 */ >> + DEVLINK_ATTR_LINECARD_STATE, /* u8 */ >> + DEVLINK_ATTR_LINECARD_TYPE, /* string */ >> + DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES, /* nested */ >> >> /* add new attributes above here, update the policy in devlink.c */ >> >> diff --git a/net/core/devlink.c b/net/core/devlink.c >> index 564e921133cf..434eecc310c3 100644 >> --- a/net/core/devlink.c >> +++ b/net/core/devlink.c >> @@ -1192,7 +1192,9 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, >> u32 seq, int flags, >> struct netlink_ext_ack *extack) >> { >> + struct nlattr *attr; >> void *hdr; >> + int i; >> >> hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd); >> if (!hdr) >> @@ -1202,6 +1204,22 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, >> goto nla_put_failure; >> if (nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX, linecard->index)) >> goto nla_put_failure; >> + if (nla_put_u8(msg, DEVLINK_ATTR_LINECARD_STATE, linecard->state)) >> + goto nla_put_failure; >> + if (linecard->state >= DEVLINK_LINECARD_STATE_PROVISIONED && > >This assumes that every state added after provisioned should report the >type. Better to check for the specific states Yes, that is correct assumption. > >> + nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, >> + linecard->provisioned_type)) >> + goto nla_put_failure; >> + >> + attr = nla_nest_start(msg, DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES); >> + if (!attr) >> + return -EMSGSIZE; >> + for (i = 0; i < linecard->ops->supported_types_count; i++) { >> + if (nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, >> + linecard->ops->supported_types[i])) >> + goto nla_put_failure; >> + } >> + nla_nest_end(msg, attr); >> >> genlmsg_end(msg, hdr); >> return 0; >> @@ -1300,6 +1318,68 @@ static int devlink_nl_cmd_linecard_get_dumpit(struct sk_buff *msg, >> return msg->len; >> } >> >> +static int devlink_nl_cmd_linecard_provision_doit(struct sk_buff *skb, >> + struct genl_info *info) >> +{ >> + struct devlink_linecard *linecard = info->user_ptr[1]; >> + const char *type; >> + int i; >> + >> + if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) { >> + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being provisioned"); >> + return -EBUSY; >> + } >> + if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) { >> + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being unprovisioned"); >> + return -EBUSY; >> + } >> + if (linecard->state != DEVLINK_LINECARD_STATE_UNPROVISIONED) { >> + NL_SET_ERR_MSG_MOD(info->extack, "Linecard already provisioned"); >> + return -EBUSY; >> + } >> + >> + if (!info->attrs[DEVLINK_ATTR_LINECARD_TYPE]) { >> + NL_SET_ERR_MSG_MOD(info->extack, "Provision type not provided"); >> + return -EINVAL; >> + } >> + >> + type = nla_data(info->attrs[DEVLINK_ATTR_LINECARD_TYPE]); >> + for (i = 0; i < linecard->ops->supported_types_count; i++) { >> + if (!strcmp(linecard->ops->supported_types[i], type)) { >> + linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING; >> + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); >> + return linecard->ops->provision(linecard, >> + linecard->priv, i, >> + info->extack); > >So if this fails user space will see 'provisioning' although nothing is >being provisioned... Better to set the state and notify if this call did >not fail The driver is responsible to either call provision_set/provision_clear helper. Note the async nature of this op. > >> + } >> + } >> + NL_SET_ERR_MSG_MOD(info->extack, "Unsupported provision type provided"); >> + return -EINVAL; >> +} >> + >> +static int devlink_nl_cmd_linecard_unprovision_doit(struct sk_buff *skb, >> + struct genl_info *info) >> +{ >> + struct devlink_linecard *linecard = info->user_ptr[1]; >> + >> + if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) { >> + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being provisioned"); >> + return -EBUSY; >> + } >> + if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) { >> + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being unprovisioned"); >> + return -EBUSY; >> + } >> + if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONED) { >> + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is not provisioned"); >> + return -EOPNOTSUPP; >> + } >> + linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONING; >> + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); >> + return linecard->ops->unprovision(linecard, linecard->priv, >> + info->extack); >> +} >> + >> static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink, >> struct devlink_sb *devlink_sb, >> enum devlink_command cmd, u32 portid, >> @@ -7759,6 +7839,7 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { >> DEVLINK_RELOAD_ACTION_MAX), >> [DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(DEVLINK_RELOAD_LIMITS_VALID_MASK), >> [DEVLINK_ATTR_LINECARD_INDEX] = { .type = NLA_U32 }, >> + [DEVLINK_ATTR_LINECARD_TYPE] = { .type = NLA_NUL_STRING }, >> }; >> >> static const struct genl_small_ops devlink_nl_ops[] = { >> @@ -7806,6 +7887,20 @@ static const struct genl_small_ops devlink_nl_ops[] = { >> .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD, >> /* can be retrieved by unprivileged users */ >> }, >> + { >> + .cmd = DEVLINK_CMD_LINECARD_PROVISION, >> + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, >> + .doit = devlink_nl_cmd_linecard_provision_doit, >> + .flags = GENL_ADMIN_PERM, >> + .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD, >> + }, >> + { >> + .cmd = DEVLINK_CMD_LINECARD_UNPROVISION, >> + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, >> + .doit = devlink_nl_cmd_linecard_unprovision_doit, >> + .flags = GENL_ADMIN_PERM, >> + .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD, >> + }, >> { >> .cmd = DEVLINK_CMD_SB_GET, >> .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, >> @@ -8613,11 +8708,17 @@ static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port, >> * Create devlink linecard instance with provided linecard index. >> * Caller can use any indexing, even hw-related one. >> */ >> -struct devlink_linecard *devlink_linecard_create(struct devlink *devlink, >> - unsigned int linecard_index) >> +struct devlink_linecard * >> +devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index, >> + const struct devlink_linecard_ops *ops, void *priv) >> { >> struct devlink_linecard *linecard; >> >> + if (WARN_ON(!ops || !ops->supported_types || >> + !ops->supported_types_count || >> + !ops->provision || !ops->unprovision)) >> + return ERR_PTR(-EINVAL); >> + >> mutex_lock(&devlink->lock); >> if (devlink_linecard_index_exists(devlink, linecard_index)) { >> mutex_unlock(&devlink->lock); >> @@ -8630,6 +8731,9 @@ struct devlink_linecard *devlink_linecard_create(struct devlink *devlink, >> >> linecard->devlink = devlink; >> linecard->index = linecard_index; >> + linecard->ops = ops; >> + linecard->priv = priv; >> + linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED; >> list_add_tail(&linecard->list, &devlink->linecard_list); >> mutex_unlock(&devlink->lock); >> devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); >> @@ -8653,6 +8757,39 @@ void devlink_linecard_destroy(struct devlink_linecard *linecard) >> } >> EXPORT_SYMBOL_GPL(devlink_linecard_create); >> >> +/** >> + * devlink_linecard_provision_set - Set provisioning on linecard > >'Set linecard as provisioned' maybe? Sure, why not. > >> + * >> + * @devlink_linecard: devlink linecard >> + * @type_index: index of the linecard type (in array of types in ops) >> + */ >> +void devlink_linecard_provision_set(struct devlink_linecard *linecard, >> + u32 type_index) >> +{ >> + WARN_ON(type_index >= linecard->ops->supported_types_count); > >Wouldn't this explode below when you use the index to access the array? >Maybe better to just warn and return Okay. > >> + mutex_lock(&linecard->devlink->lock); >> + linecard->state = DEVLINK_LINECARD_STATE_PROVISIONED; >> + linecard->provisioned_type = linecard->ops->supported_types[type_index]; >> + mutex_unlock(&linecard->devlink->lock); >> + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); >> +} >> +EXPORT_SYMBOL_GPL(devlink_linecard_provision_set); >> + >> +/** >> + * devlink_linecard_provision_clear - Clear provisioning on linecard > >'Set linecard as unprovisioned' maybe? Sure, why not. > >> + * >> + * @devlink_linecard: devlink linecard >> + */ >> +void devlink_linecard_provision_clear(struct devlink_linecard *linecard) >> +{ >> + mutex_lock(&linecard->devlink->lock); >> + linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED; >> + linecard->provisioned_type = NULL; >> + mutex_unlock(&linecard->devlink->lock); >> + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); >> +} >> +EXPORT_SYMBOL_GPL(devlink_linecard_provision_clear); >> + >> int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, >> u32 size, u16 ingress_pools_count, >> u16 egress_pools_count, u16 ingress_tc_count, >> -- >> 2.26.2 >>
On Fri, Jan 15, 2021 at 05:51:57PM +0100, Jiri Pirko wrote: > Fri, Jan 15, 2021 at 05:03:19PM CET, idosch@idosch.org wrote: > >On Wed, Jan 13, 2021 at 01:12:14PM +0100, Jiri Pirko wrote: > >> From: Jiri Pirko <jiri@nvidia.com> > >> > >> In order to be able to configure all needed stuff on a port/netdevice > >> of a line card without the line card being present, introduce line card > >> provisioning. Basically provisioning will create a placeholder for > >> instances (ports/netdevices) for a line card type. > >> > >> Allow the user to query the supported line card types over line card > >> get command. Then implement two netlink commands to allow user to > >> provision/unprovision the line card with selected line card type. > >> > >> On the driver API side, add provision/unprovision ops and supported > >> types array to be advertised. Upon provision op call, the driver should > >> take care of creating the instances for the particular line card type. > >> Introduce provision_set/clear() functions to be called by the driver > >> once the provisioning/unprovisioning is done on its side. > >> > >> Signed-off-by: Jiri Pirko <jiri@nvidia.com> > >> --- > >> include/net/devlink.h | 31 +++++++- > >> include/uapi/linux/devlink.h | 17 +++++ > >> net/core/devlink.c | 141 ++++++++++++++++++++++++++++++++++- > >> 3 files changed, 185 insertions(+), 4 deletions(-) > >> > >> diff --git a/include/net/devlink.h b/include/net/devlink.h > >> index 67c2547d5ef9..854abd53e9ea 100644 > >> --- a/include/net/devlink.h > >> +++ b/include/net/devlink.h > >> @@ -139,10 +139,33 @@ struct devlink_port { > >> struct mutex reporters_lock; /* Protects reporter_list */ > >> }; > >> > >> +struct devlink_linecard_ops; > >> + > >> struct devlink_linecard { > >> struct list_head list; > >> struct devlink *devlink; > >> unsigned int index; > >> + const struct devlink_linecard_ops *ops; > >> + void *priv; > >> + enum devlink_linecard_state state; > >> + const char *provisioned_type; > >> +}; > >> + > >> +/** > >> + * struct devlink_linecard_ops - Linecard operations > >> + * @supported_types: array of supported types of linecards > >> + * @supported_types_count: number of elements in the array above > >> + * @provision: callback to provision the linecard slot with certain > >> + * type of linecard > >> + * @unprovision: callback to unprovision the linecard slot > >> + */ > >> +struct devlink_linecard_ops { > >> + const char **supported_types; > >> + unsigned int supported_types_count; > >> + int (*provision)(struct devlink_linecard *linecard, void *priv, > >> + u32 type_index, struct netlink_ext_ack *extack); > >> + int (*unprovision)(struct devlink_linecard *linecard, void *priv, > >> + struct netlink_ext_ack *extack); > >> }; > >> > >> struct devlink_sb_pool_info { > >> @@ -1414,9 +1437,13 @@ void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 contro > >> u16 pf, bool external); > >> void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller, > >> u16 pf, u16 vf, bool external); > >> -struct devlink_linecard *devlink_linecard_create(struct devlink *devlink, > >> - unsigned int linecard_index); > >> +struct devlink_linecard * > >> +devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index, > >> + const struct devlink_linecard_ops *ops, void *priv); > >> void devlink_linecard_destroy(struct devlink_linecard *linecard); > >> +void devlink_linecard_provision_set(struct devlink_linecard *linecard, > >> + u32 type_index); > >> +void devlink_linecard_provision_clear(struct devlink_linecard *linecard); > >> int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, > >> u32 size, u16 ingress_pools_count, > >> u16 egress_pools_count, u16 ingress_tc_count, > >> diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h > >> index e5ed0522591f..4111ddcc000b 100644 > >> --- a/include/uapi/linux/devlink.h > >> +++ b/include/uapi/linux/devlink.h > >> @@ -131,6 +131,9 @@ enum devlink_command { > >> DEVLINK_CMD_LINECARD_NEW, > >> DEVLINK_CMD_LINECARD_DEL, > >> > >> + DEVLINK_CMD_LINECARD_PROVISION, > >> + DEVLINK_CMD_LINECARD_UNPROVISION, > > > >I do not really see the point in these two commands. Better extend > >DEVLINK_CMD_LINECARD_SET to carry these attributes. > > Yeah, I was thinking about that. Not sure it is correct though. This is > single purpose command. It really does not change "an attribute" as the > "_SET" commands are usually doing. Consider extension of "_SET" by other > attributes. Then it looks wrong. It is setting the type of the linecard, which is an attribute of the linecard. > > > > > >> + > >> /* add new commands above here */ > >> __DEVLINK_CMD_MAX, > >> DEVLINK_CMD_MAX = __DEVLINK_CMD_MAX - 1 > >> @@ -329,6 +332,17 @@ enum devlink_reload_limit { > >> > >> #define DEVLINK_RELOAD_LIMITS_VALID_MASK (_BITUL(__DEVLINK_RELOAD_LIMIT_MAX) - 1) > >> > >> +enum devlink_linecard_state { > >> + DEVLINK_LINECARD_STATE_UNSPEC, > >> + DEVLINK_LINECARD_STATE_UNPROVISIONED, > >> + DEVLINK_LINECARD_STATE_UNPROVISIONING, > >> + DEVLINK_LINECARD_STATE_PROVISIONING, > > > >Can you explain why these two states are necessary? Any reason the > >provision operation can't be synchronous? This is somewhat explained in > >patch #8, but it should really be explained here. Changelog says: > > > >"To avoid deadlock and to mimic actual HW flow, use workqueue > >to add/del ports during provisioning as the port add/del calls > >devlink_port_register/unregister() which take devlink mutex." > > > >The deadlock is not really a reason to have these states. > > It is, need to avoid recursice locking > > >'DEVLINK_CMD_PORT_SPLIT' also calls devlink_port_register() / > >devlink_port_unregister() and the deadlock is solved by: > > > >'internal_flags = DEVLINK_NL_FLAG_NO_LOCK' > > Yeah, however, there, the port_index is passed down to the driver, not > the actual object pointer. That's why it can be done like that. > > > > >A hardware flow the requires it is something else... > > Hardware flow in case of Spectrum is async too. OK, so the changelog needs to state that these states are necessary because the nature of linecard provisioning is asynchronous. > > > > > >> + DEVLINK_LINECARD_STATE_PROVISIONED, > >> + > >> + __DEVLINK_LINECARD_STATE_MAX, > >> + DEVLINK_LINECARD_STATE_MAX = __DEVLINK_LINECARD_STATE_MAX - 1 > >> +}; > >> + > >> enum devlink_attr { > >> /* don't change the order or add anything between, this is ABI! */ > >> DEVLINK_ATTR_UNSPEC, > >> @@ -535,6 +549,9 @@ enum devlink_attr { > >> DEVLINK_ATTR_RELOAD_ACTION_STATS, /* nested */ > >> > >> DEVLINK_ATTR_LINECARD_INDEX, /* u32 */ > >> + DEVLINK_ATTR_LINECARD_STATE, /* u8 */ > >> + DEVLINK_ATTR_LINECARD_TYPE, /* string */ > >> + DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES, /* nested */ > >> > >> /* add new attributes above here, update the policy in devlink.c */ > >> > >> diff --git a/net/core/devlink.c b/net/core/devlink.c > >> index 564e921133cf..434eecc310c3 100644 > >> --- a/net/core/devlink.c > >> +++ b/net/core/devlink.c > >> @@ -1192,7 +1192,9 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, > >> u32 seq, int flags, > >> struct netlink_ext_ack *extack) > >> { > >> + struct nlattr *attr; > >> void *hdr; > >> + int i; > >> > >> hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd); > >> if (!hdr) > >> @@ -1202,6 +1204,22 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, > >> goto nla_put_failure; > >> if (nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX, linecard->index)) > >> goto nla_put_failure; > >> + if (nla_put_u8(msg, DEVLINK_ATTR_LINECARD_STATE, linecard->state)) > >> + goto nla_put_failure; > >> + if (linecard->state >= DEVLINK_LINECARD_STATE_PROVISIONED && > > > >This assumes that every state added after provisioned should report the > >type. Better to check for the specific states > > Yes, that is correct assumption. It is correct now, but what if tomorrow someone adds a new state? It can't be added before the provisioned state because it will break uapi. > > > > > >> + nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, > >> + linecard->provisioned_type)) > >> + goto nla_put_failure;
Fri, Jan 15, 2021 at 07:09:44PM CET, idosch@idosch.org wrote: >On Fri, Jan 15, 2021 at 05:51:57PM +0100, Jiri Pirko wrote: >> Fri, Jan 15, 2021 at 05:03:19PM CET, idosch@idosch.org wrote: >> >On Wed, Jan 13, 2021 at 01:12:14PM +0100, Jiri Pirko wrote: >> >> From: Jiri Pirko <jiri@nvidia.com> >> >> >> >> In order to be able to configure all needed stuff on a port/netdevice >> >> of a line card without the line card being present, introduce line card >> >> provisioning. Basically provisioning will create a placeholder for >> >> instances (ports/netdevices) for a line card type. >> >> >> >> Allow the user to query the supported line card types over line card >> >> get command. Then implement two netlink commands to allow user to >> >> provision/unprovision the line card with selected line card type. >> >> >> >> On the driver API side, add provision/unprovision ops and supported >> >> types array to be advertised. Upon provision op call, the driver should >> >> take care of creating the instances for the particular line card type. >> >> Introduce provision_set/clear() functions to be called by the driver >> >> once the provisioning/unprovisioning is done on its side. >> >> >> >> Signed-off-by: Jiri Pirko <jiri@nvidia.com> >> >> --- >> >> include/net/devlink.h | 31 +++++++- >> >> include/uapi/linux/devlink.h | 17 +++++ >> >> net/core/devlink.c | 141 ++++++++++++++++++++++++++++++++++- >> >> 3 files changed, 185 insertions(+), 4 deletions(-) >> >> >> >> diff --git a/include/net/devlink.h b/include/net/devlink.h >> >> index 67c2547d5ef9..854abd53e9ea 100644 >> >> --- a/include/net/devlink.h >> >> +++ b/include/net/devlink.h >> >> @@ -139,10 +139,33 @@ struct devlink_port { >> >> struct mutex reporters_lock; /* Protects reporter_list */ >> >> }; >> >> >> >> +struct devlink_linecard_ops; >> >> + >> >> struct devlink_linecard { >> >> struct list_head list; >> >> struct devlink *devlink; >> >> unsigned int index; >> >> + const struct devlink_linecard_ops *ops; >> >> + void *priv; >> >> + enum devlink_linecard_state state; >> >> + const char *provisioned_type; >> >> +}; >> >> + >> >> +/** >> >> + * struct devlink_linecard_ops - Linecard operations >> >> + * @supported_types: array of supported types of linecards >> >> + * @supported_types_count: number of elements in the array above >> >> + * @provision: callback to provision the linecard slot with certain >> >> + * type of linecard >> >> + * @unprovision: callback to unprovision the linecard slot >> >> + */ >> >> +struct devlink_linecard_ops { >> >> + const char **supported_types; >> >> + unsigned int supported_types_count; >> >> + int (*provision)(struct devlink_linecard *linecard, void *priv, >> >> + u32 type_index, struct netlink_ext_ack *extack); >> >> + int (*unprovision)(struct devlink_linecard *linecard, void *priv, >> >> + struct netlink_ext_ack *extack); >> >> }; >> >> >> >> struct devlink_sb_pool_info { >> >> @@ -1414,9 +1437,13 @@ void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 contro >> >> u16 pf, bool external); >> >> void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller, >> >> u16 pf, u16 vf, bool external); >> >> -struct devlink_linecard *devlink_linecard_create(struct devlink *devlink, >> >> - unsigned int linecard_index); >> >> +struct devlink_linecard * >> >> +devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index, >> >> + const struct devlink_linecard_ops *ops, void *priv); >> >> void devlink_linecard_destroy(struct devlink_linecard *linecard); >> >> +void devlink_linecard_provision_set(struct devlink_linecard *linecard, >> >> + u32 type_index); >> >> +void devlink_linecard_provision_clear(struct devlink_linecard *linecard); >> >> int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, >> >> u32 size, u16 ingress_pools_count, >> >> u16 egress_pools_count, u16 ingress_tc_count, >> >> diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h >> >> index e5ed0522591f..4111ddcc000b 100644 >> >> --- a/include/uapi/linux/devlink.h >> >> +++ b/include/uapi/linux/devlink.h >> >> @@ -131,6 +131,9 @@ enum devlink_command { >> >> DEVLINK_CMD_LINECARD_NEW, >> >> DEVLINK_CMD_LINECARD_DEL, >> >> >> >> + DEVLINK_CMD_LINECARD_PROVISION, >> >> + DEVLINK_CMD_LINECARD_UNPROVISION, >> > >> >I do not really see the point in these two commands. Better extend >> >DEVLINK_CMD_LINECARD_SET to carry these attributes. >> >> Yeah, I was thinking about that. Not sure it is correct though. This is >> single purpose command. It really does not change "an attribute" as the >> "_SET" commands are usually doing. Consider extension of "_SET" by other >> attributes. Then it looks wrong. > >It is setting the type of the linecard, which is an attribute of the >linecard. Hmm. Still, consider the async nature. Do you have any example of attr set with async nature? I expect the attr to be set when cmd returns 0. IDK. Does not feel correct... > >> >> >> > >> >> + >> >> /* add new commands above here */ >> >> __DEVLINK_CMD_MAX, >> >> DEVLINK_CMD_MAX = __DEVLINK_CMD_MAX - 1 >> >> @@ -329,6 +332,17 @@ enum devlink_reload_limit { >> >> >> >> #define DEVLINK_RELOAD_LIMITS_VALID_MASK (_BITUL(__DEVLINK_RELOAD_LIMIT_MAX) - 1) >> >> >> >> +enum devlink_linecard_state { >> >> + DEVLINK_LINECARD_STATE_UNSPEC, >> >> + DEVLINK_LINECARD_STATE_UNPROVISIONED, >> >> + DEVLINK_LINECARD_STATE_UNPROVISIONING, >> >> + DEVLINK_LINECARD_STATE_PROVISIONING, >> > >> >Can you explain why these two states are necessary? Any reason the >> >provision operation can't be synchronous? This is somewhat explained in >> >patch #8, but it should really be explained here. Changelog says: >> > >> >"To avoid deadlock and to mimic actual HW flow, use workqueue >> >to add/del ports during provisioning as the port add/del calls >> >devlink_port_register/unregister() which take devlink mutex." >> > >> >The deadlock is not really a reason to have these states. >> >> It is, need to avoid recursice locking >> >> >'DEVLINK_CMD_PORT_SPLIT' also calls devlink_port_register() / >> >devlink_port_unregister() and the deadlock is solved by: >> > >> >'internal_flags = DEVLINK_NL_FLAG_NO_LOCK' >> >> Yeah, however, there, the port_index is passed down to the driver, not >> the actual object pointer. That's why it can be done like that. >> >> > >> >A hardware flow the requires it is something else... >> >> Hardware flow in case of Spectrum is async too. > >OK, so the changelog needs to state that these states are necessary >because the nature of linecard provisioning is asynchronous. Ok. > >> >> >> > >> >> + DEVLINK_LINECARD_STATE_PROVISIONED, >> >> + >> >> + __DEVLINK_LINECARD_STATE_MAX, >> >> + DEVLINK_LINECARD_STATE_MAX = __DEVLINK_LINECARD_STATE_MAX - 1 >> >> +}; >> >> + >> >> enum devlink_attr { >> >> /* don't change the order or add anything between, this is ABI! */ >> >> DEVLINK_ATTR_UNSPEC, >> >> @@ -535,6 +549,9 @@ enum devlink_attr { >> >> DEVLINK_ATTR_RELOAD_ACTION_STATS, /* nested */ >> >> >> >> DEVLINK_ATTR_LINECARD_INDEX, /* u32 */ >> >> + DEVLINK_ATTR_LINECARD_STATE, /* u8 */ >> >> + DEVLINK_ATTR_LINECARD_TYPE, /* string */ >> >> + DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES, /* nested */ >> >> >> >> /* add new attributes above here, update the policy in devlink.c */ >> >> >> >> diff --git a/net/core/devlink.c b/net/core/devlink.c >> >> index 564e921133cf..434eecc310c3 100644 >> >> --- a/net/core/devlink.c >> >> +++ b/net/core/devlink.c >> >> @@ -1192,7 +1192,9 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, >> >> u32 seq, int flags, >> >> struct netlink_ext_ack *extack) >> >> { >> >> + struct nlattr *attr; >> >> void *hdr; >> >> + int i; >> >> >> >> hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd); >> >> if (!hdr) >> >> @@ -1202,6 +1204,22 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, >> >> goto nla_put_failure; >> >> if (nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX, linecard->index)) >> >> goto nla_put_failure; >> >> + if (nla_put_u8(msg, DEVLINK_ATTR_LINECARD_STATE, linecard->state)) >> >> + goto nla_put_failure; >> >> + if (linecard->state >= DEVLINK_LINECARD_STATE_PROVISIONED && >> > >> >This assumes that every state added after provisioned should report the >> >type. Better to check for the specific states >> >> Yes, that is correct assumption. > >It is correct now, but what if tomorrow someone adds a new state? It >can't be added before the provisioned state because it will break uapi. Then this check will need to be changed... > >> >> >> > >> >> + nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, >> >> + linecard->provisioned_type)) >> >> + goto nla_put_failure;
diff --git a/include/net/devlink.h b/include/net/devlink.h index 67c2547d5ef9..854abd53e9ea 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -139,10 +139,33 @@ struct devlink_port { struct mutex reporters_lock; /* Protects reporter_list */ }; +struct devlink_linecard_ops; + struct devlink_linecard { struct list_head list; struct devlink *devlink; unsigned int index; + const struct devlink_linecard_ops *ops; + void *priv; + enum devlink_linecard_state state; + const char *provisioned_type; +}; + +/** + * struct devlink_linecard_ops - Linecard operations + * @supported_types: array of supported types of linecards + * @supported_types_count: number of elements in the array above + * @provision: callback to provision the linecard slot with certain + * type of linecard + * @unprovision: callback to unprovision the linecard slot + */ +struct devlink_linecard_ops { + const char **supported_types; + unsigned int supported_types_count; + int (*provision)(struct devlink_linecard *linecard, void *priv, + u32 type_index, struct netlink_ext_ack *extack); + int (*unprovision)(struct devlink_linecard *linecard, void *priv, + struct netlink_ext_ack *extack); }; struct devlink_sb_pool_info { @@ -1414,9 +1437,13 @@ void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 contro u16 pf, bool external); void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller, u16 pf, u16 vf, bool external); -struct devlink_linecard *devlink_linecard_create(struct devlink *devlink, - unsigned int linecard_index); +struct devlink_linecard * +devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index, + const struct devlink_linecard_ops *ops, void *priv); void devlink_linecard_destroy(struct devlink_linecard *linecard); +void devlink_linecard_provision_set(struct devlink_linecard *linecard, + u32 type_index); +void devlink_linecard_provision_clear(struct devlink_linecard *linecard); int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, u32 size, u16 ingress_pools_count, u16 egress_pools_count, u16 ingress_tc_count, diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h index e5ed0522591f..4111ddcc000b 100644 --- a/include/uapi/linux/devlink.h +++ b/include/uapi/linux/devlink.h @@ -131,6 +131,9 @@ enum devlink_command { DEVLINK_CMD_LINECARD_NEW, DEVLINK_CMD_LINECARD_DEL, + DEVLINK_CMD_LINECARD_PROVISION, + DEVLINK_CMD_LINECARD_UNPROVISION, + /* add new commands above here */ __DEVLINK_CMD_MAX, DEVLINK_CMD_MAX = __DEVLINK_CMD_MAX - 1 @@ -329,6 +332,17 @@ enum devlink_reload_limit { #define DEVLINK_RELOAD_LIMITS_VALID_MASK (_BITUL(__DEVLINK_RELOAD_LIMIT_MAX) - 1) +enum devlink_linecard_state { + DEVLINK_LINECARD_STATE_UNSPEC, + DEVLINK_LINECARD_STATE_UNPROVISIONED, + DEVLINK_LINECARD_STATE_UNPROVISIONING, + DEVLINK_LINECARD_STATE_PROVISIONING, + DEVLINK_LINECARD_STATE_PROVISIONED, + + __DEVLINK_LINECARD_STATE_MAX, + DEVLINK_LINECARD_STATE_MAX = __DEVLINK_LINECARD_STATE_MAX - 1 +}; + enum devlink_attr { /* don't change the order or add anything between, this is ABI! */ DEVLINK_ATTR_UNSPEC, @@ -535,6 +549,9 @@ enum devlink_attr { DEVLINK_ATTR_RELOAD_ACTION_STATS, /* nested */ DEVLINK_ATTR_LINECARD_INDEX, /* u32 */ + DEVLINK_ATTR_LINECARD_STATE, /* u8 */ + DEVLINK_ATTR_LINECARD_TYPE, /* string */ + DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES, /* nested */ /* add new attributes above here, update the policy in devlink.c */ diff --git a/net/core/devlink.c b/net/core/devlink.c index 564e921133cf..434eecc310c3 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -1192,7 +1192,9 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, u32 seq, int flags, struct netlink_ext_ack *extack) { + struct nlattr *attr; void *hdr; + int i; hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd); if (!hdr) @@ -1202,6 +1204,22 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, goto nla_put_failure; if (nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX, linecard->index)) goto nla_put_failure; + if (nla_put_u8(msg, DEVLINK_ATTR_LINECARD_STATE, linecard->state)) + goto nla_put_failure; + if (linecard->state >= DEVLINK_LINECARD_STATE_PROVISIONED && + nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, + linecard->provisioned_type)) + goto nla_put_failure; + + attr = nla_nest_start(msg, DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES); + if (!attr) + return -EMSGSIZE; + for (i = 0; i < linecard->ops->supported_types_count; i++) { + if (nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, + linecard->ops->supported_types[i])) + goto nla_put_failure; + } + nla_nest_end(msg, attr); genlmsg_end(msg, hdr); return 0; @@ -1300,6 +1318,68 @@ static int devlink_nl_cmd_linecard_get_dumpit(struct sk_buff *msg, return msg->len; } +static int devlink_nl_cmd_linecard_provision_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct devlink_linecard *linecard = info->user_ptr[1]; + const char *type; + int i; + + if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) { + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being provisioned"); + return -EBUSY; + } + if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) { + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being unprovisioned"); + return -EBUSY; + } + if (linecard->state != DEVLINK_LINECARD_STATE_UNPROVISIONED) { + NL_SET_ERR_MSG_MOD(info->extack, "Linecard already provisioned"); + return -EBUSY; + } + + if (!info->attrs[DEVLINK_ATTR_LINECARD_TYPE]) { + NL_SET_ERR_MSG_MOD(info->extack, "Provision type not provided"); + return -EINVAL; + } + + type = nla_data(info->attrs[DEVLINK_ATTR_LINECARD_TYPE]); + for (i = 0; i < linecard->ops->supported_types_count; i++) { + if (!strcmp(linecard->ops->supported_types[i], type)) { + linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING; + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); + return linecard->ops->provision(linecard, + linecard->priv, i, + info->extack); + } + } + NL_SET_ERR_MSG_MOD(info->extack, "Unsupported provision type provided"); + return -EINVAL; +} + +static int devlink_nl_cmd_linecard_unprovision_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct devlink_linecard *linecard = info->user_ptr[1]; + + if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) { + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being provisioned"); + return -EBUSY; + } + if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) { + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is currently being unprovisioned"); + return -EBUSY; + } + if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONED) { + NL_SET_ERR_MSG_MOD(info->extack, "Linecard is not provisioned"); + return -EOPNOTSUPP; + } + linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONING; + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); + return linecard->ops->unprovision(linecard, linecard->priv, + info->extack); +} + static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink, struct devlink_sb *devlink_sb, enum devlink_command cmd, u32 portid, @@ -7759,6 +7839,7 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { DEVLINK_RELOAD_ACTION_MAX), [DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(DEVLINK_RELOAD_LIMITS_VALID_MASK), [DEVLINK_ATTR_LINECARD_INDEX] = { .type = NLA_U32 }, + [DEVLINK_ATTR_LINECARD_TYPE] = { .type = NLA_NUL_STRING }, }; static const struct genl_small_ops devlink_nl_ops[] = { @@ -7806,6 +7887,20 @@ static const struct genl_small_ops devlink_nl_ops[] = { .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD, /* can be retrieved by unprivileged users */ }, + { + .cmd = DEVLINK_CMD_LINECARD_PROVISION, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .doit = devlink_nl_cmd_linecard_provision_doit, + .flags = GENL_ADMIN_PERM, + .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD, + }, + { + .cmd = DEVLINK_CMD_LINECARD_UNPROVISION, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .doit = devlink_nl_cmd_linecard_unprovision_doit, + .flags = GENL_ADMIN_PERM, + .internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD, + }, { .cmd = DEVLINK_CMD_SB_GET, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, @@ -8613,11 +8708,17 @@ static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port, * Create devlink linecard instance with provided linecard index. * Caller can use any indexing, even hw-related one. */ -struct devlink_linecard *devlink_linecard_create(struct devlink *devlink, - unsigned int linecard_index) +struct devlink_linecard * +devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index, + const struct devlink_linecard_ops *ops, void *priv) { struct devlink_linecard *linecard; + if (WARN_ON(!ops || !ops->supported_types || + !ops->supported_types_count || + !ops->provision || !ops->unprovision)) + return ERR_PTR(-EINVAL); + mutex_lock(&devlink->lock); if (devlink_linecard_index_exists(devlink, linecard_index)) { mutex_unlock(&devlink->lock); @@ -8630,6 +8731,9 @@ struct devlink_linecard *devlink_linecard_create(struct devlink *devlink, linecard->devlink = devlink; linecard->index = linecard_index; + linecard->ops = ops; + linecard->priv = priv; + linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED; list_add_tail(&linecard->list, &devlink->linecard_list); mutex_unlock(&devlink->lock); devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); @@ -8653,6 +8757,39 @@ void devlink_linecard_destroy(struct devlink_linecard *linecard) } EXPORT_SYMBOL_GPL(devlink_linecard_create); +/** + * devlink_linecard_provision_set - Set provisioning on linecard + * + * @devlink_linecard: devlink linecard + * @type_index: index of the linecard type (in array of types in ops) + */ +void devlink_linecard_provision_set(struct devlink_linecard *linecard, + u32 type_index) +{ + WARN_ON(type_index >= linecard->ops->supported_types_count); + mutex_lock(&linecard->devlink->lock); + linecard->state = DEVLINK_LINECARD_STATE_PROVISIONED; + linecard->provisioned_type = linecard->ops->supported_types[type_index]; + mutex_unlock(&linecard->devlink->lock); + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); +} +EXPORT_SYMBOL_GPL(devlink_linecard_provision_set); + +/** + * devlink_linecard_provision_clear - Clear provisioning on linecard + * + * @devlink_linecard: devlink linecard + */ +void devlink_linecard_provision_clear(struct devlink_linecard *linecard) +{ + mutex_lock(&linecard->devlink->lock); + linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED; + linecard->provisioned_type = NULL; + mutex_unlock(&linecard->devlink->lock); + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); +} +EXPORT_SYMBOL_GPL(devlink_linecard_provision_clear); + int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, u32 size, u16 ingress_pools_count, u16 egress_pools_count, u16 ingress_tc_count,