aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJack Morgenstein <jackm@dev.mellanox.co.il>2014-06-08 06:49:45 -0400
committerDavid S. Miller <davem@davemloft.net>2014-06-11 03:32:46 -0400
commit95646373c9bb8e7706e0ae3c07e741b682fc9c2c (patch)
tree446efd619ed7c5243fd264f37cbdd76e2c245492
parentf05d435bde588ac26ba75f7f6893b2a63ae71381 (diff)
net/mlx4_core: Fix SRIOV free-pool management when enforcing resource quotas
The Hypervisor driver tracks free slots and reserved slots at the global level and tracks allocated slots and guaranteed slots per VF. Guaranteed slots are treated as reserved by the driver, so the total reserved slots is the sum of all guaranteed slots over all the VFs. As VFs allocate resources, free (global) is decremented and allocated (per VF) is incremented for those resources. However, reserved (global) is never changed. This means that effectively, when a VF allocates a resource from its guaranteed pool, it is actually reducing that resource's free pool (since the global reserved count was not also reduced). The fix for this problem is the following: For each resource, as long as a VF's allocated count is <= its guaranteed number, when allocating for that VF, the reserved count (global) should be reduced by the allocation as well. When the global reserved count reaches zero, the remaining global free count is still accessible as the free pool for that resource. When the VF frees resources, the reverse happens: the global reserved count for a resource is incremented only once the VFs allocated number falls below its guaranteed number. This fix was developed by Rick Kready <kready@us.ibm.com> Reported-by: Rick Kready <kready@us.ibm.com> Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il> Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com> Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/resource_tracker.c41
1 files changed, 37 insertions, 4 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
index b6cddef24391..fcf3689c93b8 100644
--- a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
+++ b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
@@ -279,7 +279,7 @@ enum qp_transition {
279}; 279};
280 280
281/* For Debug uses */ 281/* For Debug uses */
282static const char *ResourceType(enum mlx4_resource rt) 282static const char *resource_str(enum mlx4_resource rt)
283{ 283{
284 switch (rt) { 284 switch (rt) {
285 case RES_QP: return "RES_QP"; 285 case RES_QP: return "RES_QP";
@@ -307,6 +307,7 @@ static inline int mlx4_grant_resource(struct mlx4_dev *dev, int slave,
307 &priv->mfunc.master.res_tracker.res_alloc[res_type]; 307 &priv->mfunc.master.res_tracker.res_alloc[res_type];
308 int err = -EINVAL; 308 int err = -EINVAL;
309 int allocated, free, reserved, guaranteed, from_free; 309 int allocated, free, reserved, guaranteed, from_free;
310 int from_rsvd;
310 311
311 if (slave > dev->num_vfs) 312 if (slave > dev->num_vfs)
312 return -EINVAL; 313 return -EINVAL;
@@ -321,11 +322,16 @@ static inline int mlx4_grant_resource(struct mlx4_dev *dev, int slave,
321 res_alloc->res_reserved; 322 res_alloc->res_reserved;
322 guaranteed = res_alloc->guaranteed[slave]; 323 guaranteed = res_alloc->guaranteed[slave];
323 324
324 if (allocated + count > res_alloc->quota[slave]) 325 if (allocated + count > res_alloc->quota[slave]) {
326 mlx4_warn(dev, "VF %d port %d res %s: quota exceeded, count %d alloc %d quota %d\n",
327 slave, port, resource_str(res_type), count,
328 allocated, res_alloc->quota[slave]);
325 goto out; 329 goto out;
330 }
326 331
327 if (allocated + count <= guaranteed) { 332 if (allocated + count <= guaranteed) {
328 err = 0; 333 err = 0;
334 from_rsvd = count;
329 } else { 335 } else {
330 /* portion may need to be obtained from free area */ 336 /* portion may need to be obtained from free area */
331 if (guaranteed - allocated > 0) 337 if (guaranteed - allocated > 0)
@@ -333,8 +339,14 @@ static inline int mlx4_grant_resource(struct mlx4_dev *dev, int slave,
333 else 339 else
334 from_free = count; 340 from_free = count;
335 341
336 if (free - from_free > reserved) 342 from_rsvd = count - from_free;
343
344 if (free - from_free >= reserved)
337 err = 0; 345 err = 0;
346 else
347 mlx4_warn(dev, "VF %d port %d res %s: free pool empty, free %d from_free %d rsvd %d\n",
348 slave, port, resource_str(res_type), free,
349 from_free, reserved);
338 } 350 }
339 351
340 if (!err) { 352 if (!err) {
@@ -342,9 +354,11 @@ static inline int mlx4_grant_resource(struct mlx4_dev *dev, int slave,
342 if (port > 0) { 354 if (port > 0) {
343 res_alloc->allocated[(port - 1) * (dev->num_vfs + 1) + slave] += count; 355 res_alloc->allocated[(port - 1) * (dev->num_vfs + 1) + slave] += count;
344 res_alloc->res_port_free[port - 1] -= count; 356 res_alloc->res_port_free[port - 1] -= count;
357 res_alloc->res_port_rsvd[port - 1] -= from_rsvd;
345 } else { 358 } else {
346 res_alloc->allocated[slave] += count; 359 res_alloc->allocated[slave] += count;
347 res_alloc->res_free -= count; 360 res_alloc->res_free -= count;
361 res_alloc->res_reserved -= from_rsvd;
348 } 362 }
349 } 363 }
350 364
@@ -360,17 +374,36 @@ static inline void mlx4_release_resource(struct mlx4_dev *dev, int slave,
360 struct mlx4_priv *priv = mlx4_priv(dev); 374 struct mlx4_priv *priv = mlx4_priv(dev);
361 struct resource_allocator *res_alloc = 375 struct resource_allocator *res_alloc =
362 &priv->mfunc.master.res_tracker.res_alloc[res_type]; 376 &priv->mfunc.master.res_tracker.res_alloc[res_type];
377 int allocated, guaranteed, from_rsvd;
363 378
364 if (slave > dev->num_vfs) 379 if (slave > dev->num_vfs)
365 return; 380 return;
366 381
367 spin_lock(&res_alloc->alloc_lock); 382 spin_lock(&res_alloc->alloc_lock);
383
384 allocated = (port > 0) ?
385 res_alloc->allocated[(port - 1) * (dev->num_vfs + 1) + slave] :
386 res_alloc->allocated[slave];
387 guaranteed = res_alloc->guaranteed[slave];
388
389 if (allocated - count >= guaranteed) {
390 from_rsvd = 0;
391 } else {
392 /* portion may need to be returned to reserved area */
393 if (allocated - guaranteed > 0)
394 from_rsvd = count - (allocated - guaranteed);
395 else
396 from_rsvd = count;
397 }
398
368 if (port > 0) { 399 if (port > 0) {
369 res_alloc->allocated[(port - 1) * (dev->num_vfs + 1) + slave] -= count; 400 res_alloc->allocated[(port - 1) * (dev->num_vfs + 1) + slave] -= count;
370 res_alloc->res_port_free[port - 1] += count; 401 res_alloc->res_port_free[port - 1] += count;
402 res_alloc->res_port_rsvd[port - 1] += from_rsvd;
371 } else { 403 } else {
372 res_alloc->allocated[slave] -= count; 404 res_alloc->allocated[slave] -= count;
373 res_alloc->res_free += count; 405 res_alloc->res_free += count;
406 res_alloc->res_reserved += from_rsvd;
374 } 407 }
375 408
376 spin_unlock(&res_alloc->alloc_lock); 409 spin_unlock(&res_alloc->alloc_lock);
@@ -4135,7 +4168,7 @@ static int _move_all_busy(struct mlx4_dev *dev, int slave,
4135 if (print) 4168 if (print)
4136 mlx4_dbg(dev, 4169 mlx4_dbg(dev,
4137 "%s id 0x%llx is busy\n", 4170 "%s id 0x%llx is busy\n",
4138 ResourceType(type), 4171 resource_str(type),
4139 r->res_id); 4172 r->res_id);
4140 ++busy; 4173 ++busy;
4141 } else { 4174 } else {