NTB: Alter MW API to support multi-ports devices

Multi-port NTB devices permit to share a memory between all accessible peers.
Memory Windows API is altered to correspondingly initialize and map memory
windows for such devices:
 ntb_mw_count(pidx); - number of inbound memory windows, which can be allocated
for shared buffer with specified peer device.
 ntb_mw_get_align(pidx, widx); - get alignment and size restriction parameters
to properly allocate inbound memory region.
 ntb_peer_mw_count(); - get number of outbound memory windows.
 ntb_peer_mw_get_addr(widx); - get mapping address of an outbound memory window

If hardware supports inbound translation configured on the local ntb port:
 ntb_mw_set_trans(pidx, widx); - set translation address of allocated inbound
memory window so a peer device could access it.
 ntb_mw_clear_trans(pidx, widx); - clear the translation address of an inbound
memory window.

If hardware supports outbound translation configured on the peer ntb port:
 ntb_peer_mw_set_trans(pidx, widx); - set translation address of a memory
window retrieved from a peer device
 ntb_peer_mw_clear_trans(pidx, widx); - clear the translation address of an
outbound memory window

Signed-off-by: Serge Semin <fancer.lancer@gmail.com>
Acked-by: Allen Hubbe <Allen.Hubbe@dell.com>
Signed-off-by: Jon Mason <jdmason@kudzu.us>

1 files changed, 163 insertions, 45 deletions

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index b2b2924f5f43..2ea83f91a236 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -5,6 +5,7 @@
  *   GPL LICENSE SUMMARY
  *
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -18,6 +19,7 @@
  *   BSD LICENSE
  *
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -201,9 +203,13 @@ static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops)
  * @link_enable:        See ntb_link_enable().
  * @link_disable:       See ntb_link_disable().
  * @mw_count:           See ntb_mw_count().
- * @mw_get_range:       See ntb_mw_get_range().
+ * @mw_get_align:       See ntb_mw_get_align().
  * @mw_set_trans:       See ntb_mw_set_trans().
  * @mw_clear_trans:     See ntb_mw_clear_trans().
+ * @peer_mw_count:      See ntb_peer_mw_count().
+ * @peer_mw_get_addr:   See ntb_peer_mw_get_addr().
+ * @peer_mw_set_trans:  See ntb_peer_mw_set_trans().
+ * @peer_mw_clear_trans:See ntb_peer_mw_clear_trans().
  * @db_is_unsafe:       See ntb_db_is_unsafe().
  * @db_valid_mask:      See ntb_db_valid_mask().
  * @db_vector_count:    See ntb_db_vector_count().
@@ -241,13 +247,20 @@ struct ntb_dev_ops {
                            enum ntb_speed max_speed, enum ntb_width max_width);
         int (*link_disable)(struct ntb_dev *ntb);
 
-        int (*mw_count)(struct ntb_dev *ntb);
-        int (*mw_get_range)(struct ntb_dev *ntb, int idx,
-                            phys_addr_t *base, resource_size_t *size,
-                        resource_size_t *align, resource_size_t *align_size);
-        int (*mw_set_trans)(struct ntb_dev *ntb, int idx,
+        int (*mw_count)(struct ntb_dev *ntb, int pidx);
+        int (*mw_get_align)(struct ntb_dev *ntb, int pidx, int widx,
+                            resource_size_t *addr_align,
+                            resource_size_t *size_align,
+                            resource_size_t *size_max);
+        int (*mw_set_trans)(struct ntb_dev *ntb, int pidx, int widx,
                             dma_addr_t addr, resource_size_t size);
-        int (*mw_clear_trans)(struct ntb_dev *ntb, int idx);
+        int (*mw_clear_trans)(struct ntb_dev *ntb, int pidx, int widx);
+        int (*peer_mw_count)(struct ntb_dev *ntb);
+        int (*peer_mw_get_addr)(struct ntb_dev *ntb, int widx,
+                                phys_addr_t *base, resource_size_t *size);
+        int (*peer_mw_set_trans)(struct ntb_dev *ntb, int pidx, int widx,
+                                 u64 addr, resource_size_t size);
+        int (*peer_mw_clear_trans)(struct ntb_dev *ntb, int pidx, int widx);
 
         int (*db_is_unsafe)(struct ntb_dev *ntb);
         u64 (*db_valid_mask)(struct ntb_dev *ntb);
@@ -295,9 +308,13 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
                 ops->link_enable                        &&
                 ops->link_disable                       &&
                 ops->mw_count                           &&
-                ops->mw_get_range                       &&
-                ops->mw_set_trans                       &&
+                ops->mw_get_align                       &&
+                (ops->mw_set_trans                      ||
+                 ops->peer_mw_set_trans)                &&
                 /* ops->mw_clear_trans                  && */
+                ops->peer_mw_count                      &&
+                ops->peer_mw_get_addr                   &&
+                /* ops->peer_mw_clear_trans             && */
 
                 /* ops->db_is_unsafe                    && */
                 ops->db_valid_mask                      &&
@@ -655,79 +672,180 @@ static inline int ntb_link_disable(struct ntb_dev *ntb)
 }
 
 /**
- * ntb_mw_count() - get the number of memory windows
+ * ntb_mw_count() - get the number of inbound memory windows, which could
+ *                  be created for a specified peer device
  * @ntb:        NTB device context.
+ * @pidx:       Port index of peer device.
  *
  * Hardware and topology may support a different number of memory windows.
+ * Moreover different peer devices can support different number of memory
+ * windows. Simply speaking this method returns the number of possible inbound
+ * memory windows to share with specified peer device.
  *
  * Return: the number of memory windows.
  */
-static inline int ntb_mw_count(struct ntb_dev *ntb)
+static inline int ntb_mw_count(struct ntb_dev *ntb, int pidx)
 {
-        return ntb->ops->mw_count(ntb);
+        return ntb->ops->mw_count(ntb, pidx);
 }
 
 /**
- * ntb_mw_get_range() - get the range of a memory window
+ * ntb_mw_get_align() - get the restriction parameters of inbound memory window
  * @ntb:        NTB device context.
- * @idx:        Memory window number.
- * @base:       OUT - the base address for mapping the memory window
- * @size:       OUT - the size for mapping the memory window
- * @align:      OUT - the base alignment for translating the memory window
- * @align_size: OUT - the size alignment for translating the memory window
- *
- * Get the range of a memory window.  NULL may be given for any output
- * parameter if the value is not needed.  The base and size may be used for
- * mapping the memory window, to access the peer memory.  The alignment and
- * size may be used for translating the memory window, for the peer to access
- * memory on the local system.
- *
- * Return: Zero on success, otherwise an error number.
+ * @pidx:       Port index of peer device.
+ * @widx:       Memory window index.
+ * @addr_align: OUT - the base alignment for translating the memory window
+ * @size_align: OUT - the size alignment for translating the memory window
+ * @size_max:   OUT - the maximum size of the memory window
+ *
+ * Get the alignments of an inbound memory window with specified index.
+ * NULL may be given for any output parameter if the value is not needed.
+ * The alignment and size parameters may be used for allocation of proper
+ * shared memory.
+ *
+ * Return: Zero on success, otherwise a negative error number.
  */
-static inline int ntb_mw_get_range(struct ntb_dev *ntb, int idx,
-                                   phys_addr_t *base, resource_size_t *size,
-                resource_size_t *align, resource_size_t *align_size)
+static inline int ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int widx,
+                                   resource_size_t *addr_align,
+                                   resource_size_t *size_align,
+                                   resource_size_t *size_max)
 {
-        return ntb->ops->mw_get_range(ntb, idx, base, size,
-                        align, align_size);
+        return ntb->ops->mw_get_align(ntb, pidx, widx, addr_align, size_align,
+                                      size_max);
 }
 
 /**
- * ntb_mw_set_trans() - set the translation of a memory window
+ * ntb_mw_set_trans() - set the translation of an inbound memory window
  * @ntb:        NTB device context.
- * @idx:        Memory window number.
- * @addr:       The dma address local memory to expose to the peer.
+ * @pidx:       Port index of peer device.
+ * @widx:       Memory window index.
+ * @addr:       The dma address of local memory to expose to the peer.
  * @size:       The size of the local memory to expose to the peer.
  *
  * Set the translation of a memory window.  The peer may access local memory
  * through the window starting at the address, up to the size.  The address
- * must be aligned to the alignment specified by ntb_mw_get_range().  The size
- * must be aligned to the size alignment specified by ntb_mw_get_range().
+ * and size must be aligned in compliance with restrictions of
+ * ntb_mw_get_align(). The region size should not exceed the size_max parameter
+ * of that method.
+ *
+ * This method may not be implemented due to the hardware specific memory
+ * windows interface.
  *
  * Return: Zero on success, otherwise an error number.
  */
-static inline int ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
+static inline int ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
                                    dma_addr_t addr, resource_size_t size)
 {
-        return ntb->ops->mw_set_trans(ntb, idx, addr, size);
+        if (!ntb->ops->mw_set_trans)
+                return 0;
+
+        return ntb->ops->mw_set_trans(ntb, pidx, widx, addr, size);
 }
 
 /**
- * ntb_mw_clear_trans() - clear the translation of a memory window
+ * ntb_mw_clear_trans() - clear the translation address of an inbound memory
+ *                        window
  * @ntb:        NTB device context.
- * @idx:        Memory window number.
+ * @pidx:       Port index of peer device.
+ * @widx:       Memory window index.
  *
- * Clear the translation of a memory window.  The peer may no longer access
- * local memory through the window.
+ * Clear the translation of an inbound memory window.  The peer may no longer
+ * access local memory through the window.
  *
  * Return: Zero on success, otherwise an error number.
  */
-static inline int ntb_mw_clear_trans(struct ntb_dev *ntb, int idx)
+static inline int ntb_mw_clear_trans(struct ntb_dev *ntb, int pidx, int widx)
 {
         if (!ntb->ops->mw_clear_trans)
-                return ntb->ops->mw_set_trans(ntb, idx, 0, 0);
+                return ntb_mw_set_trans(ntb, pidx, widx, 0, 0);
+
+        return ntb->ops->mw_clear_trans(ntb, pidx, widx);
+}
+
+/**
+ * ntb_peer_mw_count() - get the number of outbound memory windows, which could
+ *                       be mapped to access a shared memory
+ * @ntb:        NTB device context.
+ *
+ * Hardware and topology may support a different number of memory windows.
+ * This method returns the number of outbound memory windows supported by
+ * local device.
+ *
+ * Return: the number of memory windows.
+ */
+static inline int ntb_peer_mw_count(struct ntb_dev *ntb)
+{
+        return ntb->ops->peer_mw_count(ntb);
+}
+
+/**
+ * ntb_peer_mw_get_addr() - get map address of an outbound memory window
+ * @ntb:        NTB device context.
+ * @widx:       Memory window index (within ntb_peer_mw_count() return value).
+ * @base:       OUT - the base address of mapping region.
+ * @size:       OUT - the size of mapping region.
+ *
+ * Get base and size of memory region to map.  NULL may be given for any output
+ * parameter if the value is not needed.  The base and size may be used for
+ * mapping the memory window, to access the peer memory.
+ *
+ * Return: Zero on success, otherwise a negative error number.
+ */
+static inline int ntb_peer_mw_get_addr(struct ntb_dev *ntb, int widx,
+                                      phys_addr_t *base, resource_size_t *size)
+{
+        return ntb->ops->peer_mw_get_addr(ntb, widx, base, size);
+}
+
+/**
+ * ntb_peer_mw_set_trans() - set a translation address of a memory window
+ *                           retrieved from a peer device
+ * @ntb:        NTB device context.
+ * @pidx:       Port index of peer device the translation address received from.
+ * @widx:       Memory window index.
+ * @addr:       The dma address of the shared memory to access.
+ * @size:       The size of the shared memory to access.
+ *
+ * Set the translation of an outbound memory window.  The local device may
+ * access shared memory allocated by a peer device sent the address.
+ *
+ * This method may not be implemented due to the hardware specific memory
+ * windows interface, so a translation address can be only set on the side,
+ * where shared memory (inbound memory windows) is allocated.
+ *
+ * Return: Zero on success, otherwise an error number.
+ */
+static inline int ntb_peer_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
+                                        u64 addr, resource_size_t size)
+{
+        if (!ntb->ops->peer_mw_set_trans)
+                return 0;
+
+        return ntb->ops->peer_mw_set_trans(ntb, pidx, widx, addr, size);
+}
+
+/**
+ * ntb_peer_mw_clear_trans() - clear the translation address of an outbound
+ *                             memory window
+ * @ntb:        NTB device context.
+ * @pidx:       Port index of peer device.
+ * @widx:       Memory window index.
+ *
+ * Clear the translation of a outbound memory window.  The local device may no
+ * longer access a shared memory through the window.
+ *
+ * This method may not be implemented due to the hardware specific memory
+ * windows interface.
+ *
+ * Return: Zero on success, otherwise an error number.
+ */
+static inline int ntb_peer_mw_clear_trans(struct ntb_dev *ntb, int pidx,
+                                          int widx)
+{
+        if (!ntb->ops->peer_mw_clear_trans)
+                return ntb_peer_mw_set_trans(ntb, pidx, widx, 0, 0);
 
-        return ntb->ops->mw_clear_trans(ntb, idx);
+        return ntb->ops->peer_mw_clear_trans(ntb, pidx, widx);
 }
 
 /**