SELinux: improve performance when AVC misses.

* We add ebitmap_for_each_positive_bit() which enables to walk on
  any positive bit on the given ebitmap, to improve its performance
  using common bit-operations defined in linux/bitops.h.
  In the previous version, this logic was implemented using a combination
  of ebitmap_for_each_bit() and ebitmap_node_get_bit(), but is was worse
  in performance aspect.
  This logic is most frequestly used to compute a new AVC entry,
  so this patch can improve SELinux performance when AVC misses are happen.
* struct ebitmap_node is redefined as an array of "unsigned long", to get
  suitable for using find_next_bit() which is fasted than iteration of
  shift and logical operation, and to maximize memory usage allocated
  from general purpose slab.
* Any ebitmap_for_each_bit() are repleced by the new implementation
  in ss/service.c and ss/mls.c. Some of related implementation are
  changed, however, there is no incompatibility with the previous
  version.
* The width of any new line are less or equal than 80-chars.

The following benchmark shows the effect of this patch, when we
access many files which have different security context one after
another. The number is more than /selinux/avc/cache_threshold, so
any access always causes AVC misses.

      selinux-2.6      selinux-2.6-ebitmap
AVG:   22.763 [s]          8.750 [s]
STD:    0.265              0.019
------------------------------------------
1st:   22.558 [s]          8.786 [s]
2nd:   22.458 [s]          8.750 [s]
3rd:   22.478 [s]          8.754 [s]
4th:   22.724 [s]          8.745 [s]
5th:   22.918 [s]          8.748 [s]
6th:   22.905 [s]          8.764 [s]
7th:   23.238 [s]          8.726 [s]
8th:   22.822 [s]          8.729 [s]

Signed-off-by: KaiGai Kohei <kaigai@ak.jp.nec.com>
Acked-by: Stephen Smalley <sds@tycho.nsa.gov>
Signed-off-by: James Morris <jmorris@namei.org>

1 files changed, 66 insertions, 21 deletions

diff --git a/security/selinux/ss/ebitmap.h b/security/selinux/ss/ebitmap.h
index 1270e34b61c1..e38a327dd703 100644
--- a/security/selinux/ss/ebitmap.h
+++ b/security/selinux/ss/ebitmap.h
@@ -16,14 +16,16 @@
 
 #include <net/netlabel.h>
 
-#define MAPTYPE u64                     /* portion of bitmap in each node */
-#define MAPSIZE (sizeof(MAPTYPE) * 8)   /* number of bits in node bitmap */
-#define MAPBIT  1ULL                    /* a bit in the node bitmap */
+#define EBITMAP_UNIT_NUMS       ((32 - sizeof(void *) - sizeof(u32))    \
+                                        / sizeof(unsigned long))
+#define EBITMAP_UNIT_SIZE       BITS_PER_LONG
+#define EBITMAP_SIZE            (EBITMAP_UNIT_NUMS * EBITMAP_UNIT_SIZE)
+#define EBITMAP_BIT             1ULL
 
 struct ebitmap_node {
-        u32 startbit;           /* starting position in the total bitmap */
-        MAPTYPE map;            /* this node's portion of the bitmap */
         struct ebitmap_node *next;
+        unsigned long maps[EBITMAP_UNIT_NUMS];
+        u32 startbit;
 };
 
 struct ebitmap {
@@ -34,11 +36,17 @@ struct ebitmap {
 #define ebitmap_length(e) ((e)->highbit)
 #define ebitmap_startbit(e) ((e)->node ? (e)->node->startbit : 0)
 
-static inline unsigned int ebitmap_start(struct ebitmap *e,
-                                         struct ebitmap_node **n)
+static inline unsigned int ebitmap_start_positive(struct ebitmap *e,
+                                                  struct ebitmap_node **n)
 {
-        *n = e->node;
-        return ebitmap_startbit(e);
+        unsigned int ofs;
+
+        for (*n = e->node; *n; *n = (*n)->next) {
+                ofs = find_first_bit((*n)->maps, EBITMAP_SIZE);
+                if (ofs < EBITMAP_SIZE)
+                        return (*n)->startbit + ofs;
+        }
+        return ebitmap_length(e);
 }
 
 static inline void ebitmap_init(struct ebitmap *e)
@@ -46,28 +54,65 @@ static inline void ebitmap_init(struct ebitmap *e)
         memset(e, 0, sizeof(*e));
 }
 
-static inline unsigned int ebitmap_next(struct ebitmap_node **n,
-                                        unsigned int bit)
+static inline unsigned int ebitmap_next_positive(struct ebitmap *e,
+                                                 struct ebitmap_node **n,
+                                                 unsigned int bit)
 {
-        if ((bit == ((*n)->startbit + MAPSIZE - 1)) &&
-            (*n)->next) {
-                *n = (*n)->next;
-                return (*n)->startbit;
-        }
+        unsigned int ofs;
+
+        ofs = find_next_bit((*n)->maps, EBITMAP_SIZE, bit - (*n)->startbit + 1);
+        if (ofs < EBITMAP_SIZE)
+                return ofs + (*n)->startbit;
 
-        return (bit+1);
+        for (*n = (*n)->next; *n; *n = (*n)->next) {
+                ofs = find_first_bit((*n)->maps, EBITMAP_SIZE);
+                if (ofs < EBITMAP_SIZE)
+                        return ofs + (*n)->startbit;
+        }
+        return ebitmap_length(e);
 }
 
-static inline int ebitmap_node_get_bit(struct ebitmap_node * n,
+#define EBITMAP_NODE_INDEX(node, bit)   \
+        (((bit) - (node)->startbit) / EBITMAP_UNIT_SIZE)
+#define EBITMAP_NODE_OFFSET(node, bit)  \
+        (((bit) - (node)->startbit) % EBITMAP_UNIT_SIZE)
+
+static inline int ebitmap_node_get_bit(struct ebitmap_node *n,
                                        unsigned int bit)
 {
-        if (n->map & (MAPBIT << (bit - n->startbit)))
+        unsigned int index = EBITMAP_NODE_INDEX(n, bit);
+        unsigned int ofs = EBITMAP_NODE_OFFSET(n, bit);
+
+        BUG_ON(index >= EBITMAP_UNIT_NUMS);
+        if ((n->maps[index] & (EBITMAP_BIT << ofs)))
                 return 1;
         return 0;
 }
 
-#define ebitmap_for_each_bit(e, n, bit) \
-        for (bit = ebitmap_start(e, &n); bit < ebitmap_length(e); bit = ebitmap_next(&n, bit)) \
+static inline void ebitmap_node_set_bit(struct ebitmap_node *n,
+                                        unsigned int bit)
+{
+        unsigned int index = EBITMAP_NODE_INDEX(n, bit);
+        unsigned int ofs = EBITMAP_NODE_OFFSET(n, bit);
+
+        BUG_ON(index >= EBITMAP_UNIT_NUMS);
+        n->maps[index] |= (EBITMAP_BIT << ofs);
+}
+
+static inline void ebitmap_node_clr_bit(struct ebitmap_node *n,
+                                        unsigned int bit)
+{
+        unsigned int index = EBITMAP_NODE_INDEX(n, bit);
+        unsigned int ofs = EBITMAP_NODE_OFFSET(n, bit);
+
+        BUG_ON(index >= EBITMAP_UNIT_NUMS);
+        n->maps[index] &= ~(EBITMAP_BIT << ofs);
+}
+
+#define ebitmap_for_each_positive_bit(e, n, bit)        \
+        for (bit = ebitmap_start_positive(e, &n);       \
+             bit < ebitmap_length(e);                   \
+             bit = ebitmap_next_positive(e, &n, bit))   \
 
 int ebitmap_cmp(struct ebitmap *e1, struct ebitmap *e2);
 int ebitmap_cpy(struct ebitmap *dst, struct ebitmap *src);