1 files changed, 88 insertions, 68 deletions

diff --git a/net/rxrpc/peer_event.c b/net/rxrpc/peer_event.c
index 0ed8b651cec2..4f9da2f51c69 100644
--- a/net/rxrpc/peer_event.c
+++ b/net/rxrpc/peer_event.c
@@ -350,97 +350,117 @@ void rxrpc_peer_add_rtt(struct rxrpc_call *call, enum rxrpc_rtt_rx_trace why,
 }
 
 /*
- * Perform keep-alive pings with VERSION packets to keep any NAT alive.
+ * Perform keep-alive pings.
  */
-void rxrpc_peer_keepalive_worker(struct work_struct *work)
+static void rxrpc_peer_keepalive_dispatch(struct rxrpc_net *rxnet,
+                                          struct list_head *collector,
+                                          time64_t base,
+                                          u8 cursor)
 {
-        struct rxrpc_net *rxnet =
-                container_of(work, struct rxrpc_net, peer_keepalive_work);
         struct rxrpc_peer *peer;
-        unsigned long delay;
-        ktime_t base, now = ktime_get_real();
-        s64 diff;
-        u8 cursor, slot;
+        const u8 mask = ARRAY_SIZE(rxnet->peer_keepalive) - 1;
+        time64_t keepalive_at;
+        int slot;
 
-        base = rxnet->peer_keepalive_base;
-        cursor = rxnet->peer_keepalive_cursor;
+        spin_lock_bh(&rxnet->peer_hash_lock);
 
-        _enter("%u,%lld", cursor, ktime_sub(now, base));
+        while (!list_empty(collector)) {
+                peer = list_entry(collector->next,
+                                  struct rxrpc_peer, keepalive_link);
 
-next_bucket:
-        diff = ktime_to_ns(ktime_sub(now, base));
-        if (diff < 0)
-                goto resched;
+                list_del_init(&peer->keepalive_link);
+                if (!rxrpc_get_peer_maybe(peer))
+                        continue;
 
-        _debug("at %u", cursor);
-        spin_lock_bh(&rxnet->peer_hash_lock);
-next_peer:
-        if (!rxnet->live) {
                 spin_unlock_bh(&rxnet->peer_hash_lock);
-                goto out;
-        }
 
-        /* Everything in the bucket at the cursor is processed this second; the
-         * bucket at cursor + 1 goes now + 1s and so on...
-         */
-        if (hlist_empty(&rxnet->peer_keepalive[cursor])) {
-                if (hlist_empty(&rxnet->peer_keepalive_new)) {
-                        spin_unlock_bh(&rxnet->peer_hash_lock);
-                        goto emptied_bucket;
+                keepalive_at = peer->last_tx_at + RXRPC_KEEPALIVE_TIME;
+                slot = keepalive_at - base;
+                _debug("%02x peer %u t=%d {%pISp}",
+                       cursor, peer->debug_id, slot, &peer->srx.transport);
+
+                if (keepalive_at <= base ||
+                    keepalive_at > base + RXRPC_KEEPALIVE_TIME) {
+                        rxrpc_send_keepalive(peer);
+                        slot = RXRPC_KEEPALIVE_TIME;
                 }
 
-                hlist_move_list(&rxnet->peer_keepalive_new,
-                                &rxnet->peer_keepalive[cursor]);
+                /* A transmission to this peer occurred since last we examined
+                 * it so put it into the appropriate future bucket.
+                 */
+                slot += cursor;
+                slot &= mask;
+                spin_lock_bh(&rxnet->peer_hash_lock);
+                list_add_tail(&peer->keepalive_link,
+                              &rxnet->peer_keepalive[slot & mask]);
+                rxrpc_put_peer(peer);
         }
 
-        peer = hlist_entry(rxnet->peer_keepalive[cursor].first,
-                           struct rxrpc_peer, keepalive_link);
-        hlist_del_init(&peer->keepalive_link);
-        if (!rxrpc_get_peer_maybe(peer))
-                goto next_peer;
-
         spin_unlock_bh(&rxnet->peer_hash_lock);
+}
 
-        _debug("peer %u {%pISp}", peer->debug_id, &peer->srx.transport);
+/*
+ * Perform keep-alive pings with VERSION packets to keep any NAT alive.
+ */
+void rxrpc_peer_keepalive_worker(struct work_struct *work)
+{
+        struct rxrpc_net *rxnet =
+                container_of(work, struct rxrpc_net, peer_keepalive_work);
+        const u8 mask = ARRAY_SIZE(rxnet->peer_keepalive) - 1;
+        time64_t base, now, delay;
+        u8 cursor, stop;
+        LIST_HEAD(collector);
 
-recalc:
-        diff = ktime_divns(ktime_sub(peer->last_tx_at, base), NSEC_PER_SEC);
-        if (diff < -30 || diff > 30)
-                goto send; /* LSW of 64-bit time probably wrapped on 32-bit */
-        diff += RXRPC_KEEPALIVE_TIME - 1;
-        if (diff < 0)
-                goto send;
+        now = ktime_get_seconds();
+        base = rxnet->peer_keepalive_base;
+        cursor = rxnet->peer_keepalive_cursor;
+        _enter("%lld,%u", base - now, cursor);
 
-        slot = (diff > RXRPC_KEEPALIVE_TIME - 1) ? RXRPC_KEEPALIVE_TIME - 1 : diff;
-        if (slot == 0)
-                goto send;
+        if (!rxnet->live)
+                return;
 
-        /* A transmission to this peer occurred since last we examined it so
-         * put it into the appropriate future bucket.
+        /* Remove to a temporary list all the peers that are currently lodged
+         * in expired buckets plus all new peers.
+         *
+         * Everything in the bucket at the cursor is processed this
+         * second; the bucket at cursor + 1 goes at now + 1s and so
+         * on...
          */
-        slot = (slot + cursor) % ARRAY_SIZE(rxnet->peer_keepalive);
         spin_lock_bh(&rxnet->peer_hash_lock);
-        hlist_add_head(&peer->keepalive_link, &rxnet->peer_keepalive[slot]);
-        rxrpc_put_peer(peer);
-        goto next_peer;
-
-send:
-        rxrpc_send_keepalive(peer);
-        now = ktime_get_real();
-        goto recalc;
+        list_splice_init(&rxnet->peer_keepalive_new, &collector);
+
+        stop = cursor + ARRAY_SIZE(rxnet->peer_keepalive);
+        while (base <= now && (s8)(cursor - stop) < 0) {
+                list_splice_tail_init(&rxnet->peer_keepalive[cursor & mask],
+                                      &collector);
+                base++;
+                cursor++;
+        }
 
-emptied_bucket:
-        cursor++;
-        if (cursor >= ARRAY_SIZE(rxnet->peer_keepalive))
-                cursor = 0;
-        base = ktime_add_ns(base, NSEC_PER_SEC);
-        goto next_bucket;
+        base = now;
+        spin_unlock_bh(&rxnet->peer_hash_lock);
 
-resched:
         rxnet->peer_keepalive_base = base;
         rxnet->peer_keepalive_cursor = cursor;
-        delay = nsecs_to_jiffies(-diff) + 1;
-        timer_reduce(&rxnet->peer_keepalive_timer, jiffies + delay);
-out:
+        rxrpc_peer_keepalive_dispatch(rxnet, &collector, base, cursor);
+        ASSERT(list_empty(&collector));
+
+        /* Schedule the timer for the next occupied timeslot. */
+        cursor = rxnet->peer_keepalive_cursor;
+        stop = cursor + RXRPC_KEEPALIVE_TIME - 1;
+        for (; (s8)(cursor - stop) < 0; cursor++) {
+                if (!list_empty(&rxnet->peer_keepalive[cursor & mask]))
+                        break;
+                base++;
+        }
+
+        now = ktime_get_seconds();
+        delay = base - now;
+        if (delay < 1)
+                delay = 1;
+        delay *= HZ;
+        if (rxnet->live)
+                timer_reduce(&rxnet->peer_keepalive_timer, jiffies + delay);
+
         _leave("");
 }