--- net/core/skbuff.c.orig      2020-11-21 11:01:44.569912940 +0800
+++ net/core/skbuff.c   2020-11-21 11:14:20.009912940 +0800
@@ -607,6 +607,17 @@ fastpath:
        kmem_cache_free(skbuff_fclone_cache, fclones);
}

+static void __save_stack_trace(unsigned long *trace)
+{
+       struct stack_trace stack_trace;
+
+       stack_trace.max_entries = 5;
+       stack_trace.nr_entries = 0;
+       stack_trace.entries = trace;
+       stack_trace.skip = 1;
+       save_stack_trace(&stack_trace);
+}
+
void skb_release_head_state(struct sk_buff *skb)
{
        skb_dst_drop(skb);
@@ -615,6 +626,7 @@ void skb_release_head_state(struct sk_bu
                WARN_ON(in_irq());
                skb->destructor(skb);
        }
+       __save_stack_trace((unsigned long *)skb->cb);
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        nf_conntrack_put(skb_nfct(skb));
#endif

cpu0
--------------------------------------------------
net_rx_action
  -->mlx5e_napi_poll
    -->mlx5e_poll_rx_cq
      -->mlx5_handle_rx_cqe
        -->napi_gro_receive
          -->netif_receive_skb_internal
            -->__netif_receive_skb
              -->__netif_receive_skb_one_core
                -->ip_rcv
                  -->ip_rcv_finish
                    -->ip_local_deliver
                      -->ip_local_deliver_finish
                        -->tcp_v4_rcv
                          -->tcp_v4_do_rcv
                            -->tcp_rcv_state_process
                              -->tcp_data_queue
                                -->tcp_fin
                                  -->tcp_time_wait
                                    -->tcp_done
                                      -->inet_csk_destroy_sock
                                        -->sk_stream_kill_queues
                                          // 释放报文
                                          -->__skb_queue_purge(&sk_receive_queue)



cpu1
--------------------------------------------------
net_rx_action
  -->mlx5e_napi_poll
    -->mlx5e_poll_tx_cq
      -->napi_consume_skb
        -->skb_release_all
          -->skb_release_head_state
            -->skb->destructor
              -->__sock_wfree
                -->if(refcount_sub_and_test(skb->truesize, &sk_sk_wmem_alloc))
                  -->__sk_free
                    -->sk_destruct
                      -->inet_sock_destruct
                        -->__skb_queue_purge(&sk->sk_receive_queue)
                        // cpu0在处理释放ack报文，sk_receive_queue在cpu0已经被清空了，同时cpu1从同一个sock获取数据到receive队列，此时再次释放报文而发生线程同步问题
                        // 问题原因cpu0释放内存后，未刷入内存导致cpu1上获取到了之前的数据

void __sock_wfree(struct sk_buff *skb)
{
        struct sock *sk = skb->sk;

        if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) {
                smp_rmb(); // 添加内存屏障验证问题解决
                __sk_free(sk);
        }
}

commit 47b8f3ab9c49daa824af848f9e02889662d8638f
Author: Elena Reshetova <elena.reshetova@intel.com>
Date:   Wed Jan 30 13:18:51 2019 +0200

    refcount_t: Add ACQUIRE ordering on success for dec(sub)_and_test() variants

    This adds an smp_acquire__after_ctrl_dep() barrier on successful
    decrease of refcounter value from 1 to 0 for refcount_dec(sub)_and_test
    variants and therefore gives stronger memory ordering guarantees than
    prior versions of these functions.

    Co-developed-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    Signed-off-by: Elena Reshetova <elena.reshetova@intel.com>
    Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    Reviewed-by: Andrea Parri <andrea.parri@amarulasolutions.com>
    Cc: Andrew Morton <akpm@linux-foundation.org>
    Cc: Linus Torvalds <torvalds@linux-foundation.org>
    Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
    Cc: Peter Zijlstra <peterz@infradead.org>
    Cc: Thomas Gleixner <tglx@linutronix.de>
    Cc: Will Deacon <will.deacon@arm.com>
    Cc: dvyukov@google.com
    Cc: keescook@chromium.org
    Cc: stern@rowland.harvard.edu
    Link: https://lkml.kernel.org/r/1548847131-27854-2-git-send-email-elena.reshetova@intel.com
    Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/lib/refcount.c b/lib/refcount.c
index ebcf8cd49e05..6e904af0fb3e 100644
--- a/lib/refcount.c
+++ b/lib/refcount.c
@@ -33,6 +33,9 @@
  * Note that the allocator is responsible for ordering things between free()
  * and alloc().
  *
+ * The decrements dec_and_test() and sub_and_test() also provide acquire
+ * ordering on success.
+ *
  */

 #include <linux/mutex.h>
@@ -164,8 +167,8 @@ EXPORT_SYMBOL(refcount_inc_checked);
  * at UINT_MAX.
  *
  * Provides release memory ordering, such that prior loads and stores are done
- * before, and provides a control dependency such that free() must come after.
- * See the comment on top.
+ * before, and provides an acquire ordering on success such that free()
+ * must come after.
  *
  * Use of this function is not recommended for the normal reference counting
  * use case in which references are taken and released one at a time.  In these
@@ -190,7 +193,12 @@ bool refcount_sub_and_test_checked(unsigned int i, refcount_t *r)

        } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));

-       return !new;
+       if (!new) {
+               smp_acquire__after_ctrl_dep();
+               return true;
+       }
+       return false;
+
}
EXPORT_SYMBOL(refcount_sub_and_test_checked);