4 files changed, 46 insertions, 24 deletions

diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index cb4e43bce98a..4fabcdf1cfa7 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -262,4 +262,6 @@ static inline void perf_check_microcode(void) { }
  static inline void amd_pmu_disable_virt(void) { }
 #endif
 
+#define arch_perf_out_copy_user copy_from_user_nmi
+
 #endif /* _ASM_X86_PERF_EVENT_H */
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 3d4d84745f07..d41394a1af36 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1319,7 +1319,7 @@ static inline bool has_branch_stack(struct perf_event *event)
 extern int perf_output_begin(struct perf_output_handle *handle,
                              struct perf_event *event, unsigned int size);
 extern void perf_output_end(struct perf_output_handle *handle);
-extern void perf_output_copy(struct perf_output_handle *handle,
+extern unsigned int perf_output_copy(struct perf_output_handle *handle,
                              const void *buf, unsigned int len);
 extern int perf_swevent_get_recursion_context(void);
 extern void perf_swevent_put_recursion_context(int rctx);
diff --git a/kernel/events/internal.h b/kernel/events/internal.h
index a096c19f2c2a..7fd5408493d2 100644
--- a/kernel/events/internal.h
+++ b/kernel/events/internal.h
@@ -2,6 +2,7 @@
 #define _KERNEL_EVENTS_INTERNAL_H
 
 #include <linux/hardirq.h>
+#include <linux/uaccess.h>
 
 /* Buffer handling */
 
@@ -76,30 +77,49 @@ static inline unsigned long perf_data_size(struct ring_buffer *rb)
         return rb->nr_pages << (PAGE_SHIFT + page_order(rb));
 }
 
-static inline void
-__output_copy(struct perf_output_handle *handle,
-                   const void *buf, unsigned int len)
+#define DEFINE_OUTPUT_COPY(func_name, memcpy_func)                      \
+static inline unsigned int                                              \
+func_name(struct perf_output_handle *handle,                            \
+          const void *buf, unsigned int len)                            \
+{                                                                       \
+        unsigned long size, written;                                    \
+                                                                        \
+        do {                                                            \
+                size = min_t(unsigned long, handle->size, len);         \
+                                                                        \
+                written = memcpy_func(handle->addr, buf, size);         \
+                                                                        \
+                len -= written;                                         \
+                handle->addr += written;                                \
+                buf += written;                                         \
+                handle->size -= written;                                \
+                if (!handle->size) {                                    \
+                        struct ring_buffer *rb = handle->rb;            \
+                                                                        \
+                        handle->page++;                                 \
+                        handle->page &= rb->nr_pages - 1;               \
+                        handle->addr = rb->data_pages[handle->page];    \
+                        handle->size = PAGE_SIZE << page_order(rb);     \
+                }                                                       \
+        } while (len && written == size);                               \
+                                                                        \
+        return len;                                                     \
+}
+
+static inline int memcpy_common(void *dst, const void *src, size_t n)
 {
-        do {
-                unsigned long size = min_t(unsigned long, handle->size, len);
-
-                memcpy(handle->addr, buf, size);
-
-                len -= size;
-                handle->addr += size;
-                buf += size;
-                handle->size -= size;
-                if (!handle->size) {
-                        struct ring_buffer *rb = handle->rb;
-
-                        handle->page++;
-                        handle->page &= rb->nr_pages - 1;
-                        handle->addr = rb->data_pages[handle->page];
-                        handle->size = PAGE_SIZE << page_order(rb);
-                }
-        } while (len);
+        memcpy(dst, src, n);
+        return n;
 }
 
+DEFINE_OUTPUT_COPY(__output_copy, memcpy_common)
+
+#ifndef arch_perf_out_copy_user
+#define arch_perf_out_copy_user __copy_from_user_inatomic
+#endif
+
+DEFINE_OUTPUT_COPY(__output_copy_user, arch_perf_out_copy_user)
+
 /* Callchain handling */
 extern struct perf_callchain_entry *
 perf_callchain(struct perf_event *event, struct pt_regs *regs);
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index 6ddaba43fb7a..b4c2ad3dee7a 100644
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -182,10 +182,10 @@ out:
         return -ENOSPC;
 }
 
-void perf_output_copy(struct perf_output_handle *handle,
+unsigned int perf_output_copy(struct perf_output_handle *handle,
                       const void *buf, unsigned int len)
 {
-        __output_copy(handle, buf, len);
+        return __output_copy(handle, buf, len);
 }
 
 void perf_output_end(struct perf_output_handle *handle)