1 files changed, 111 insertions, 107 deletions
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 7876df00695f..424129eb20a4 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -1159,6 +1159,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
                  unsigned type, unsigned long length, u64 *ts)
 {
        struct buffer_page *tail_page, *head_page, *reader_page, *commit_page;
+        struct buffer_page *next_page;
        unsigned long tail, write;
        struct ring_buffer *buffer = cpu_buffer->buffer;
        struct ring_buffer_event *event;
@@ -1173,137 +1174,140 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
        tail = write - length;
        /* See if we shot pass the end of this buffer page */
-        if (write > BUF_PAGE_SIZE) {
+        if (write > BUF_PAGE_SIZE)
-                struct buffer_page *next_page = tail_page;
+                goto next_page;
-                local_irq_save(flags);
+        /* We reserved something on the buffer */
-                /*
-                 * Since the write to the buffer is still not
-                 * fully lockless, we must be careful with NMIs.
-                 * The locks in the writers are taken when a write
-                 * crosses to a new page. The locks protect against
-                 * races with the readers (this will soon be fixed
-                 * with a lockless solution).
-                 *
-                 * Because we can not protect against NMIs, and we
-                 * want to keep traces reentrant, we need to manage
-                 * what happens when we are in an NMI.
-                 *
-                 * NMIs can happen after we take the lock.
-                 * If we are in an NMI, only take the lock
-                 * if it is not already taken. Otherwise
-                 * simply fail.
-                 */
-                if (unlikely(in_nmi())) {
-                        if (!__raw_spin_trylock(&cpu_buffer->lock)) {
-                                cpu_buffer->nmi_dropped++;
-                                goto out_reset;
-                        }
-                } else
-                        __raw_spin_lock(&cpu_buffer->lock);
-                lock_taken = true;
-                rb_inc_page(cpu_buffer, &next_page);
+        if (RB_WARN_ON(cpu_buffer, write > BUF_PAGE_SIZE))
+                return NULL;
-                head_page = cpu_buffer->head_page;
+        event = __rb_page_index(tail_page, tail);
-                reader_page = cpu_buffer->reader_page;
+        rb_update_event(event, type, length);
-                /* we grabbed the lock before incrementing */
+        /* The passed in type is zero for DATA */
-                if (RB_WARN_ON(cpu_buffer, next_page == reader_page))
+        if (likely(!type))
-                        goto out_reset;
+                local_inc(&tail_page->entries);
-                /*
+        /*
-                 * If for some reason, we had an interrupt storm that made
+         * If this is a commit and the tail is zero, then update
-                 * it all the way around the buffer, bail, and warn
+         * this page's time stamp.
-                 * about it.
+         */
-                 */
+        if (!tail && rb_is_commit(cpu_buffer, event))
-                if (unlikely(next_page == commit_page)) {
+                cpu_buffer->commit_page->page->time_stamp = *ts;
-                        cpu_buffer->commit_overrun++;
-                        goto out_reset;
-                }
-                if (next_page == head_page) {
+        return event;
-                        if (!(buffer->flags & RB_FL_OVERWRITE))
-                                goto out_reset;
-                        /* tail_page has not moved yet? */
+ next_page:
-                        if (tail_page == cpu_buffer->tail_page) {
-                                /* count overflows */
-                                cpu_buffer->overrun +=
-                                        local_read(&head_page->entries);
-                                rb_inc_page(cpu_buffer, &head_page);
+        next_page = tail_page;
-                                cpu_buffer->head_page = head_page;
-                                cpu_buffer->head_page->read = 0;
-                        }
-                }
-                /*
+        local_irq_save(flags);
-                 * If the tail page is still the same as what we think
+        /*
-                 * it is, then it is up to us to update the tail
+         * Since the write to the buffer is still not
-                 * pointer.
+         * fully lockless, we must be careful with NMIs.
-                 */
+         * The locks in the writers are taken when a write
-                if (tail_page == cpu_buffer->tail_page) {
+         * crosses to a new page. The locks protect against
-                        local_set(&next_page->write, 0);
+         * races with the readers (this will soon be fixed
-                        local_set(&next_page->entries, 0);
+         * with a lockless solution).
-                        local_set(&next_page->page->commit, 0);
+         *
-                        cpu_buffer->tail_page = next_page;
+         * Because we can not protect against NMIs, and we
+         * want to keep traces reentrant, we need to manage
-                        /* reread the time stamp */
+         * what happens when we are in an NMI.
-                        *ts = ring_buffer_time_stamp(buffer, cpu_buffer->cpu);
+         *
-                        cpu_buffer->tail_page->page->time_stamp = *ts;
+         * NMIs can happen after we take the lock.
+         * If we are in an NMI, only take the lock
+         * if it is not already taken. Otherwise
+         * simply fail.
+         */
+        if (unlikely(in_nmi())) {
+                if (!__raw_spin_trylock(&cpu_buffer->lock)) {
+                        cpu_buffer->nmi_dropped++;
+                        goto out_reset;
                }
+        } else
+                __raw_spin_lock(&cpu_buffer->lock);
-                /*
+        lock_taken = true;
-                 * The actual tail page has moved forward.
-                 */
-                if (tail < BUF_PAGE_SIZE) {
-                        /* Mark the rest of the page with padding */
-                        event = __rb_page_index(tail_page, tail);
-                        rb_event_set_padding(event);
-                }
-                if (tail <= BUF_PAGE_SIZE)
+        rb_inc_page(cpu_buffer, &next_page);
-                        /* Set the write back to the previous setting */
-                        local_set(&tail_page->write, tail);
-                /*
+        head_page = cpu_buffer->head_page;
-                 * If this was a commit entry that failed,
+        reader_page = cpu_buffer->reader_page;
-                 * increment that too
-                 */
-                if (tail_page == cpu_buffer->commit_page &&
-                    tail == rb_commit_index(cpu_buffer)) {
-                        rb_set_commit_to_write(cpu_buffer);
-                }
-                __raw_spin_unlock(&cpu_buffer->lock);
+        /* we grabbed the lock before incrementing */
-                local_irq_restore(flags);
+        if (RB_WARN_ON(cpu_buffer, next_page == reader_page))
+                goto out_reset;
-                /* fail and let the caller try again */
+        /*
-                return ERR_PTR(-EAGAIN);
+         * If for some reason, we had an interrupt storm that made
+         * it all the way around the buffer, bail, and warn
+         * about it.
+         */
+        if (unlikely(next_page == commit_page)) {
+                cpu_buffer->commit_overrun++;
+                goto out_reset;
        }
-        /* We reserved something on the buffer */
+        if (next_page == head_page) {
+                if (!(buffer->flags & RB_FL_OVERWRITE))
+                        goto out_reset;
-        if (RB_WARN_ON(cpu_buffer, write > BUF_PAGE_SIZE))
+                /* tail_page has not moved yet? */
-                return NULL;
+                if (tail_page == cpu_buffer->tail_page) {
+                        /* count overflows */
+                        cpu_buffer->overrun +=
+                                local_read(&head_page->entries);
-        event = __rb_page_index(tail_page, tail);
+                        rb_inc_page(cpu_buffer, &head_page);
-        rb_update_event(event, type, length);
+                        cpu_buffer->head_page = head_page;
+                        cpu_buffer->head_page->read = 0;
+                }
+        }
-        /* The passed in type is zero for DATA */
+        /*
-        if (likely(!type))
+         * If the tail page is still the same as what we think
-                local_inc(&tail_page->entries);
+         * it is, then it is up to us to update the tail
+         * pointer.
+         */
+        if (tail_page == cpu_buffer->tail_page) {
+                local_set(&next_page->write, 0);
+                local_set(&next_page->entries, 0);
+                local_set(&next_page->page->commit, 0);
+                cpu_buffer->tail_page = next_page;
+                /* reread the time stamp */
+                *ts = ring_buffer_time_stamp(buffer, cpu_buffer->cpu);
+                cpu_buffer->tail_page->page->time_stamp = *ts;
+        }
        /*
-         * If this is a commit and the tail is zero, then update
+         * The actual tail page has moved forward.
-         * this page's time stamp.
         */
-        if (!tail && rb_is_commit(cpu_buffer, event))
+        if (tail < BUF_PAGE_SIZE) {
-                cpu_buffer->commit_page->page->time_stamp = *ts;
+                /* Mark the rest of the page with padding */
+                event = __rb_page_index(tail_page, tail);
+                rb_event_set_padding(event);
+        }
-        return event;
+        if (tail <= BUF_PAGE_SIZE)
+                /* Set the write back to the previous setting */
+                local_set(&tail_page->write, tail);
+        /*
+         * If this was a commit entry that failed,
+         * increment that too
+         */
+        if (tail_page == cpu_buffer->commit_page &&
+            tail == rb_commit_index(cpu_buffer)) {
+                rb_set_commit_to_write(cpu_buffer);
+        }
+        __raw_spin_unlock(&cpu_buffer->lock);
+        local_irq_restore(flags);
+        /* fail and let the caller try again */
+        return ERR_PTR(-EAGAIN);
 out_reset:
        /* reset write */

diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 7876df00695f..424129eb20a4 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c
@@ -1159,6 +1159,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1159	unsigned type, unsigned long length, u64 *ts)	1159	unsigned type, unsigned long length, u64 *ts)
1160	{	1160	{
1161	struct buffer_page tail_page, head_page, reader_page, commit_page;	1161	struct buffer_page tail_page, head_page, reader_page, commit_page;
		1162	struct buffer_page *next_page;
1162	unsigned long tail, write;	1163	unsigned long tail, write;
1163	struct ring_buffer *buffer = cpu_buffer->buffer;	1164	struct ring_buffer *buffer = cpu_buffer->buffer;
1164	struct ring_buffer_event *event;	1165	struct ring_buffer_event *event;
@@ -1173,137 +1174,140 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1173	tail = write - length;	1174	tail = write - length;
1174		1175
1175	/* See if we shot pass the end of this buffer page */	1176	/* See if we shot pass the end of this buffer page */
1176	if (write > BUF_PAGE_SIZE) {	1177	if (write > BUF_PAGE_SIZE)
1177	struct buffer_page *next_page = tail_page;	1178	goto next_page;
1178		1179
1179	local_irq_save(flags);	1180	/* We reserved something on the buffer */
1180	/*
1181	* Since the write to the buffer is still not
1182	* fully lockless, we must be careful with NMIs.
1183	* The locks in the writers are taken when a write
1184	* crosses to a new page. The locks protect against
1185	* races with the readers (this will soon be fixed
1186	* with a lockless solution).
1187	*
1188	* Because we can not protect against NMIs, and we
1189	* want to keep traces reentrant, we need to manage
1190	* what happens when we are in an NMI.
1191	*
1192	* NMIs can happen after we take the lock.
1193	* If we are in an NMI, only take the lock
1194	* if it is not already taken. Otherwise
1195	* simply fail.
1196	*/
1197	if (unlikely(in_nmi())) {
1198	if (!__raw_spin_trylock(&cpu_buffer->lock)) {
1199	cpu_buffer->nmi_dropped++;
1200	goto out_reset;
1201	}
1202	} else
1203	__raw_spin_lock(&cpu_buffer->lock);
1204
1205	lock_taken = true;
1206		1181
1207	rb_inc_page(cpu_buffer, &next_page);	1182	if (RB_WARN_ON(cpu_buffer, write > BUF_PAGE_SIZE))
		1183	return NULL;
1208		1184
1209	head_page = cpu_buffer->head_page;	1185	event = __rb_page_index(tail_page, tail);
1210	reader_page = cpu_buffer->reader_page;	1186	rb_update_event(event, type, length);
1211		1187
1212	/* we grabbed the lock before incrementing */	1188	/* The passed in type is zero for DATA */
1213	if (RB_WARN_ON(cpu_buffer, next_page == reader_page))	1189	if (likely(!type))
1214	goto out_reset;	1190	local_inc(&tail_page->entries);
1215		1191
1216	/*	1192	/*
1217	* If for some reason, we had an interrupt storm that made	1193	* If this is a commit and the tail is zero, then update
1218	* it all the way around the buffer, bail, and warn	1194	* this page's time stamp.
1219	* about it.	1195	*/
1220	*/	1196	if (!tail && rb_is_commit(cpu_buffer, event))
1221	if (unlikely(next_page == commit_page)) {	1197	cpu_buffer->commit_page->page->time_stamp = *ts;
1222	cpu_buffer->commit_overrun++;
1223	goto out_reset;
1224	}
1225		1198
1226	if (next_page == head_page) {	1199	return event;
1227	if (!(buffer->flags & RB_FL_OVERWRITE))
1228	goto out_reset;
1229		1200
1230	/* tail_page has not moved yet? */	1201	next_page:
1231	if (tail_page == cpu_buffer->tail_page) {
1232	/* count overflows */
1233	cpu_buffer->overrun +=
1234	local_read(&head_page->entries);
1235		1202
1236	rb_inc_page(cpu_buffer, &head_page);	1203	next_page = tail_page;
1237	cpu_buffer->head_page = head_page;
1238	cpu_buffer->head_page->read = 0;
1239	}
1240	}
1241		1204
1242	/*	1205	local_irq_save(flags);
1243	* If the tail page is still the same as what we think	1206	/*
1244	* it is, then it is up to us to update the tail	1207	* Since the write to the buffer is still not
1245	* pointer.	1208	* fully lockless, we must be careful with NMIs.
1246	*/	1209	* The locks in the writers are taken when a write
1247	if (tail_page == cpu_buffer->tail_page) {	1210	* crosses to a new page. The locks protect against
1248	local_set(&next_page->write, 0);	1211	* races with the readers (this will soon be fixed
1249	local_set(&next_page->entries, 0);	1212	* with a lockless solution).
1250	local_set(&next_page->page->commit, 0);	1213	*
1251	cpu_buffer->tail_page = next_page;	1214	* Because we can not protect against NMIs, and we
1252		1215	* want to keep traces reentrant, we need to manage
1253	/* reread the time stamp */	1216	* what happens when we are in an NMI.
1254	*ts = ring_buffer_time_stamp(buffer, cpu_buffer->cpu);	1217	*
1255	cpu_buffer->tail_page->page->time_stamp = *ts;	1218	* NMIs can happen after we take the lock.
		1219	* If we are in an NMI, only take the lock
		1220	* if it is not already taken. Otherwise
		1221	* simply fail.
		1222	*/
		1223	if (unlikely(in_nmi())) {
		1224	if (!__raw_spin_trylock(&cpu_buffer->lock)) {
		1225	cpu_buffer->nmi_dropped++;
		1226	goto out_reset;
1256	}	1227	}
		1228	} else
		1229	__raw_spin_lock(&cpu_buffer->lock);
1257		1230
1258	/*	1231	lock_taken = true;
1259	* The actual tail page has moved forward.
1260	*/
1261	if (tail < BUF_PAGE_SIZE) {
1262	/* Mark the rest of the page with padding */
1263	event = __rb_page_index(tail_page, tail);
1264	rb_event_set_padding(event);
1265	}
1266		1232
1267	if (tail <= BUF_PAGE_SIZE)	1233	rb_inc_page(cpu_buffer, &next_page);
1268	/* Set the write back to the previous setting */
1269	local_set(&tail_page->write, tail);
1270		1234
1271	/*	1235	head_page = cpu_buffer->head_page;
1272	* If this was a commit entry that failed,	1236	reader_page = cpu_buffer->reader_page;
1273	* increment that too
1274	*/
1275	if (tail_page == cpu_buffer->commit_page &&
1276	tail == rb_commit_index(cpu_buffer)) {
1277	rb_set_commit_to_write(cpu_buffer);
1278	}
1279		1237
1280	__raw_spin_unlock(&cpu_buffer->lock);	1238	/* we grabbed the lock before incrementing */
1281	local_irq_restore(flags);	1239	if (RB_WARN_ON(cpu_buffer, next_page == reader_page))
		1240	goto out_reset;
1282		1241
1283	/* fail and let the caller try again */	1242	/*
1284	return ERR_PTR(-EAGAIN);	1243	* If for some reason, we had an interrupt storm that made
		1244	* it all the way around the buffer, bail, and warn
		1245	* about it.
		1246	*/
		1247	if (unlikely(next_page == commit_page)) {
		1248	cpu_buffer->commit_overrun++;
		1249	goto out_reset;
1285	}	1250	}
1286		1251
1287	/* We reserved something on the buffer */	1252	if (next_page == head_page) {
		1253	if (!(buffer->flags & RB_FL_OVERWRITE))
		1254	goto out_reset;
1288		1255
1289	if (RB_WARN_ON(cpu_buffer, write > BUF_PAGE_SIZE))	1256	/* tail_page has not moved yet? */
1290	return NULL;	1257	if (tail_page == cpu_buffer->tail_page) {
		1258	/* count overflows */
		1259	cpu_buffer->overrun +=
		1260	local_read(&head_page->entries);
1291		1261
1292	event = __rb_page_index(tail_page, tail);	1262	rb_inc_page(cpu_buffer, &head_page);
1293	rb_update_event(event, type, length);	1263	cpu_buffer->head_page = head_page;
		1264	cpu_buffer->head_page->read = 0;
		1265	}
		1266	}
1294		1267
1295	/* The passed in type is zero for DATA */	1268	/*
1296	if (likely(!type))	1269	* If the tail page is still the same as what we think
1297	local_inc(&tail_page->entries);	1270	* it is, then it is up to us to update the tail
		1271	* pointer.
		1272	*/
		1273	if (tail_page == cpu_buffer->tail_page) {
		1274	local_set(&next_page->write, 0);
		1275	local_set(&next_page->entries, 0);
		1276	local_set(&next_page->page->commit, 0);
		1277	cpu_buffer->tail_page = next_page;
		1278
		1279	/* reread the time stamp */
		1280	*ts = ring_buffer_time_stamp(buffer, cpu_buffer->cpu);
		1281	cpu_buffer->tail_page->page->time_stamp = *ts;
		1282	}
1298		1283
1299	/*	1284	/*
1300	* If this is a commit and the tail is zero, then update	1285	* The actual tail page has moved forward.
1301	* this page's time stamp.
1302	*/	1286	*/
1303	if (!tail && rb_is_commit(cpu_buffer, event))	1287	if (tail < BUF_PAGE_SIZE) {
1304	cpu_buffer->commit_page->page->time_stamp = *ts;	1288	/* Mark the rest of the page with padding */
		1289	event = __rb_page_index(tail_page, tail);
		1290	rb_event_set_padding(event);
		1291	}
1305		1292
1306	return event;	1293	if (tail <= BUF_PAGE_SIZE)
		1294	/* Set the write back to the previous setting */
		1295	local_set(&tail_page->write, tail);
		1296
		1297	/*
		1298	* If this was a commit entry that failed,
		1299	* increment that too
		1300	*/
		1301	if (tail_page == cpu_buffer->commit_page &&
		1302	tail == rb_commit_index(cpu_buffer)) {
		1303	rb_set_commit_to_write(cpu_buffer);
		1304	}
		1305
		1306	__raw_spin_unlock(&cpu_buffer->lock);
		1307	local_irq_restore(flags);
		1308
		1309	/* fail and let the caller try again */
		1310	return ERR_PTR(-EAGAIN);
1307		1311
1308	out_reset:	1312	out_reset:
1309	/* reset write */	1313	/* reset write */