1 files changed, 136 insertions, 0 deletions
diff --git a/include/asm-i386/mutex.h b/include/asm-i386/mutex.h
new file mode 100644
index 000000000000..9b2199e829f3
--- /dev/null
+++ b/include/asm-i386/mutex.h
@@ -0,0 +1,136 @@
+/*
+ * Assembly implementation of the mutex fastpath, based on atomic
+ * decrement/increment.
+ *
+ * started by Ingo Molnar:
+ *
+ *  Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
+ */
+#ifndef _ASM_MUTEX_H
+#define _ASM_MUTEX_H
+/**
+ *  __mutex_fastpath_lock - try to take the lock by moving the count
+ *                          from 1 to a 0 value
+ *  @count: pointer of type atomic_t
+ *  @fn: function to call if the original value was not 1
+ *
+ * Change the count from 1 to a value lower than 1, and call <fn> if it
+ * wasn't 1 originally. This function MUST leave the value lower than 1
+ * even when the "1" assertion wasn't true.
+ */
+#define __mutex_fastpath_lock(count, fail_fn)                           \
+do {                                                                    \
+        unsigned int dummy;                                             \
+                                                                        \
+        typecheck(atomic_t *, count);                                   \
+        typecheck_fn(fastcall void (*)(atomic_t *), fail_fn);           \
+                                                                        \
+        __asm__ __volatile__(                                           \
+                LOCK    "   decl (%%eax)        \n"                     \
+                        "   js 2f               \n"                     \
+                        "1:                     \n"                     \
+                                                                        \
+                LOCK_SECTION_START("")                                  \
+                        "2: call "#fail_fn"     \n"                     \
+                        "   jmp 1b              \n"                     \
+                LOCK_SECTION_END                                        \
+                                                                        \
+                :"=a" (dummy)                                           \
+                : "a" (count)                                           \
+                : "memory", "ecx", "edx");                              \
+} while (0)
+/**
+ *  __mutex_fastpath_lock_retval - try to take the lock by moving the count
+ *                                 from 1 to a 0 value
+ *  @count: pointer of type atomic_t
+ *  @fail_fn: function to call if the original value was not 1
+ *
+ * Change the count from 1 to a value lower than 1, and call <fail_fn> if it
+ * wasn't 1 originally. This function returns 0 if the fastpath succeeds,
+ * or anything the slow path function returns
+ */
+static inline int
+__mutex_fastpath_lock_retval(atomic_t *count,
+                             int fastcall (*fail_fn)(atomic_t *))
+{
+        if (unlikely(atomic_dec_return(count) < 0))
+                return fail_fn(count);
+        else
+                return 0;
+}
+/**
+ *  __mutex_fastpath_unlock - try to promote the mutex from 0 to 1
+ *  @count: pointer of type atomic_t
+ *  @fail_fn: function to call if the original value was not 0
+ *
+ * try to promote the mutex from 0 to 1. if it wasn't 0, call <fail_fn>.
+ * In the failure case, this function is allowed to either set the value
+ * to 1, or to set it to a value lower than 1.
+ *
+ * If the implementation sets it to a value of lower than 1, the
+ * __mutex_slowpath_needs_to_unlock() macro needs to return 1, it needs
+ * to return 0 otherwise.
+ */
+#define __mutex_fastpath_unlock(count, fail_fn)                         \
+do {                                                                    \
+        unsigned int dummy;                                             \
+                                                                        \
+        typecheck(atomic_t *, count);                                   \
+        typecheck_fn(fastcall void (*)(atomic_t *), fail_fn);           \
+                                                                        \
+        __asm__ __volatile__(                                           \
+                LOCK    "   incl (%%eax)        \n"                     \
+                        "   jle 2f              \n"                     \
+                        "1:                     \n"                     \
+                                                                        \
+                LOCK_SECTION_START("")                                  \
+                        "2: call "#fail_fn"     \n"                     \
+                        "   jmp 1b              \n"                     \
+                LOCK_SECTION_END                                        \
+                                                                        \
+                :"=a" (dummy)                                           \
+                : "a" (count)                                           \
+                : "memory", "ecx", "edx");                              \
+} while (0)
+#define __mutex_slowpath_needs_to_unlock()      1
+/**
+ * __mutex_fastpath_trylock - try to acquire the mutex, without waiting
+ *
+ *  @count: pointer of type atomic_t
+ *  @fail_fn: fallback function
+ *
+ * Change the count from 1 to a value lower than 1, and return 0 (failure)
+ * if it wasn't 1 originally, or return 1 (success) otherwise. This function
+ * MUST leave the value lower than 1 even when the "1" assertion wasn't true.
+ * Additionally, if the value was < 0 originally, this function must not leave
+ * it to 0 on failure.
+ */
+static inline int
+__mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))
+{
+        /*
+         * We have two variants here. The cmpxchg based one is the best one
+         * because it never induce a false contention state.  It is included
+         * here because architectures using the inc/dec algorithms over the
+         * xchg ones are much more likely to support cmpxchg natively.
+         *
+         * If not we fall back to the spinlock based variant - that is
+         * just as efficient (and simpler) as a 'destructive' probing of
+         * the mutex state would be.
+         */
+#ifdef __HAVE_ARCH_CMPXCHG
+        if (likely(atomic_cmpxchg(count, 1, 0) == 1))
+                return 1;
+        return 0;
+#else
+        return fail_fn(count);
+#endif
+}
+#endif

diff --git a/include/asm-i386/mutex.h b/include/asm-i386/mutex.h new file mode 100644 index 000000000000..9b2199e829f3 --- /dev/null +++ b/include/asm-i386/mutex.h
@@ -0,0 +1,136 @@
	1	/*
	2	* Assembly implementation of the mutex fastpath, based on atomic
	3	* decrement/increment.
	4	*
	5	* started by Ingo Molnar:
	6	*
	7	* Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
	8	*/
	9	#ifndef _ASM_MUTEX_H
	10	#define _ASM_MUTEX_H
	11
	12	/**
	13	* __mutex_fastpath_lock - try to take the lock by moving the count
	14	* from 1 to a 0 value
	15	* @count: pointer of type atomic_t
	16	* @fn: function to call if the original value was not 1
	17	*
	18	* Change the count from 1 to a value lower than 1, and call <fn> if it
	19	* wasn't 1 originally. This function MUST leave the value lower than 1
	20	* even when the "1" assertion wasn't true.
	21	*/
	22	#define __mutex_fastpath_lock(count, fail_fn) \
	23	do { \
	24	unsigned int dummy; \
	25	\
	26	typecheck(atomic_t *, count); \
	27	typecheck_fn(fastcall void ()(atomic_t ), fail_fn); \
	28	\
	29	__asm__ __volatile__( \
	30	LOCK " decl (%%eax) \n" \
	31	" js 2f \n" \
	32	"1: \n" \
	33	\
	34	LOCK_SECTION_START("") \
	35	"2: call "#fail_fn" \n" \
	36	" jmp 1b \n" \
	37	LOCK_SECTION_END \
	38	\
	39	:"=a" (dummy) \
	40	: "a" (count) \
	41	: "memory", "ecx", "edx"); \
	42	} while (0)
	43
	44
	45	/**
	46	* __mutex_fastpath_lock_retval - try to take the lock by moving the count
	47	* from 1 to a 0 value
	48	* @count: pointer of type atomic_t
	49	* @fail_fn: function to call if the original value was not 1
	50	*
	51	* Change the count from 1 to a value lower than 1, and call <fail_fn> if it
	52	* wasn't 1 originally. This function returns 0 if the fastpath succeeds,
	53	* or anything the slow path function returns
	54	*/
	55	static inline int
	56	__mutex_fastpath_lock_retval(atomic_t *count,
	57	int fastcall (fail_fn)(atomic_t ))
	58	{
	59	if (unlikely(atomic_dec_return(count) < 0))
	60	return fail_fn(count);
	61	else
	62	return 0;
	63	}
	64
	65	/**
	66	* __mutex_fastpath_unlock - try to promote the mutex from 0 to 1
	67	* @count: pointer of type atomic_t
	68	* @fail_fn: function to call if the original value was not 0
	69	*
	70	* try to promote the mutex from 0 to 1. if it wasn't 0, call <fail_fn>.
	71	* In the failure case, this function is allowed to either set the value
	72	* to 1, or to set it to a value lower than 1.
	73	*
	74	* If the implementation sets it to a value of lower than 1, the
	75	* __mutex_slowpath_needs_to_unlock() macro needs to return 1, it needs
	76	* to return 0 otherwise.
	77	*/
	78	#define __mutex_fastpath_unlock(count, fail_fn) \
	79	do { \
	80	unsigned int dummy; \
	81	\
	82	typecheck(atomic_t *, count); \
	83	typecheck_fn(fastcall void ()(atomic_t ), fail_fn); \
	84	\
	85	__asm__ __volatile__( \
	86	LOCK " incl (%%eax) \n" \
	87	" jle 2f \n" \
	88	"1: \n" \
	89	\
	90	LOCK_SECTION_START("") \
	91	"2: call "#fail_fn" \n" \
	92	" jmp 1b \n" \
	93	LOCK_SECTION_END \
	94	\
	95	:"=a" (dummy) \
	96	: "a" (count) \
	97	: "memory", "ecx", "edx"); \
	98	} while (0)
	99
	100	#define __mutex_slowpath_needs_to_unlock() 1
	101
	102	/**
	103	* __mutex_fastpath_trylock - try to acquire the mutex, without waiting
	104	*
	105	* @count: pointer of type atomic_t
	106	* @fail_fn: fallback function
	107	*
	108	* Change the count from 1 to a value lower than 1, and return 0 (failure)
	109	* if it wasn't 1 originally, or return 1 (success) otherwise. This function
	110	* MUST leave the value lower than 1 even when the "1" assertion wasn't true.
	111	* Additionally, if the value was < 0 originally, this function must not leave
	112	* it to 0 on failure.
	113	*/
	114	static inline int
	115	__mutex_fastpath_trylock(atomic_t count, int (fail_fn)(atomic_t *))
	116	{
	117	/*
	118	* We have two variants here. The cmpxchg based one is the best one
	119	* because it never induce a false contention state. It is included
	120	* here because architectures using the inc/dec algorithms over the
	121	* xchg ones are much more likely to support cmpxchg natively.
	122	*
	123	* If not we fall back to the spinlock based variant - that is
	124	* just as efficient (and simpler) as a 'destructive' probing of
	125	* the mutex state would be.
	126	*/
	127	#ifdef __HAVE_ARCH_CMPXCHG
	128	if (likely(atomic_cmpxchg(count, 1, 0) == 1))
	129	return 1;
	130	return 0;
	131	#else
	132	return fail_fn(count);
	133	#endif
	134	}
	135
	136	#endif