aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorPauli Nieminen <suokkos@gmail.com>2010-04-01 08:45:01 -0400
committerDave Airlie <airlied@redhat.com>2010-04-05 21:36:06 -0400
commit4f64625412be120cef9e9b97e88c406ec2c78027 (patch)
treed26eb670482e2b3adfed309044294c2edb529942 /arch
parentbf62acdef89cb5b294668a6a747f7411dfe2ea7d (diff)
arch/x86: Add array variants for setting memory to wc caching.
Setting single memory pages at a time to wc takes a lot time in cache flush. To reduce number of cache flush set_pages_array_wc and set_memory_array_wc can be used to set multiple pages to WC with single cache flush. This improves allocation performance for wc cached pages in drm/ttm. CC: Suresh Siddha <suresh.b.siddha@intel.com> CC: Venkatesh Pallipadi <venkatesh.pallipadi@gmail.com> Signed-off-by: Pauli Nieminen <suokkos@gmail.com> Signed-off-by: Dave Airlie <airlied@redhat.com>
Diffstat (limited to 'arch')
-rw-r--r--arch/x86/include/asm/cacheflush.h2
-rw-r--r--arch/x86/mm/pageattr.c53
2 files changed, 47 insertions, 8 deletions
diff --git a/arch/x86/include/asm/cacheflush.h b/arch/x86/include/asm/cacheflush.h
index 634c40a739a..d92d63a6286 100644
--- a/arch/x86/include/asm/cacheflush.h
+++ b/arch/x86/include/asm/cacheflush.h
@@ -139,9 +139,11 @@ int set_memory_np(unsigned long addr, int numpages);
139int set_memory_4k(unsigned long addr, int numpages); 139int set_memory_4k(unsigned long addr, int numpages);
140 140
141int set_memory_array_uc(unsigned long *addr, int addrinarray); 141int set_memory_array_uc(unsigned long *addr, int addrinarray);
142int set_memory_array_wc(unsigned long *addr, int addrinarray);
142int set_memory_array_wb(unsigned long *addr, int addrinarray); 143int set_memory_array_wb(unsigned long *addr, int addrinarray);
143 144
144int set_pages_array_uc(struct page **pages, int addrinarray); 145int set_pages_array_uc(struct page **pages, int addrinarray);
146int set_pages_array_wc(struct page **pages, int addrinarray);
145int set_pages_array_wb(struct page **pages, int addrinarray); 147int set_pages_array_wb(struct page **pages, int addrinarray);
146 148
147/* 149/*
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index cf07c26d9a4..0c98a7583bf 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -997,7 +997,8 @@ out_err:
997} 997}
998EXPORT_SYMBOL(set_memory_uc); 998EXPORT_SYMBOL(set_memory_uc);
999 999
1000int set_memory_array_uc(unsigned long *addr, int addrinarray) 1000int _set_memory_array(unsigned long *addr, int addrinarray,
1001 unsigned long new_type)
1001{ 1002{
1002 int i, j; 1003 int i, j;
1003 int ret; 1004 int ret;
@@ -1007,13 +1008,19 @@ int set_memory_array_uc(unsigned long *addr, int addrinarray)
1007 */ 1008 */
1008 for (i = 0; i < addrinarray; i++) { 1009 for (i = 0; i < addrinarray; i++) {
1009 ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE, 1010 ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE,
1010 _PAGE_CACHE_UC_MINUS, NULL); 1011 new_type, NULL);
1011 if (ret) 1012 if (ret)
1012 goto out_free; 1013 goto out_free;
1013 } 1014 }
1014 1015
1015 ret = change_page_attr_set(addr, addrinarray, 1016 ret = change_page_attr_set(addr, addrinarray,
1016 __pgprot(_PAGE_CACHE_UC_MINUS), 1); 1017 __pgprot(_PAGE_CACHE_UC_MINUS), 1);
1018
1019 if (!ret && new_type == _PAGE_CACHE_WC)
1020 ret = change_page_attr_set_clr(addr, addrinarray,
1021 __pgprot(_PAGE_CACHE_WC),
1022 __pgprot(_PAGE_CACHE_MASK),
1023 0, CPA_ARRAY, NULL);
1017 if (ret) 1024 if (ret)
1018 goto out_free; 1025 goto out_free;
1019 1026
@@ -1025,8 +1032,19 @@ out_free:
1025 1032
1026 return ret; 1033 return ret;
1027} 1034}
1035
1036int set_memory_array_uc(unsigned long *addr, int addrinarray)
1037{
1038 return _set_memory_array(addr, addrinarray, _PAGE_CACHE_UC_MINUS);
1039}
1028EXPORT_SYMBOL(set_memory_array_uc); 1040EXPORT_SYMBOL(set_memory_array_uc);
1029 1041
1042int set_memory_array_wc(unsigned long *addr, int addrinarray)
1043{
1044 return _set_memory_array(addr, addrinarray, _PAGE_CACHE_WC);
1045}
1046EXPORT_SYMBOL(set_memory_array_wc);
1047
1030int _set_memory_wc(unsigned long addr, int numpages) 1048int _set_memory_wc(unsigned long addr, int numpages)
1031{ 1049{
1032 int ret; 1050 int ret;
@@ -1153,26 +1171,34 @@ int set_pages_uc(struct page *page, int numpages)
1153} 1171}
1154EXPORT_SYMBOL(set_pages_uc); 1172EXPORT_SYMBOL(set_pages_uc);
1155 1173
1156int set_pages_array_uc(struct page **pages, int addrinarray) 1174static int _set_pages_array(struct page **pages, int addrinarray,
1175 unsigned long new_type)
1157{ 1176{
1158 unsigned long start; 1177 unsigned long start;
1159 unsigned long end; 1178 unsigned long end;
1160 int i; 1179 int i;
1161 int free_idx; 1180 int free_idx;
1181 int ret;
1162 1182
1163 for (i = 0; i < addrinarray; i++) { 1183 for (i = 0; i < addrinarray; i++) {
1164 if (PageHighMem(pages[i])) 1184 if (PageHighMem(pages[i]))
1165 continue; 1185 continue;
1166 start = page_to_pfn(pages[i]) << PAGE_SHIFT; 1186 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
1167 end = start + PAGE_SIZE; 1187 end = start + PAGE_SIZE;
1168 if (reserve_memtype(start, end, _PAGE_CACHE_UC_MINUS, NULL)) 1188 if (reserve_memtype(start, end, new_type, NULL))
1169 goto err_out; 1189 goto err_out;
1170 } 1190 }
1171 1191
1172 if (cpa_set_pages_array(pages, addrinarray, 1192 ret = cpa_set_pages_array(pages, addrinarray,
1173 __pgprot(_PAGE_CACHE_UC_MINUS)) == 0) { 1193 __pgprot(_PAGE_CACHE_UC_MINUS));
1174 return 0; /* Success */ 1194 if (!ret && new_type == _PAGE_CACHE_WC)
1175 } 1195 ret = change_page_attr_set_clr(NULL, addrinarray,
1196 __pgprot(_PAGE_CACHE_WC),
1197 __pgprot(_PAGE_CACHE_MASK),
1198 0, CPA_PAGES_ARRAY, pages);
1199 if (ret)
1200 goto err_out;
1201 return 0; /* Success */
1176err_out: 1202err_out:
1177 free_idx = i; 1203 free_idx = i;
1178 for (i = 0; i < free_idx; i++) { 1204 for (i = 0; i < free_idx; i++) {
@@ -1184,8 +1210,19 @@ err_out:
1184 } 1210 }
1185 return -EINVAL; 1211 return -EINVAL;
1186} 1212}
1213
1214int set_pages_array_uc(struct page **pages, int addrinarray)
1215{
1216 return _set_pages_array(pages, addrinarray, _PAGE_CACHE_UC_MINUS);
1217}
1187EXPORT_SYMBOL(set_pages_array_uc); 1218EXPORT_SYMBOL(set_pages_array_uc);
1188 1219
1220int set_pages_array_wc(struct page **pages, int addrinarray)
1221{
1222 return _set_pages_array(pages, addrinarray, _PAGE_CACHE_WC);
1223}
1224EXPORT_SYMBOL(set_pages_array_wc);
1225
1189int set_pages_wb(struct page *page, int numpages) 1226int set_pages_wb(struct page *page, int numpages)
1190{ 1227{
1191 unsigned long addr = (unsigned long)page_address(page); 1228 unsigned long addr = (unsigned long)page_address(page);