diff options
Diffstat (limited to 'mm/highmem.c')
-rw-r--r-- | mm/highmem.c | 281 |
1 files changed, 0 insertions, 281 deletions
diff --git a/mm/highmem.c b/mm/highmem.c index ee5519b176ee..0206e7e5018c 100644 --- a/mm/highmem.c +++ b/mm/highmem.c | |||
@@ -29,13 +29,6 @@ | |||
29 | #include <linux/blktrace_api.h> | 29 | #include <linux/blktrace_api.h> |
30 | #include <asm/tlbflush.h> | 30 | #include <asm/tlbflush.h> |
31 | 31 | ||
32 | static mempool_t *page_pool, *isa_page_pool; | ||
33 | |||
34 | static void *mempool_alloc_pages_isa(gfp_t gfp_mask, void *data) | ||
35 | { | ||
36 | return mempool_alloc_pages(gfp_mask | GFP_DMA, data); | ||
37 | } | ||
38 | |||
39 | /* | 32 | /* |
40 | * Virtual_count is not a pure "count". | 33 | * Virtual_count is not a pure "count". |
41 | * 0 means that it is not mapped, and has not been mapped | 34 | * 0 means that it is not mapped, and has not been mapped |
@@ -217,282 +210,8 @@ void fastcall kunmap_high(struct page *page) | |||
217 | } | 210 | } |
218 | 211 | ||
219 | EXPORT_SYMBOL(kunmap_high); | 212 | EXPORT_SYMBOL(kunmap_high); |
220 | |||
221 | #define POOL_SIZE 64 | ||
222 | |||
223 | static __init int init_emergency_pool(void) | ||
224 | { | ||
225 | struct sysinfo i; | ||
226 | si_meminfo(&i); | ||
227 | si_swapinfo(&i); | ||
228 | |||
229 | if (!i.totalhigh) | ||
230 | return 0; | ||
231 | |||
232 | page_pool = mempool_create_page_pool(POOL_SIZE, 0); | ||
233 | BUG_ON(!page_pool); | ||
234 | printk("highmem bounce pool size: %d pages\n", POOL_SIZE); | ||
235 | |||
236 | return 0; | ||
237 | } | ||
238 | |||
239 | __initcall(init_emergency_pool); | ||
240 | |||
241 | /* | ||
242 | * highmem version, map in to vec | ||
243 | */ | ||
244 | static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom) | ||
245 | { | ||
246 | unsigned long flags; | ||
247 | unsigned char *vto; | ||
248 | |||
249 | local_irq_save(flags); | ||
250 | vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ); | ||
251 | memcpy(vto + to->bv_offset, vfrom, to->bv_len); | ||
252 | kunmap_atomic(vto, KM_BOUNCE_READ); | ||
253 | local_irq_restore(flags); | ||
254 | } | ||
255 | |||
256 | #else /* CONFIG_HIGHMEM */ | ||
257 | |||
258 | #define bounce_copy_vec(to, vfrom) \ | ||
259 | memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len) | ||
260 | |||
261 | #endif | 213 | #endif |
262 | 214 | ||
263 | #define ISA_POOL_SIZE 16 | ||
264 | |||
265 | /* | ||
266 | * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA | ||
267 | * as the max address, so check if the pool has already been created. | ||
268 | */ | ||
269 | int init_emergency_isa_pool(void) | ||
270 | { | ||
271 | if (isa_page_pool) | ||
272 | return 0; | ||
273 | |||
274 | isa_page_pool = mempool_create(ISA_POOL_SIZE, mempool_alloc_pages_isa, | ||
275 | mempool_free_pages, (void *) 0); | ||
276 | BUG_ON(!isa_page_pool); | ||
277 | |||
278 | printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE); | ||
279 | return 0; | ||
280 | } | ||
281 | |||
282 | /* | ||
283 | * Simple bounce buffer support for highmem pages. Depending on the | ||
284 | * queue gfp mask set, *to may or may not be a highmem page. kmap it | ||
285 | * always, it will do the Right Thing | ||
286 | */ | ||
287 | static void copy_to_high_bio_irq(struct bio *to, struct bio *from) | ||
288 | { | ||
289 | unsigned char *vfrom; | ||
290 | struct bio_vec *tovec, *fromvec; | ||
291 | int i; | ||
292 | |||
293 | __bio_for_each_segment(tovec, to, i, 0) { | ||
294 | fromvec = from->bi_io_vec + i; | ||
295 | |||
296 | /* | ||
297 | * not bounced | ||
298 | */ | ||
299 | if (tovec->bv_page == fromvec->bv_page) | ||
300 | continue; | ||
301 | |||
302 | /* | ||
303 | * fromvec->bv_offset and fromvec->bv_len might have been | ||
304 | * modified by the block layer, so use the original copy, | ||
305 | * bounce_copy_vec already uses tovec->bv_len | ||
306 | */ | ||
307 | vfrom = page_address(fromvec->bv_page) + tovec->bv_offset; | ||
308 | |||
309 | flush_dcache_page(tovec->bv_page); | ||
310 | bounce_copy_vec(tovec, vfrom); | ||
311 | } | ||
312 | } | ||
313 | |||
314 | static void bounce_end_io(struct bio *bio, mempool_t *pool, int err) | ||
315 | { | ||
316 | struct bio *bio_orig = bio->bi_private; | ||
317 | struct bio_vec *bvec, *org_vec; | ||
318 | int i; | ||
319 | |||
320 | if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags)) | ||
321 | set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags); | ||
322 | |||
323 | /* | ||
324 | * free up bounce indirect pages used | ||
325 | */ | ||
326 | __bio_for_each_segment(bvec, bio, i, 0) { | ||
327 | org_vec = bio_orig->bi_io_vec + i; | ||
328 | if (bvec->bv_page == org_vec->bv_page) | ||
329 | continue; | ||
330 | |||
331 | dec_zone_page_state(bvec->bv_page, NR_BOUNCE); | ||
332 | mempool_free(bvec->bv_page, pool); | ||
333 | } | ||
334 | |||
335 | bio_endio(bio_orig, bio_orig->bi_size, err); | ||
336 | bio_put(bio); | ||
337 | } | ||
338 | |||
339 | static int bounce_end_io_write(struct bio *bio, unsigned int bytes_done, int err) | ||
340 | { | ||
341 | if (bio->bi_size) | ||
342 | return 1; | ||
343 | |||
344 | bounce_end_io(bio, page_pool, err); | ||
345 | return 0; | ||
346 | } | ||
347 | |||
348 | static int bounce_end_io_write_isa(struct bio *bio, unsigned int bytes_done, int err) | ||
349 | { | ||
350 | if (bio->bi_size) | ||
351 | return 1; | ||
352 | |||
353 | bounce_end_io(bio, isa_page_pool, err); | ||
354 | return 0; | ||
355 | } | ||
356 | |||
357 | static void __bounce_end_io_read(struct bio *bio, mempool_t *pool, int err) | ||
358 | { | ||
359 | struct bio *bio_orig = bio->bi_private; | ||
360 | |||
361 | if (test_bit(BIO_UPTODATE, &bio->bi_flags)) | ||
362 | copy_to_high_bio_irq(bio_orig, bio); | ||
363 | |||
364 | bounce_end_io(bio, pool, err); | ||
365 | } | ||
366 | |||
367 | static int bounce_end_io_read(struct bio *bio, unsigned int bytes_done, int err) | ||
368 | { | ||
369 | if (bio->bi_size) | ||
370 | return 1; | ||
371 | |||
372 | __bounce_end_io_read(bio, page_pool, err); | ||
373 | return 0; | ||
374 | } | ||
375 | |||
376 | static int bounce_end_io_read_isa(struct bio *bio, unsigned int bytes_done, int err) | ||
377 | { | ||
378 | if (bio->bi_size) | ||
379 | return 1; | ||
380 | |||
381 | __bounce_end_io_read(bio, isa_page_pool, err); | ||
382 | return 0; | ||
383 | } | ||
384 | |||
385 | static void __blk_queue_bounce(request_queue_t *q, struct bio **bio_orig, | ||
386 | mempool_t *pool) | ||
387 | { | ||
388 | struct page *page; | ||
389 | struct bio *bio = NULL; | ||
390 | int i, rw = bio_data_dir(*bio_orig); | ||
391 | struct bio_vec *to, *from; | ||
392 | |||
393 | bio_for_each_segment(from, *bio_orig, i) { | ||
394 | page = from->bv_page; | ||
395 | |||
396 | /* | ||
397 | * is destination page below bounce pfn? | ||
398 | */ | ||
399 | if (page_to_pfn(page) < q->bounce_pfn) | ||
400 | continue; | ||
401 | |||
402 | /* | ||
403 | * irk, bounce it | ||
404 | */ | ||
405 | if (!bio) | ||
406 | bio = bio_alloc(GFP_NOIO, (*bio_orig)->bi_vcnt); | ||
407 | |||
408 | to = bio->bi_io_vec + i; | ||
409 | |||
410 | to->bv_page = mempool_alloc(pool, q->bounce_gfp); | ||
411 | to->bv_len = from->bv_len; | ||
412 | to->bv_offset = from->bv_offset; | ||
413 | inc_zone_page_state(to->bv_page, NR_BOUNCE); | ||
414 | |||
415 | if (rw == WRITE) { | ||
416 | char *vto, *vfrom; | ||
417 | |||
418 | flush_dcache_page(from->bv_page); | ||
419 | vto = page_address(to->bv_page) + to->bv_offset; | ||
420 | vfrom = kmap(from->bv_page) + from->bv_offset; | ||
421 | memcpy(vto, vfrom, to->bv_len); | ||
422 | kunmap(from->bv_page); | ||
423 | } | ||
424 | } | ||
425 | |||
426 | /* | ||
427 | * no pages bounced | ||
428 | */ | ||
429 | if (!bio) | ||
430 | return; | ||
431 | |||
432 | /* | ||
433 | * at least one page was bounced, fill in possible non-highmem | ||
434 | * pages | ||
435 | */ | ||
436 | __bio_for_each_segment(from, *bio_orig, i, 0) { | ||
437 | to = bio_iovec_idx(bio, i); | ||
438 | if (!to->bv_page) { | ||
439 | to->bv_page = from->bv_page; | ||
440 | to->bv_len = from->bv_len; | ||
441 | to->bv_offset = from->bv_offset; | ||
442 | } | ||
443 | } | ||
444 | |||
445 | bio->bi_bdev = (*bio_orig)->bi_bdev; | ||
446 | bio->bi_flags |= (1 << BIO_BOUNCED); | ||
447 | bio->bi_sector = (*bio_orig)->bi_sector; | ||
448 | bio->bi_rw = (*bio_orig)->bi_rw; | ||
449 | |||
450 | bio->bi_vcnt = (*bio_orig)->bi_vcnt; | ||
451 | bio->bi_idx = (*bio_orig)->bi_idx; | ||
452 | bio->bi_size = (*bio_orig)->bi_size; | ||
453 | |||
454 | if (pool == page_pool) { | ||
455 | bio->bi_end_io = bounce_end_io_write; | ||
456 | if (rw == READ) | ||
457 | bio->bi_end_io = bounce_end_io_read; | ||
458 | } else { | ||
459 | bio->bi_end_io = bounce_end_io_write_isa; | ||
460 | if (rw == READ) | ||
461 | bio->bi_end_io = bounce_end_io_read_isa; | ||
462 | } | ||
463 | |||
464 | bio->bi_private = *bio_orig; | ||
465 | *bio_orig = bio; | ||
466 | } | ||
467 | |||
468 | void blk_queue_bounce(request_queue_t *q, struct bio **bio_orig) | ||
469 | { | ||
470 | mempool_t *pool; | ||
471 | |||
472 | /* | ||
473 | * for non-isa bounce case, just check if the bounce pfn is equal | ||
474 | * to or bigger than the highest pfn in the system -- in that case, | ||
475 | * don't waste time iterating over bio segments | ||
476 | */ | ||
477 | if (!(q->bounce_gfp & GFP_DMA)) { | ||
478 | if (q->bounce_pfn >= blk_max_pfn) | ||
479 | return; | ||
480 | pool = page_pool; | ||
481 | } else { | ||
482 | BUG_ON(!isa_page_pool); | ||
483 | pool = isa_page_pool; | ||
484 | } | ||
485 | |||
486 | blk_add_trace_bio(q, *bio_orig, BLK_TA_BOUNCE); | ||
487 | |||
488 | /* | ||
489 | * slow path | ||
490 | */ | ||
491 | __blk_queue_bounce(q, bio_orig, pool); | ||
492 | } | ||
493 | |||
494 | EXPORT_SYMBOL(blk_queue_bounce); | ||
495 | |||
496 | #if defined(HASHED_PAGE_VIRTUAL) | 215 | #if defined(HASHED_PAGE_VIRTUAL) |
497 | 216 | ||
498 | #define PA_HASH_ORDER 7 | 217 | #define PA_HASH_ORDER 7 |