diff options
Diffstat (limited to 'mm/highmem.c')
| -rw-r--r-- | mm/highmem.c | 281 |
1 files changed, 0 insertions, 281 deletions
diff --git a/mm/highmem.c b/mm/highmem.c index ee5519b176ee..0206e7e5018c 100644 --- a/mm/highmem.c +++ b/mm/highmem.c | |||
| @@ -29,13 +29,6 @@ | |||
| 29 | #include <linux/blktrace_api.h> | 29 | #include <linux/blktrace_api.h> |
| 30 | #include <asm/tlbflush.h> | 30 | #include <asm/tlbflush.h> |
| 31 | 31 | ||
| 32 | static mempool_t *page_pool, *isa_page_pool; | ||
| 33 | |||
| 34 | static void *mempool_alloc_pages_isa(gfp_t gfp_mask, void *data) | ||
| 35 | { | ||
| 36 | return mempool_alloc_pages(gfp_mask | GFP_DMA, data); | ||
| 37 | } | ||
| 38 | |||
| 39 | /* | 32 | /* |
| 40 | * Virtual_count is not a pure "count". | 33 | * Virtual_count is not a pure "count". |
| 41 | * 0 means that it is not mapped, and has not been mapped | 34 | * 0 means that it is not mapped, and has not been mapped |
| @@ -217,282 +210,8 @@ void fastcall kunmap_high(struct page *page) | |||
| 217 | } | 210 | } |
| 218 | 211 | ||
| 219 | EXPORT_SYMBOL(kunmap_high); | 212 | EXPORT_SYMBOL(kunmap_high); |
| 220 | |||
| 221 | #define POOL_SIZE 64 | ||
| 222 | |||
| 223 | static __init int init_emergency_pool(void) | ||
| 224 | { | ||
| 225 | struct sysinfo i; | ||
| 226 | si_meminfo(&i); | ||
| 227 | si_swapinfo(&i); | ||
| 228 | |||
| 229 | if (!i.totalhigh) | ||
| 230 | return 0; | ||
| 231 | |||
| 232 | page_pool = mempool_create_page_pool(POOL_SIZE, 0); | ||
| 233 | BUG_ON(!page_pool); | ||
| 234 | printk("highmem bounce pool size: %d pages\n", POOL_SIZE); | ||
| 235 | |||
| 236 | return 0; | ||
| 237 | } | ||
| 238 | |||
| 239 | __initcall(init_emergency_pool); | ||
| 240 | |||
| 241 | /* | ||
| 242 | * highmem version, map in to vec | ||
| 243 | */ | ||
| 244 | static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom) | ||
| 245 | { | ||
| 246 | unsigned long flags; | ||
| 247 | unsigned char *vto; | ||
| 248 | |||
| 249 | local_irq_save(flags); | ||
| 250 | vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ); | ||
| 251 | memcpy(vto + to->bv_offset, vfrom, to->bv_len); | ||
| 252 | kunmap_atomic(vto, KM_BOUNCE_READ); | ||
| 253 | local_irq_restore(flags); | ||
| 254 | } | ||
| 255 | |||
| 256 | #else /* CONFIG_HIGHMEM */ | ||
| 257 | |||
| 258 | #define bounce_copy_vec(to, vfrom) \ | ||
| 259 | memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len) | ||
| 260 | |||
| 261 | #endif | 213 | #endif |
| 262 | 214 | ||
| 263 | #define ISA_POOL_SIZE 16 | ||
| 264 | |||
| 265 | /* | ||
| 266 | * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA | ||
| 267 | * as the max address, so check if the pool has already been created. | ||
| 268 | */ | ||
| 269 | int init_emergency_isa_pool(void) | ||
| 270 | { | ||
| 271 | if (isa_page_pool) | ||
| 272 | return 0; | ||
| 273 | |||
| 274 | isa_page_pool = mempool_create(ISA_POOL_SIZE, mempool_alloc_pages_isa, | ||
| 275 | mempool_free_pages, (void *) 0); | ||
| 276 | BUG_ON(!isa_page_pool); | ||
| 277 | |||
| 278 | printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE); | ||
| 279 | return 0; | ||
| 280 | } | ||
| 281 | |||
| 282 | /* | ||
| 283 | * Simple bounce buffer support for highmem pages. Depending on the | ||
| 284 | * queue gfp mask set, *to may or may not be a highmem page. kmap it | ||
| 285 | * always, it will do the Right Thing | ||
| 286 | */ | ||
| 287 | static void copy_to_high_bio_irq(struct bio *to, struct bio *from) | ||
| 288 | { | ||
| 289 | unsigned char *vfrom; | ||
| 290 | struct bio_vec *tovec, *fromvec; | ||
| 291 | int i; | ||
| 292 | |||
| 293 | __bio_for_each_segment(tovec, to, i, 0) { | ||
| 294 | fromvec = from->bi_io_vec + i; | ||
| 295 | |||
| 296 | /* | ||
| 297 | * not bounced | ||
| 298 | */ | ||
| 299 | if (tovec->bv_page == fromvec->bv_page) | ||
| 300 | continue; | ||
| 301 | |||
| 302 | /* | ||
| 303 | * fromvec->bv_offset and fromvec->bv_len might have been | ||
| 304 | * modified by the block layer, so use the original copy, | ||
| 305 | * bounce_copy_vec already uses tovec->bv_len | ||
| 306 | */ | ||
| 307 | vfrom = page_address(fromvec->bv_page) + tovec->bv_offset; | ||
| 308 | |||
| 309 | flush_dcache_page(tovec->bv_page); | ||
| 310 | bounce_copy_vec(tovec, vfrom); | ||
| 311 | } | ||
| 312 | } | ||
| 313 | |||
| 314 | static void bounce_end_io(struct bio *bio, mempool_t *pool, int err) | ||
| 315 | { | ||
| 316 | struct bio *bio_orig = bio->bi_private; | ||
| 317 | struct bio_vec *bvec, *org_vec; | ||
| 318 | int i; | ||
| 319 | |||
| 320 | if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags)) | ||
| 321 | set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags); | ||
| 322 | |||
| 323 | /* | ||
| 324 | * free up bounce indirect pages used | ||
| 325 | */ | ||
| 326 | __bio_for_each_segment(bvec, bio, i, 0) { | ||
| 327 | org_vec = bio_orig->bi_io_vec + i; | ||
| 328 | if (bvec->bv_page == org_vec->bv_page) | ||
| 329 | continue; | ||
| 330 | |||
| 331 | dec_zone_page_state(bvec->bv_page, NR_BOUNCE); | ||
| 332 | mempool_free(bvec->bv_page, pool); | ||
| 333 | } | ||
| 334 | |||
| 335 | bio_endio(bio_orig, bio_orig->bi_size, err); | ||
| 336 | bio_put(bio); | ||
| 337 | } | ||
| 338 | |||
| 339 | static int bounce_end_io_write(struct bio *bio, unsigned int bytes_done, int err) | ||
| 340 | { | ||
| 341 | if (bio->bi_size) | ||
| 342 | return 1; | ||
| 343 | |||
| 344 | bounce_end_io(bio, page_pool, err); | ||
| 345 | return 0; | ||
| 346 | } | ||
| 347 | |||
| 348 | static int bounce_end_io_write_isa(struct bio *bio, unsigned int bytes_done, int err) | ||
| 349 | { | ||
| 350 | if (bio->bi_size) | ||
| 351 | return 1; | ||
| 352 | |||
| 353 | bounce_end_io(bio, isa_page_pool, err); | ||
| 354 | return 0; | ||
| 355 | } | ||
| 356 | |||
| 357 | static void __bounce_end_io_read(struct bio *bio, mempool_t *pool, int err) | ||
| 358 | { | ||
| 359 | struct bio *bio_orig = bio->bi_private; | ||
| 360 | |||
| 361 | if (test_bit(BIO_UPTODATE, &bio->bi_flags)) | ||
| 362 | copy_to_high_bio_irq(bio_orig, bio); | ||
| 363 | |||
| 364 | bounce_end_io(bio, pool, err); | ||
| 365 | } | ||
| 366 | |||
| 367 | static int bounce_end_io_read(struct bio *bio, unsigned int bytes_done, int err) | ||
| 368 | { | ||
| 369 | if (bio->bi_size) | ||
| 370 | return 1; | ||
| 371 | |||
| 372 | __bounce_end_io_read(bio, page_pool, err); | ||
| 373 | return 0; | ||
| 374 | } | ||
| 375 | |||
| 376 | static int bounce_end_io_read_isa(struct bio *bio, unsigned int bytes_done, int err) | ||
| 377 | { | ||
| 378 | if (bio->bi_size) | ||
| 379 | return 1; | ||
| 380 | |||
| 381 | __bounce_end_io_read(bio, isa_page_pool, err); | ||
| 382 | return 0; | ||
| 383 | } | ||
| 384 | |||
| 385 | static void __blk_queue_bounce(request_queue_t *q, struct bio **bio_orig, | ||
| 386 | mempool_t *pool) | ||
| 387 | { | ||
| 388 | struct page *page; | ||
| 389 | struct bio *bio = NULL; | ||
| 390 | int i, rw = bio_data_dir(*bio_orig); | ||
| 391 | struct bio_vec *to, *from; | ||
| 392 | |||
| 393 | bio_for_each_segment(from, *bio_orig, i) { | ||
| 394 | page = from->bv_page; | ||
| 395 | |||
| 396 | /* | ||
| 397 | * is destination page below bounce pfn? | ||
| 398 | */ | ||
| 399 | if (page_to_pfn(page) < q->bounce_pfn) | ||
| 400 | continue; | ||
| 401 | |||
| 402 | /* | ||
| 403 | * irk, bounce it | ||
| 404 | */ | ||
| 405 | if (!bio) | ||
| 406 | bio = bio_alloc(GFP_NOIO, (*bio_orig)->bi_vcnt); | ||
| 407 | |||
| 408 | to = bio->bi_io_vec + i; | ||
| 409 | |||
| 410 | to->bv_page = mempool_alloc(pool, q->bounce_gfp); | ||
| 411 | to->bv_len = from->bv_len; | ||
| 412 | to->bv_offset = from->bv_offset; | ||
| 413 | inc_zone_page_state(to->bv_page, NR_BOUNCE); | ||
| 414 | |||
| 415 | if (rw == WRITE) { | ||
| 416 | char *vto, *vfrom; | ||
| 417 | |||
| 418 | flush_dcache_page(from->bv_page); | ||
| 419 | vto = page_address(to->bv_page) + to->bv_offset; | ||
| 420 | vfrom = kmap(from->bv_page) + from->bv_offset; | ||
| 421 | memcpy(vto, vfrom, to->bv_len); | ||
| 422 | kunmap(from->bv_page); | ||
| 423 | } | ||
| 424 | } | ||
| 425 | |||
| 426 | /* | ||
| 427 | * no pages bounced | ||
| 428 | */ | ||
| 429 | if (!bio) | ||
| 430 | return; | ||
| 431 | |||
| 432 | /* | ||
| 433 | * at least one page was bounced, fill in possible non-highmem | ||
| 434 | * pages | ||
| 435 | */ | ||
| 436 | __bio_for_each_segment(from, *bio_orig, i, 0) { | ||
| 437 | to = bio_iovec_idx(bio, i); | ||
| 438 | if (!to->bv_page) { | ||
| 439 | to->bv_page = from->bv_page; | ||
| 440 | to->bv_len = from->bv_len; | ||
| 441 | to->bv_offset = from->bv_offset; | ||
| 442 | } | ||
| 443 | } | ||
| 444 | |||
| 445 | bio->bi_bdev = (*bio_orig)->bi_bdev; | ||
| 446 | bio->bi_flags |= (1 << BIO_BOUNCED); | ||
| 447 | bio->bi_sector = (*bio_orig)->bi_sector; | ||
| 448 | bio->bi_rw = (*bio_orig)->bi_rw; | ||
| 449 | |||
| 450 | bio->bi_vcnt = (*bio_orig)->bi_vcnt; | ||
| 451 | bio->bi_idx = (*bio_orig)->bi_idx; | ||
| 452 | bio->bi_size = (*bio_orig)->bi_size; | ||
| 453 | |||
| 454 | if (pool == page_pool) { | ||
| 455 | bio->bi_end_io = bounce_end_io_write; | ||
| 456 | if (rw == READ) | ||
| 457 | bio->bi_end_io = bounce_end_io_read; | ||
| 458 | } else { | ||
| 459 | bio->bi_end_io = bounce_end_io_write_isa; | ||
| 460 | if (rw == READ) | ||
| 461 | bio->bi_end_io = bounce_end_io_read_isa; | ||
| 462 | } | ||
| 463 | |||
| 464 | bio->bi_private = *bio_orig; | ||
| 465 | *bio_orig = bio; | ||
| 466 | } | ||
| 467 | |||
| 468 | void blk_queue_bounce(request_queue_t *q, struct bio **bio_orig) | ||
| 469 | { | ||
| 470 | mempool_t *pool; | ||
| 471 | |||
| 472 | /* | ||
| 473 | * for non-isa bounce case, just check if the bounce pfn is equal | ||
| 474 | * to or bigger than the highest pfn in the system -- in that case, | ||
| 475 | * don't waste time iterating over bio segments | ||
| 476 | */ | ||
| 477 | if (!(q->bounce_gfp & GFP_DMA)) { | ||
| 478 | if (q->bounce_pfn >= blk_max_pfn) | ||
| 479 | return; | ||
| 480 | pool = page_pool; | ||
| 481 | } else { | ||
| 482 | BUG_ON(!isa_page_pool); | ||
| 483 | pool = isa_page_pool; | ||
| 484 | } | ||
| 485 | |||
| 486 | blk_add_trace_bio(q, *bio_orig, BLK_TA_BOUNCE); | ||
| 487 | |||
| 488 | /* | ||
| 489 | * slow path | ||
| 490 | */ | ||
| 491 | __blk_queue_bounce(q, bio_orig, pool); | ||
| 492 | } | ||
| 493 | |||
| 494 | EXPORT_SYMBOL(blk_queue_bounce); | ||
| 495 | |||
| 496 | #if defined(HASHED_PAGE_VIRTUAL) | 215 | #if defined(HASHED_PAGE_VIRTUAL) |
| 497 | 216 | ||
| 498 | #define PA_HASH_ORDER 7 | 217 | #define PA_HASH_ORDER 7 |
