litmus-rt.git - The LITMUS^RT kernel.

The LITMUS^RT kernel.

about summary refs log tree commit diff stats

path: root/tools/perf/scripts/python/netdev-times.py

diff options

author	Brian Norris <computersforpeace@gmail.com>	2012-02-21 13:38:43 -0500
committer	Jeff Garzik <jgarzik@redhat.com>	2012-03-13 16:35:47 -0400
commit	55d5ec316627b64c3764e4c1b4b8e1988e272c1f (patch)
tree	02143de94a97d4c56c94d30acf7f544ec233ed15 /tools/perf/scripts/python/netdev-times.py
parent	66583c9fa63d05d5580e409f9a58d3cad6d76d17 (diff)

ahci: move AHCI_HFLAGS() macro to ahci.h

We will need this macro in both ahci.c and ahci_platform.c, so just move it to the header. Signed-off-by: Brian Norris <computersforpeace@gmail.com> Signed-off-by: Jeff Garzik <jgarzik@redhat.com> Cc: stable@kernel.org

Diffstat (limited to 'tools/perf/scripts/python/netdev-times.py')

0 files changed, 0 insertions, 0 deletions

s='sha1'>1da177e4c3f4

001281881067

1da177e4c3f4

72c4783210f7

1da177e4c3f4

3
4

26
27

38
39

49
50

57
58
59

63
64

77
78

80
81

91
92

103
104

156
157

158
159
160

186
187
188

189
190

202
203

205
206

208
209
210

222
223

224
225

244
245

247
248
249

289
290
291

293
294

319
320
321

322
323

344
345
346

352
353
354

377
378

380
381

382
383
384

385
386

387
388

395
396

409
410

412
413

430
431
432

439
440
441

448
449

628
629

630
631
632

640
641
642

643
644
645

646
647
648

760
761
762

769
770

885
886

887
888

892
893
894

924
925
926

935
936
937

943
944

945
946
947

1002
1003

1005
1006

1011
1012

1013
1014

1016
1017
1018

1027
1028

1035
1036

1046
1047
1048

1049
1050

1079
1080
1081

1084
1085

1094
1095
1096

1098
1099

1104
1105
1106

1123
1124

1127
1128

1144
1145

1163
1164

1165
1166

1203
1204

1231
1232

1235
1236

1238
1239
1240

1265
1266
1267

1273
1274

1275
1276

1285
1286
1287

1297
1298
1299

1301
1302
1303

1308
1309
1310

1314
1315
1316

1323
1324

1364
1365

1372
1373

1374
1375

1377
1378

1381
1382

1405
1406
1407

1408
1409

1412
1413
1414

1415
1416

1434
1435
1436

1448
1449

1455
1456

1457
1458

1475
1476
1477

1479
1480

1481
1482

1484
1485

1491
1492

1508
1509
1510

1515
1516
1517

1519
1520
1521

1544
1545

1572
1573

1593
1594
1595

1616
1617
1618

1622
1623

1624
1625

1627
1628

1655
1656
1657

1667
1668
1669

1670
1671

1672
1673

1674
1675
1676

1678
1679

1685
1686

1743
1744
1745

1747
1748

1773
1774
1775

1780
1781
1782

1784
1785
1786

1789
1790
1791

1793
1794
1795

1823
1824
1825

1896
1897

1923
1924
1925

1938
1939

1955
1956

1960
1961
1962

1965
1966

1969
1970
1971

1974
1975

1986
1987

1991
1992
1993

1995
1996

2001
2002
2003

2005
2006
2007

2022
2023

/*
 * mm/page-writeback.c
 *
 * Copyright (C) 2002, Linus Torvalds.
 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
 *
 * Contains functions related to writing back dirty pages at the
 * address_space level.
 *
 * 10Apr2002	Andrew Morton
 *		Initial version
 */

#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/slab.h>
#include <linux/pagemap.h>
#include <linux/writeback.h>
#include <linux/init.h>
#include <linux/backing-dev.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/blkdev.h>
#include <linux/mpage.h>
#include <linux/rmap.h>
#include <linux/percpu.h>
#include <linux/notifier.h>
#include <linux/smp.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/syscalls.h>
#include <linux/buffer_head.h>
#include <linux/pagevec.h>
#include <trace/events/writeback.h>

/*
 * Sleep at most 200ms at a time in balance_dirty_pages().
 */
#define MAX_PAUSE		max(HZ/5, 1)

/*
 * Estimate write bandwidth at 200ms intervals.
 */
#define BANDWIDTH_INTERVAL	max(HZ/5, 1)

#define RATELIMIT_CALC_SHIFT	10

/*
 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
 * will look to see if it needs to force writeback or throttling.
 */
static long ratelimit_pages = 32;

/* The following parameters are exported via /proc/sys/vm */

/*
 * Start background writeback (via writeback threads) at this percentage
 */
int dirty_background_ratio = 10;

/*
 * dirty_background_bytes starts at 0 (disabled) so that it is a function of
 * dirty_background_ratio * the amount of dirtyable memory
 */
unsigned long dirty_background_bytes;

/*
 * free highmem will not be subtracted from the total free memory
 * for calculating free ratios if vm_highmem_is_dirtyable is true
 */
int vm_highmem_is_dirtyable;

/*
 * The generator of dirty data starts writeback at this percentage
 */
int vm_dirty_ratio = 20;

/*
 * vm_dirty_bytes starts at 0 (disabled) so that it is a function of
 * vm_dirty_ratio * the amount of dirtyable memory
 */
unsigned long vm_dirty_bytes;

/*
 * The interval between `kupdate'-style writebacks
 */
unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */

/*
 * The longest time for which data is allowed to remain dirty
 */
unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */

/*
 * Flag that makes the machine dump writes/reads and block dirtyings.
 */
int block_dump;

/*
 * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies:
 * a full sync is triggered after this time elapses without any disk activity.
 */
int laptop_mode;

EXPORT_SYMBOL(laptop_mode);

/* End of sysctl-exported parameters */

unsigned long global_dirty_limit;

/*
 * Scale the writeback cache size proportional to the relative writeout speeds.
 *
 * We do this by keeping a floating proportion between BDIs, based on page
 * writeback completions [end_page_writeback()]. Those devices that write out
 * pages fastest will get the larger share, while the slower will get a smaller
 * share.
 *
 * We use page writeout completions because we are interested in getting rid of
 * dirty pages. Having them written out is the primary goal.
 *
 * We introduce a concept of time, a period over which we measure these events,
 * because demand can/will vary over time. The length of this period itself is
 * measured in page writeback completions.
 *
 */
static struct prop_descriptor vm_completions;

/*
 * couple the period to the dirty_ratio:
 *
 *   period/2 ~ roundup_pow_of_two(dirty limit)
 */
static int calc_period_shift(void)
{
	unsigned long dirty_total;

	if (vm_dirty_bytes)
		dirty_total = vm_dirty_bytes / PAGE_SIZE;
	else
		dirty_total = (vm_dirty_ratio * determine_dirtyable_memory()) /
				100;
	return 2 + ilog2(dirty_total - 1);
}

/*
 * update the period when the dirty threshold changes.
 */
static void update_completion_period(void)
{
	int shift = calc_period_shift();
	prop_change_shift(&vm_completions, shift);

	writeback_set_ratelimit();
}

int dirty_background_ratio_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int ret;

	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
	if (ret == 0 && write)
		dirty_background_bytes = 0;
	return ret;
}

int dirty_background_bytes_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int ret;

	ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
	if (ret == 0 && write)
		dirty_background_ratio = 0;
	return ret;
}

int dirty_ratio_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int old_ratio = vm_dirty_ratio;
	int ret;

	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
	if (ret == 0 && write && vm_dirty_ratio != old_ratio) {
		update_completion_period();
		vm_dirty_bytes = 0;
	}
	return ret;
}


int dirty_bytes_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	unsigned long old_bytes = vm_dirty_bytes;
	int ret;

	ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
	if (ret == 0 && write && vm_dirty_bytes != old_bytes) {
		update_completion_period();
		vm_dirty_ratio = 0;
	}
	return ret;
}

/*
 * Increment the BDI's writeout completion count and the global writeout
 * completion count. Called from test_clear_page_writeback().
 */
static inline void __bdi_writeout_inc(struct backing_dev_info *bdi)
{
	__inc_bdi_stat(bdi, BDI_WRITTEN);
	__prop_inc_percpu_max(&vm_completions, &bdi->completions,
			      bdi->max_prop_frac);
}

void bdi_writeout_inc(struct backing_dev_info *bdi)
{
	unsigned long flags;

	local_irq_save(flags);
	__bdi_writeout_inc(bdi);
	local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(bdi_writeout_inc);

/*
 * Obtain an accurate fraction of the BDI's portion.
 */
static void bdi_writeout_fraction(struct backing_dev_info *bdi,
		long *numerator, long *denominator)
{
	prop_fraction_percpu(&vm_completions, &bdi->completions,
				numerator, denominator);
}

/*
 * bdi_min_ratio keeps the sum of the minimum dirty shares of all
 * registered backing devices, which, for obvious reasons, can not
 * exceed 100%.
 */
static unsigned int bdi_min_ratio;

int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
{
	int ret = 0;

	spin_lock_bh(&bdi_lock);
	if (min_ratio > bdi->max_ratio) {
		ret = -EINVAL;
	} else {
		min_ratio -= bdi->min_ratio;
		if (bdi_min_ratio + min_ratio < 100) {
			bdi_min_ratio += min_ratio;
			bdi->min_ratio += min_ratio;
		} else {
			ret = -EINVAL;
		}
	}
	spin_unlock_bh(&bdi_lock);

	return ret;
}

int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio)
{
	int ret = 0;

	if (max_ratio > 100)
		return -EINVAL;

	spin_lock_bh(&bdi_lock);
	if (bdi->min_ratio > max_ratio) {
		ret = -EINVAL;
	} else {
		bdi->max_ratio = max_ratio;
		bdi->max_prop_frac = (PROP_FRAC_BASE * max_ratio) / 100;
	}
	spin_unlock_bh(&bdi_lock);

	return ret;
}
EXPORT_SYMBOL(bdi_set_max_ratio);

/*
 * Work out the current dirty-memory clamping and background writeout
 * thresholds.
 *
 * The main aim here is to lower them aggressively if there is a lot of mapped
 * memory around.  To avoid stressing page reclaim with lots of unreclaimable
 * pages.  It is better to clamp down on writers than to start swapping, and
 * performing lots of scanning.
 *
 * We only allow 1/2 of the currently-unmapped memory to be dirtied.
 *
 * We don't permit the clamping level to fall below 5% - that is getting rather
 * excessive.
 *
 * We make sure that the background writeout level is below the adjusted
 * clamping level.
 */

static unsigned long highmem_dirtyable_memory(unsigned long total)
{
#ifdef CONFIG_HIGHMEM
	int node;
	unsigned long x = 0;

	for_each_node_state(node, N_HIGH_MEMORY) {
		struct zone *z =
			&NODE_DATA(node)->node_zones[ZONE_HIGHMEM];

		x += zone_page_state(z, NR_FREE_PAGES) +
		     zone_reclaimable_pages(z);
	}
	/*
	 * Make sure that the number of highmem pages is never larger
	 * than the number of the total dirtyable memory. This can only
	 * occur in very strange VM situations but we want to make sure
	 * that this does not occur.
	 */
	return min(x, total);
#else
	return 0;
#endif
}

/**
 * determine_dirtyable_memory - amount of memory that may be used
 *
 * Returns the numebr of pages that can currently be freed and used
 * by the kernel for direct mappings.
 */
unsigned long determine_dirtyable_memory(void)
{
	unsigned long x;

	x = global_page_state(NR_FREE_PAGES) + global_reclaimable_pages();

	if (!vm_highmem_is_dirtyable)
		x -= highmem_dirtyable_memory(x);

	return x + 1;	/* Ensure that we never return 0 */
}

static unsigned long dirty_freerun_ceiling(unsigned long thresh,
					   unsigned long bg_thresh)
{
	return (thresh + bg_thresh) / 2;
}

static unsigned long hard_dirty_limit(unsigned long thresh)
{
	return max(thresh, global_dirty_limit);
}

/*
 * global_dirty_limits - background-writeback and dirty-throttling thresholds
 *
 * Calculate the dirty thresholds based on sysctl parameters
 * - vm.dirty_background_ratio  or  vm.dirty_background_bytes
 * - vm.dirty_ratio             or  vm.dirty_bytes
 * The dirty limits will be lifted by 1/4 for PF_LESS_THROTTLE (ie. nfsd) and
 * real-time tasks.
 */
void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
{
	unsigned long background;
	unsigned long dirty;
	unsigned long uninitialized_var(available_memory);
	struct task_struct *tsk;

	if (!vm_dirty_bytes || !dirty_background_bytes)
		available_memory = determine_dirtyable_memory();

	if (vm_dirty_bytes)
		dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE);
	else
		dirty = (vm_dirty_ratio * available_memory) / 100;

	if (dirty_background_bytes)
		background = DIV_ROUND_UP(dirty_background_bytes, PAGE_SIZE);
	else
		background = (dirty_background_ratio * available_memory) / 100;

	if (background >= dirty)
		background = dirty / 2;
	tsk = current;
	if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) {
		background += background / 4;
		dirty += dirty / 4;
	}
	*pbackground = background;
	*pdirty = dirty;
	trace_global_dirty_state(background, dirty);
}

/**
 * bdi_dirty_limit - @bdi's share of dirty throttling threshold
 * @bdi: the backing_dev_info to query
 * @dirty: global dirty limit in pages
 *
 * Returns @bdi's dirty limit in pages. The term "dirty" in the context of
 * dirty balancing includes all PG_dirty, PG_writeback and NFS unstable pages.
 *
 * Note that balance_dirty_pages() will only seriously take it as a hard limit
 * when sleeping max_pause per page is not enough to keep the dirty pages under
 * control. For example, when the device is completely stalled due to some error
 * conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key.
 * In the other normal situations, it acts more gently by throttling the tasks
 * more (rather than completely block them) when the bdi dirty pages go high.
 *
 * It allocates high/low dirty limits to fast/slow devices, in order to prevent
 * - starving fast devices
 * - piling up dirty pages (that will take long time to sync) on slow devices
 *
 * The bdi's share of dirty limit will be adapting to its throughput and
 * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.
 */
unsigned long bdi_dirty_limit(struct backing_dev_info *bdi, unsigned long dirty)
{
	u64 bdi_dirty;
	long numerator, denominator;

	/*
	 * Calculate this BDI's share of the dirty ratio.
	 */
	bdi_writeout_fraction(bdi, &numerator, &denominator);

	bdi_dirty = (dirty * (100 - bdi_min_ratio)) / 100;
	bdi_dirty *= numerator;
	do_div(bdi_dirty, denominator);

	bdi_dirty += (dirty * bdi->min_ratio) / 100;
	if (bdi_dirty > (dirty * bdi->max_ratio) / 100)
		bdi_dirty = dirty * bdi->max_ratio / 100;

	return bdi_dirty;
}

/*
 * Dirty position control.
 *
 * (o) global/bdi setpoints
 *
 * We want the dirty pages be balanced around the global/bdi setpoints.
 * When the number of dirty pages is higher/lower than the setpoint, the
 * dirty position control ratio (and hence task dirty ratelimit) will be
 * decreased/increased to bring the dirty pages back to the setpoint.
 *
 *     pos_ratio = 1 << RATELIMIT_CALC_SHIFT
 *
 *     if (dirty < setpoint) scale up   pos_ratio
 *     if (dirty > setpoint) scale down pos_ratio
 *
 *     if (bdi_dirty < bdi_setpoint) scale up   pos_ratio
 *     if (bdi_dirty > bdi_setpoint) scale down pos_ratio
 *
 *     task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT
 *
 * (o) global control line
 *
 *     ^ pos_ratio
 *     |
 *     |            |<===== global dirty control scope ======>|
 * 2.0 .............*
 *     |            .*
 *     |            . *
 *     |            .   *
 *     |            .     *
 *     |            .        *
 *     |            .            *
 * 1.0 ................................*
 *     |            .                  .     *
 *     |            .                  .          *
 *     |            .                  .              *
 *     |            .                  .                 *
 *     |            .                  .                    *
 *   0 +------------.------------------.----------------------*------------->
 *           freerun^          setpoint^                 limit^   dirty pages
 *
 * (o) bdi control line
 *
 *     ^ pos_ratio
 *     |
 *     |            *
 *     |              *
 *     |                *
 *     |                  *
 *     |                    * |<=========== span ============>|
 * 1.0 .......................*
 *     |                      . *
 *     |                      .   *
 *     |                      .     *
 *     |                      .       *
 *     |                      .         *
 *     |                      .           *
 *     |                      .             *
 *     |                      .               *
 *     |                      .                 *
 *     |                      .                   *
 *     |                      .                     *
 * 1/4 ...............................................* * * * * * * * * * * *
 *     |                      .                         .
 *     |                      .                           .
 *     |                      .                             .
 *   0 +----------------------.-------------------------------.------------->
 *                bdi_setpoint^                    x_intercept^
 *
 * The bdi control line won't drop below pos_ratio=1/4, so that bdi_dirty can
 * be smoothly throttled down to normal if it starts high in situations like
 * - start writing to a slow SD card and a fast disk at the same time. The SD
 *   card's bdi_dirty may rush to many times higher than bdi_setpoint.
 * - the bdi dirty thresh drops quickly due to change of JBOD workload
 */
static unsigned long bdi_position_ratio(struct backing_dev_info *bdi,
					unsigned long thresh,
					unsigned long bg_thresh,
					unsigned long dirty,
					unsigned long bdi_thresh,
					unsigned long bdi_dirty)
{
	unsigned long write_bw = bdi->avg_write_bandwidth;
	unsigned long freerun = dirty_freerun_ceiling(thresh, bg_thresh);
	unsigned long limit = hard_dirty_limit(thresh);
	unsigned long x_intercept;
	unsigned long setpoint;		/* dirty pages' target balance point */
	unsigned long bdi_setpoint;
	unsigned long span;
	long long pos_ratio;		/* for scaling up/down the rate limit */
	long x;

	if (unlikely(dirty >= limit))
		return 0;

	/*
	 * global setpoint
	 *
	 *                           setpoint - dirty 3
	 *        f(dirty) := 1.0 + (----------------)
	 *                           limit - setpoint
	 *
	 * it's a 3rd order polynomial that subjects to
	 *
	 * (1) f(freerun)  = 2.0 => rampup dirty_ratelimit reasonably fast
	 * (2) f(setpoint) = 1.0 => the balance point
	 * (3) f(limit)    = 0   => the hard limit
	 * (4) df/dx      <= 0	 => negative feedback control
	 * (5) the closer to setpoint, the smaller |df/dx| (and the reverse)
	 *     => fast response on large errors; small oscillation near setpoint
	 */
	setpoint = (freerun + limit) / 2;
	x = div_s64((setpoint - dirty) << RATELIMIT_CALC_SHIFT,
		    limit - setpoint + 1);
	pos_ratio = x;
	pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
	pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
	pos_ratio += 1 << RATELIMIT_CALC_SHIFT;

	/*
	 * We have computed basic pos_ratio above based on global situation. If
	 * the bdi is over/under its share of dirty pages, we want to scale
	 * pos_ratio further down/up. That is done by the following mechanism.
	 */

	/*
	 * bdi setpoint
	 *
	 *        f(bdi_dirty) := 1.0 + k * (bdi_dirty - bdi_setpoint)
	 *
	 *                        x_intercept - bdi_dirty
	 *                     := --------------------------
	 *                        x_intercept - bdi_setpoint
	 *
	 * The main bdi control line is a linear function that subjects to
	 *
	 * (1) f(bdi_setpoint) = 1.0
	 * (2) k = - 1 / (8 * write_bw)  (in single bdi case)
	 *     or equally: x_intercept = bdi_setpoint + 8 * write_bw
	 *
	 * For single bdi case, the dirty pages are observed to fluctuate
	 * regularly within range
	 *        [bdi_setpoint - write_bw/2, bdi_setpoint + write_bw/2]
	 * for various filesystems, where (2) can yield in a reasonable 12.5%
	 * fluctuation range for pos_ratio.
	 *
	 * For JBOD case, bdi_thresh (not bdi_dirty!) could fluctuate up to its
	 * own size, so move the slope over accordingly and choose a slope that
	 * yields 100% pos_ratio fluctuation on suddenly doubled bdi_thresh.
	 */
	if (unlikely(bdi_thresh > thresh))
		bdi_thresh = thresh;
	/*
	 * It's very possible that bdi_thresh is close to 0 not because the
	 * device is slow, but that it has remained inactive for long time.
	 * Honour such devices a reasonable good (hopefully IO efficient)
	 * threshold, so that the occasional writes won't be blocked and active
	 * writes can rampup the threshold quickly.
	 */
	bdi_thresh = max(bdi_thresh, (limit - dirty) / 8);
	/*
	 * scale global setpoint to bdi's:
	 *	bdi_setpoint = setpoint * bdi_thresh / thresh
	 */
	x = div_u64((u64)bdi_thresh << 16, thresh + 1);
	bdi_setpoint = setpoint * (u64)x >> 16;
	/*
	 * Use span=(8*write_bw) in single bdi case as indicated by
	 * (thresh - bdi_thresh ~= 0) and transit to bdi_thresh in JBOD case.
	 *
	 *        bdi_thresh                    thresh - bdi_thresh
	 * span = ---------- * (8 * write_bw) + ------------------- * bdi_thresh
	 *          thresh                            thresh
	 */
	span = (thresh - bdi_thresh + 8 * write_bw) * (u64)x >> 16;
	x_intercept = bdi_setpoint + span;

	if (bdi_dirty < x_intercept - span / 4) {
		pos_ratio = div_u64(pos_ratio * (x_intercept - bdi_dirty),
				    x_intercept - bdi_setpoint + 1);
	} else
		pos_ratio /= 4;

	/*
	 * bdi reserve area, safeguard against dirty pool underrun and disk idle
	 * It may push the desired control point of global dirty pages higher
	 * than setpoint.
	 */
	x_intercept = bdi_thresh / 2;
	if (bdi_dirty < x_intercept) {
		if (bdi_dirty > x_intercept / 8)
			pos_ratio = div_u64(pos_ratio * x_intercept, bdi_dirty);
		else
			pos_ratio *= 8;
	}

	return pos_ratio;
}

static void bdi_update_write_bandwidth(struct backing_dev_info *bdi,
				       unsigned long elapsed,
				       unsigned long written)
{
	const unsigned long period = roundup_pow_of_two(3 * HZ);
	unsigned long avg = bdi->avg_write_bandwidth;
	unsigned long old = bdi->write_bandwidth;
	u64 bw;

	/*
	 * bw = written * HZ / elapsed
	 *
	 *                   bw * elapsed + write_bandwidth * (period - elapsed)
	 * write_bandwidth = ---------------------------------------------------
	 *                                          period
	 */
	bw = written - bdi->written_stamp;
	bw *= HZ;
	if (unlikely(elapsed > period)) {
		do_div(bw, elapsed);
		avg = bw;
		goto out;
	}
	bw += (u64)bdi->write_bandwidth * (period - elapsed);
	bw >>= ilog2(period);

	/*
	 * one more level of smoothing, for filtering out sudden spikes
	 */
	if (avg > old && old >= (unsigned long)bw)
		avg -= (avg - old) >> 3;

	if (avg < old && old <= (unsigned long)bw)
		avg += (old - avg) >> 3;

out:
	bdi->write_bandwidth = bw;
	bdi->avg_write_bandwidth = avg;
}

/*
 * The global dirtyable memory and dirty threshold could be suddenly knocked
 * down by a large amount (eg. on the startup of KVM in a swapless system).
 * This may throw the system into deep dirty exceeded state and throttle
 * heavy/light dirtiers alike. To retain good responsiveness, maintain
 * global_dirty_limit for tracking slowly down to the knocked down dirty
 * threshold.
 */
static void update_dirty_limit(unsigned long thresh, unsigned long dirty)
{
	unsigned long limit = global_dirty_limit;

	/*
	 * Follow up in one step.
	 */
	if (limit < thresh) {
		limit = thresh;
		goto update;
	}

	/*
	 * Follow down slowly. Use the higher one as the target, because thresh
	 * may drop below dirty. This is exactly the reason to introduce
	 * global_dirty_limit which is guaranteed to lie above the dirty pages.
	 */
	thresh = max(thresh, dirty);
	if (limit > thresh) {
		limit -= (limit - thresh) >> 5;
		goto update;
	}
	return;
update:
	global_dirty_limit = limit;
}

static void global_update_bandwidth(unsigned long thresh,
				    unsigned long dirty,
				    unsigned long now)
{
	static DEFINE_SPINLOCK(dirty_lock);
	static unsigned long update_time;

	/*
	 * check locklessly first to optimize away locking for the most time
	 */
	if (time_before(now, update_time + BANDWIDTH_INTERVAL))
		return;

	spin_lock(&dirty_lock);
	if (time_after_eq(now, update_time + BANDWIDTH_INTERVAL)) {
		update_dirty_limit(thresh, dirty);
		update_time = now;
	}
	spin_unlock(&dirty_lock);
}

/*
 * Maintain bdi->dirty_ratelimit, the base dirty throttle rate.
 *
 * Normal bdi tasks will be curbed at or below it in long term.
 * Obviously it should be around (write_bw / N) when there are N dd tasks.
 */
static void bdi_update_dirty_ratelimit(struct backing_dev_info *bdi,
				       unsigned long thresh,
				       unsigned long bg_thresh,
				       unsigned long dirty,
				       unsigned long bdi_thresh,
				       unsigned long bdi_dirty,
				       unsigned long dirtied,
				       unsigned long elapsed)
{
	unsigned long freerun = dirty_freerun_ceiling(thresh, bg_thresh);
	unsigned long limit = hard_dirty_limit(thresh);
	unsigned long setpoint = (freerun + limit) / 2;
	unsigned long write_bw = bdi->avg_write_bandwidth;
	unsigned long dirty_ratelimit = bdi->dirty_ratelimit;
	unsigned long dirty_rate;
	unsigned long task_ratelimit;
	unsigned long balanced_dirty_ratelimit;
	unsigned long pos_ratio;
	unsigned long step;
	unsigned long x;

	/*
	 * The dirty rate will match the writeout rate in long term, except
	 * when dirty pages are truncated by userspace or re-dirtied by FS.
	 */
	dirty_rate = (dirtied - bdi->dirtied_stamp) * HZ / elapsed;

	pos_ratio = bdi_position_ratio(bdi, thresh, bg_thresh, dirty,
				       bdi_thresh, bdi_dirty);
	/*
	 * task_ratelimit reflects each dd's dirty rate for the past 200ms.
	 */
	task_ratelimit = (u64)dirty_ratelimit *
					pos_ratio >> RATELIMIT_CALC_SHIFT;
	task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */

	/*
	 * A linear estimation of the "balanced" throttle rate. The theory is,
	 * if there are N dd tasks, each throttled at task_ratelimit, the bdi's
	 * dirty_rate will be measured to be (N * task_ratelimit). So the below
	 * formula will yield the balanced rate limit (write_bw / N).
	 *
	 * Note that the expanded form is not a pure rate feedback:
	 *	rate_(i+1) = rate_(i) * (write_bw / dirty_rate)		     (1)
	 * but also takes pos_ratio into account:
	 *	rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio  (2)
	 *
	 * (1) is not realistic because pos_ratio also takes part in balancing
	 * the dirty rate.  Consider the state
	 *	pos_ratio = 0.5						     (3)
	 *	rate = 2 * (write_bw / N)				     (4)
	 * If (1) is used, it will stuck in that state! Because each dd will
	 * be throttled at
	 *	task_ratelimit = pos_ratio * rate = (write_bw / N)	     (5)
	 * yielding
	 *	dirty_rate = N * task_ratelimit = write_bw		     (6)
	 * put (6) into (1) we get
	 *	rate_(i+1) = rate_(i)					     (7)
	 *
	 * So we end up using (2) to always keep
	 *	rate_(i+1) ~= (write_bw / N)				     (8)
	 * regardless of the value of pos_ratio. As long as (8) is satisfied,
	 * pos_ratio is able to drive itself to 1.0, which is not only where
	 * the dirty count meet the setpoint, but also where the slope of
	 * pos_ratio is most flat and hence task_ratelimit is least fluctuated.
	 */
	balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw,
					   dirty_rate | 1);

	/*
	 * We could safely do this and return immediately:
	 *
	 *	bdi->dirty_ratelimit = balanced_dirty_ratelimit;
	 *
	 * However to get a more stable dirty_ratelimit, the below elaborated
	 * code makes use of task_ratelimit to filter out sigular points and
	 * limit the step size.
	 *


context:
space:
mode: