469 files changed, 19442 insertions, 10689 deletions

diff --git a/Documentation/SubmittingPatches b/Documentation/SubmittingPatches
index dcadffcab2dc..0a523c9a5ff4 100644
--- a/Documentation/SubmittingPatches
+++ b/Documentation/SubmittingPatches
@@ -84,18 +84,42 @@ is another popular alternative.
 
 2) Describe your changes.
 
-Describe the technical detail of the change(s) your patch includes.
-
-Be as specific as possible.  The WORST descriptions possible include
-things like "update driver X", "bug fix for driver X", or "this patch
-includes updates for subsystem X.  Please apply."
+Describe your problem.  Whether your patch is a one-line bug fix or
+5000 lines of a new feature, there must be an underlying problem that
+motivated you to do this work.  Convince the reviewer that there is a
+problem worth fixing and that it makes sense for them to read past the
+first paragraph.
+
+Describe user-visible impact.  Straight up crashes and lockups are
+pretty convincing, but not all bugs are that blatant.  Even if the
+problem was spotted during code review, describe the impact you think
+it can have on users.  Keep in mind that the majority of Linux
+installations run kernels from secondary stable trees or
+vendor/product-specific trees that cherry-pick only specific patches
+from upstream, so include anything that could help route your change
+downstream: provoking circumstances, excerpts from dmesg, crash
+descriptions, performance regressions, latency spikes, lockups, etc.
+
+Quantify optimizations and trade-offs.  If you claim improvements in
+performance, memory consumption, stack footprint, or binary size,
+include numbers that back them up.  But also describe non-obvious
+costs.  Optimizations usually aren't free but trade-offs between CPU,
+memory, and readability; or, when it comes to heuristics, between
+different workloads.  Describe the expected downsides of your
+optimization so that the reviewer can weigh costs against benefits.
+
+Once the problem is established, describe what you are actually doing
+about it in technical detail.  It's important to describe the change
+in plain English for the reviewer to verify that the code is behaving
+as you intend it to.
 
 The maintainer will thank you if you write your patch description in a
 form which can be easily pulled into Linux's source code management
 system, git, as a "commit log".  See #15, below.
 
-If your description starts to get long, that's a sign that you probably
-need to split up your patch.  See #3, next.
+Solve only one problem per patch.  If your description starts to get
+long, that's a sign that you probably need to split up your patch.
+See #3, next.
 
 When you submit or resubmit a patch or patch series, include the
 complete patch description and justification for it.  Don't just
diff --git a/Documentation/devicetree/bindings/i2c/i2c-efm32.txt b/Documentation/devicetree/bindings/i2c/i2c-efm32.txt
index fc15ac519437..50b25c3da186 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-efm32.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-efm32.txt
@@ -10,7 +10,7 @@ Required properties :
 Recommended properties :
 
  - clock-frequency : maximal I2C bus clock frequency in Hz.
- - efm32,location : Decides the location of the USART I/O pins.
+ - energymicro,location : Decides the location of the USART I/O pins.
    Allowed range : [0 .. 6]
 
 Example:
@@ -23,7 +23,7 @@ Example:
                 clocks = <&cmu clk_HFPERCLKI2C0>;
                 clock-frequency = <100000>;
                 status = "ok";
-                efm32,location = <3>;
+                energymicro,location = <3>;
 
                 eeprom@50 {
                         compatible = "microchip,24c02";
diff --git a/Documentation/filesystems/cifs/AUTHORS b/Documentation/filesystems/cifs/AUTHORS
index ca4a67a0bb1e..c98800df677f 100644
--- a/Documentation/filesystems/cifs/AUTHORS
+++ b/Documentation/filesystems/cifs/AUTHORS
@@ -40,6 +40,7 @@ Gunter Kukkukk (testing and suggestions for support of old servers)
 Igor Mammedov (DFS support)
 Jeff Layton (many, many fixes, as well as great work on the cifs Kerberos code)
 Scott Lovenberg
+Pavel Shilovsky (for great work adding SMB2 support, and various SMB3 features)
 
 Test case and Bug Report contributors
 -------------------------------------
diff --git a/Documentation/filesystems/cifs/TODO b/Documentation/filesystems/cifs/TODO
index 355abcdcda98..066ffddc3964 100644
--- a/Documentation/filesystems/cifs/TODO
+++ b/Documentation/filesystems/cifs/TODO
@@ -1,4 +1,4 @@
-Version 1.53 May 20, 2008
+Version 2.03 August 1, 2014
 
 A Partial List of Missing Features
 ==================================
@@ -7,63 +7,49 @@ Contributions are welcome.  There are plenty of opportunities
 for visible, important contributions to this module.  Here
 is a partial list of the known problems and missing features:
 
-a) Support for SecurityDescriptors(Windows/CIFS ACLs) for chmod/chgrp/chown
-so that these operations can be supported to Windows servers
+a) SMB3 (and SMB3.02) missing optional features:
+   - RDMA
+   - multichannel (started)
+   - directory leases (improved metadata caching)
+   - T10 copy offload (copy chunk is only mechanism supported)
+   - encrypted shares
 
-b) Mapping POSIX ACLs (and eventually NFSv4 ACLs) to CIFS
-SecurityDescriptors
+b) improved sparse file support
 
-c) Better pam/winbind integration (e.g. to handle uid mapping
-better)
-
-d) Cleanup now unneeded SessSetup code in
-fs/cifs/connect.c and add back in NTLMSSP code if any servers
-need it
-
-e) fix NTLMv2 signing when two mounts with different users to same
-server.
-
-f) Directory entry caching relies on a 1 second timer, rather than 
+c) Directory entry caching relies on a 1 second timer, rather than
 using FindNotify or equivalent.  - (started)
 
-g) quota support (needs minor kernel change since quota calls
+d) quota support (needs minor kernel change since quota calls
 to make it to network filesystems or deviceless filesystems)
 
-h) investigate sync behavior (including syncpage) and check  
-for proper behavior of intr/nointr
-
-i) improve support for very old servers (OS/2 and Win9x for example)
+e) improve support for very old servers (OS/2 and Win9x for example)
 Including support for changing the time remotely (utimes command).
 
-j) hook lower into the sockets api (as NFS/SunRPC does) to avoid the
+f) hook lower into the sockets api (as NFS/SunRPC does) to avoid the
 extra copy in/out of the socket buffers in some cases.
 
-k) Better optimize open (and pathbased setfilesize) to reduce the
+g) Better optimize open (and pathbased setfilesize) to reduce the
 oplock breaks coming from windows srv.  Piggyback identical file
 opens on top of each other by incrementing reference count rather
 than resending (helps reduce server resource utilization and avoid
 spurious oplock breaks).
 
-l) Improve performance of readpages by sending more than one read
-at a time when 8 pages or more are requested. In conjuntion
-add support for async_cifs_readpages.
-
-m) Add support for storing symlink info to Windows servers 
+h) Add support for storing symlink info to Windows servers
 in the Extended Attribute format their SFU clients would recognize.
 
-n) Finish fcntl D_NOTIFY support so kde and gnome file list windows
+i) Finish inotify support so kde and gnome file list windows
 will autorefresh (partially complete by Asser). Needs minor kernel
 vfs change to support removing D_NOTIFY on a file.   
 
-o) Add GUI tool to configure /proc/fs/cifs settings and for display of
+j) Add GUI tool to configure /proc/fs/cifs settings and for display of
 the CIFS statistics (started)
 
-p) implement support for security and trusted categories of xattrs
+k) implement support for security and trusted categories of xattrs
 (requires minor protocol extension) to enable better support for SELINUX
 
-q) Implement O_DIRECT flag on open (already supported on mount)
+l) Implement O_DIRECT flag on open (already supported on mount)
 
-r) Create UID mapping facility so server UIDs can be mapped on a per
+m) Create UID mapping facility so server UIDs can be mapped on a per
 mount or a per server basis to client UIDs or nobody if no mapping
 exists.  This is helpful when Unix extensions are negotiated to
 allow better permission checking when UIDs differ on the server
@@ -71,28 +57,29 @@ and client.  Add new protocol request to the CIFS protocol
 standard for asking the server for the corresponding name of a
 particular uid.
 
-s) Add support for CIFS Unix and also the newer POSIX extensions to the
-server side for Samba 4.
+n) DOS attrs - returned as pseudo-xattr in Samba format (check VFAT and NTFS for this too)
+
+o) mount check for unmatched uids
 
-t) In support for OS/2 (LANMAN 1.2 and LANMAN2.1 based SMB servers) 
-need to add ability to set time to server (utimes command)
+p) Add support for new vfs entry point for fallocate
 
-u) DOS attrs - returned as pseudo-xattr in Samba format (check VFAT and NTFS for this too)
+q) Add tools to take advantage of cifs/smb3 specific ioctls and features
+such as "CopyChunk" (fast server side file copy)
 
-v) mount check for unmatched uids
+r) encrypted file support
 
-w) Add support for new vfs entry point for fallocate
+s) improved stats gathering, tools (perhaps integration with nfsometer?)
 
-x) Fix Samba 3 server to handle Linux kernel aio so dbench with lots of 
-processes can proceed better in parallel (on the server)
+t) allow setting more NTFS/SMB3 file attributes remotely (currently limited to compressed
+file attribute via chflags)
 
-y) Fix Samba 3 to handle reads/writes over 127K (and remove the cifs mount
-restriction of wsize max being 127K) 
+u) mount helper GUI (to simplify the various configuration options on mount)
 
-KNOWN BUGS (updated April 24, 2007)
+
+KNOWN BUGS
 ====================================
 See http://bugzilla.samba.org - search on product "CifsVFS" for
-current bug list.
+current bug list.  Also check http://bugzilla.kernel.org (Product = File System, Component = CIFS)
 
 1) existing symbolic links (Windows reparse points) are recognized but
 can not be created remotely. They are implemented for Samba and those that
@@ -100,30 +87,18 @@ support the CIFS Unix extensions, although earlier versions of Samba
 overly restrict the pathnames.
 2) follow_link and readdir code does not follow dfs junctions
 but recognizes them
-3) create of new files to FAT partitions on Windows servers can
-succeed but still return access denied (appears to be Windows 
-server not cifs client problem) and has not been reproduced recently.
-NTFS partitions do not have this problem.
-4) Unix/POSIX capabilities are reset after reconnection, and affect
-a few fields in the tree connection but we do do not know which
-superblocks to apply these changes to.  We should probably walk
-the list of superblocks to set these.  Also need to check the
-flags on the second mount to the same share, and see if we
-can do the same trick that NFS does to remount duplicate shares.
 
 Misc testing to do
 ==================
 1) check out max path names and max path name components against various server
 types. Try nested symlinks (8 deep). Return max path name in stat -f information
 
-2) Modify file portion of ltp so it can run against a mounted network
-share and run it against cifs vfs in automated fashion.
+2) Improve xfstest's cifs enablement and adapt xfstests where needed to test
+cifs better
 
 3) Additional performance testing and optimization using iozone and similar - 
 there are some easy changes that can be done to parallelize sequential writes,
 and when signing is disabled to request larger read sizes (larger than 
 negotiated size) and send larger write sizes to modern servers.
 
-4) More exhaustively test against less common servers.  More testing
-against Windows 9x, Windows ME servers.
-
+4) More exhaustively test against less common servers
diff --git a/Documentation/i2c/busses/i2c-i801 b/Documentation/i2c/busses/i2c-i801
index adf5e33e8312..e9c803ea306d 100644
--- a/Documentation/i2c/busses/i2c-i801
+++ b/Documentation/i2c/busses/i2c-i801
@@ -25,6 +25,7 @@ Supported adapters:
   * Intel Avoton (SOC)
   * Intel Wellsburg (PCH)
   * Intel Coleto Creek (PCH)
+  * Intel Wildcat Point (PCH)
   * Intel Wildcat Point-LP (PCH)
   * Intel BayTrail (SOC)
    Datasheets: Publicly available at the Intel website
diff --git a/Documentation/i2c/i2c-stub b/Documentation/i2c/i2c-stub
index fa4b669c166b..a16924fbd289 100644
--- a/Documentation/i2c/i2c-stub
+++ b/Documentation/i2c/i2c-stub
@@ -2,9 +2,9 @@ MODULE: i2c-stub
 
 DESCRIPTION:
 
-This module is a very simple fake I2C/SMBus driver.  It implements five
+This module is a very simple fake I2C/SMBus driver.  It implements six
 types of SMBus commands: write quick, (r/w) byte, (r/w) byte data, (r/w)
-word data, and (r/w) I2C block data.
+word data, (r/w) I2C block data, and (r/w) SMBus block data.
 
 You need to provide chip addresses as a module parameter when loading this
 driver, which will then only react to SMBus commands to these addresses.
@@ -19,6 +19,14 @@ A pointer register with auto-increment is implemented for all byte
 operations.  This allows for continuous byte reads like those supported by
 EEPROMs, among others.
 
+SMBus block command support is disabled by default, and must be enabled
+explicitly by setting the respective bits (0x03000000) in the functionality
+module parameter.
+
+SMBus block commands must be written to configure an SMBus command for
+SMBus block operations. Writes can be partial. Block read commands always
+return the number of bytes selected with the largest write so far.
+
 The typical use-case is like this:
         1. load this module
         2. use i2cset (from the i2c-tools project) to pre-load some data
@@ -39,15 +47,18 @@ unsigned long functionality:
         value 0x1f0000 would only enable the quick, byte and byte data
         commands.
 
+u8 bank_reg[10]
+u8 bank_mask[10]
+u8 bank_start[10]
+u8 bank_end[10]:
+        Optional bank settings. They tell which bits in which register
+        select the active bank, as well as the range of banked registers.
+
 CAVEATS:
 
 If your target driver polls some byte or word waiting for it to change, the
 stub could lock it up.  Use i2cset to unlock it.
 
-If the hardware for your driver has banked registers (e.g. Winbond sensors
-chips) this module will not work well - although it could be extended to
-support that pretty easily.
-
 If you spam it hard enough, printk can be lossy.  This module really wants
 something like relayfs.
 
diff --git a/Documentation/ioctl/00-INDEX b/Documentation/ioctl/00-INDEX
index d2fe4d4729ef..c1a925787950 100644
--- a/Documentation/ioctl/00-INDEX
+++ b/Documentation/ioctl/00-INDEX
@@ -1,5 +1,7 @@
 00-INDEX
         - this file
+botching-up-ioctls.txt
+        - how to avoid botching up ioctls
 cdrom.txt
         - summary of CDROM ioctl calls
 hdio.txt
diff --git a/Documentation/ioctl/botching-up-ioctls.txt b/Documentation/ioctl/botching-up-ioctls.txt
new file mode 100644
index 000000000000..45fe78c58019
--- /dev/null
+++ b/Documentation/ioctl/botching-up-ioctls.txt
@@ -0,0 +1,219 @@
+(How to avoid) Botching up ioctls
+=================================
+
+From: http://blog.ffwll.ch/2013/11/botching-up-ioctls.html
+
+By: Daniel Vetter, Copyright © 2013 Intel Corporation
+
+One clear insight kernel graphics hackers gained in the past few years is that
+trying to come up with a unified interface to manage the execution units and
+memory on completely different GPUs is a futile effort. So nowadays every
+driver has its own set of ioctls to allocate memory and submit work to the GPU.
+Which is nice, since there's no more insanity in the form of fake-generic, but
+actually only used once interfaces. But the clear downside is that there's much
+more potential to screw things up.
+
+To avoid repeating all the same mistakes again I've written up some of the
+lessons learned while botching the job for the drm/i915 driver. Most of these
+only cover technicalities and not the big-picture issues like what the command
+submission ioctl exactly should look like. Learning these lessons is probably
+something every GPU driver has to do on its own.
+
+
+Prerequisites
+-------------
+
+First the prerequisites. Without these you have already failed, because you
+will need to add a a 32-bit compat layer:
+
+ * Only use fixed sized integers. To avoid conflicts with typedefs in userspace
+   the kernel has special types like __u32, __s64. Use them.
+
+ * Align everything to the natural size and use explicit padding. 32-bit
+   platforms don't necessarily align 64-bit values to 64-bit boundaries, but
+   64-bit platforms do. So we always need padding to the natural size to get
+   this right.
+
+ * Pad the entire struct to a multiple of 64-bits - the structure size will
+   otherwise differ on 32-bit versus 64-bit. Having a different structure size
+   hurts when passing arrays of structures to the kernel, or if the kernel
+   checks the structure size, which e.g. the drm core does.
+
+ * Pointers are __u64, cast from/to a uintprt_t on the userspace side and
+   from/to a void __user * in the kernel. Try really hard not to delay this
+   conversion or worse, fiddle the raw __u64 through your code since that
+   diminishes the checking tools like sparse can provide.
+
+
+Basics
+------
+
+With the joys of writing a compat layer avoided we can take a look at the basic
+fumbles. Neglecting these will make backward and forward compatibility a real
+pain. And since getting things wrong on the first attempt is guaranteed you
+will have a second iteration or at least an extension for any given interface.
+
+ * Have a clear way for userspace to figure out whether your new ioctl or ioctl
+   extension is supported on a given kernel. If you can't rely on old kernels
+   rejecting the new flags/modes or ioctls (since doing that was botched in the
+   past) then you need a driver feature flag or revision number somewhere.
+
+ * Have a plan for extending ioctls with new flags or new fields at the end of
+   the structure. The drm core checks the passed-in size for each ioctl call
+   and zero-extends any mismatches between kernel and userspace. That helps,
+   but isn't a complete solution since newer userspace on older kernels won't
+   notice that the newly added fields at the end get ignored. So this still
+   needs a new driver feature flags.
+
+ * Check all unused fields and flags and all the padding for whether it's 0,
+   and reject the ioctl if that's not the case. Otherwise your nice plan for
+   future extensions is going right down the gutters since someone will submit
+   an ioctl struct with random stack garbage in the yet unused parts. Which
+   then bakes in the ABI that those fields can never be used for anything else
+   but garbage.
+
+ * Have simple testcases for all of the above.
+
+
+Fun with Error Paths
+--------------------
+
+Nowadays we don't have any excuse left any more for drm drivers being neat
+little root exploits. This means we both need full input validation and solid
+error handling paths - GPUs will die eventually in the oddmost corner cases
+anyway:
+
+ * The ioctl must check for array overflows. Also it needs to check for
+   over/underflows and clamping issues of integer values in general. The usual
+   example is sprite positioning values fed directly into the hardware with the
+   hardware just having 12 bits or so. Works nicely until some odd display
+   server doesn't bother with clamping itself and the cursor wraps around the
+   screen.
+
+ * Have simple testcases for every input validation failure case in your ioctl.
+   Check that the error code matches your expectations. And finally make sure
+   that you only test for one single error path in each subtest by submitting
+   otherwise perfectly valid data. Without this an earlier check might reject
+   the ioctl already and shadow the codepath you actually want to test, hiding
+   bugs and regressions.
+
+ * Make all your ioctls restartable. First X really loves signals and second
+   this will allow you to test 90% of all error handling paths by just
+   interrupting your main test suite constantly with signals. Thanks to X's
+   love for signal you'll get an excellent base coverage of all your error
+   paths pretty much for free for graphics drivers. Also, be consistent with
+   how you handle ioctl restarting - e.g. drm has a tiny drmIoctl helper in its
+   userspace library. The i915 driver botched this with the set_tiling ioctl,
+   now we're stuck forever with some arcane semantics in both the kernel and
+   userspace.
+
+ * If you can't make a given codepath restartable make a stuck task at least
+   killable. GPUs just die and your users won't like you more if you hang their
+   entire box (by means of an unkillable X process). If the state recovery is
+   still too tricky have a timeout or hangcheck safety net as a last-ditch
+   effort in case the hardware has gone bananas.
+
+ * Have testcases for the really tricky corner cases in your error recovery code
+   - it's way too easy to create a deadlock between your hangcheck code and
+   waiters.
+
+
+Time, Waiting and Missing it
+----------------------------
+
+GPUs do most everything asynchronously, so we have a need to time operations and
+wait for oustanding ones. This is really tricky business; at the moment none of
+the ioctls supported by the drm/i915 get this fully right, which means there's
+still tons more lessons to learn here.
+
+ * Use CLOCK_MONOTONIC as your reference time, always. It's what alsa, drm and
+   v4l use by default nowadays. But let userspace know which timestamps are
+   derived from different clock domains like your main system clock (provided
+   by the kernel) or some independent hardware counter somewhere else. Clocks
+   will mismatch if you look close enough, but if performance measuring tools
+   have this information they can at least compensate. If your userspace can
+   get at the raw values of some clocks (e.g. through in-command-stream
+   performance counter sampling instructions) consider exposing those also.
+
+ * Use __s64 seconds plus __u64 nanoseconds to specify time. It's not the most
+   convenient time specification, but it's mostly the standard.
+
+ * Check that input time values are normalized and reject them if not. Note
+   that the kernel native struct ktime has a signed integer for both seconds
+   and nanoseconds, so beware here.
+
+ * For timeouts, use absolute times. If you're a good fellow and made your
+   ioctl restartable relative timeouts tend to be too coarse and can
+   indefinitely extend your wait time due to rounding on each restart.
+   Especially if your reference clock is something really slow like the display
+   frame counter. With a spec laywer hat on this isn't a bug since timeouts can
+   always be extended - but users will surely hate you if their neat animations
+   starts to stutter due to this.
+
+ * Consider ditching any synchronous wait ioctls with timeouts and just deliver
+   an asynchronous event on a pollable file descriptor. It fits much better
+   into event driven applications' main loop.
+
+ * Have testcases for corner-cases, especially whether the return values for
+   already-completed events, successful waits and timed-out waits are all sane
+   and suiting to your needs.
+
+
+Leaking Resources, Not
+----------------------
+
+A full-blown drm driver essentially implements a little OS, but specialized to
+the given GPU platforms. This means a driver needs to expose tons of handles
+for different objects and other resources to userspace. Doing that right
+entails its own little set of pitfalls:
+
+ * Always attach the lifetime of your dynamically created resources to the
+   lifetime of a file descriptor. Consider using a 1:1 mapping if your resource
+   needs to be shared across processes -  fd-passing over unix domain sockets
+   also simplifies lifetime management for userspace.
+
+ * Always have O_CLOEXEC support.
+
+ * Ensure that you have sufficient insulation between different clients. By
+   default pick a private per-fd namespace which forces any sharing to be done
+   explictly. Only go with a more global per-device namespace if the objects
+   are truly device-unique. One counterexample in the drm modeset interfaces is
+   that the per-device modeset objects like connectors share a namespace with
+   framebuffer objects, which mostly are not shared at all. A separate
+   namespace, private by default, for framebuffers would have been more
+   suitable.
+
+ * Think about uniqueness requirements for userspace handles. E.g. for most drm
+   drivers it's a userspace bug to submit the same object twice in the same
+   command submission ioctl. But then if objects are shareable userspace needs
+   to know whether it has seen an imported object from a different process
+   already or not. I haven't tried this myself yet due to lack of a new class
+   of objects, but consider using inode numbers on your shared file descriptors
+   as unique identifiers - it's how real files are told apart, too.
+   Unfortunately this requires a full-blown virtual filesystem in the kernel.
+
+
+Last, but not Least
+-------------------
+
+Not every problem needs a new ioctl:
+
+ * Think hard whether you really want a driver-private interface. Of course
+   it's much quicker to push a driver-private interface than engaging in
+   lengthy discussions for a more generic solution. And occasionally doing a
+   private interface to spearhead a new concept is what's required. But in the
+   end, once the generic interface comes around you'll end up maintainer two
+   interfaces. Indefinitely.
+
+ * Consider other interfaces than ioctls. A sysfs attribute is much better for
+   per-device settings, or for child objects with fairly static lifetimes (like
+   output connectors in drm with all the detection override attributes). Or
+   maybe only your testsuite needs this interface, and then debugfs with its
+   disclaimer of not having a stable ABI would be better.
+
+Finally, the name of the game is to get it right on the first attempt, since if
+your driver proves popular and your hardware platforms long-lived then you'll
+be stuck with a given ioctl essentially forever. You can try to deprecate
+horrible ioctls on newer iterations of your hardware, but generally it takes
+years to accomplish this. And then again years until the last user able to
+complain about regressions disappears, too.
diff --git a/Documentation/laptops/freefall.c b/Documentation/laptops/freefall.c
index d4173186555c..5e44b20b1848 100644
--- a/Documentation/laptops/freefall.c
+++ b/Documentation/laptops/freefall.c
@@ -29,16 +29,12 @@ static const char app_name[] = "FREE FALL";
 
 static int set_unload_heads_path(char *device)
 {
-        char devname[64];
-
         if (strlen(device) <= 5 || strncmp(device, "/dev/", 5) != 0)
                 return -EINVAL;
-        strncpy(devname, device + 5, sizeof(devname) - 1);
-        devname[sizeof(devname) - 1] = '\0';
         strncpy(device_path, device, sizeof(device_path) - 1);
 
         snprintf(unload_heads_path, sizeof(unload_heads_path) - 1,
-                                "/sys/block/%s/device/unload_heads", devname);
+                                "/sys/block/%s/device/unload_heads", device+5);
         return 0;
 }
 
diff --git a/arch/arc/include/asm/arcregs.h b/arch/arc/include/asm/arcregs.h
index 355cb470c2a4..372466b371bf 100644
--- a/arch/arc/include/asm/arcregs.h
+++ b/arch/arc/include/asm/arcregs.h
@@ -296,7 +296,7 @@ struct cpuinfo_arc_mmu {
 };
 
 struct cpuinfo_arc_cache {
-        unsigned int sz, line_len, assoc, ver;
+        unsigned int sz_k:8, line_len:8, assoc:4, ver:4, alias:1, vipt:1, pad:6;
 };
 
 struct cpuinfo_arc_ccm {
diff --git a/arch/arc/include/asm/irq.h b/arch/arc/include/asm/irq.h
index fb4efb648971..f38652fb2ed7 100644
--- a/arch/arc/include/asm/irq.h
+++ b/arch/arc/include/asm/irq.h
@@ -16,9 +16,13 @@
 #define TIMER0_IRQ      3
 #define TIMER1_IRQ      4
 
+#include <linux/interrupt.h>
 #include <asm-generic/irq.h>
 
 extern void arc_init_IRQ(void);
 void arc_local_timer_setup(void);
+void arc_request_percpu_irq(int irq, int cpu,
+                            irqreturn_t (*isr)(int irq, void *dev),
+                            const char *irq_nm, void *percpu_dev);
 
 #endif
diff --git a/arch/arc/include/asm/irqflags.h b/arch/arc/include/asm/irqflags.h
index cb7efc29f16f..587df8236e8b 100644
--- a/arch/arc/include/asm/irqflags.h
+++ b/arch/arc/include/asm/irqflags.h
@@ -131,24 +131,6 @@ static inline int arch_irqs_disabled(void)
         return arch_irqs_disabled_flags(arch_local_save_flags());
 }
 
-static inline void arch_mask_irq(unsigned int irq)
-{
-        unsigned int ienb;
-
-        ienb = read_aux_reg(AUX_IENABLE);
-        ienb &= ~(1 << irq);
-        write_aux_reg(AUX_IENABLE, ienb);
-}
-
-static inline void arch_unmask_irq(unsigned int irq)
-{
-        unsigned int ienb;
-
-        ienb = read_aux_reg(AUX_IENABLE);
-        ienb |= (1 << irq);
-        write_aux_reg(AUX_IENABLE, ienb);
-}
-
 #else
 
 #ifdef CONFIG_TRACE_IRQFLAGS
diff --git a/arch/arc/kernel/irq.c b/arch/arc/kernel/irq.c
index 7d653c0d0773..620ec2fe32a9 100644
--- a/arch/arc/kernel/irq.c
+++ b/arch/arc/kernel/irq.c
@@ -19,21 +19,16 @@
 
 /*
  * Early Hardware specific Interrupt setup
+ * -Platform independent, needed for each CPU (not foldable into init_IRQ)
  * -Called very early (start_kernel -> setup_arch -> setup_processor)
- * -Platform Independent (must for any ARC700)
- * -Needed for each CPU (hence not foldable into init_IRQ)
  *
  * what it does ?
- * -Disable all IRQs (on CPU side)
  * -Optionally, setup the High priority Interrupts as Level 2 IRQs
  */
 void arc_init_IRQ(void)
 {
         int level_mask = 0;
 
-        /* Disable all IRQs: enable them as devices request */
-        write_aux_reg(AUX_IENABLE, 0);
-
        /* setup any high priority Interrupts (Level2 in ARCompact jargon) */
         level_mask |= IS_ENABLED(CONFIG_ARC_IRQ3_LV2) << 3;
         level_mask |= IS_ENABLED(CONFIG_ARC_IRQ5_LV2) << 5;
@@ -60,20 +55,28 @@ void arc_init_IRQ(void)
  * below, per IRQ.
  */
 
-static void arc_mask_irq(struct irq_data *data)
+static void arc_irq_mask(struct irq_data *data)
 {
-        arch_mask_irq(data->irq);
+        unsigned int ienb;
+
+        ienb = read_aux_reg(AUX_IENABLE);
+        ienb &= ~(1 << data->irq);
+        write_aux_reg(AUX_IENABLE, ienb);
 }
 
-static void arc_unmask_irq(struct irq_data *data)
+static void arc_irq_unmask(struct irq_data *data)
 {
-        arch_unmask_irq(data->irq);
+        unsigned int ienb;
+
+        ienb = read_aux_reg(AUX_IENABLE);
+        ienb |= (1 << data->irq);
+        write_aux_reg(AUX_IENABLE, ienb);
 }
 
 static struct irq_chip onchip_intc = {
         .name           = "ARC In-core Intc",
-        .irq_mask       = arc_mask_irq,
-        .irq_unmask     = arc_unmask_irq,
+        .irq_mask       = arc_irq_mask,
+        .irq_unmask     = arc_irq_unmask,
 };
 
 static int arc_intc_domain_map(struct irq_domain *d, unsigned int irq,
@@ -150,6 +153,32 @@ void arch_do_IRQ(unsigned int irq, struct pt_regs *regs)
         set_irq_regs(old_regs);
 }
 
+void arc_request_percpu_irq(int irq, int cpu,
+                            irqreturn_t (*isr)(int irq, void *dev),
+                            const char *irq_nm,
+                            void *percpu_dev)
+{
+        /* Boot cpu calls request, all call enable */
+        if (!cpu) {
+                int rc;
+
+                /*
+                 * These 2 calls are essential to making percpu IRQ APIs work
+                 * Ideally these details could be hidden in irq chip map function
+                 * but the issue is IPIs IRQs being static (non-DT) and platform
+                 * specific, so we can't identify them there.
+                 */
+                irq_set_percpu_devid(irq);
+                irq_modify_status(irq, IRQ_NOAUTOEN, 0);  /* @irq, @clr, @set */
+
+                rc = request_percpu_irq(irq, isr, irq_nm, percpu_dev);
+                if (rc)
+                        panic("Percpu IRQ request failed for %d\n", irq);
+        }
+
+        enable_percpu_irq(irq, 0);
+}
+
 /*
  * arch_local_irq_enable - Enable interrupts.
  *
diff --git a/arch/arc/kernel/signal.c b/arch/arc/kernel/signal.c
index 7e95e1a86510..cb3142a2d40b 100644
--- a/arch/arc/kernel/signal.c
+++ b/arch/arc/kernel/signal.c
@@ -141,17 +141,13 @@ badframe:
 /*
  * Determine which stack to use..
  */
-static inline void __user *get_sigframe(struct k_sigaction *ka,
+static inline void __user *get_sigframe(struct ksignal *ksig,
                                         struct pt_regs *regs,
                                         unsigned long framesize)
 {
-        unsigned long sp = regs->sp;
+        unsigned long sp = sigsp(regs->sp, ksig);
         void __user *frame;
 
-        /* This is the X/Open sanctioned signal stack switching */
-        if ((ka->sa.sa_flags & SA_ONSTACK) && !sas_ss_flags(sp))
-                sp = current->sas_ss_sp + current->sas_ss_size;
-
         /* No matter what happens, 'sp' must be word
          * aligned otherwise nasty things could happen
          */
@@ -179,14 +175,13 @@ static inline int map_sig(int sig)
 }
 
 static int
-setup_rt_frame(int signo, struct k_sigaction *ka, siginfo_t *info,
-               sigset_t *set, struct pt_regs *regs)
+setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs)
 {
         struct rt_sigframe __user *sf;
         unsigned int magic = 0;
         int err = 0;
 
-        sf = get_sigframe(ka, regs, sizeof(struct rt_sigframe));
+        sf = get_sigframe(ksig, regs, sizeof(struct rt_sigframe));
         if (!sf)
                 return 1;
 
@@ -205,8 +200,8 @@ setup_rt_frame(int signo, struct k_sigaction *ka, siginfo_t *info,
          *  #2: struct siginfo
          *  #3: struct ucontext (completely populated)
          */
-        if (unlikely(ka->sa.sa_flags & SA_SIGINFO)) {
-                err |= copy_siginfo_to_user(&sf->info, info);
+        if (unlikely(ksig->ka.sa.sa_flags & SA_SIGINFO)) {
+                err |= copy_siginfo_to_user(&sf->info, &ksig->info);
                 err |= __put_user(0, &sf->uc.uc_flags);
                 err |= __put_user(NULL, &sf->uc.uc_link);
                 err |= __save_altstack(&sf->uc.uc_stack, regs->sp);
@@ -227,16 +222,16 @@ setup_rt_frame(int signo, struct k_sigaction *ka, siginfo_t *info,
                 return err;
 
         /* #1 arg to the user Signal handler */
-        regs->r0 = map_sig(signo);
+        regs->r0 = map_sig(ksig->sig);
 
         /* setup PC of user space signal handler */
-        regs->ret = (unsigned long)ka->sa.sa_handler;
+        regs->ret = (unsigned long)ksig->ka.sa.sa_handler;
 
         /*
          * handler returns using sigreturn stub provided already by userpsace
          */
-        BUG_ON(!(ka->sa.sa_flags & SA_RESTORER));
-        regs->blink = (unsigned long)ka->sa.sa_restorer;
+        BUG_ON(!(ksig->ka.sa.sa_flags & SA_RESTORER));
+        regs->blink = (unsigned long)ksig->ka.sa.sa_restorer;
 
         /* User Stack for signal handler will be above the frame just carved */
         regs->sp = (unsigned long)sf;
@@ -298,38 +293,30 @@ static void arc_restart_syscall(struct k_sigaction *ka, struct pt_regs *regs)
  * OK, we're invoking a handler
  */
 static void
-handle_signal(unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
-              struct pt_regs *regs)
+handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
         sigset_t *oldset = sigmask_to_save();
         int ret;
 
         /* Set up the stack frame */
-        ret = setup_rt_frame(sig, ka, info, oldset, regs);
+        ret = setup_rt_frame(ksig, oldset, regs);
 
-        if (ret)
-                force_sigsegv(sig, current);
-        else
-                signal_delivered(sig, info, ka, regs, 0);
+        signal_setup_done(ret, ksig, 0);
 }
 
 void do_signal(struct pt_regs *regs)
 {
-        struct k_sigaction ka;
-        siginfo_t info;
-        int signr;
+        struct ksignal ksig;
         int restart_scall;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-
         restart_scall = in_syscall(regs) && syscall_restartable(regs);
 
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 if (restart_scall) {
-                        arc_restart_syscall(&ka, regs);
+                        arc_restart_syscall(&ksig.ka, regs);
                         syscall_wont_restart(regs);     /* No more restarts */
                 }
-                handle_signal(signr, &ka, &info, regs);
+                handle_signal(&ksig, regs);
                 return;
         }
 
diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c
index c802bb500602..dcd317c47d09 100644
--- a/arch/arc/kernel/smp.c
+++ b/arch/arc/kernel/smp.c
@@ -12,23 +12,15 @@
  *    -- Initial Write (Borrowed heavily from ARM)
  */
 
-#include <linux/module.h>
-#include <linux/init.h>
 #include <linux/spinlock.h>
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 #include <linux/profile.h>
-#include <linux/errno.h>
-#include <linux/err.h>
 #include <linux/mm.h>
 #include <linux/cpu.h>
-#include <linux/smp.h>
 #include <linux/irq.h>
-#include <linux/delay.h>
 #include <linux/atomic.h>
-#include <linux/percpu.h>
 #include <linux/cpumask.h>
-#include <linux/spinlock_types.h>
 #include <linux/reboot.h>
 #include <asm/processor.h>
 #include <asm/setup.h>
@@ -136,7 +128,7 @@ void start_kernel_secondary(void)
         pr_info("## CPU%u LIVE ##: Executing Code...\n", cpu);
 
         if (machine_desc->init_smp)
-                machine_desc->init_smp(smp_processor_id());
+                machine_desc->init_smp(cpu);
 
         arc_local_timer_setup();
 
@@ -338,18 +330,11 @@ irqreturn_t do_IPI(int irq, void *dev_id)
  */
 static DEFINE_PER_CPU(int, ipi_dev);
 
-static struct irqaction arc_ipi_irq = {
-        .name    = "IPI Interrupt",
-        .flags   = IRQF_PERCPU,
-        .handler = do_IPI,
-};
-
 int smp_ipi_irq_setup(int cpu, int irq)
 {
-        if (!cpu)
-                return setup_irq(irq, &arc_ipi_irq);
-        else
-                arch_unmask_irq(irq);
+        int *dev = per_cpu_ptr(&ipi_dev, cpu);
+
+        arc_request_percpu_irq(irq, cpu, do_IPI, "IPI Interrupt", dev);
 
         return 0;
 }
diff --git a/arch/arc/kernel/time.c b/arch/arc/kernel/time.c
index 36c2aa99436f..dbe74f418019 100644
--- a/arch/arc/kernel/time.c
+++ b/arch/arc/kernel/time.c
@@ -144,12 +144,12 @@ static struct clocksource arc_counter = {
 /********** Clock Event Device *********/
 
 /*
- * Arm the timer to interrupt after @limit cycles
+ * Arm the timer to interrupt after @cycles
  * The distinction for oneshot/periodic is done in arc_event_timer_ack() below
  */
-static void arc_timer_event_setup(unsigned int limit)
+static void arc_timer_event_setup(unsigned int cycles)
 {
-        write_aux_reg(ARC_REG_TIMER0_LIMIT, limit);
+        write_aux_reg(ARC_REG_TIMER0_LIMIT, cycles);
         write_aux_reg(ARC_REG_TIMER0_CNT, 0);   /* start from 0 */
 
         write_aux_reg(ARC_REG_TIMER0_CTRL, TIMER_CTRL_IE | TIMER_CTRL_NH);
@@ -168,6 +168,10 @@ static void arc_clkevent_set_mode(enum clock_event_mode mode,
 {
         switch (mode) {
         case CLOCK_EVT_MODE_PERIODIC:
+                /*
+                 * At X Hz, 1 sec = 1000ms -> X cycles;
+                 *                    10ms -> X / 100 cycles
+                 */
                 arc_timer_event_setup(arc_get_core_freq() / HZ);
                 break;
         case CLOCK_EVT_MODE_ONESHOT:
@@ -210,12 +214,6 @@ static irqreturn_t timer_irq_handler(int irq, void *dev_id)
         return IRQ_HANDLED;
 }
 
-static struct irqaction arc_timer_irq = {
-        .name    = "Timer0 (clock-evt-dev)",
-        .flags   = IRQF_TIMER | IRQF_PERCPU,
-        .handler = timer_irq_handler,
-};
-
 /*
  * Setup the local event timer for @cpu
  */
@@ -228,15 +226,9 @@ void arc_local_timer_setup()
         clockevents_config_and_register(evt, arc_get_core_freq(),
                                         0, ARC_TIMER_MAX);
 
-        /*
-         * setup the per-cpu timer IRQ handler - for all cpus
-         * For non boot CPU explicitly unmask at intc
-         * setup_irq() -> .. -> irq_startup() already does this on boot-cpu
-         */
-        if (!cpu)
-                setup_irq(TIMER0_IRQ, &arc_timer_irq);
-        else
-                arch_unmask_irq(TIMER0_IRQ);
+        /* setup the per-cpu timer IRQ handler - for all cpus */
+        arc_request_percpu_irq(TIMER0_IRQ, cpu, timer_irq_handler,
+                               "Timer0 (per-cpu-tick)", evt);
 }
 
 /*
diff --git a/arch/arc/mm/cache_arc700.c b/arch/arc/mm/cache_arc700.c
index 353b202c37c9..4670afc3b971 100644
--- a/arch/arc/mm/cache_arc700.c
+++ b/arch/arc/mm/cache_arc700.c
@@ -77,21 +77,19 @@ char *arc_cache_mumbojumbo(int c, char *buf, int len)
 {
         int n = 0;
 
-#define PR_CACHE(p, enb, str)                                           \
-{                                                                       \
+#define PR_CACHE(p, cfg, str)                                           \
         if (!(p)->ver)                                                  \
                 n += scnprintf(buf + n, len - n, str"\t\t: N/A\n");     \
         else                                                            \
                 n += scnprintf(buf + n, len - n,                        \
-                        str"\t\t: (%uK) VIPT, %dway set-asc, %ub Line %s\n", \
-                        TO_KB((p)->sz), (p)->assoc, (p)->line_len,      \
-                        enb ?  "" : "DISABLED (kernel-build)");         \
-}
+                        str"\t\t: %uK, %dway/set, %uB Line, %s%s%s\n",  \
+                        (p)->sz_k, (p)->assoc, (p)->line_len,           \
+                        (p)->vipt ? "VIPT" : "PIPT",                    \
+                        (p)->alias ? " aliasing" : "",                  \
+                        IS_ENABLED(cfg) ? "" : " (not used)");
 
-        PR_CACHE(&cpuinfo_arc700[c].icache, IS_ENABLED(CONFIG_ARC_HAS_ICACHE),
-                        "I-Cache");
-        PR_CACHE(&cpuinfo_arc700[c].dcache, IS_ENABLED(CONFIG_ARC_HAS_DCACHE),
-                        "D-Cache");
+        PR_CACHE(&cpuinfo_arc700[c].icache, CONFIG_ARC_HAS_ICACHE, "I-Cache");
+        PR_CACHE(&cpuinfo_arc700[c].dcache, CONFIG_ARC_HAS_DCACHE, "D-Cache");
 
         return buf;
 }
@@ -116,20 +114,31 @@ void read_decode_cache_bcr(void)
         p_ic = &cpuinfo_arc700[cpu].icache;
         READ_BCR(ARC_REG_IC_BCR, ibcr);
 
+        if (!ibcr.ver)
+                goto dc_chk;
+
         BUG_ON(ibcr.config != 3);
         p_ic->assoc = 2;                /* Fixed to 2w set assoc */
         p_ic->line_len = 8 << ibcr.line_len;
-        p_ic->sz = 0x200 << ibcr.sz;
+        p_ic->sz_k = 1 << (ibcr.sz - 1);
         p_ic->ver = ibcr.ver;
+        p_ic->vipt = 1;
+        p_ic->alias = p_ic->sz_k/p_ic->assoc/TO_KB(PAGE_SIZE) > 1;
 
+dc_chk:
         p_dc = &cpuinfo_arc700[cpu].dcache;
         READ_BCR(ARC_REG_DC_BCR, dbcr);
 
+        if (!dbcr.ver)
+                return;
+
         BUG_ON(dbcr.config != 2);
         p_dc->assoc = 4;                /* Fixed to 4w set assoc */
         p_dc->line_len = 16 << dbcr.line_len;
-        p_dc->sz = 0x200 << dbcr.sz;
+        p_dc->sz_k = 1 << (dbcr.sz - 1);
         p_dc->ver = dbcr.ver;
+        p_dc->vipt = 1;
+        p_dc->alias = p_dc->sz_k/p_dc->assoc/TO_KB(PAGE_SIZE) > 1;
 }
 
 /*
@@ -142,14 +151,16 @@ void read_decode_cache_bcr(void)
 void arc_cache_init(void)
 {
         unsigned int __maybe_unused cpu = smp_processor_id();
-        struct cpuinfo_arc_cache __maybe_unused *ic, __maybe_unused *dc;
         char str[256];
 
         printk(arc_cache_mumbojumbo(0, str, sizeof(str)));
 
-#ifdef CONFIG_ARC_HAS_ICACHE
-        ic = &cpuinfo_arc700[cpu].icache;
-        if (ic->ver) {
+        if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) {
+                struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache;
+
+                if (!ic->ver)
+                        panic("cache support enabled but non-existent cache\n");
+
                 if (ic->line_len != L1_CACHE_BYTES)
                         panic("ICache line [%d] != kernel Config [%d]",
                               ic->line_len, L1_CACHE_BYTES);
@@ -158,26 +169,26 @@ void arc_cache_init(void)
                         panic("Cache ver [%d] doesn't match MMU ver [%d]\n",
                               ic->ver, CONFIG_ARC_MMU_VER);
         }
-#endif
 
-#ifdef CONFIG_ARC_HAS_DCACHE
-        dc = &cpuinfo_arc700[cpu].dcache;
-        if (dc->ver) {
-                unsigned int dcache_does_alias;
+        if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) {
+                struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache;
+                int handled;
+
+                if (!dc->ver)
+                        panic("cache support enabled but non-existent cache\n");
 
                 if (dc->line_len != L1_CACHE_BYTES)
                         panic("DCache line [%d] != kernel Config [%d]",
                               dc->line_len, L1_CACHE_BYTES);
 
                 /* check for D-Cache aliasing */
-                dcache_does_alias = (dc->sz / dc->assoc) > PAGE_SIZE;
+                handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING);
 
-                if (dcache_does_alias && !cache_is_vipt_aliasing())
+                if (dc->alias && !handled)
                         panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n");
-                else if (!dcache_does_alias && cache_is_vipt_aliasing())
+                else if (!dc->alias && handled)
                         panic("Don't need CONFIG_ARC_CACHE_VIPT_ALIASING\n");
         }
-#endif
 }
 
 #define OP_INV          0x1
@@ -255,10 +266,32 @@ static inline void __cache_line_loop(unsigned long paddr, unsigned long vaddr,
  * Machine specific helpers for Entire D-Cache or Per Line ops
  */
 
-static inline void wait_for_flush(void)
+static unsigned int __before_dc_op(const int op)
+{
+        unsigned int reg = reg;
+
+        if (op == OP_FLUSH_N_INV) {
+                /* Dcache provides 2 cmd: FLUSH or INV
+                 * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE
+                 * flush-n-inv is achieved by INV cmd but with IM=1
+                 * So toggle INV sub-mode depending on op request and default
+                 */
+                reg = read_aux_reg(ARC_REG_DC_CTRL);
+                write_aux_reg(ARC_REG_DC_CTRL, reg | DC_CTRL_INV_MODE_FLUSH)
+                        ;
+        }
+
+        return reg;
+}
+
+static void __after_dc_op(const int op, unsigned int reg)
 {
-        while (read_aux_reg(ARC_REG_DC_CTRL) & DC_CTRL_FLUSH_STATUS)
-                ;
+        if (op & OP_FLUSH)      /* flush / flush-n-inv both wait */
+                while (read_aux_reg(ARC_REG_DC_CTRL) & DC_CTRL_FLUSH_STATUS);
+
+        /* Switch back to default Invalidate mode */
+        if (op == OP_FLUSH_N_INV)
+                write_aux_reg(ARC_REG_DC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH);
 }
 
 /*
@@ -269,18 +302,10 @@ static inline void wait_for_flush(void)
  */
 static inline void __dc_entire_op(const int cacheop)
 {
-        unsigned int tmp = tmp;
+        unsigned int ctrl_reg;
         int aux;
 
-        if (cacheop == OP_FLUSH_N_INV) {
-                /* Dcache provides 2 cmd: FLUSH or INV
-                 * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE
-                 * flush-n-inv is achieved by INV cmd but with IM=1
-                 * Default INV sub-mode is DISCARD, which needs to be toggled
-                 */
-                tmp = read_aux_reg(ARC_REG_DC_CTRL);
-                write_aux_reg(ARC_REG_DC_CTRL, tmp | DC_CTRL_INV_MODE_FLUSH);
-        }
+        ctrl_reg = __before_dc_op(cacheop);
 
         if (cacheop & OP_INV)   /* Inv or flush-n-inv use same cmd reg */
                 aux = ARC_REG_DC_IVDC;
@@ -289,12 +314,7 @@ static inline void __dc_entire_op(const int cacheop)
 
         write_aux_reg(aux, 0x1);
 
-        if (cacheop & OP_FLUSH) /* flush / flush-n-inv both wait */
-                wait_for_flush();
-
-        /* Switch back the DISCARD ONLY Invalidate mode */
-        if (cacheop == OP_FLUSH_N_INV)
-                write_aux_reg(ARC_REG_DC_CTRL, tmp & ~DC_CTRL_INV_MODE_FLUSH);
+        __after_dc_op(cacheop, ctrl_reg);
 }
 
 /* For kernel mappings cache operation: index is same as paddr */
@@ -306,29 +326,16 @@ static inline void __dc_entire_op(const int cacheop)
 static inline void __dc_line_op(unsigned long paddr, unsigned long vaddr,
                                 unsigned long sz, const int cacheop)
 {
-        unsigned long flags, tmp = tmp;
+        unsigned long flags;
+        unsigned int ctrl_reg;
 
         local_irq_save(flags);
 
-        if (cacheop == OP_FLUSH_N_INV) {
-                /*
-                 * Dcache provides 2 cmd: FLUSH or INV
-                 * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE
-                 * flush-n-inv is achieved by INV cmd but with IM=1
-                 * Default INV sub-mode is DISCARD, which needs to be toggled
-                 */
-                tmp = read_aux_reg(ARC_REG_DC_CTRL);
-                write_aux_reg(ARC_REG_DC_CTRL, tmp | DC_CTRL_INV_MODE_FLUSH);
-        }
+        ctrl_reg = __before_dc_op(cacheop);
 
         __cache_line_loop(paddr, vaddr, sz, cacheop);
 
-        if (cacheop & OP_FLUSH) /* flush / flush-n-inv both wait */
-                wait_for_flush();
-
-        /* Switch back the DISCARD ONLY Invalidate mode */
-        if (cacheop == OP_FLUSH_N_INV)
-                write_aux_reg(ARC_REG_DC_CTRL, tmp & ~DC_CTRL_INV_MODE_FLUSH);
+        __after_dc_op(cacheop, ctrl_reg);
 
         local_irq_restore(flags);
 }
@@ -389,8 +396,16 @@ static inline void __dc_line_op(unsigned long paddr, unsigned long vaddr,
 /***********************************************************
  * Machine specific helper for per line I-Cache invalidate.
  */
-static void __ic_line_inv_vaddr_local(unsigned long paddr, unsigned long vaddr,
-                                unsigned long sz)
+
+static inline void __ic_entire_inv(void)
+{
+        write_aux_reg(ARC_REG_IC_IVIC, 1);
+        read_aux_reg(ARC_REG_IC_CTRL);  /* blocks */
+}
+
+static inline void
+__ic_line_inv_vaddr_local(unsigned long paddr, unsigned long vaddr,
+                          unsigned long sz)
 {
         unsigned long flags;
 
@@ -399,30 +414,39 @@ static void __ic_line_inv_vaddr_local(unsigned long paddr, unsigned long vaddr,
         local_irq_restore(flags);
 }
 
-static inline void __ic_entire_inv(void)
-{
-        write_aux_reg(ARC_REG_IC_IVIC, 1);
-        read_aux_reg(ARC_REG_IC_CTRL);  /* blocks */
-}
+#ifndef CONFIG_SMP
+
+#define __ic_line_inv_vaddr(p, v, s)    __ic_line_inv_vaddr_local(p, v, s)
 
-struct ic_line_inv_vaddr_ipi {
+#else
+
+struct ic_inv_args {
         unsigned long paddr, vaddr;
         int sz;
 };
 
 static void __ic_line_inv_vaddr_helper(void *info)
 {
-        struct ic_line_inv_vaddr_ipi *ic_inv = (struct ic_line_inv_vaddr_ipi*) info;
+        struct ic_inv *ic_inv_args = (struct ic_inv_args *) info;
+
         __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz);
 }
 
 static void __ic_line_inv_vaddr(unsigned long paddr, unsigned long vaddr,
                                 unsigned long sz)
 {
-        struct ic_line_inv_vaddr_ipi ic_inv = { paddr, vaddr , sz};
+        struct ic_inv_args ic_inv = {
+                .paddr = paddr,
+                .vaddr = vaddr,
+                .sz    = sz
+        };
+
         on_each_cpu(__ic_line_inv_vaddr_helper, &ic_inv, 1);
 }
-#else
+
+#endif  /* CONFIG_SMP */
+
+#else   /* !CONFIG_ARC_HAS_ICACHE */
 
 #define __ic_entire_inv()
 #define __ic_line_inv_vaddr(pstart, vstart, sz)
diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c
index 9c69552350c4..6f7e3a68803a 100644
--- a/arch/arc/mm/fault.c
+++ b/arch/arc/mm/fault.c
@@ -159,7 +159,6 @@ good_area:
                 return;
         }
 
-        /* TBD: switch to pagefault_out_of_memory() */
         if (fault & VM_FAULT_OOM)
                 goto out_of_memory;
         else if (fault & VM_FAULT_SIGBUS)
diff --git a/arch/arc/mm/tlbex.S b/arch/arc/mm/tlbex.S
index 79bfc81358c9..d572f1c2c724 100644
--- a/arch/arc/mm/tlbex.S
+++ b/arch/arc/mm/tlbex.S
@@ -220,9 +220,9 @@ ex_saved_reg1:
 
 .macro CONV_PTE_TO_TLB
         and    r3, r0, PTE_BITS_RWX     ;       r w x
-        lsl    r2, r3, 3                ; r w x 0 0 0
+        lsl    r2, r3, 3                ; r w x 0 0 0 (GLOBAL, kernel only)
         and.f  0,  r0, _PAGE_GLOBAL
-        or.z   r2, r2, r3               ; r w x r w x
+        or.z   r2, r2, r3               ; r w x r w x (!GLOBAL, user page)
 
         and r3, r0, PTE_BITS_NON_RWX_IN_PD1 ; Extract PFN+cache bits from PTE
         or  r3, r3, r2
diff --git a/arch/arc/plat-arcfpga/Makefile b/arch/arc/plat-arcfpga/Makefile
index 4d1bddc34b5b..66fd0ecd68b3 100644
--- a/arch/arc/plat-arcfpga/Makefile
+++ b/arch/arc/plat-arcfpga/Makefile
@@ -8,5 +8,5 @@
 
 KBUILD_CFLAGS   += -Iarch/arc/plat-arcfpga/include
 
-obj-y := platform.o irq.o
+obj-y := platform.o
 obj-$(CONFIG_ISS_SMP_EXTN)              += smp.o
diff --git a/arch/arc/plat-arcfpga/include/plat/irq.h b/arch/arc/plat-arcfpga/include/plat/irq.h
index 6adbc53c3a5b..2c9dea690ac4 100644
--- a/arch/arc/plat-arcfpga/include/plat/irq.h
+++ b/arch/arc/plat-arcfpga/include/plat/irq.h
@@ -24,6 +24,4 @@
 #define IDU_INTERRUPT_0 16
 #endif
 
-extern void __init plat_fpga_init_IRQ(void);
-
 #endif
diff --git a/arch/arc/plat-arcfpga/irq.c b/arch/arc/plat-arcfpga/irq.c
deleted file mode 100644
index d2215fd889c2..000000000000
--- a/arch/arc/plat-arcfpga/irq.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * ARC FPGA Platform IRQ hookups
- *
- * Copyright (C) 2012 Synopsys, Inc. (www.synopsys.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/interrupt.h>
-#include <plat/irq.h>
-
-void __init plat_fpga_init_IRQ(void)
-{
-        /*
-         * SMP Hack because UART IRQ hardwired to cpu0 (boot-cpu) but if the
-         * request_irq() comes from any other CPU, the low level IRQ unamsking
-         * essential for getting Interrupts won't be enabled on cpu0, locking
-         * up the UART state machine.
-         */
-#ifdef CONFIG_SMP
-        arch_unmask_irq(UART0_IRQ);
-#endif
-}
diff --git a/arch/arc/plat-arcfpga/platform.c b/arch/arc/plat-arcfpga/platform.c
index b8d0d456627f..1038949a99a1 100644
--- a/arch/arc/plat-arcfpga/platform.c
+++ b/arch/arc/plat-arcfpga/platform.c
@@ -57,7 +57,6 @@ MACHINE_START(ANGEL4, "angel4")
         .dt_compat      = aa4_compat,
         .init_early     = plat_fpga_early_init,
         .init_machine   = plat_fpga_populate_dev,
-        .init_irq       = plat_fpga_init_IRQ,
 #ifdef CONFIG_ISS_SMP_EXTN
         .init_smp       = iss_model_init_smp,
 #endif
@@ -72,7 +71,6 @@ MACHINE_START(ML509, "ml509")
         .dt_compat      = ml509_compat,
         .init_early     = plat_fpga_early_init,
         .init_machine   = plat_fpga_populate_dev,
-        .init_irq       = plat_fpga_init_IRQ,
 #ifdef CONFIG_SMP
         .init_smp       = iss_model_init_smp,
 #endif
@@ -87,5 +85,4 @@ MACHINE_START(NSIMOSCI, "nsimosci")
         .dt_compat      = nsimosci_compat,
         .init_early     = NULL,
         .init_machine   = plat_fpga_populate_dev,
-        .init_irq       = NULL,
 MACHINE_END
diff --git a/arch/arm/include/asm/unistd.h b/arch/arm/include/asm/unistd.h
index 21ca0cebcab0..32640c431a08 100644
--- a/arch/arm/include/asm/unistd.h
+++ b/arch/arm/include/asm/unistd.h
@@ -19,7 +19,7 @@
  * This may need to be greater than __NR_last_syscall+1 in order to
  * account for the padding in the syscall table
  */
-#define __NR_syscalls  (384)
+#define __NR_syscalls  (388)
 
 /*
  * *NOTE*: This is a ghost syscall private to the kernel.  Only the
diff --git a/arch/arm/include/uapi/asm/unistd.h b/arch/arm/include/uapi/asm/unistd.h
index 767ea204334e..3aaa75cae90c 100644
--- a/arch/arm/include/uapi/asm/unistd.h
+++ b/arch/arm/include/uapi/asm/unistd.h
@@ -410,6 +410,8 @@
 #define __NR_sched_getattr              (__NR_SYSCALL_BASE+381)
 #define __NR_renameat2                  (__NR_SYSCALL_BASE+382)
 #define __NR_seccomp                    (__NR_SYSCALL_BASE+383)
+#define __NR_getrandom                  (__NR_SYSCALL_BASE+384)
+#define __NR_memfd_create               (__NR_SYSCALL_BASE+385)
 
 /*
  * The following SWIs are ARM private.
diff --git a/arch/arm/kernel/calls.S b/arch/arm/kernel/calls.S
index bea85f97f363..9f899d8fdcca 100644
--- a/arch/arm/kernel/calls.S
+++ b/arch/arm/kernel/calls.S
@@ -393,6 +393,8 @@
                 CALL(sys_sched_getattr)
                 CALL(sys_renameat2)
                 CALL(sys_seccomp)
+                CALL(sys_getrandom)
+/* 385 */       CALL(sys_memfd_create)
 #ifndef syscalls_counted
 .equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls
 #define syscalls_counted
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 7c4fada440f0..9388a3d479e1 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -47,6 +47,9 @@
 #include <asm/mach/arch.h>
 #include <asm/mpu.h>
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/ipi.h>
+
 /*
  * as from 2.5, kernels no longer have an init_tasks structure
  * so we need some other way of telling a new secondary core
@@ -430,38 +433,15 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
         }
 }
 
-static void (*smp_cross_call)(const struct cpumask *, unsigned int);
+static void (*__smp_cross_call)(const struct cpumask *, unsigned int);
 
 void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
 {
-        if (!smp_cross_call)
-                smp_cross_call = fn;
-}
-
-void arch_send_call_function_ipi_mask(const struct cpumask *mask)
-{
-        smp_cross_call(mask, IPI_CALL_FUNC);
-}
-
-void arch_send_wakeup_ipi_mask(const struct cpumask *mask)
-{
-        smp_cross_call(mask, IPI_WAKEUP);
-}
-
-void arch_send_call_function_single_ipi(int cpu)
-{
-        smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
+        if (!__smp_cross_call)
+                __smp_cross_call = fn;
 }
 
-#ifdef CONFIG_IRQ_WORK
-void arch_irq_work_raise(void)
-{
-        if (is_smp())
-                smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
-}
-#endif
-
-static const char *ipi_types[NR_IPI] = {
+static const char *ipi_types[NR_IPI] __tracepoint_string = {
 #define S(x,s)  [x] = s
         S(IPI_WAKEUP, "CPU wakeup interrupts"),
         S(IPI_TIMER, "Timer broadcast interrupts"),
@@ -473,6 +453,12 @@ static const char *ipi_types[NR_IPI] = {
         S(IPI_COMPLETION, "completion interrupts"),
 };
 
+static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
+{
+        trace_ipi_raise(target, ipi_types[ipinr]);
+        __smp_cross_call(target, ipinr);
+}
+
 void show_ipi_list(struct seq_file *p, int prec)
 {
         unsigned int cpu, i;
@@ -499,6 +485,29 @@ u64 smp_irq_stat_cpu(unsigned int cpu)
         return sum;
 }
 
+void arch_send_call_function_ipi_mask(const struct cpumask *mask)
+{
+        smp_cross_call(mask, IPI_CALL_FUNC);
+}
+
+void arch_send_wakeup_ipi_mask(const struct cpumask *mask)
+{
+        smp_cross_call(mask, IPI_WAKEUP);
+}
+
+void arch_send_call_function_single_ipi(int cpu)
+{
+        smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
+}
+
+#ifdef CONFIG_IRQ_WORK
+void arch_irq_work_raise(void)
+{
+        if (is_smp())
+                smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
+}
+#endif
+
 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 void tick_broadcast(const struct cpumask *mask)
 {
@@ -556,8 +565,10 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
         unsigned int cpu = smp_processor_id();
         struct pt_regs *old_regs = set_irq_regs(regs);
 
-        if (ipinr < NR_IPI)
+        if ((unsigned)ipinr < NR_IPI) {
+                trace_ipi_entry(ipi_types[ipinr]);
                 __inc_irq_stat(cpu, ipi_irqs[ipinr]);
+        }
 
         switch (ipinr) {
         case IPI_WAKEUP:
@@ -612,6 +623,9 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
                        cpu, ipinr);
                 break;
         }
+
+        if ((unsigned)ipinr < NR_IPI)
+                trace_ipi_exit(ipi_types[ipinr]);
         set_irq_regs(old_regs);
 }
 
diff --git a/arch/arm/mm/proc-v7-3level.S b/arch/arm/mm/proc-v7-3level.S
index e4c8acfc1323..1a24e9232ec8 100644
--- a/arch/arm/mm/proc-v7-3level.S
+++ b/arch/arm/mm/proc-v7-3level.S
@@ -146,12 +146,11 @@ ENDPROC(cpu_v7_set_pte_ext)
         mov     \tmp, \ttbr1, lsr #(32 - ARCH_PGD_SHIFT)        @ upper bits
         mov     \ttbr1, \ttbr1, lsl #ARCH_PGD_SHIFT             @ lower bits
         addls   \ttbr1, \ttbr1, #TTBR1_OFFSET
-        mcrr    p15, 1, \ttbr1, \zero, c2                       @ load TTBR1
+        adcls   \tmp, \tmp, #0
+        mcrr    p15, 1, \ttbr1, \tmp, c2                        @ load TTBR1
         mov     \tmp, \ttbr0, lsr #(32 - ARCH_PGD_SHIFT)        @ upper bits
         mov     \ttbr0, \ttbr0, lsl #ARCH_PGD_SHIFT             @ lower bits
-        mcrr    p15, 0, \ttbr0, \zero, c2                       @ load TTBR0
-        mcrr    p15, 1, \ttbr1, \zero, c2                       @ load TTBR1
-        mcrr    p15, 0, \ttbr0, \zero, c2                       @ load TTBR0
+        mcrr    p15, 0, \ttbr0, \tmp, c2                        @ load TTBR0
         .endm
 
         /*
diff --git a/arch/arm64/include/asm/signal32.h b/arch/arm64/include/asm/signal32.h
index 7c275e3b640f..eeaa97559bab 100644
--- a/arch/arm64/include/asm/signal32.h
+++ b/arch/arm64/include/asm/signal32.h
@@ -24,22 +24,21 @@
 
 extern const compat_ulong_t aarch32_sigret_code[6];
 
-int compat_setup_frame(int usig, struct k_sigaction *ka, sigset_t *set,
+int compat_setup_frame(int usig, struct ksignal *ksig, sigset_t *set,
                        struct pt_regs *regs);
-int compat_setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
-                          sigset_t *set, struct pt_regs *regs);
+int compat_setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs);
 
 void compat_setup_restart_syscall(struct pt_regs *regs);
 #else
 
-static inline int compat_setup_frame(int usid, struct k_sigaction *ka,
+static inline int compat_setup_frame(int usid, struct ksignal *ksig,
                                      sigset_t *set, struct pt_regs *regs)
 {
         return -ENOSYS;
 }
 
-static inline int compat_setup_rt_frame(int usig, struct k_sigaction *ka,
-                                        siginfo_t *info, sigset_t *set,
+static inline int compat_setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set,
                                         struct pt_regs *regs)
 {
         return -ENOSYS;
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 6357b9c6c90e..6fa792137eda 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -209,19 +209,13 @@ static int setup_sigframe(struct rt_sigframe __user *sf,
         return err;
 }
 
-static struct rt_sigframe __user *get_sigframe(struct k_sigaction *ka,
+static struct rt_sigframe __user *get_sigframe(struct ksignal *ksig,
                                                struct pt_regs *regs)
 {
         unsigned long sp, sp_top;
         struct rt_sigframe __user *frame;
 
-        sp = sp_top = regs->sp;
-
-        /*
-         * This is the X/Open sanctioned signal stack switching.
-         */
-        if ((ka->sa.sa_flags & SA_ONSTACK) && !sas_ss_flags(sp))
-                sp = sp_top = current->sas_ss_sp + current->sas_ss_size;
+        sp = sp_top = sigsp(regs->sp, ksig);
 
         sp = (sp - sizeof(struct rt_sigframe)) & ~15;
         frame = (struct rt_sigframe __user *)sp;
@@ -253,13 +247,13 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka,
         regs->regs[30] = (unsigned long)sigtramp;
 }
 
-static int setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
-                          sigset_t *set, struct pt_regs *regs)
+static int setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
         int err = 0;
 
-        frame = get_sigframe(ka, regs);
+        frame = get_sigframe(ksig, regs);
         if (!frame)
                 return 1;
 
@@ -269,9 +263,9 @@ static int setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
         err |= __save_altstack(&frame->uc.uc_stack, regs->sp);
         err |= setup_sigframe(frame, regs, set);
         if (err == 0) {
-                setup_return(regs, ka, frame, usig);
-                if (ka->sa.sa_flags & SA_SIGINFO) {
-                        err |= copy_siginfo_to_user(&frame->info, info);
+                setup_return(regs, &ksig->ka, frame, usig);
+                if (ksig->ka.sa.sa_flags & SA_SIGINFO) {
+                        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
                         regs->regs[1] = (unsigned long)&frame->info;
                         regs->regs[2] = (unsigned long)&frame->uc;
                 }
@@ -291,13 +285,12 @@ static void setup_restart_syscall(struct pt_regs *regs)
 /*
  * OK, we're invoking a handler
  */
-static void handle_signal(unsigned long sig, struct k_sigaction *ka,
-                          siginfo_t *info, struct pt_regs *regs)
+static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
         struct thread_info *thread = current_thread_info();
         struct task_struct *tsk = current;
         sigset_t *oldset = sigmask_to_save();
-        int usig = sig;
+        int usig = ksig->sig;
         int ret;
 
         /*
@@ -310,13 +303,12 @@ static void handle_signal(unsigned long sig, struct k_sigaction *ka,
          * Set up the stack frame
          */
         if (is_compat_task()) {
-                if (ka->sa.sa_flags & SA_SIGINFO)
-                        ret = compat_setup_rt_frame(usig, ka, info, oldset,
-                                                    regs);
+                if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                        ret = compat_setup_rt_frame(usig, ksig, oldset, regs);
                 else
-                        ret = compat_setup_frame(usig, ka, oldset, regs);
+                        ret = compat_setup_frame(usig, ksig, oldset, regs);
         } else {
-                ret = setup_rt_frame(usig, ka, info, oldset, regs);
+                ret = setup_rt_frame(usig, ksig, oldset, regs);
         }
 
         /*
@@ -324,18 +316,14 @@ static void handle_signal(unsigned long sig, struct k_sigaction *ka,
          */
         ret |= !valid_user_regs(&regs->user_regs);
 
-        if (ret != 0) {
-                force_sigsegv(sig, tsk);
-                return;
-        }
-
         /*
          * Fast forward the stepping logic so we step into the signal
          * handler.
          */
-        user_fastforward_single_step(tsk);
+        if (!ret)
+                user_fastforward_single_step(tsk);
 
-        signal_delivered(sig, info, ka, regs, 0);
+        signal_setup_done(ret, ksig, 0);
 }
 
 /*
@@ -350,10 +338,9 @@ static void handle_signal(unsigned long sig, struct k_sigaction *ka,
 static void do_signal(struct pt_regs *regs)
 {
         unsigned long continue_addr = 0, restart_addr = 0;
-        struct k_sigaction ka;
-        siginfo_t info;
-        int signr, retval = 0;
+        int retval = 0;
         int syscall = (int)regs->syscallno;
+        struct ksignal ksig;
 
         /*
          * If we were from a system call, check for system call restarting...
@@ -387,8 +374,7 @@ static void do_signal(struct pt_regs *regs)
          * Get the signal to deliver. When running under ptrace, at this point
          * the debugger may change all of our registers.
          */
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /*
                  * Depending on the signal settings, we may need to revert the
                  * decision to restart the system call, but skip this if a
@@ -398,12 +384,12 @@ static void do_signal(struct pt_regs *regs)
                     (retval == -ERESTARTNOHAND ||
                      retval == -ERESTART_RESTARTBLOCK ||
                      (retval == -ERESTARTSYS &&
-                      !(ka.sa.sa_flags & SA_RESTART)))) {
+                      !(ksig.ka.sa.sa_flags & SA_RESTART)))) {
                         regs->regs[0] = -EINTR;
                         regs->pc = continue_addr;
                 }
 
-                handle_signal(signr, &ka, &info, regs);
+                handle_signal(&ksig, regs);
                 return;
         }
 
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index c5ee208321c3..1b9ad02837cf 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -407,20 +407,14 @@ badframe:
         return 0;
 }
 
-static void __user *compat_get_sigframe(struct k_sigaction *ka,
+static void __user *compat_get_sigframe(struct ksignal *ksig,
                                         struct pt_regs *regs,
                                         int framesize)
 {
-        compat_ulong_t sp = regs->compat_sp;
+        compat_ulong_t sp = sigsp(regs->compat_sp, ksig);
         void __user *frame;
 
         /*
-         * This is the X/Open sanctioned signal stack switching.
-         */
-        if ((ka->sa.sa_flags & SA_ONSTACK) && !sas_ss_flags(sp))
-                sp = current->sas_ss_sp + current->sas_ss_size;
-
-        /*
          * ATPCS B01 mandates 8-byte alignment
          */
         frame = compat_ptr((compat_uptr_t)((sp - framesize) & ~7));
@@ -520,18 +514,18 @@ static int compat_setup_sigframe(struct compat_sigframe __user *sf,
 /*
  * 32-bit signal handling routines called from signal.c
  */
-int compat_setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
+int compat_setup_rt_frame(int usig, struct ksignal *ksig,
                           sigset_t *set, struct pt_regs *regs)
 {
         struct compat_rt_sigframe __user *frame;
         int err = 0;
 
-        frame = compat_get_sigframe(ka, regs, sizeof(*frame));
+        frame = compat_get_sigframe(ksig, regs, sizeof(*frame));
 
         if (!frame)
                 return 1;
 
-        err |= copy_siginfo_to_user32(&frame->info, info);
+        err |= copy_siginfo_to_user32(&frame->info, &ksig->info);
 
         __put_user_error(0, &frame->sig.uc.uc_flags, err);
         __put_user_error(0, &frame->sig.uc.uc_link, err);
@@ -541,7 +535,7 @@ int compat_setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
         err |= compat_setup_sigframe(&frame->sig, regs, set);
 
         if (err == 0) {
-                compat_setup_return(regs, ka, frame->sig.retcode, frame, usig);
+                compat_setup_return(regs, &ksig->ka, frame->sig.retcode, frame, usig);
                 regs->regs[1] = (compat_ulong_t)(unsigned long)&frame->info;
                 regs->regs[2] = (compat_ulong_t)(unsigned long)&frame->sig.uc;
         }
@@ -549,13 +543,13 @@ int compat_setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
         return err;
 }
 
-int compat_setup_frame(int usig, struct k_sigaction *ka, sigset_t *set,
+int compat_setup_frame(int usig, struct ksignal *ksig, sigset_t *set,
                        struct pt_regs *regs)
 {
         struct compat_sigframe __user *frame;
         int err = 0;
 
-        frame = compat_get_sigframe(ka, regs, sizeof(*frame));
+        frame = compat_get_sigframe(ksig, regs, sizeof(*frame));
 
         if (!frame)
                 return 1;
@@ -564,7 +558,7 @@ int compat_setup_frame(int usig, struct k_sigaction *ka, sigset_t *set,
 
         err |= compat_setup_sigframe(frame, regs, set);
         if (err == 0)
-                compat_setup_return(regs, ka, frame->retcode, frame, usig);
+                compat_setup_return(regs, &ksig->ka, frame->retcode, frame, usig);
 
         return err;
 }
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 3e2f5ebbf63e..474339718105 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -51,6 +51,9 @@
 #include <asm/tlbflush.h>
 #include <asm/ptrace.h>
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/ipi.h>
+
 /*
  * as from 2.5, kernels no longer have an init_tasks structure
  * so we need some other way of telling a new secondary core
@@ -313,8 +316,6 @@ void __init smp_prepare_boot_cpu(void)
         set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
 }
 
-static void (*smp_cross_call)(const struct cpumask *, unsigned int);
-
 /*
  * Enumerate the possible CPU set from the device tree and build the
  * cpu logical map array containing MPIDR values related to logical
@@ -469,32 +470,15 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
         }
 }
 
+static void (*__smp_cross_call)(const struct cpumask *, unsigned int);
 
 void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
 {
-        smp_cross_call = fn;
+        __smp_cross_call = fn;
 }
 
-void arch_send_call_function_ipi_mask(const struct cpumask *mask)
-{
-        smp_cross_call(mask, IPI_CALL_FUNC);
-}
-
-void arch_send_call_function_single_ipi(int cpu)
-{
-        smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
-}
-
-#ifdef CONFIG_IRQ_WORK
-void arch_irq_work_raise(void)
-{
-        if (smp_cross_call)
-                smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
-}
-#endif
-
-static const char *ipi_types[NR_IPI] = {
-#define S(x,s)  [x - IPI_RESCHEDULE] = s
+static const char *ipi_types[NR_IPI] __tracepoint_string = {
+#define S(x,s)  [x] = s
         S(IPI_RESCHEDULE, "Rescheduling interrupts"),
         S(IPI_CALL_FUNC, "Function call interrupts"),
         S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
@@ -503,12 +487,18 @@ static const char *ipi_types[NR_IPI] = {
         S(IPI_IRQ_WORK, "IRQ work interrupts"),
 };
 
+static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
+{
+        trace_ipi_raise(target, ipi_types[ipinr]);
+        __smp_cross_call(target, ipinr);
+}
+
 void show_ipi_list(struct seq_file *p, int prec)
 {
         unsigned int cpu, i;
 
         for (i = 0; i < NR_IPI; i++) {
-                seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i + IPI_RESCHEDULE,
+                seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
                            prec >= 4 ? " " : "");
                 for_each_online_cpu(cpu)
                         seq_printf(p, "%10u ",
@@ -528,6 +518,24 @@ u64 smp_irq_stat_cpu(unsigned int cpu)
         return sum;
 }
 
+void arch_send_call_function_ipi_mask(const struct cpumask *mask)
+{
+        smp_cross_call(mask, IPI_CALL_FUNC);
+}
+
+void arch_send_call_function_single_ipi(int cpu)
+{
+        smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
+}
+
+#ifdef CONFIG_IRQ_WORK
+void arch_irq_work_raise(void)
+{
+        if (__smp_cross_call)
+                smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
+}
+#endif
+
 static DEFINE_RAW_SPINLOCK(stop_lock);
 
 /*
@@ -559,8 +567,10 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
         unsigned int cpu = smp_processor_id();
         struct pt_regs *old_regs = set_irq_regs(regs);
 
-        if (ipinr >= IPI_RESCHEDULE && ipinr < IPI_RESCHEDULE + NR_IPI)
-                __inc_irq_stat(cpu, ipi_irqs[ipinr - IPI_RESCHEDULE]);
+        if ((unsigned)ipinr < NR_IPI) {
+                trace_ipi_entry(ipi_types[ipinr]);
+                __inc_irq_stat(cpu, ipi_irqs[ipinr]);
+        }
 
         switch (ipinr) {
         case IPI_RESCHEDULE:
@@ -605,6 +615,9 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
                 pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
                 break;
         }
+
+        if ((unsigned)ipinr < NR_IPI)
+                trace_ipi_exit(ipi_types[ipinr]);
         set_irq_regs(old_regs);
 }
 
diff --git a/arch/avr32/kernel/signal.c b/arch/avr32/kernel/signal.c
index b80c0b3d2bab..d309fbcc3bd6 100644
--- a/arch/avr32/kernel/signal.c
+++ b/arch/avr32/kernel/signal.c
@@ -127,24 +127,20 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs)
 }
 
 static inline void __user *
-get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, int framesize)
+get_sigframe(struct ksignal *ksig, struct pt_regs *regs, int framesize)
 {
-        unsigned long sp = regs->sp;
-
-        if ((ka->sa.sa_flags & SA_ONSTACK) && !sas_ss_flags(sp))
-                sp = current->sas_ss_sp + current->sas_ss_size;
+        unsigned long sp = sigsp(regs->sp, ksig);
 
         return (void __user *)((sp - framesize) & ~3);
 }
 
 static int
-setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-               sigset_t *set, struct pt_regs *regs)
+setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, regs, sizeof(*frame));
         err = -EFAULT;
         if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame)))
                 goto out;
@@ -164,7 +160,7 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         err = __put_user(0x3008d733 | (__NR_rt_sigreturn << 20),
                          &frame->retcode);
 
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
 
         /* Set up the ucontext */
         err |= __put_user(0, &frame->uc.uc_flags);
@@ -176,12 +172,12 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         if (err)
                 goto out;
 
-        regs->r12 = sig;
+        regs->r12 = ksig->sig;
         regs->r11 = (unsigned long) &frame->info;
         regs->r10 = (unsigned long) &frame->uc;
         regs->sp = (unsigned long) frame;
-        if (ka->sa.sa_flags & SA_RESTORER)
-                regs->lr = (unsigned long)ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER)
+                regs->lr = (unsigned long)ksig->ka.sa.sa_restorer;
         else {
                 printk(KERN_NOTICE "[%s:%d] did not set SA_RESTORER\n",
                        current->comm, current->pid);
@@ -189,10 +185,10 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         }
 
         pr_debug("SIG deliver [%s:%d]: sig=%d sp=0x%lx pc=0x%lx->0x%p lr=0x%lx\n",
-                 current->comm, current->pid, sig, regs->sp,
-                 regs->pc, ka->sa.sa_handler, regs->lr);
+                 current->comm, current->pid, ksig->sig, regs->sp,
+                 regs->pc, ksig->ka.sa.sa_handler, regs->lr);
 
-        regs->pc = (unsigned long) ka->sa.sa_handler;
+        regs->pc = (unsigned long)ksig->ka.sa.sa_handler;
 
 out:
         return err;
@@ -208,15 +204,14 @@ static inline void setup_syscall_restart(struct pt_regs *regs)
 }
 
 static inline void
-handle_signal(unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
-              struct pt_regs *regs, int syscall)
+handle_signal(struct ksignal *ksig, struct pt_regs *regs, int syscall)
 {
         int ret;
 
         /*
          * Set up the stack frame
          */
-        ret = setup_rt_frame(sig, ka, info, sigmask_to_save(), regs);
+        ret = setup_rt_frame(ksig, sigmask_to_save(), regs);
 
         /*
          * Check that the resulting registers are sane
@@ -226,10 +221,7 @@ handle_signal(unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
         /*
          * Block the signal if we were successful.
          */
-        if (ret != 0)
-                force_sigsegv(sig, current);
-        else
-                signal_delivered(sig, info, ka, regs, 0);
+        signal_setup_done(ret, ksig, 0);
 }
 
 /*
@@ -239,9 +231,7 @@ handle_signal(unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
  */
 static void do_signal(struct pt_regs *regs, int syscall)
 {
-        siginfo_t info;
-        int signr;
-        struct k_sigaction ka;
+        struct ksignal ksig;
 
         /*
          * We want the common case to go fast, which is why we may in
@@ -251,18 +241,18 @@ static void do_signal(struct pt_regs *regs, int syscall)
         if (!user_mode(regs))
                 return;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
+        get_signal(&ksig);
         if (syscall) {
                 switch (regs->r12) {
                 case -ERESTART_RESTARTBLOCK:
                 case -ERESTARTNOHAND:
-                        if (signr > 0) {
+                        if (ksig.sig > 0) {
                                 regs->r12 = -EINTR;
                                 break;
                         }
                         /* fall through */
                 case -ERESTARTSYS:
-                        if (signr > 0 && !(ka.sa.sa_flags & SA_RESTART)) {
+                        if (ksig.sig > 0 && !(ksig.ka.sa.sa_flags & SA_RESTART)) {
                                 regs->r12 = -EINTR;
                                 break;
                         }
@@ -272,13 +262,13 @@ static void do_signal(struct pt_regs *regs, int syscall)
                 }
         }
 
-        if (signr == 0) {
+        if (!ksig.sig) {
                 /* No signal to deliver -- put the saved sigmask back */
                 restore_saved_sigmask();
                 return;
         }
 
-        handle_signal(signr, &ka, &info, regs, syscall);
+        handle_signal(&ksig, regs, syscall);
 }
 
 asmlinkage void do_notify_resume(struct pt_regs *regs, struct thread_info *ti)
diff --git a/arch/blackfin/kernel/signal.c b/arch/blackfin/kernel/signal.c
index b022af6c48f8..ef275571d885 100644
--- a/arch/blackfin/kernel/signal.c
+++ b/arch/blackfin/kernel/signal.c
@@ -135,40 +135,31 @@ static inline int rt_setup_sigcontext(struct sigcontext *sc, struct pt_regs *reg
         return err;
 }
 
-static inline void *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
+static inline void *get_sigframe(struct ksignal *ksig,
                                  size_t frame_size)
 {
-        unsigned long usp;
+        unsigned long usp = sigsp(rdusp(), ksig);
 
-        /* Default to using normal stack.  */
-        usp = rdusp();
-
-        /* This is the X/Open sanctioned signal stack switching.  */
-        if (ka->sa.sa_flags & SA_ONSTACK) {
-                if (!on_sig_stack(usp))
-                        usp = current->sas_ss_sp + current->sas_ss_size;
-        }
         return (void *)((usp - frame_size) & -8UL);
 }
 
 static int
-setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t * info,
-               sigset_t * set, struct pt_regs *regs)
+setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs)
 {
         struct rt_sigframe *frame;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, sizeof(*frame));
 
         err |= __put_user((current_thread_info()->exec_domain
                            && current_thread_info()->exec_domain->signal_invmap
-                           && sig < 32
+                           && ksig->sig < 32
                            ? current_thread_info()->exec_domain->
-                           signal_invmap[sig] : sig), &frame->sig);
+                           signal_invmap[ksig->sig] : ksig->sig), &frame->sig);
 
         err |= __put_user(&frame->info, &frame->pinfo);
         err |= __put_user(&frame->uc, &frame->puc);
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
 
         /* Create the ucontext.  */
         err |= __put_user(0, &frame->uc.uc_flags);
@@ -183,7 +174,7 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t * info,
         /* Set up registers for signal handler */
         if (current->personality & FDPIC_FUNCPTRS) {
                 struct fdpic_func_descriptor __user *funcptr =
-                        (struct fdpic_func_descriptor *) ka->sa.sa_handler;
+                        (struct fdpic_func_descriptor *) ksig->ka.sa.sa_handler;
                 u32 pc, p3;
                 err |= __get_user(pc, &funcptr->text);
                 err |= __get_user(p3, &funcptr->GOT);
@@ -192,7 +183,7 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t * info,
                 regs->pc = pc;
                 regs->p3 = p3;
         } else
-                regs->pc = (unsigned long)ka->sa.sa_handler;
+                regs->pc = (unsigned long)ksig->ka.sa.sa_handler;
         wrusp((unsigned long)frame);
         regs->rets = SIGRETURN_STUB;
 
@@ -237,20 +228,19 @@ handle_restart(struct pt_regs *regs, struct k_sigaction *ka, int has_handler)
  * OK, we're invoking a handler
  */
 static void
-handle_signal(int sig, siginfo_t *info, struct k_sigaction *ka,
-              struct pt_regs *regs)
+handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
+        int ret;
+
         /* are we from a system call? to see pt_regs->orig_p0 */
         if (regs->orig_p0 >= 0)
                 /* If so, check system call restarting.. */
-                handle_restart(regs, ka, 1);
+                handle_restart(regs, &ksig->ka, 1);
 
         /* set up the stack frame */
-        if (setup_rt_frame(sig, ka, info, sigmask_to_save(), regs) < 0)
-                force_sigsegv(sig, current);
-        else 
-                signal_delivered(sig, info, ka, regs,
-                                test_thread_flag(TIF_SINGLESTEP));
+        ret = setup_rt_frame(ksig, sigmask_to_save(), regs);
+
+        signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLESTEP));
 }
 
 /*
@@ -264,16 +254,13 @@ handle_signal(int sig, siginfo_t *info, struct k_sigaction *ka,
  */
 asmlinkage void do_signal(struct pt_regs *regs)
 {
-        siginfo_t info;
-        int signr;
-        struct k_sigaction ka;
+        struct ksignal ksig;
 
         current->thread.esp0 = (unsigned long)regs;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /* Whee!  Actually deliver the signal.  */
-                handle_signal(signr, &info, &ka, regs);
+                handle_signal(&ksig, regs);
                 return;
         }
 
diff --git a/arch/c6x/kernel/signal.c b/arch/c6x/kernel/signal.c
index 3998b24e26f2..fe68226f6c4d 100644
--- a/arch/c6x/kernel/signal.c
+++ b/arch/c6x/kernel/signal.c
@@ -127,17 +127,11 @@ static int setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs,
         return err;
 }
 
-static inline void __user *get_sigframe(struct k_sigaction *ka,
+static inline void __user *get_sigframe(struct ksignal *ksig,
                                         struct pt_regs *regs,
                                         unsigned long framesize)
 {
-        unsigned long sp = regs->sp;
-
-        /*
-         * This is the X/Open sanctioned signal stack switching.
-         */
-        if ((ka->sa.sa_flags & SA_ONSTACK) && sas_ss_flags(sp) == 0)
-                sp = current->sas_ss_sp + current->sas_ss_size;
+        unsigned long sp = sigsp(regs->sp, ksig);
 
         /*
          * No matter what happens, 'sp' must be dword
@@ -146,21 +140,21 @@ static inline void __user *get_sigframe(struct k_sigaction *ka,
         return (void __user *)((sp - framesize) & ~7);
 }
 
-static int setup_rt_frame(int signr, struct k_sigaction *ka, siginfo_t *info,
-                           sigset_t *set, struct pt_regs *regs)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
         unsigned long __user *retcode;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, regs, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto segv_and_exit;
+                return -EFAULT;
 
         err |= __put_user(&frame->info, &frame->pinfo);
         err |= __put_user(&frame->uc, &frame->puc);
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
 
         /* Clear all the bits of the ucontext we don't use.  */
         err |= __clear_user(&frame->uc, offsetof(struct ucontext, uc_mcontext));
@@ -188,7 +182,7 @@ static int setup_rt_frame(int signr, struct k_sigaction *ka, siginfo_t *info,
 #undef COPY
 
         if (err)
-                goto segv_and_exit;
+                return -EFAULT;
 
         flush_icache_range((unsigned long) &frame->retcode,
                            (unsigned long) &frame->retcode + RETCODE_SIZE);
@@ -198,10 +192,10 @@ static int setup_rt_frame(int signr, struct k_sigaction *ka, siginfo_t *info,
         /* Change user context to branch to signal handler */
         regs->sp = (unsigned long) frame - 8;
         regs->b3 = (unsigned long) retcode;
-        regs->pc = (unsigned long) ka->sa.sa_handler;
+        regs->pc = (unsigned long) ksig->ka.sa.sa_handler;
 
         /* Give the signal number to the handler */
-        regs->a4 = signr;
+        regs->a4 = ksig->sig;
 
         /*
          * For realtime signals we must also set the second and third
@@ -212,10 +206,6 @@ static int setup_rt_frame(int signr, struct k_sigaction *ka, siginfo_t *info,
         regs->a6 = (unsigned long)&frame->uc;
 
         return 0;
-
-segv_and_exit:
-        force_sigsegv(signr, current);
-        return -EFAULT;
 }
 
 static inline void
@@ -245,10 +235,11 @@ do_restart:
 /*
  * handle the actual delivery of a signal to userspace
  */
-static void handle_signal(int sig,
-                         siginfo_t *info, struct k_sigaction *ka,
-                         struct pt_regs *regs, int syscall)
+static void handle_signal(struct ksignal *ksig, struct pt_regs *regs,
+                          int syscall)
 {
+        int ret;
+
         /* Are we from a system call? */
         if (syscall) {
                 /* If so, check system call restarting.. */
@@ -259,7 +250,7 @@ static void handle_signal(int sig,
                         break;
 
                 case -ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
+                        if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                 regs->a4 = -EINTR;
                                 break;
                         }
@@ -272,9 +263,8 @@ static void handle_signal(int sig,
         }
 
         /* Set up the stack frame */
-        if (setup_rt_frame(sig, ka, info, sigmask_to_save(), regs) < 0)
-                return;
-        signal_delivered(sig, info, ka, regs, 0);
+        ret = setup_rt_frame(ksig, sigmask_to_save(), regs);
+        signal_setup_done(ret, ksig, 0);
 }
 
 /*
@@ -282,18 +272,15 @@ static void handle_signal(int sig,
  */
 static void do_signal(struct pt_regs *regs, int syscall)
 {
-        struct k_sigaction ka;
-        siginfo_t info;
-        int signr;
+        struct ksignal ksig;
 
         /* we want the common case to go fast, which is why we may in certain
          * cases get here from kernel mode */
         if (!user_mode(regs))
                 return;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
-                handle_signal(signr, &info, &ka, regs, syscall);
+        if (get_signal(&ksig)) {
+                handle_signal(&ksig, regs, syscall);
                 return;
         }
 
diff --git a/arch/cris/arch-v10/kernel/signal.c b/arch/cris/arch-v10/kernel/signal.c
index 61ce6273a895..9b32d338838b 100644
--- a/arch/cris/arch-v10/kernel/signal.c
+++ b/arch/cris/arch-v10/kernel/signal.c
@@ -203,15 +203,9 @@ static int setup_sigcontext(struct sigcontext __user *sc,
  * - usually on the stack. */
 
 static inline void __user *
-get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size)
+get_sigframe(struct ksignal *ksig, size_t frame_size)
 {
-        unsigned long sp = rdusp();
-
-        /* This is the X/Open sanctioned signal stack switching.  */
-        if (ka->sa.sa_flags & SA_ONSTACK) {
-                if (! on_sig_stack(sp))
-                        sp = current->sas_ss_sp + current->sas_ss_size;
-        }
+        unsigned long sp = sigsp(rdusp(), ksig);
 
         /* make sure the frame is dword-aligned */
 
@@ -228,33 +222,33 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size)
  * user-mode trampoline.
  */
 
-static int setup_frame(int sig, struct k_sigaction *ka,
-                       sigset_t *set, struct pt_regs *regs)
+static int setup_frame(struct ksignal *ksig, sigset_t *set,
+                       struct pt_regs *regs)
 {
         struct sigframe __user *frame;
         unsigned long return_ip;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         err |= setup_sigcontext(&frame->sc, regs, set->sig[0]);
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (_NSIG_WORDS > 1) {
                 err |= __copy_to_user(frame->extramask, &set->sig[1],
                                       sizeof(frame->extramask));
         }
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up to return from userspace.  If provided, use a stub
            already in userspace.  */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                return_ip = (unsigned long)ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                return_ip = (unsigned long)ksig->ka.sa.sa_restorer;
         } else {
                 /* trampoline - the desired return ip is the retcode itself */
                 return_ip = (unsigned long)&frame->retcode;
@@ -265,42 +259,38 @@ static int setup_frame(int sig, struct k_sigaction *ka,
         }
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up registers for signal handler */
 
-        regs->irp = (unsigned long) ka->sa.sa_handler;  /* what we enter NOW   */
+        regs->irp = (unsigned long) ksig->ka.sa.sa_handler;  /* what we enter NOW   */
         regs->srp = return_ip;                          /* what we enter LATER */
-        regs->r10 = sig;                                /* first argument is signo */
+        regs->r10 = ksig->sig;                                /* first argument is signo */
 
         /* actually move the usp to reflect the stacked frame */
 
         wrusp((unsigned long)frame);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
-static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-        sigset_t *set, struct pt_regs *regs)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
         unsigned long return_ip;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         err |= __put_user(&frame->info, &frame->pinfo);
         err |= __put_user(&frame->uc, &frame->puc);
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Clear all the bits of the ucontext we don't use.  */
         err |= __clear_user(&frame->uc, offsetof(struct ucontext, uc_mcontext));
@@ -312,12 +302,12 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         err |= __save_altstack(&frame->uc.uc_stack, rdusp());
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up to return from userspace.  If provided, use a stub
            already in userspace.  */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                return_ip = (unsigned long)ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                return_ip = (unsigned long)ksig->ka.sa.sa_restorer;
         } else {
                 /* trampoline - the desired return ip is the retcode itself */
                 return_ip = (unsigned long)&frame->retcode;
@@ -329,18 +319,18 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         }
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* TODO what is the current->exec_domain stuff and invmap ? */
 
         /* Set up registers for signal handler */
 
         /* What we enter NOW   */
-        regs->irp = (unsigned long) ka->sa.sa_handler;
+        regs->irp = (unsigned long) ksig->ka.sa.sa_handler;
         /* What we enter LATER */
         regs->srp = return_ip;
         /* First argument is signo */
-        regs->r10 = sig;
+        regs->r10 = ksig->sig;
         /* Second argument is (siginfo_t *) */
         regs->r11 = (unsigned long)&frame->info;
         /* Third argument is unused */
@@ -350,19 +340,14 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         wrusp((unsigned long)frame);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
 /*
  * OK, we're invoking a handler
  */
 
-static inline void handle_signal(int canrestart, unsigned long sig,
-        siginfo_t *info, struct k_sigaction *ka,
-        struct pt_regs *regs)
+static inline void handle_signal(int canrestart, struct ksignal *ksig,
+                                 struct pt_regs *regs)
 {
         sigset_t *oldset = sigmask_to_save();
         int ret;
@@ -383,7 +368,7 @@ static inline void handle_signal(int canrestart, unsigned long sig,
                         /* ERESTARTSYS means to restart the syscall if
                          * there is no handler or the handler was
                          * registered with SA_RESTART */
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
+                        if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                 regs->r10 = -EINTR;
                                 break;
                         }
@@ -396,13 +381,12 @@ static inline void handle_signal(int canrestart, unsigned long sig,
         }
 
         /* Set up the stack frame */
-        if (ka->sa.sa_flags & SA_SIGINFO)
-                ret = setup_rt_frame(sig, ka, info, oldset, regs);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                ret = setup_rt_frame(ksig, oldset, regs);
         else
-                ret = setup_frame(sig, ka, oldset, regs);
+                ret = setup_frame(ksig, oldset, regs);
 
-        if (ret == 0)
-                signal_delivered(sig, info, ka, regs, 0);
+        signal_setup_done(ret, ksig, 0);
 }
 
 /*
@@ -419,9 +403,7 @@ static inline void handle_signal(int canrestart, unsigned long sig,
 
 void do_signal(int canrestart, struct pt_regs *regs)
 {
-        siginfo_t info;
-        int signr;
-        struct k_sigaction ka;
+        struct ksignal ksig;
 
         /*
          * We want the common case to go fast, which
@@ -432,10 +414,9 @@ void do_signal(int canrestart, struct pt_regs *regs)
         if (!user_mode(regs))
                 return;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /* Whee!  Actually deliver the signal.  */
-                handle_signal(canrestart, signr, &info, &ka, regs);
+                handle_signal(canrestart, &ksig, regs);
                 return;
         }
 
diff --git a/arch/cris/arch-v32/kernel/signal.c b/arch/cris/arch-v32/kernel/signal.c
index 01d1375c9004..78ce3b1c9bcb 100644
--- a/arch/cris/arch-v32/kernel/signal.c
+++ b/arch/cris/arch-v32/kernel/signal.c
@@ -189,17 +189,9 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs,
 
 /* Figure out where to put the new signal frame - usually on the stack. */
 static inline void __user *
-get_sigframe(struct k_sigaction *ka, struct pt_regs * regs, size_t frame_size)
+get_sigframe(struct ksignal *ksig, size_t frame_size)
 {
-        unsigned long sp;
-
-        sp = rdusp();
-
-        /* This is the X/Open sanctioned signal stack switching. */
-        if (ka->sa.sa_flags & SA_ONSTACK) {
-                if (!on_sig_stack(sp))
-                        sp = current->sas_ss_sp + current->sas_ss_size;
-        }
+        unsigned long sp = sigsp(rdusp(), ksig);
 
         /* Make sure the frame is dword-aligned. */
         sp &= ~3;
@@ -215,23 +207,22 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs * regs, size_t frame_size)
  * trampoline.
   */
 static int
-setup_frame(int sig, struct k_sigaction *ka,  sigset_t *set,
-            struct pt_regs * regs)
+setup_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs)
 {
         int err;
         unsigned long return_ip;
         struct signal_frame __user *frame;
 
         err = 0;
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         err |= setup_sigcontext(&frame->sc, regs, set->sig[0]);
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (_NSIG_WORDS > 1) {
                 err |= __copy_to_user(frame->extramask, &set->sig[1],
@@ -239,14 +230,14 @@ setup_frame(int sig, struct k_sigaction *ka,  sigset_t *set,
         }
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /*
          * Set up to return from user-space. If provided, use a stub
          * already located in user-space.
          */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                return_ip = (unsigned long)ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                return_ip = (unsigned long)ksig->ka.sa.sa_restorer;
         } else {
                 /* Trampoline - the desired return ip is in the signal return page. */
                 return_ip = cris_signal_return_page;
@@ -264,7 +255,7 @@ setup_frame(int sig, struct k_sigaction *ka,  sigset_t *set,
         }
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /*
          * Set up registers for signal handler.
@@ -273,42 +264,37 @@ setup_frame(int sig, struct k_sigaction *ka,  sigset_t *set,
          * Where the code enter later.
          * First argument, signo.
          */
-        regs->erp = (unsigned long) ka->sa.sa_handler;
+        regs->erp = (unsigned long) ksig->ka.sa.sa_handler;
         regs->srp = return_ip;
-        regs->r10 = sig;
+        regs->r10 = ksig->sig;
 
         /* Actually move the USP to reflect the stacked frame. */
         wrusp((unsigned long)frame);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
 static int
-setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-               sigset_t *set, struct pt_regs * regs)
+setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs)
 {
         int err;
         unsigned long return_ip;
         struct rt_signal_frame __user *frame;
 
         err = 0;
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* TODO: what is the current->exec_domain stuff and invmap ? */
 
         err |= __put_user(&frame->info, &frame->pinfo);
         err |= __put_user(&frame->uc, &frame->puc);
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Clear all the bits of the ucontext we don't use.  */
         err |= __clear_user(&frame->uc, offsetof(struct ucontext, uc_mcontext));
@@ -317,14 +303,14 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         err |= __save_altstack(&frame->uc.uc_stack, rdusp());
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /*
          * Set up to return from user-space. If provided, use a stub
          * already located in user-space.
          */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                return_ip = (unsigned long) ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                return_ip = (unsigned long) ksig->ka.sa.sa_restorer;
         } else {
                 /* Trampoline - the desired return ip is in the signal return page. */
                 return_ip = cris_signal_return_page + 6;
@@ -345,7 +331,7 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         }
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /*
          * Set up registers for signal handler.
@@ -356,9 +342,9 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
          * Second argument is (siginfo_t *).
          * Third argument is unused.
          */
-        regs->erp = (unsigned long) ka->sa.sa_handler;
+        regs->erp = (unsigned long) ksig->ka.sa.sa_handler;
         regs->srp = return_ip;
-        regs->r10 = sig;
+        regs->r10 = ksig->sig;
         regs->r11 = (unsigned long) &frame->info;
         regs->r12 = 0;
 
@@ -366,17 +352,11 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         wrusp((unsigned long)frame);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
 /* Invoke a signal handler to, well, handle the signal. */
 static inline void
-handle_signal(int canrestart, unsigned long sig,
-              siginfo_t *info, struct k_sigaction *ka,
-              struct pt_regs * regs)
+handle_signal(int canrestart, struct ksignal *ksig, struct pt_regs *regs)
 {
         sigset_t *oldset = sigmask_to_save();
         int ret;
@@ -404,7 +384,7 @@ handle_signal(int canrestart, unsigned long sig,
                                  * there is no handler, or the handler
                                  * was registered with SA_RESTART.
                                  */
-                                if (!(ka->sa.sa_flags & SA_RESTART)) {
+                                if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                         regs->r10 = -EINTR;
                                         break;
                                 }
@@ -423,13 +403,12 @@ handle_signal(int canrestart, unsigned long sig,
         }
 
         /* Set up the stack frame. */
-        if (ka->sa.sa_flags & SA_SIGINFO)
-                ret = setup_rt_frame(sig, ka, info, oldset, regs);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                ret = setup_rt_frame(ksig, oldset, regs);
         else
-                ret = setup_frame(sig, ka, oldset, regs);
+                ret = setup_frame(ksig, oldset, regs);
 
-        if (ret == 0)
-                signal_delivered(sig, info, ka, regs, 0);
+        signal_setup_done(ret, ksig, 0);
 }
 
 /*
@@ -446,9 +425,7 @@ handle_signal(int canrestart, unsigned long sig,
 void
 do_signal(int canrestart, struct pt_regs *regs)
 {
-        int signr;
-        siginfo_t info;
-        struct k_sigaction ka;
+        struct ksignal ksig;
 
         /*
          * The common case should go fast, which is why this point is
@@ -458,11 +435,9 @@ do_signal(int canrestart, struct pt_regs *regs)
         if (!user_mode(regs))
                 return;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /* Whee!  Actually deliver the signal.  */
-                handle_signal(canrestart, signr, &info, &ka, regs);
+                handle_signal(canrestart, &ksig, regs);
                 return;
         }
 
diff --git a/arch/frv/kernel/signal.c b/arch/frv/kernel/signal.c
index d822700d4f15..dc3d59de0870 100644
--- a/arch/frv/kernel/signal.c
+++ b/arch/frv/kernel/signal.c
@@ -158,19 +158,10 @@ static int setup_sigcontext(struct sigcontext __user *sc, unsigned long mask)
 /*
  * Determine which stack to use..
  */
-static inline void __user *get_sigframe(struct k_sigaction *ka,
+static inline void __user *get_sigframe(struct ksignal *ksig,
                                         size_t frame_size)
 {
-        unsigned long sp;
-
-        /* Default to using normal stack */
-        sp = __frame->sp;
-
-        /* This is the X/Open sanctioned signal stack switching.  */
-        if (ka->sa.sa_flags & SA_ONSTACK) {
-                if (! sas_ss_flags(sp))
-                        sp = current->sas_ss_sp + current->sas_ss_size;
-        }
+        unsigned long sp = sigsp(__frame->sp, ksig);
 
         return (void __user *) ((sp - frame_size) & ~7UL);
 
@@ -180,17 +171,17 @@ static inline void __user *get_sigframe(struct k_sigaction *ka,
 /*
  *
  */
-static int setup_frame(int sig, struct k_sigaction *ka, sigset_t *set)
+static int setup_frame(struct ksignal *ksig, sigset_t *set)
 {
         struct sigframe __user *frame;
-        int rsig;
+        int rsig, sig = ksig->sig;
 
         set_fs(USER_DS);
 
-        frame = get_sigframe(ka, sizeof(*frame));
+        frame = get_sigframe(ksig, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         rsig = sig;
         if (sig < 32 &&
@@ -199,22 +190,22 @@ static int setup_frame(int sig, struct k_sigaction *ka, sigset_t *set)
                 rsig = __current_thread_info->exec_domain->signal_invmap[sig];
 
         if (__put_user(rsig, &frame->sig) < 0)
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (setup_sigcontext(&frame->sc, set->sig[0]))
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (_NSIG_WORDS > 1) {
                 if (__copy_to_user(frame->extramask, &set->sig[1],
                                    sizeof(frame->extramask)))
-                        goto give_sigsegv;
+                        return -EFAULT;
         }
 
         /* Set up to return from userspace.  If provided, use a stub
          * already in userspace.  */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                if (__put_user(ka->sa.sa_restorer, &frame->pretcode) < 0)
-                        goto give_sigsegv;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                if (__put_user(ksig->ka.sa.sa_restorer, &frame->pretcode) < 0)
+                        return -EFAULT;
         }
         else {
                 /* Set up the following code on the stack:
@@ -224,7 +215,7 @@ static int setup_frame(int sig, struct k_sigaction *ka, sigset_t *set)
                 if (__put_user((__sigrestore_t)frame->retcode, &frame->pretcode) ||
                     __put_user(0x8efc0000|__NR_sigreturn, &frame->retcode[0]) ||
                     __put_user(0xc0700000, &frame->retcode[1]))
-                        goto give_sigsegv;
+                        return -EFAULT;
 
                 flush_icache_range((unsigned long) frame->retcode,
                                    (unsigned long) (frame->retcode + 2));
@@ -233,14 +224,14 @@ static int setup_frame(int sig, struct k_sigaction *ka, sigset_t *set)
         /* Set up registers for the signal handler */
         if (current->personality & FDPIC_FUNCPTRS) {
                 struct fdpic_func_descriptor __user *funcptr =
-                        (struct fdpic_func_descriptor __user *) ka->sa.sa_handler;
+                        (struct fdpic_func_descriptor __user *) ksig->ka.sa.sa_handler;
                 struct fdpic_func_descriptor desc;
                 if (copy_from_user(&desc, funcptr, sizeof(desc)))
-                        goto give_sigsegv;
+                        return -EFAULT;
                 __frame->pc = desc.text;
                 __frame->gr15 = desc.GOT;
         } else {
-                __frame->pc   = (unsigned long) ka->sa.sa_handler;
+                __frame->pc   = (unsigned long) ksig->ka.sa.sa_handler;
                 __frame->gr15 = 0;
         }
 
@@ -255,29 +246,23 @@ static int setup_frame(int sig, struct k_sigaction *ka, sigset_t *set)
 #endif
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
-
 } /* end setup_frame() */
 
 /*****************************************************************************/
 /*
  *
  */
-static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                          sigset_t *set)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set)
 {
         struct rt_sigframe __user *frame;
-        int rsig;
+        int rsig, sig = ksig->sig;
 
         set_fs(USER_DS);
 
-        frame = get_sigframe(ka, sizeof(*frame));
+        frame = get_sigframe(ksig, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         rsig = sig;
         if (sig < 32 &&
@@ -288,28 +273,28 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         if (__put_user(rsig,            &frame->sig) ||
             __put_user(&frame->info,    &frame->pinfo) ||
             __put_user(&frame->uc,      &frame->puc))
-                goto give_sigsegv;
+                return -EFAULT;
 
-        if (copy_siginfo_to_user(&frame->info, info))
-                goto give_sigsegv;
+        if (copy_siginfo_to_user(&frame->info, &ksig->info))
+                return -EFAULT;
 
         /* Create the ucontext.  */
         if (__put_user(0, &frame->uc.uc_flags) ||
             __put_user(NULL, &frame->uc.uc_link) ||
             __save_altstack(&frame->uc.uc_stack, __frame->sp))
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (setup_sigcontext(&frame->uc.uc_mcontext, set->sig[0]))
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (__copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up to return from userspace.  If provided, use a stub
          * already in userspace.  */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                if (__put_user(ka->sa.sa_restorer, &frame->pretcode))
-                        goto give_sigsegv;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                if (__put_user(ksig->ka.sa.sa_restorer, &frame->pretcode))
+                        return -EFAULT;
         }
         else {
                 /* Set up the following code on the stack:
@@ -319,7 +304,7 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 if (__put_user((__sigrestore_t)frame->retcode, &frame->pretcode) ||
                     __put_user(0x8efc0000|__NR_rt_sigreturn, &frame->retcode[0]) ||
                     __put_user(0xc0700000, &frame->retcode[1]))
-                        goto give_sigsegv;
+                        return -EFAULT;
 
                 flush_icache_range((unsigned long) frame->retcode,
                                    (unsigned long) (frame->retcode + 2));
@@ -328,14 +313,14 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         /* Set up registers for signal handler */
         if (current->personality & FDPIC_FUNCPTRS) {
                 struct fdpic_func_descriptor __user *funcptr =
-                        (struct fdpic_func_descriptor __user *) ka->sa.sa_handler;
+                        (struct fdpic_func_descriptor __user *) ksig->ka.sa.sa_handler;
                 struct fdpic_func_descriptor desc;
                 if (copy_from_user(&desc, funcptr, sizeof(desc)))
-                        goto give_sigsegv;
+                        return -EFAULT;
                 __frame->pc = desc.text;
                 __frame->gr15 = desc.GOT;
         } else {
-                __frame->pc   = (unsigned long) ka->sa.sa_handler;
+                __frame->pc   = (unsigned long) ksig->ka.sa.sa_handler;
                 __frame->gr15 = 0;
         }
 
@@ -349,21 +334,15 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                sig, current->comm, current->pid, frame, __frame->pc,
                frame->pretcode);
 #endif
-
         return 0;
 
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
-
 } /* end setup_rt_frame() */
 
 /*****************************************************************************/
 /*
  * OK, we're invoking a handler
  */
-static void handle_signal(unsigned long sig, siginfo_t *info,
-                         struct k_sigaction *ka)
+static void handle_signal(struct ksignal *ksig)
 {
         sigset_t *oldset = sigmask_to_save();
         int ret;
@@ -378,7 +357,7 @@ static void handle_signal(unsigned long sig, siginfo_t *info,
                         break;
 
                 case -ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
+                        if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                 __frame->gr8 = -EINTR;
                                 break;
                         }
@@ -392,16 +371,12 @@ static void handle_signal(unsigned long sig, siginfo_t *info,
         }
 
         /* Set up the stack frame */
-        if (ka->sa.sa_flags & SA_SIGINFO)
-                ret = setup_rt_frame(sig, ka, info, oldset);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                ret = setup_rt_frame(ksig, oldset);
         else
-                ret = setup_frame(sig, ka, oldset);
-
-        if (ret)
-                return;
+                ret = setup_frame(ksig, oldset);
 
-        signal_delivered(sig, info, ka, __frame,
-                                 test_thread_flag(TIF_SINGLESTEP));
+        signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLESTEP));
 } /* end handle_signal() */
 
 /*****************************************************************************/
@@ -412,13 +387,10 @@ static void handle_signal(unsigned long sig, siginfo_t *info,
  */
 static void do_signal(void)
 {
-        struct k_sigaction ka;
-        siginfo_t info;
-        int signr;
+        struct ksignal ksig;
 
-        signr = get_signal_to_deliver(&info, &ka, __frame, NULL);
-        if (signr > 0) {
-                handle_signal(signr, &info, &ka);
+        if (get_signal(&ksig)) {
+                handle_signal(&ksig);
                 return;
         }
 
diff --git a/arch/hexagon/kernel/signal.c b/arch/hexagon/kernel/signal.c
index d7c73874b515..eadd70e47e7e 100644
--- a/arch/hexagon/kernel/signal.c
+++ b/arch/hexagon/kernel/signal.c
@@ -36,18 +36,10 @@ struct rt_sigframe {
         struct ucontext uc;
 };
 
-static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
+static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs,
                           size_t frame_size)
 {
-        unsigned long sp = regs->r29;
-
-        /* check if we would overflow the alt stack */
-        if (on_sig_stack(sp) && !likely(on_sig_stack(sp - frame_size)))
-                return (void __user __force *)-1UL;
-
-        /* Switch to signal stack if appropriate */
-        if ((ka->sa.sa_flags & SA_ONSTACK) && (sas_ss_flags(sp) == 0))
-                sp = current->sas_ss_sp + current->sas_ss_size;
+        unsigned long sp = sigsp(regs->r29, ksig);
 
         return (void __user *)((sp - frame_size) & ~(sizeof(long long) - 1));
 }
@@ -112,20 +104,20 @@ static int restore_sigcontext(struct pt_regs *regs,
 /*
  * Setup signal stack frame with siginfo structure
  */
-static int setup_rt_frame(int signr, struct k_sigaction *ka, siginfo_t *info,
-                          sigset_t *set,  struct pt_regs *regs)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         int err = 0;
         struct rt_sigframe __user *frame;
         struct hexagon_vdso *vdso = current->mm->context.vdso;
 
-        frame = get_sigframe(ka, regs, sizeof(struct rt_sigframe));
+        frame = get_sigframe(ksig, regs, sizeof(struct rt_sigframe));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(struct rt_sigframe)))
-                goto    sigsegv;
+                return -EFAULT;
 
-        if (copy_siginfo_to_user(&frame->info, info))
-                goto    sigsegv;
+        if (copy_siginfo_to_user(&frame->info, &ksig->info))
+                return -EFAULT;
 
         /* The on-stack signal trampoline is no longer executed;
          * however, the libgcc signal frame unwinding code checks for
@@ -137,29 +129,26 @@ static int setup_rt_frame(int signr, struct k_sigaction *ka, siginfo_t *info,
         err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
         err |= __save_altstack(&frame->uc.uc_stack, user_stack_pointer(regs));
         if (err)
-                goto sigsegv;
+                return -EFAULT;
 
         /* Load r0/r1 pair with signumber/siginfo pointer... */
         regs->r0100 = ((unsigned long long)((unsigned long)&frame->info) << 32)
-                | (unsigned long long)signr;
+                | (unsigned long long)ksig->sig;
         regs->r02 = (unsigned long) &frame->uc;
         regs->r31 = (unsigned long) vdso->rt_signal_trampoline;
         pt_psp(regs) = (unsigned long) frame;
-        pt_set_elr(regs, (unsigned long)ka->sa.sa_handler);
+        pt_set_elr(regs, (unsigned long)ksig->ka.sa.sa_handler);
 
         return 0;
-
-sigsegv:
-        force_sigsegv(signr, current);
-        return -EFAULT;
 }
 
 /*
  * Setup invocation of signal handler
  */
-static void handle_signal(int sig, siginfo_t *info, struct k_sigaction *ka,
-                         struct pt_regs *regs)
+static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
+        int ret;
+
         /*
          * If we're handling a signal that aborted a system call,
          * set up the error return value before adding the signal
@@ -173,7 +162,7 @@ static void handle_signal(int sig, siginfo_t *info, struct k_sigaction *ka,
                         regs->r00 = -EINTR;
                         break;
                 case -ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
+                        if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                 regs->r00 = -EINTR;
                                 break;
                         }
@@ -193,11 +182,9 @@ static void handle_signal(int sig, siginfo_t *info, struct k_sigaction *ka,
          * only set up the rt_frame flavor.
          */
         /* If there was an error on setup, no signal was delivered. */
-        if (setup_rt_frame(sig, ka, info, sigmask_to_save(), regs) < 0)
-                return;
+        ret = setup_rt_frame(ksig, sigmask_to_save(), regs);
 
-        signal_delivered(sig, info, ka, regs,
-                        test_thread_flag(TIF_SINGLESTEP));
+        signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLESTEP));
 }
 
 /*
@@ -205,17 +192,13 @@ static void handle_signal(int sig, siginfo_t *info, struct k_sigaction *ka,
  */
 void do_signal(struct pt_regs *regs)
 {
-        struct k_sigaction sigact;
-        siginfo_t info;
-        int signo;
+        struct ksignal ksig;
 
         if (!user_mode(regs))
                 return;
 
-        signo = get_signal_to_deliver(&info, &sigact, regs, NULL);
-
-        if (signo > 0) {
-                handle_signal(signo, &info, &sigact, regs);
+        if (get_signal(&ksig)) {
+                handle_signal(&ksig, regs);
                 return;
         }
 
diff --git a/arch/ia64/include/asm/unistd.h b/arch/ia64/include/asm/unistd.h
index fb13dc5e8f8c..4254f5d3218c 100644
--- a/arch/ia64/include/asm/unistd.h
+++ b/arch/ia64/include/asm/unistd.h
@@ -11,7 +11,7 @@
 
 
 
-#define NR_syscalls                     315 /* length of syscall table */
+#define NR_syscalls                     316 /* length of syscall table */
 
 /*
  * The following defines stop scripts/checksyscalls.sh from complaining about
diff --git a/arch/ia64/include/uapi/asm/unistd.h b/arch/ia64/include/uapi/asm/unistd.h
index 7de0a2d65da4..99801c3be914 100644
--- a/arch/ia64/include/uapi/asm/unistd.h
+++ b/arch/ia64/include/uapi/asm/unistd.h
@@ -328,5 +328,6 @@
 #define __NR_sched_setattr              1336
 #define __NR_sched_getattr              1337
 #define __NR_renameat2                  1338
+#define __NR_getrandom                  1339
 
 #endif /* _UAPI_ASM_IA64_UNISTD_H */
diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S
index ba3d03503e84..4c13837a9269 100644
--- a/arch/ia64/kernel/entry.S
+++ b/arch/ia64/kernel/entry.S
@@ -1776,6 +1776,7 @@ sys_call_table:
         data8 sys_sched_setattr
         data8 sys_sched_getattr
         data8 sys_renameat2
+        data8 sys_getrandom
 
         .org sys_call_table + 8*NR_syscalls     // guard against failures to increase NR_syscalls
 #endif /* __IA64_ASM_PARAVIRTUALIZED_NATIVE */
diff --git a/arch/ia64/kernel/signal.c b/arch/ia64/kernel/signal.c
index 33cab9a8adff..6d92170be457 100644
--- a/arch/ia64/kernel/signal.c
+++ b/arch/ia64/kernel/signal.c
@@ -309,12 +309,11 @@ force_sigsegv_info (int sig, void __user *addr)
         si.si_uid = from_kuid_munged(current_user_ns(), current_uid());
         si.si_addr = addr;
         force_sig_info(SIGSEGV, &si, current);
-        return 0;
+        return 1;
 }
 
 static long
-setup_frame (int sig, struct k_sigaction *ka, siginfo_t *info, sigset_t *set,
-             struct sigscratch *scr)
+setup_frame(struct ksignal *ksig, sigset_t *set, struct sigscratch *scr)
 {
         extern char __kernel_sigtramp[];
         unsigned long tramp_addr, new_rbs = 0, new_sp;
@@ -323,7 +322,7 @@ setup_frame (int sig, struct k_sigaction *ka, siginfo_t *info, sigset_t *set,
 
         new_sp = scr->pt.r12;
         tramp_addr = (unsigned long) __kernel_sigtramp;
-        if (ka->sa.sa_flags & SA_ONSTACK) {
+        if (ksig->ka.sa.sa_flags & SA_ONSTACK) {
                 int onstack = sas_ss_flags(new_sp);
 
                 if (onstack == 0) {
@@ -347,29 +346,29 @@ setup_frame (int sig, struct k_sigaction *ka, siginfo_t *info, sigset_t *set,
                          */
                         check_sp = (new_sp - sizeof(*frame)) & -STACK_ALIGN;
                         if (!likely(on_sig_stack(check_sp)))
-                                return force_sigsegv_info(sig, (void __user *)
+                                return force_sigsegv_info(ksig->sig, (void __user *)
                                                           check_sp);
                 }
         }
         frame = (void __user *) ((new_sp - sizeof(*frame)) & -STACK_ALIGN);
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                return force_sigsegv_info(sig, frame);
+                return force_sigsegv_info(ksig->sig, frame);
 
-        err  = __put_user(sig, &frame->arg0);
+        err  = __put_user(ksig->sig, &frame->arg0);
         err |= __put_user(&frame->info, &frame->arg1);
         err |= __put_user(&frame->sc, &frame->arg2);
         err |= __put_user(new_rbs, &frame->sc.sc_rbs_base);
         err |= __put_user(0, &frame->sc.sc_loadrs);     /* initialize to zero */
-        err |= __put_user(ka->sa.sa_handler, &frame->handler);
+        err |= __put_user(ksig->ka.sa.sa_handler, &frame->handler);
 
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
 
         err |= __save_altstack(&frame->sc.sc_stack, scr->pt.r12);
         err |= setup_sigcontext(&frame->sc, set, scr);
 
         if (unlikely(err))
-                return force_sigsegv_info(sig, frame);
+                return force_sigsegv_info(ksig->sig, frame);
 
         scr->pt.r12 = (unsigned long) frame - 16;       /* new stack pointer */
         scr->pt.ar_fpsr = FPSR_DEFAULT;                 /* reset fpsr for signal handler */
@@ -394,22 +393,20 @@ setup_frame (int sig, struct k_sigaction *ka, siginfo_t *info, sigset_t *set,
 
 #if DEBUG_SIG
         printk("SIG deliver (%s:%d): sig=%d sp=%lx ip=%lx handler=%p\n",
-               current->comm, current->pid, sig, scr->pt.r12, frame->sc.sc_ip, frame->handler);
+               current->comm, current->pid, ksig->sig, scr->pt.r12, frame->sc.sc_ip, frame->handler);
 #endif
-        return 1;
+        return 0;
 }
 
 static long
-handle_signal (unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
-               struct sigscratch *scr)
+handle_signal (struct ksignal *ksig, struct sigscratch *scr)
 {
-        if (!setup_frame(sig, ka, info, sigmask_to_save(), scr))
-                return 0;
+        int ret = setup_frame(ksig, sigmask_to_save(), scr);
 
-        signal_delivered(sig, info, ka, &scr->pt,
-                                 test_thread_flag(TIF_SINGLESTEP));
+        if (!ret)
+                signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLESTEP));
 
-        return 1;
+        return ret;
 }
 
 /*
@@ -419,17 +416,16 @@ handle_signal (unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
 void
 ia64_do_signal (struct sigscratch *scr, long in_syscall)
 {
-        struct k_sigaction ka;
-        siginfo_t info;
         long restart = in_syscall;
         long errno = scr->pt.r8;
+        struct ksignal ksig;
 
         /*
          * This only loops in the rare cases of handle_signal() failing, in which case we
          * need to push through a forced SIGSEGV.
          */
         while (1) {
-                int signr = get_signal_to_deliver(&info, &ka, &scr->pt, NULL);
+                get_signal(&ksig);
 
                 /*
                  * get_signal_to_deliver() may have run a debugger (via notify_parent())
@@ -446,7 +442,7 @@ ia64_do_signal (struct sigscratch *scr, long in_syscall)
                          */
                         restart = 0;
 
-                if (signr <= 0)
+                if (ksig.sig <= 0)
                         break;
 
                 if (unlikely(restart)) {
@@ -458,7 +454,7 @@ ia64_do_signal (struct sigscratch *scr, long in_syscall)
                                 break;
 
                               case ERESTARTSYS:
-                                if ((ka.sa.sa_flags & SA_RESTART) == 0) {
+                                if ((ksig.ka.sa.sa_flags & SA_RESTART) == 0) {
                                         scr->pt.r8 = EINTR;
                                         /* note: scr->pt.r10 is already -1 */
                                         break;
@@ -473,7 +469,7 @@ ia64_do_signal (struct sigscratch *scr, long in_syscall)
                  * Whee!  Actually deliver the signal.  If the delivery failed, we need to
                  * continue to iterate in this loop so we can deliver the SIGSEGV...
                  */
-                if (handle_signal(signr, &ka, &info, scr))
+                if (handle_signal(&ksig, scr))
                         return;
         }
 
diff --git a/arch/m32r/kernel/signal.c b/arch/m32r/kernel/signal.c
index d503568cb753..95408b8f130a 100644
--- a/arch/m32r/kernel/signal.c
+++ b/arch/m32r/kernel/signal.c
@@ -162,28 +162,22 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs,
  * Determine which stack to use..
  */
 static inline void __user *
-get_sigframe(struct k_sigaction *ka, unsigned long sp, size_t frame_size)
+get_sigframe(struct ksignal *ksig, unsigned long sp, size_t frame_size)
 {
-        /* This is the X/Open sanctioned signal stack switching.  */
-        if (ka->sa.sa_flags & SA_ONSTACK) {
-                if (sas_ss_flags(sp) == 0)
-                        sp = current->sas_ss_sp + current->sas_ss_size;
-        }
-
-        return (void __user *)((sp - frame_size) & -8ul);
+        return (void __user *)((sigsp(sp, ksig) - frame_size) & -8ul);
 }
 
-static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                           sigset_t *set, struct pt_regs *regs)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
         int err = 0;
-        int signal;
+        int signal, sig = ksig->sig;
 
-        frame = get_sigframe(ka, regs->spu, sizeof(*frame));
+        frame = get_sigframe(ksig, regs->spu, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         signal = current_thread_info()->exec_domain
                 && current_thread_info()->exec_domain->signal_invmap
@@ -193,13 +187,13 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
         err |= __put_user(signal, &frame->sig);
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         err |= __put_user(&frame->info, &frame->pinfo);
         err |= __put_user(&frame->uc, &frame->puc);
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Create the ucontext.  */
         err |= __put_user(0, &frame->uc.uc_flags);
@@ -208,17 +202,17 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, set->sig[0]);
         err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up to return from userspace.  */
-        regs->lr = (unsigned long)ka->sa.sa_restorer;
+        regs->lr = (unsigned long)ksig->ka.sa.sa_restorer;
 
         /* Set up registers for signal handler */
         regs->spu = (unsigned long)frame;
         regs->r0 = signal;      /* Arg for signal handler */
         regs->r1 = (unsigned long)&frame->info;
         regs->r2 = (unsigned long)&frame->uc;
-        regs->bpc = (unsigned long)ka->sa.sa_handler;
+        regs->bpc = (unsigned long)ksig->ka.sa.sa_handler;
 
         set_fs(USER_DS);
 
@@ -228,10 +222,6 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 #endif
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
 static int prev_insn(struct pt_regs *regs)
@@ -252,9 +242,10 @@ static int prev_insn(struct pt_regs *regs)
  */
 
 static void
-handle_signal(unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
-              struct pt_regs *regs)
+handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
+        int ret;
+
         /* Are we from a system call? */
         if (regs->syscall_nr >= 0) {
                 /* If so, check system call restarting.. */
@@ -265,7 +256,7 @@ handle_signal(unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
                                 break;
 
                         case -ERESTARTSYS:
-                                if (!(ka->sa.sa_flags & SA_RESTART)) {
+                                if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                         regs->r0 = -EINTR;
                                         break;
                                 }
@@ -278,10 +269,9 @@ handle_signal(unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
         }
 
         /* Set up the stack frame */
-        if (setup_rt_frame(sig, ka, info, sigmask_to_save(), regs))
-                return;
+        ret = setup_rt_frame(ksig, sigmask_to_save(), regs);
 
-        signal_delivered(sig, info, ka, regs, 0);
+        signal_setup_done(ret, ksig, 0);
 }
 
 /*
@@ -291,9 +281,7 @@ handle_signal(unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
  */
 static void do_signal(struct pt_regs *regs)
 {
-        siginfo_t info;
-        int signr;
-        struct k_sigaction ka;
+        struct ksignal ksig;
 
         /*
          * We want the common case to go fast, which
@@ -304,8 +292,7 @@ static void do_signal(struct pt_regs *regs)
         if (!user_mode(regs))
                 return;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /* Re-enable any watchpoints before delivering the
                  * signal to user space. The processor register will
                  * have been cleared if the watchpoint triggered
@@ -313,7 +300,7 @@ static void do_signal(struct pt_regs *regs)
                  */
 
                 /* Whee!  Actually deliver the signal.  */
-                handle_signal(signr, &ka, &info, regs);
+                handle_signal(&ksig, regs);
 
                 return;
         }
diff --git a/arch/m68k/kernel/signal.c b/arch/m68k/kernel/signal.c
index 57fd286e4b0b..967a8b7e1527 100644
--- a/arch/m68k/kernel/signal.c
+++ b/arch/m68k/kernel/signal.c
@@ -835,38 +835,30 @@ static inline int rt_setup_ucontext(struct ucontext __user *uc, struct pt_regs *
 }
 
 static inline void __user *
-get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size)
+get_sigframe(struct ksignal *ksig, size_t frame_size)
 {
-        unsigned long usp;
-
-        /* Default to using normal stack.  */
-        usp = rdusp();
+        unsigned long usp = sigsp(rdusp(), ksig);
 
-        /* This is the X/Open sanctioned signal stack switching.  */
-        if (ka->sa.sa_flags & SA_ONSTACK) {
-                if (!sas_ss_flags(usp))
-                        usp = current->sas_ss_sp + current->sas_ss_size;
-        }
         return (void __user *)((usp - frame_size) & -8UL);
 }
 
-static int setup_frame (int sig, struct k_sigaction *ka,
-                         sigset_t *set, struct pt_regs *regs)
+static int setup_frame(struct ksignal *ksig, sigset_t *set,
+                        struct pt_regs *regs)
 {
         struct sigframe __user *frame;
         int fsize = frame_extra_sizes(regs->format);
         struct sigcontext context;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
 
         if (fsize < 0) {
 #ifdef DEBUG
                 printk ("setup_frame: Unknown frame format %#x\n",
                         regs->format);
 #endif
-                goto give_sigsegv;
+                return -EFAULT;
         }
 
-        frame = get_sigframe(ka, regs, sizeof(*frame) + fsize);
+        frame = get_sigframe(ksig, sizeof(*frame) + fsize);
 
         if (fsize)
                 err |= copy_to_user (frame + 1, regs + 1, fsize);
@@ -899,7 +891,7 @@ static int setup_frame (int sig, struct k_sigaction *ka,
 #endif
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         push_cache ((unsigned long) &frame->retcode);
 
@@ -908,7 +900,7 @@ static int setup_frame (int sig, struct k_sigaction *ka,
          * to destroy is successfully copied to sigframe.
          */
         wrusp ((unsigned long) frame);
-        regs->pc = (unsigned long) ka->sa.sa_handler;
+        regs->pc = (unsigned long) ksig->ka.sa.sa_handler;
         adjustformat(regs);
 
         /*
@@ -934,28 +926,24 @@ static int setup_frame (int sig, struct k_sigaction *ka,
                 tregs->sr = regs->sr;
         }
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return err;
 }
 
-static int setup_rt_frame (int sig, struct k_sigaction *ka, siginfo_t *info,
-                            sigset_t *set, struct pt_regs *regs)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                           struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
         int fsize = frame_extra_sizes(regs->format);
-        int err = 0;
+        int err = 0, sig = ksig->sig;
 
         if (fsize < 0) {
 #ifdef DEBUG
                 printk ("setup_frame: Unknown frame format %#x\n",
                         regs->format);
 #endif
-                goto give_sigsegv;
+                return -EFAULT;
         }
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, sizeof(*frame));
 
         if (fsize)
                 err |= copy_to_user (&frame->uc.uc_extra, regs + 1, fsize);
@@ -968,7 +956,7 @@ static int setup_rt_frame (int sig, struct k_sigaction *ka, siginfo_t *info,
                           &frame->sig);
         err |= __put_user(&frame->info, &frame->pinfo);
         err |= __put_user(&frame->uc, &frame->puc);
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
 
         /* Create the ucontext.  */
         err |= __put_user(0, &frame->uc.uc_flags);
@@ -996,7 +984,7 @@ static int setup_rt_frame (int sig, struct k_sigaction *ka, siginfo_t *info,
 #endif /* CONFIG_MMU */
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         push_cache ((unsigned long) &frame->retcode);
 
@@ -1005,7 +993,7 @@ static int setup_rt_frame (int sig, struct k_sigaction *ka, siginfo_t *info,
          * to destroy is successfully copied to sigframe.
          */
         wrusp ((unsigned long) frame);
-        regs->pc = (unsigned long) ka->sa.sa_handler;
+        regs->pc = (unsigned long) ksig->ka.sa.sa_handler;
         adjustformat(regs);
 
         /*
@@ -1031,10 +1019,6 @@ static int setup_rt_frame (int sig, struct k_sigaction *ka, siginfo_t *info,
                 tregs->sr = regs->sr;
         }
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return err;
 }
 
 static inline void
@@ -1074,26 +1058,22 @@ handle_restart(struct pt_regs *regs, struct k_sigaction *ka, int has_handler)
  * OK, we're invoking a handler
  */
 static void
-handle_signal(int sig, struct k_sigaction *ka, siginfo_t *info,
-              struct pt_regs *regs)
+handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
         sigset_t *oldset = sigmask_to_save();
         int err;
         /* are we from a system call? */
         if (regs->orig_d0 >= 0)
                 /* If so, check system call restarting.. */
-                handle_restart(regs, ka, 1);
+                handle_restart(regs, &ksig->ka, 1);
 
         /* set up the stack frame */
-        if (ka->sa.sa_flags & SA_SIGINFO)
-                err = setup_rt_frame(sig, ka, info, oldset, regs);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                err = setup_rt_frame(ksig, oldset, regs);
         else
-                err = setup_frame(sig, ka, oldset, regs);
-
-        if (err)
-                return;
+                err = setup_frame(ksig, oldset, regs);
 
-        signal_delivered(sig, info, ka, regs, 0);
+        signal_setup_done(err, ksig, 0);
 
         if (test_thread_flag(TIF_DELAYED_TRACE)) {
                 regs->sr &= ~0x8000;
@@ -1108,16 +1088,13 @@ handle_signal(int sig, struct k_sigaction *ka, siginfo_t *info,
  */
 static void do_signal(struct pt_regs *regs)
 {
-        siginfo_t info;
-        struct k_sigaction ka;
-        int signr;
+        struct ksignal ksig;
 
         current->thread.esp0 = (unsigned long) regs;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /* Whee!  Actually deliver the signal.  */
-                handle_signal(signr, &ka, &info, regs);
+                handle_signal(&ksig, regs);
                 return;
         }
 
diff --git a/arch/metag/kernel/signal.c b/arch/metag/kernel/signal.c
index b9e4a82d2bd4..0d100d5c1407 100644
--- a/arch/metag/kernel/signal.c
+++ b/arch/metag/kernel/signal.c
@@ -140,13 +140,9 @@ static int setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs,
 /*
  * Determine which stack to use..
  */
-static void __user *get_sigframe(struct k_sigaction *ka, unsigned long sp,
-                                 size_t frame_size)
+static void __user *get_sigframe(struct ksignal *ksig, unsigned long sp)
 {
-        /* Meta stacks grows upwards */
-        if ((ka->sa.sa_flags & SA_ONSTACK) && (sas_ss_flags(sp) == 0))
-                sp = current->sas_ss_sp;
-
+        sp = sigsp(sp, ksig);
         sp = (sp + 7) & ~7;                     /* 8byte align stack */
 
         return (void __user *)sp;
@@ -159,7 +155,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
         int err;
         unsigned long code;
 
-        frame = get_sigframe(&ksig->ka, regs->REG_SP, sizeof(*frame));
+        frame = get_sigframe(ksig, regs->REG_SP);
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
                 return -EFAULT;
 
diff --git a/arch/microblaze/kernel/signal.c b/arch/microblaze/kernel/signal.c
index 49a07a4d76d0..8955a3829cf0 100644
--- a/arch/microblaze/kernel/signal.c
+++ b/arch/microblaze/kernel/signal.c
@@ -145,22 +145,19 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs,
  * Determine which stack to use..
  */
 static inline void __user *
-get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size)
+get_sigframe(struct ksignal *ksig, struct pt_regs *regs, size_t frame_size)
 {
         /* Default to using normal stack */
-        unsigned long sp = regs->r1;
-
-        if ((ka->sa.sa_flags & SA_ONSTACK) != 0 && !on_sig_stack(sp))
-                sp = current->sas_ss_sp + current->sas_ss_size;
+        unsigned long sp = sigsp(regs->r1, ksig);
 
         return (void __user *)((sp - frame_size) & -8UL);
 }
 
-static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                        sigset_t *set, struct pt_regs *regs)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
         int signal;
         unsigned long address = 0;
 #ifdef CONFIG_MMU
@@ -168,10 +165,10 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         pte_t *ptep;
 #endif
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, regs, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         signal = current_thread_info()->exec_domain
                 && current_thread_info()->exec_domain->signal_invmap
@@ -179,8 +176,8 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 ? current_thread_info()->exec_domain->signal_invmap[sig]
                 : sig;
 
-        if (info)
-                err |= copy_siginfo_to_user(&frame->info, info);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                err |= copy_siginfo_to_user(&frame->info, &ksig->info);
 
         /* Create the ucontext. */
         err |= __put_user(0, &frame->uc.uc_flags);
@@ -227,7 +224,7 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         flush_dcache_range(address, address + 8);
 #endif
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up registers for signal handler */
         regs->r1 = (unsigned long) frame;
@@ -237,7 +234,7 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         regs->r6 = (unsigned long) &frame->info; /* arg 1: siginfo */
         regs->r7 = (unsigned long) &frame->uc; /* arg2: ucontext */
         /* Offset to handle microblaze rtid r14, 0 */
-        regs->pc = (unsigned long)ka->sa.sa_handler;
+        regs->pc = (unsigned long)ksig->ka.sa.sa_handler;
 
         set_fs(USER_DS);
 
@@ -247,10 +244,6 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 #endif
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
 /* Handle restarting system calls */
@@ -283,23 +276,15 @@ do_restart:
  */
 
 static void
-handle_signal(unsigned long sig, struct k_sigaction *ka,
-                siginfo_t *info, struct pt_regs *regs)
+handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
         sigset_t *oldset = sigmask_to_save();
         int ret;
 
         /* Set up the stack frame */
-        if (ka->sa.sa_flags & SA_SIGINFO)
-                ret = setup_rt_frame(sig, ka, info, oldset, regs);
-        else
-                ret = setup_rt_frame(sig, ka, NULL, oldset, regs);
+        ret = setup_rt_frame(ksig, oldset, regs);
 
-        if (ret)
-                return;
-
-        signal_delivered(sig, info, ka, regs,
-                        test_thread_flag(TIF_SINGLESTEP));
+        signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLESTEP));
 }
 
 /*
@@ -313,21 +298,19 @@ handle_signal(unsigned long sig, struct k_sigaction *ka,
  */
 static void do_signal(struct pt_regs *regs, int in_syscall)
 {
-        siginfo_t info;
-        int signr;
-        struct k_sigaction ka;
+        struct ksignal ksig;
+
 #ifdef DEBUG_SIG
         pr_info("do signal: %p %d\n", regs, in_syscall);
         pr_info("do signal2: %lx %lx %ld [%lx]\n", regs->pc, regs->r1,
                         regs->r12, current_thread_info()->flags);
 #endif
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /* Whee! Actually deliver the signal. */
                 if (in_syscall)
-                        handle_restart(regs, &ka, 1);
-                handle_signal(signr, &ka, &info, regs);
+                        handle_restart(regs, &ksig.ka, 1);
+                handle_signal(&ksig, regs);
                 return;
         }
 
diff --git a/arch/mips/include/asm/abi.h b/arch/mips/include/asm/abi.h
index 909bb6984866..7186bb51b89b 100644
--- a/arch/mips/include/asm/abi.h
+++ b/arch/mips/include/asm/abi.h
@@ -13,13 +13,11 @@
 #include <asm/siginfo.h>
 
 struct mips_abi {
-        int (* const setup_frame)(void *sig_return, struct k_sigaction *ka,
-                                  struct pt_regs *regs, int signr,
-                                  sigset_t *set);
+        int (* const setup_frame)(void *sig_return, struct ksignal *ksig,
+                                  struct pt_regs *regs, sigset_t *set);
         const unsigned long     signal_return_offset;
-        int (* const setup_rt_frame)(void *sig_return, struct k_sigaction *ka,
-                               struct pt_regs *regs, int signr,
-                               sigset_t *set, siginfo_t *info);
+        int (* const setup_rt_frame)(void *sig_return, struct ksignal *ksig,
+                                     struct pt_regs *regs, sigset_t *set);
         const unsigned long     rt_signal_return_offset;
         const unsigned long     restart;
 };
diff --git a/arch/mips/kernel/signal-common.h b/arch/mips/kernel/signal-common.h
index 9c60d09e62a7..06805e09bcd3 100644
--- a/arch/mips/kernel/signal-common.h
+++ b/arch/mips/kernel/signal-common.h
@@ -22,7 +22,7 @@
 /*
  * Determine which stack to use..
  */
-extern void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
+extern void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs,
                                  size_t frame_size);
 /* Check and clear pending FPU exceptions in saved CSR */
 extern int fpcsr_pending(unsigned int __user *fpcsr);
diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c
index 9e60d117e41e..1d57605e4615 100644
--- a/arch/mips/kernel/signal.c
+++ b/arch/mips/kernel/signal.c
@@ -280,7 +280,7 @@ int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc)
         return err;
 }
 
-void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
+void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs,
                           size_t frame_size)
 {
         unsigned long sp;
@@ -295,9 +295,7 @@ void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
          */
         sp -= 32;
 
-        /* This is the X/Open sanctioned signal stack switching.  */
-        if ((ka->sa.sa_flags & SA_ONSTACK) && (sas_ss_flags (sp) == 0))
-                sp = current->sas_ss_sp + current->sas_ss_size;
+        sp = sigsp(sp, ksig);
 
         return (void __user *)((sp - frame_size) & (ICACHE_REFILLS_WORKAROUND_WAR ? ~(cpu_icache_line_size()-1) : ALMASK));
 }
@@ -428,20 +426,20 @@ badframe:
 }
 
 #ifdef CONFIG_TRAD_SIGNALS
-static int setup_frame(void *sig_return, struct k_sigaction *ka,
-                       struct pt_regs *regs, int signr, sigset_t *set)
+static int setup_frame(void *sig_return, struct ksignal *ksig,
+                       struct pt_regs *regs, sigset_t *set)
 {
         struct sigframe __user *frame;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, regs, sizeof(*frame));
         if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         err |= setup_sigcontext(regs, &frame->sf_sc);
         err |= __copy_to_user(&frame->sf_mask, set, sizeof(*set));
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /*
          * Arguments to signal handler:
@@ -453,37 +451,32 @@ static int setup_frame(void *sig_return, struct k_sigaction *ka,
          * $25 and c0_epc point to the signal handler, $29 points to the
          * struct sigframe.
          */
-        regs->regs[ 4] = signr;
+        regs->regs[ 4] = ksig->sig;
         regs->regs[ 5] = 0;
         regs->regs[ 6] = (unsigned long) &frame->sf_sc;
         regs->regs[29] = (unsigned long) frame;
         regs->regs[31] = (unsigned long) sig_return;
-        regs->cp0_epc = regs->regs[25] = (unsigned long) ka->sa.sa_handler;
+        regs->cp0_epc = regs->regs[25] = (unsigned long) ksig->ka.sa.sa_handler;
 
         DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n",
                current->comm, current->pid,
                frame, regs->cp0_epc, regs->regs[31]);
         return 0;
-
-give_sigsegv:
-        force_sigsegv(signr, current);
-        return -EFAULT;
 }
 #endif
 
-static int setup_rt_frame(void *sig_return, struct k_sigaction *ka,
-                          struct pt_regs *regs, int signr, sigset_t *set,
-                          siginfo_t *info)
+static int setup_rt_frame(void *sig_return, struct ksignal *ksig,
+                          struct pt_regs *regs, sigset_t *set)
 {
         struct rt_sigframe __user *frame;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, regs, sizeof(*frame));
         if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Create siginfo.  */
-        err |= copy_siginfo_to_user(&frame->rs_info, info);
+        err |= copy_siginfo_to_user(&frame->rs_info, &ksig->info);
 
         /* Create the ucontext.  */
         err |= __put_user(0, &frame->rs_uc.uc_flags);
@@ -493,7 +486,7 @@ static int setup_rt_frame(void *sig_return, struct k_sigaction *ka,
         err |= __copy_to_user(&frame->rs_uc.uc_sigmask, set, sizeof(*set));
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /*
          * Arguments to signal handler:
@@ -505,22 +498,18 @@ static int setup_rt_frame(void *sig_return, struct k_sigaction *ka,
          * $25 and c0_epc point to the signal handler, $29 points to
          * the struct rt_sigframe.
          */
-        regs->regs[ 4] = signr;
+        regs->regs[ 4] = ksig->sig;
         regs->regs[ 5] = (unsigned long) &frame->rs_info;
         regs->regs[ 6] = (unsigned long) &frame->rs_uc;
         regs->regs[29] = (unsigned long) frame;
         regs->regs[31] = (unsigned long) sig_return;
-        regs->cp0_epc = regs->regs[25] = (unsigned long) ka->sa.sa_handler;
+        regs->cp0_epc = regs->regs[25] = (unsigned long) ksig->ka.sa.sa_handler;
 
         DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n",
                current->comm, current->pid,
                frame, regs->cp0_epc, regs->regs[31]);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(signr, current);
-        return -EFAULT;
 }
 
 struct mips_abi mips_abi = {
@@ -534,8 +523,7 @@ struct mips_abi mips_abi = {
         .restart        = __NR_restart_syscall
 };
 
-static void handle_signal(unsigned long sig, siginfo_t *info,
-        struct k_sigaction *ka, struct pt_regs *regs)
+static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
         sigset_t *oldset = sigmask_to_save();
         int ret;
@@ -557,7 +545,7 @@ static void handle_signal(unsigned long sig, siginfo_t *info,
                         regs->regs[2] = EINTR;
                         break;
                 case ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
+                        if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                 regs->regs[2] = EINTR;
                                 break;
                         }
@@ -571,29 +559,23 @@ static void handle_signal(unsigned long sig, siginfo_t *info,
                 regs->regs[0] = 0;              /* Don't deal with this again.  */
         }
 
-        if (sig_uses_siginfo(ka))
+        if (sig_uses_siginfo(&ksig->ka))
                 ret = abi->setup_rt_frame(vdso + abi->rt_signal_return_offset,
-                                          ka, regs, sig, oldset, info);
+                                          ksig, regs, oldset);
         else
-                ret = abi->setup_frame(vdso + abi->signal_return_offset,
-                                       ka, regs, sig, oldset);
-
-        if (ret)
-                return;
+                ret = abi->setup_frame(vdso + abi->signal_return_offset, ksig,
+                                       regs, oldset);
 
-        signal_delivered(sig, info, ka, regs, 0);
+        signal_setup_done(ret, ksig, 0);
 }
 
 static void do_signal(struct pt_regs *regs)
 {
-        struct k_sigaction ka;
-        siginfo_t info;
-        int signr;
+        struct ksignal ksig;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /* Whee!  Actually deliver the signal.  */
-                handle_signal(signr, &info, &ka, regs);
+                handle_signal(&ksig, regs);
                 return;
         }
 
diff --git a/arch/mips/kernel/signal32.c b/arch/mips/kernel/signal32.c
index bae2e6ee2109..d69179c0d49d 100644
--- a/arch/mips/kernel/signal32.c
+++ b/arch/mips/kernel/signal32.c
@@ -490,21 +490,21 @@ badframe:
         force_sig(SIGSEGV, current);
 }
 
-static int setup_frame_32(void *sig_return, struct k_sigaction *ka,
-                          struct pt_regs *regs, int signr, sigset_t *set)
+static int setup_frame_32(void *sig_return, struct ksignal *ksig,
+                          struct pt_regs *regs, sigset_t *set)
 {
         struct sigframe32 __user *frame;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, regs, sizeof(*frame));
         if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         err |= setup_sigcontext32(regs, &frame->sf_sc);
         err |= __copy_conv_sigset_to_user(&frame->sf_mask, set);
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /*
          * Arguments to signal handler:
@@ -516,37 +516,32 @@ static int setup_frame_32(void *sig_return, struct k_sigaction *ka,
          * $25 and c0_epc point to the signal handler, $29 points to the
          * struct sigframe.
          */
-        regs->regs[ 4] = signr;
+        regs->regs[ 4] = ksig->sig;
         regs->regs[ 5] = 0;
         regs->regs[ 6] = (unsigned long) &frame->sf_sc;
         regs->regs[29] = (unsigned long) frame;
         regs->regs[31] = (unsigned long) sig_return;
-        regs->cp0_epc = regs->regs[25] = (unsigned long) ka->sa.sa_handler;
+        regs->cp0_epc = regs->regs[25] = (unsigned long) ksig->ka.sa.sa_handler;
 
         DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n",
                current->comm, current->pid,
                frame, regs->cp0_epc, regs->regs[31]);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(signr, current);
-        return -EFAULT;
 }
 
-static int setup_rt_frame_32(void *sig_return, struct k_sigaction *ka,
-                             struct pt_regs *regs, int signr, sigset_t *set,
-                             siginfo_t *info)
+static int setup_rt_frame_32(void *sig_return, struct ksignal *ksig,
+                             struct pt_regs *regs, sigset_t *set)
 {
         struct rt_sigframe32 __user *frame;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, regs, sizeof(*frame));
         if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Convert (siginfo_t -> compat_siginfo_t) and copy to user. */
-        err |= copy_siginfo_to_user32(&frame->rs_info, info);
+        err |= copy_siginfo_to_user32(&frame->rs_info, &ksig->info);
 
         /* Create the ucontext.  */
         err |= __put_user(0, &frame->rs_uc.uc_flags);
@@ -556,7 +551,7 @@ static int setup_rt_frame_32(void *sig_return, struct k_sigaction *ka,
         err |= __copy_conv_sigset_to_user(&frame->rs_uc.uc_sigmask, set);
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /*
          * Arguments to signal handler:
@@ -568,22 +563,18 @@ static int setup_rt_frame_32(void *sig_return, struct k_sigaction *ka,
          * $25 and c0_epc point to the signal handler, $29 points to
          * the struct rt_sigframe32.
          */
-        regs->regs[ 4] = signr;
+        regs->regs[ 4] = ksig->sig;
         regs->regs[ 5] = (unsigned long) &frame->rs_info;
         regs->regs[ 6] = (unsigned long) &frame->rs_uc;
         regs->regs[29] = (unsigned long) frame;
         regs->regs[31] = (unsigned long) sig_return;
-        regs->cp0_epc = regs->regs[25] = (unsigned long) ka->sa.sa_handler;
+        regs->cp0_epc = regs->regs[25] = (unsigned long) ksig->ka.sa.sa_handler;
 
         DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n",
                current->comm, current->pid,
                frame, regs->cp0_epc, regs->regs[31]);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(signr, current);
-        return -EFAULT;
 }
 
 /*
diff --git a/arch/mips/kernel/signal_n32.c b/arch/mips/kernel/signal_n32.c
index b2241bb9cac1..f1d4751eead0 100644
--- a/arch/mips/kernel/signal_n32.c
+++ b/arch/mips/kernel/signal_n32.c
@@ -102,18 +102,18 @@ badframe:
         force_sig(SIGSEGV, current);
 }
 
-static int setup_rt_frame_n32(void *sig_return, struct k_sigaction *ka,
-        struct pt_regs *regs, int signr, sigset_t *set, siginfo_t *info)
+static int setup_rt_frame_n32(void *sig_return, struct ksignal *ksig,
+                              struct pt_regs *regs, sigset_t *set)
 {
         struct rt_sigframe_n32 __user *frame;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, regs, sizeof(*frame));
         if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Create siginfo.  */
-        err |= copy_siginfo_to_user32(&frame->rs_info, info);
+        err |= copy_siginfo_to_user32(&frame->rs_info, &ksig->info);
 
         /* Create the ucontext.  */
         err |= __put_user(0, &frame->rs_uc.uc_flags);
@@ -123,7 +123,7 @@ static int setup_rt_frame_n32(void *sig_return, struct k_sigaction *ka,
         err |= __copy_conv_sigset_to_user(&frame->rs_uc.uc_sigmask, set);
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /*
          * Arguments to signal handler:
@@ -135,22 +135,18 @@ static int setup_rt_frame_n32(void *sig_return, struct k_sigaction *ka,
          * $25 and c0_epc point to the signal handler, $29 points to
          * the struct rt_sigframe.
          */
-        regs->regs[ 4] = signr;
+        regs->regs[ 4] = ksig->sig;
         regs->regs[ 5] = (unsigned long) &frame->rs_info;
         regs->regs[ 6] = (unsigned long) &frame->rs_uc;
         regs->regs[29] = (unsigned long) frame;
         regs->regs[31] = (unsigned long) sig_return;
-        regs->cp0_epc = regs->regs[25] = (unsigned long) ka->sa.sa_handler;
+        regs->cp0_epc = regs->regs[25] = (unsigned long) ksig->ka.sa.sa_handler;
 
         DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n",
                current->comm, current->pid,
                frame, regs->cp0_epc, regs->regs[31]);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(signr, current);
-        return -EFAULT;
 }
 
 struct mips_abi mips_abi_n32 = {
diff --git a/arch/mn10300/kernel/signal.c b/arch/mn10300/kernel/signal.c
index 9dfac5cd16e6..a6c0858592c3 100644
--- a/arch/mn10300/kernel/signal.c
+++ b/arch/mn10300/kernel/signal.c
@@ -186,20 +186,11 @@ static int setup_sigcontext(struct sigcontext __user *sc,
 /*
  * determine which stack to use..
  */
-static inline void __user *get_sigframe(struct k_sigaction *ka,
+static inline void __user *get_sigframe(struct ksignal *ksig,
                                         struct pt_regs *regs,
                                         size_t frame_size)
 {
-        unsigned long sp;
-
-        /* default to using normal stack */
-        sp = regs->sp;
-
-        /* this is the X/Open sanctioned signal stack switching.  */
-        if (ka->sa.sa_flags & SA_ONSTACK) {
-                if (sas_ss_flags(sp) == 0)
-                        sp = current->sas_ss_sp + current->sas_ss_size;
-        }
+        unsigned long sp = sigsp(regs->sp, ksig);
 
         return (void __user *) ((sp - frame_size) & ~7UL);
 }
@@ -207,16 +198,16 @@ static inline void __user *get_sigframe(struct k_sigaction *ka,
 /*
  * set up a normal signal frame
  */
-static int setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
+static int setup_frame(struct ksignal *ksig, sigset_t *set,
                        struct pt_regs *regs)
 {
         struct sigframe __user *frame;
-        int rsig;
+        int rsig, sig = ksig->sig;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, regs, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         rsig = sig;
         if (sig < 32 &&
@@ -226,40 +217,40 @@ static int setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
 
         if (__put_user(rsig, &frame->sig) < 0 ||
             __put_user(&frame->sc, &frame->psc) < 0)
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (setup_sigcontext(&frame->sc, &frame->fpuctx, regs, set->sig[0]))
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (_NSIG_WORDS > 1) {
                 if (__copy_to_user(frame->extramask, &set->sig[1],
                                    sizeof(frame->extramask)))
-                        goto give_sigsegv;
+                        return -EFAULT;
         }
 
         /* set up to return from userspace.  If provided, use a stub already in
          * userspace */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                if (__put_user(ka->sa.sa_restorer, &frame->pretcode))
-                        goto give_sigsegv;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                if (__put_user(ksig->ka.sa.sa_restorer, &frame->pretcode))
+                        return -EFAULT;
         } else {
                 if (__put_user((void (*)(void))frame->retcode,
                                &frame->pretcode))
-                        goto give_sigsegv;
+                        return -EFAULT;
                 /* this is mov $,d0; syscall 0 */
                 if (__put_user(0x2c, (char *)(frame->retcode + 0)) ||
                     __put_user(__NR_sigreturn, (char *)(frame->retcode + 1)) ||
                     __put_user(0x00, (char *)(frame->retcode + 2)) ||
                     __put_user(0xf0, (char *)(frame->retcode + 3)) ||
                     __put_user(0xe0, (char *)(frame->retcode + 4)))
-                        goto give_sigsegv;
+                        return -EFAULT;
                 flush_icache_range((unsigned long) frame->retcode,
                                    (unsigned long) frame->retcode + 5);
         }
 
         /* set up registers for signal handler */
         regs->sp = (unsigned long) frame;
-        regs->pc = (unsigned long) ka->sa.sa_handler;
+        regs->pc = (unsigned long) ksig->ka.sa.sa_handler;
         regs->d0 = sig;
         regs->d1 = (unsigned long) &frame->sc;
 
@@ -270,25 +261,21 @@ static int setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
 #endif
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
 /*
  * set up a realtime signal frame
  */
-static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                          sigset_t *set, struct pt_regs *regs)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
-        int rsig;
+        int rsig, sig = ksig->sig;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, regs, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         rsig = sig;
         if (sig < 32 &&
@@ -299,8 +286,8 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         if (__put_user(rsig, &frame->sig) ||
             __put_user(&frame->info, &frame->pinfo) ||
             __put_user(&frame->uc, &frame->puc) ||
-            copy_siginfo_to_user(&frame->info, info))
-                goto give_sigsegv;
+            copy_siginfo_to_user(&frame->info, &ksig->info))
+                return -EFAULT;
 
         /* create the ucontext.  */
         if (__put_user(0, &frame->uc.uc_flags) ||
@@ -309,13 +296,14 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
             setup_sigcontext(&frame->uc.uc_mcontext,
                              &frame->fpuctx, regs, set->sig[0]) ||
             __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* set up to return from userspace.  If provided, use a stub already in
          * userspace */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                if (__put_user(ka->sa.sa_restorer, &frame->pretcode))
-                        goto give_sigsegv;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                if (__put_user(ksig->ka.sa.sa_restorer, &frame->pretcode))
+                        return -EFAULT;
+
         } else {
                 if (__put_user((void(*)(void))frame->retcode,
                                &frame->pretcode) ||
@@ -326,7 +314,7 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                     __put_user(0x00, (char *)(frame->retcode + 2)) ||
                     __put_user(0xf0, (char *)(frame->retcode + 3)) ||
                     __put_user(0xe0, (char *)(frame->retcode + 4)))
-                        goto give_sigsegv;
+                        return -EFAULT;
 
                 flush_icache_range((u_long) frame->retcode,
                                    (u_long) frame->retcode + 5);
@@ -334,7 +322,7 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
         /* Set up registers for signal handler */
         regs->sp = (unsigned long) frame;
-        regs->pc = (unsigned long) ka->sa.sa_handler;
+        regs->pc = (unsigned long) ksig->ka.sa.sa_handler;
         regs->d0 = sig;
         regs->d1 = (long) &frame->info;
 
@@ -345,10 +333,6 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 #endif
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
 static inline void stepback(struct pt_regs *regs)
@@ -360,9 +344,7 @@ static inline void stepback(struct pt_regs *regs)
 /*
  * handle the actual delivery of a signal to userspace
  */
-static int handle_signal(int sig,
-                         siginfo_t *info, struct k_sigaction *ka,
-                         struct pt_regs *regs)
+static int handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
         sigset_t *oldset = sigmask_to_save();
         int ret;
@@ -377,7 +359,7 @@ static int handle_signal(int sig,
                         break;
 
                 case -ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
+                        if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                 regs->d0 = -EINTR;
                                 break;
                         }
@@ -390,15 +372,12 @@ static int handle_signal(int sig,
         }
 
         /* Set up the stack frame */
-        if (ka->sa.sa_flags & SA_SIGINFO)
-                ret = setup_rt_frame(sig, ka, info, oldset, regs);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                ret = setup_rt_frame(ksig, oldset, regs);
         else
-                ret = setup_frame(sig, ka, oldset, regs);
-        if (ret)
-                return ret;
+                ret = setup_frame(ksig, oldset, regs);
 
-        signal_delivered(sig, info, ka, regs,
-                         test_thread_flag(TIF_SINGLESTEP));
+        signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLESTEP));
         return 0;
 }
 
@@ -407,15 +386,10 @@ static int handle_signal(int sig,
  */
 static void do_signal(struct pt_regs *regs)
 {
-        struct k_sigaction ka;
-        siginfo_t info;
-        int signr;
-
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
-                if (handle_signal(signr, &info, &ka, regs) == 0) {
-                }
+        struct ksignal ksig;
 
+        if (get_signal(&ksig)) {
+                handle_signal(&ksig, regs);
                 return;
         }
 
diff --git a/arch/openrisc/kernel/signal.c b/arch/openrisc/kernel/signal.c
index 66775bc07a8e..7d1b8235bf90 100644
--- a/arch/openrisc/kernel/signal.c
+++ b/arch/openrisc/kernel/signal.c
@@ -132,30 +132,16 @@ static inline unsigned long align_sigframe(unsigned long sp)
  * or the alternate stack.
  */
 
-static inline void __user *get_sigframe(struct k_sigaction *ka,
+static inline void __user *get_sigframe(struct ksignal *ksig,
                                         struct pt_regs *regs, size_t frame_size)
 {
         unsigned long sp = regs->sp;
-        int onsigstack = on_sig_stack(sp);
 
         /* redzone */
         sp -= STACK_FRAME_OVERHEAD;
-
-        /* This is the X/Open sanctioned signal stack switching.  */
-        if ((ka->sa.sa_flags & SA_ONSTACK) && !onsigstack) {
-                if (current->sas_ss_size)
-                        sp = current->sas_ss_sp + current->sas_ss_size;
-        }
-
+        sp = sigsp(sp, ksig);
         sp = align_sigframe(sp - frame_size);
 
-        /*
-         * If we are on the alternate signal stack and would overflow it, don't.
-         * Return an always-bogus address instead so we will die with SIGSEGV.
-         */
-        if (onsigstack && !likely(on_sig_stack(sp)))
-                return (void __user *)-1L;
-
         return (void __user *)sp;
 }
 
@@ -173,7 +159,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
         unsigned long return_ip;
         int err = 0;
 
-        frame = get_sigframe(&ksig->ka, regs, sizeof(*frame));
+        frame = get_sigframe(ksig, regs, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
                 return -EFAULT;
diff --git a/arch/parisc/kernel/signal.c b/arch/parisc/kernel/signal.c
index 1cba8f29bb49..012d4fa63d97 100644
--- a/arch/parisc/kernel/signal.c
+++ b/arch/parisc/kernel/signal.c
@@ -227,8 +227,8 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, int in_sysc
 }
 
 static long
-setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-               sigset_t *set, struct pt_regs *regs, int in_syscall)
+setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs,
+               int in_syscall)
 {
         struct rt_sigframe __user *frame;
         unsigned long rp, usp;
@@ -241,10 +241,10 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         
         usp = (regs->gr[30] & ~(0x01UL));
         /*FIXME: frame_size parameter is unused, remove it. */
-        frame = get_sigframe(ka, usp, sizeof(*frame));
+        frame = get_sigframe(&ksig->ka, usp, sizeof(*frame));
 
         DBG(1,"SETUP_RT_FRAME: START\n");
-        DBG(1,"setup_rt_frame: frame %p info %p\n", frame, info);
+        DBG(1,"setup_rt_frame: frame %p info %p\n", frame, ksig->info);
 
         
 #ifdef CONFIG_64BIT
@@ -253,7 +253,7 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         
         if (is_compat_task()) {
                 DBG(1,"setup_rt_frame: frame->info = 0x%p\n", &compat_frame->info);
-                err |= copy_siginfo_to_user32(&compat_frame->info, info);
+                err |= copy_siginfo_to_user32(&compat_frame->info, &ksig->info);
                 err |= __compat_save_altstack( &compat_frame->uc.uc_stack, regs->gr[30]);
                 DBG(1,"setup_rt_frame: frame->uc = 0x%p\n", &compat_frame->uc);
                 DBG(1,"setup_rt_frame: frame->uc.uc_mcontext = 0x%p\n", &compat_frame->uc.uc_mcontext);
@@ -265,7 +265,7 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 #endif
         {       
                 DBG(1,"setup_rt_frame: frame->info = 0x%p\n", &frame->info);
-                err |= copy_siginfo_to_user(&frame->info, info);
+                err |= copy_siginfo_to_user(&frame->info, &ksig->info);
                 err |= __save_altstack(&frame->uc.uc_stack, regs->gr[30]);
                 DBG(1,"setup_rt_frame: frame->uc = 0x%p\n", &frame->uc);
                 DBG(1,"setup_rt_frame: frame->uc.uc_mcontext = 0x%p\n", &frame->uc.uc_mcontext);
@@ -275,7 +275,7 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         }
         
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up to return from userspace.  If provided, use a stub
            already in userspace. The first words of tramp are used to
@@ -312,9 +312,9 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         rp = (unsigned long) &frame->tramp[SIGRESTARTBLOCK_TRAMP];
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
-        haddr = A(ka->sa.sa_handler);
+        haddr = A(ksig->ka.sa.sa_handler);
         /* The sa_handler may be a pointer to a function descriptor */
 #ifdef CONFIG_64BIT
         if (is_compat_task()) {
@@ -326,7 +326,7 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                         err = __copy_from_user(&fdesc, ufdesc, sizeof(fdesc));
 
                         if (err)
-                                goto give_sigsegv;
+                                return -EFAULT;
 
                         haddr = fdesc.addr;
                         regs->gr[19] = fdesc.gp;
@@ -339,7 +339,7 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 err = __copy_from_user(&fdesc, ufdesc, sizeof(fdesc));
                 
                 if (err)
-                        goto give_sigsegv;
+                        return -EFAULT;
                 
                 haddr = fdesc.addr;
                 regs->gr[19] = fdesc.gp;
@@ -386,7 +386,7 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         }
 
         regs->gr[2]  = rp;                /* userland return pointer */
-        regs->gr[26] = sig;               /* signal number */
+        regs->gr[26] = ksig->sig;               /* signal number */
         
 #ifdef CONFIG_64BIT
         if (is_compat_task()) {
@@ -410,11 +410,6 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                current->comm, current->pid, frame, regs->gr[30],
                regs->iaoq[0], regs->iaoq[1], rp);
 
-        return 1;
-
-give_sigsegv:
-        DBG(1,"setup_rt_frame: sending SIGSEGV\n");
-        force_sigsegv(sig, current);
         return 0;
 }
 
@@ -423,20 +418,19 @@ give_sigsegv:
  */     
 
 static void
-handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
-                struct pt_regs *regs, int in_syscall)
+handle_signal(struct ksignal *ksig, struct pt_regs *regs, int in_syscall)
 {
+        int ret;
         sigset_t *oldset = sigmask_to_save();
+
         DBG(1,"handle_signal: sig=%ld, ka=%p, info=%p, oldset=%p, regs=%p\n",
-               sig, ka, info, oldset, regs);
+               ksig->sig, ksig->ka, ksig->info, oldset, regs);
         
         /* Set up the stack frame */
-        if (!setup_rt_frame(sig, ka, info, oldset, regs, in_syscall))
-                return;
+        ret = setup_rt_frame(ksig, oldset, regs, in_syscall);
 
-        signal_delivered(sig, info, ka, regs, 
-                test_thread_flag(TIF_SINGLESTEP) ||
-                test_thread_flag(TIF_BLOCKSTEP));
+        signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLESTEP) ||
+                          test_thread_flag(TIF_BLOCKSTEP));
 
         DBG(1,KERN_DEBUG "do_signal: Exit (success), regs->gr[28] = %ld\n",
                 regs->gr[28]);
@@ -544,22 +538,18 @@ insert_restart_trampoline(struct pt_regs *regs)
 asmlinkage void
 do_signal(struct pt_regs *regs, long in_syscall)
 {
-        siginfo_t info;
-        struct k_sigaction ka;
-        int signr;
+        struct ksignal ksig;
 
         DBG(1,"\ndo_signal: regs=0x%p, sr7 %#lx, in_syscall=%d\n",
                regs, regs->sr[7], in_syscall);
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        DBG(3,"do_signal: signr = %d, regs->gr[28] = %ld\n", signr, regs->gr[28]); 
-        
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
+                DBG(3,"do_signal: signr = %d, regs->gr[28] = %ld\n", signr, regs->gr[28]);
                 /* Restart a system call if necessary. */
                 if (in_syscall)
-                        syscall_restart(regs, &ka);
+                        syscall_restart(regs, &ksig.ka);
 
-                handle_signal(signr, &info, &ka, regs, in_syscall);
+                handle_signal(&ksig, regs, in_syscall);
                 return;
         }
 
diff --git a/arch/powerpc/kernel/signal.c b/arch/powerpc/kernel/signal.c
index 1c794cef2883..cf8c7e4e0b21 100644
--- a/arch/powerpc/kernel/signal.c
+++ b/arch/powerpc/kernel/signal.c
@@ -31,20 +31,14 @@ int show_unhandled_signals = 1;
 /*
  * Allocate space for the signal frame
  */
-void __user * get_sigframe(struct k_sigaction *ka, unsigned long sp,
+void __user *get_sigframe(struct ksignal *ksig, unsigned long sp,
                            size_t frame_size, int is_32)
 {
         unsigned long oldsp, newsp;
 
         /* Default to using normal stack */
         oldsp = get_clean_sp(sp, is_32);
-
-        /* Check for alt stack */
-        if ((ka->sa.sa_flags & SA_ONSTACK) &&
-            current->sas_ss_size && !on_sig_stack(oldsp))
-                oldsp = (current->sas_ss_sp + current->sas_ss_size);
-
-        /* Get aligned frame */
+        oldsp = sigsp(oldsp, ksig);
         newsp = (oldsp - frame_size) & ~0xFUL;
 
         /* Check access */
@@ -105,25 +99,23 @@ static void check_syscall_restart(struct pt_regs *regs, struct k_sigaction *ka,
         }
 }
 
-static int do_signal(struct pt_regs *regs)
+static void do_signal(struct pt_regs *regs)
 {
         sigset_t *oldset = sigmask_to_save();
-        siginfo_t info;
-        int signr;
-        struct k_sigaction ka;
+        struct ksignal ksig;
         int ret;
         int is32 = is_32bit_task();
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
+        get_signal(&ksig);
 
         /* Is there any syscall restart business here ? */
-        check_syscall_restart(regs, &ka, signr > 0);
+        check_syscall_restart(regs, &ksig.ka, ksig.sig > 0);
 
-        if (signr <= 0) {
+        if (ksig.sig <= 0) {
                 /* No signal to deliver -- put the saved sigmask back */
                 restore_saved_sigmask();
                 regs->trap = 0;
-                return 0;               /* no signals delivered */
+                return;               /* no signals delivered */
         }
 
 #ifndef CONFIG_PPC_ADV_DEBUG_REGS
@@ -140,23 +132,16 @@ static int do_signal(struct pt_regs *regs)
         thread_change_pc(current, regs);
 
         if (is32) {
-                if (ka.sa.sa_flags & SA_SIGINFO)
-                        ret = handle_rt_signal32(signr, &ka, &info, oldset,
-                                        regs);
+                if (ksig.ka.sa.sa_flags & SA_SIGINFO)
+                        ret = handle_rt_signal32(&ksig, oldset, regs);
                 else
-                        ret = handle_signal32(signr, &ka, &info, oldset,
-                                        regs);
+                        ret = handle_signal32(&ksig, oldset, regs);
         } else {
-                ret = handle_rt_signal64(signr, &ka, &info, oldset, regs);
+                ret = handle_rt_signal64(&ksig, oldset, regs);
         }
 
         regs->trap = 0;
-        if (ret) {
-                signal_delivered(signr, &info, &ka, regs,
-                                         test_thread_flag(TIF_SINGLESTEP));
-        }
-
-        return ret;
+        signal_setup_done(ret, &ksig, test_thread_flag(TIF_SINGLESTEP));
 }
 
 void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags)
diff --git a/arch/powerpc/kernel/signal.h b/arch/powerpc/kernel/signal.h
index c69b9aeb9f23..51b274199dd9 100644
--- a/arch/powerpc/kernel/signal.h
+++ b/arch/powerpc/kernel/signal.h
@@ -12,15 +12,13 @@
 
 extern void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags);
 
-extern void __user * get_sigframe(struct k_sigaction *ka, unsigned long sp,
+extern void __user *get_sigframe(struct ksignal *ksig, unsigned long sp,
                                   size_t frame_size, int is_32);
 
-extern int handle_signal32(unsigned long sig, struct k_sigaction *ka,
-                           siginfo_t *info, sigset_t *oldset,
+extern int handle_signal32(struct ksignal *ksig, sigset_t *oldset,
                            struct pt_regs *regs);
 
-extern int handle_rt_signal32(unsigned long sig, struct k_sigaction *ka,
-                              siginfo_t *info, sigset_t *oldset,
+extern int handle_rt_signal32(struct ksignal *ksig, sigset_t *oldset,
                               struct pt_regs *regs);
 
 extern unsigned long copy_fpr_to_user(void __user *to,
@@ -44,14 +42,12 @@ extern unsigned long copy_transact_vsx_from_user(struct task_struct *task,
 
 #ifdef CONFIG_PPC64
 
-extern int handle_rt_signal64(int signr, struct k_sigaction *ka,
-                              siginfo_t *info, sigset_t *set,
+extern int handle_rt_signal64(struct ksignal *ksig, sigset_t *set,
                               struct pt_regs *regs);
 
 #else /* CONFIG_PPC64 */
 
-static inline int handle_rt_signal64(int signr, struct k_sigaction *ka,
-                                     siginfo_t *info, sigset_t *set,
+static inline int handle_rt_signal64(struct ksignal *ksig, sigset_t *set,
                                      struct pt_regs *regs)
 {
         return -EFAULT;
diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c
index 1bc5a1755ed4..b171001698ff 100644
--- a/arch/powerpc/kernel/signal_32.c
+++ b/arch/powerpc/kernel/signal_32.c
@@ -981,9 +981,8 @@ int copy_siginfo_from_user32(siginfo_t *to, struct compat_siginfo __user *from)
  * Set up a signal frame for a "real-time" signal handler
  * (one which gets siginfo).
  */
-int handle_rt_signal32(unsigned long sig, struct k_sigaction *ka,
-                siginfo_t *info, sigset_t *oldset,
-                struct pt_regs *regs)
+int handle_rt_signal32(struct ksignal *ksig, sigset_t *oldset,
+                       struct pt_regs *regs)
 {
         struct rt_sigframe __user *rt_sf;
         struct mcontext __user *frame;
@@ -995,13 +994,13 @@ int handle_rt_signal32(unsigned long sig, struct k_sigaction *ka,
 
         /* Set up Signal Frame */
         /* Put a Real Time Context onto stack */
-        rt_sf = get_sigframe(ka, get_tm_stackpointer(regs), sizeof(*rt_sf), 1);
+        rt_sf = get_sigframe(ksig, get_tm_stackpointer(regs), sizeof(*rt_sf), 1);
         addr = rt_sf;
         if (unlikely(rt_sf == NULL))
                 goto badframe;
 
         /* Put the siginfo & fill in most of the ucontext */
-        if (copy_siginfo_to_user(&rt_sf->info, info)
+        if (copy_siginfo_to_user(&rt_sf->info, &ksig->info)
             || __put_user(0, &rt_sf->uc.uc_flags)
             || __save_altstack(&rt_sf->uc.uc_stack, regs->gpr[1])
             || __put_user(to_user_ptr(&rt_sf->uc.uc_mcontext),
@@ -1051,15 +1050,15 @@ int handle_rt_signal32(unsigned long sig, struct k_sigaction *ka,
 
         /* Fill registers for signal handler */
         regs->gpr[1] = newsp;
-        regs->gpr[3] = sig;
+        regs->gpr[3] = ksig->sig;
         regs->gpr[4] = (unsigned long) &rt_sf->info;
         regs->gpr[5] = (unsigned long) &rt_sf->uc;
         regs->gpr[6] = (unsigned long) rt_sf;
-        regs->nip = (unsigned long) ka->sa.sa_handler;
+        regs->nip = (unsigned long) ksig->ka.sa.sa_handler;
         /* enter the signal handler in native-endian mode */
         regs->msr &= ~MSR_LE;
         regs->msr |= (MSR_KERNEL & MSR_LE);
-        return 1;
+        return 0;
 
 badframe:
         if (show_unhandled_signals)
@@ -1069,8 +1068,7 @@ badframe:
                                    current->comm, current->pid,
                                    addr, regs->nip, regs->link);
 
-        force_sigsegv(sig, current);
-        return 0;
+        return 1;
 }
 
 static int do_setcontext(struct ucontext __user *ucp, struct pt_regs *regs, int sig)
@@ -1409,8 +1407,7 @@ int sys_debug_setcontext(struct ucontext __user *ctx,
 /*
  * OK, we're invoking a handler
  */
-int handle_signal32(unsigned long sig, struct k_sigaction *ka,
-                    siginfo_t *info, sigset_t *oldset, struct pt_regs *regs)
+int handle_signal32(struct ksignal *ksig, sigset_t *oldset, struct pt_regs *regs)
 {
         struct sigcontext __user *sc;
         struct sigframe __user *frame;
@@ -1420,7 +1417,7 @@ int handle_signal32(unsigned long sig, struct k_sigaction *ka,
         unsigned long tramp;
 
         /* Set up Signal Frame */
-        frame = get_sigframe(ka, get_tm_stackpointer(regs), sizeof(*frame), 1);
+        frame = get_sigframe(ksig, get_tm_stackpointer(regs), sizeof(*frame), 1);
         if (unlikely(frame == NULL))
                 goto badframe;
         sc = (struct sigcontext __user *) &frame->sctx;
@@ -1428,7 +1425,7 @@ int handle_signal32(unsigned long sig, struct k_sigaction *ka,
 #if _NSIG != 64
 #error "Please adjust handle_signal()"
 #endif
-        if (__put_user(to_user_ptr(ka->sa.sa_handler), &sc->handler)
+        if (__put_user(to_user_ptr(ksig->ka.sa.sa_handler), &sc->handler)
             || __put_user(oldset->sig[0], &sc->oldmask)
 #ifdef CONFIG_PPC64
             || __put_user((oldset->sig[0] >> 32), &sc->_unused[3])
@@ -1436,7 +1433,7 @@ int handle_signal32(unsigned long sig, struct k_sigaction *ka,
             || __put_user(oldset->sig[1], &sc->_unused[3])
 #endif
             || __put_user(to_user_ptr(&frame->mctx), &sc->regs)
-            || __put_user(sig, &sc->signal))
+            || __put_user(ksig->sig, &sc->signal))
                 goto badframe;
 
         if (vdso32_sigtramp && current->mm->context.vdso_base) {
@@ -1471,12 +1468,12 @@ int handle_signal32(unsigned long sig, struct k_sigaction *ka,
                 goto badframe;
 
         regs->gpr[1] = newsp;
-        regs->gpr[3] = sig;
+        regs->gpr[3] = ksig->sig;
         regs->gpr[4] = (unsigned long) sc;
-        regs->nip = (unsigned long) ka->sa.sa_handler;
+        regs->nip = (unsigned long) (unsigned long)ksig->ka.sa.sa_handler;
         /* enter the signal handler in big-endian mode */
         regs->msr &= ~MSR_LE;
-        return 1;
+        return 0;
 
 badframe:
         if (show_unhandled_signals)
@@ -1486,8 +1483,7 @@ badframe:
                                    current->comm, current->pid,
                                    frame, regs->nip, regs->link);
 
-        force_sigsegv(sig, current);
-        return 0;
+        return 1;
 }
 
 /*
diff --git a/arch/powerpc/kernel/signal_64.c b/arch/powerpc/kernel/signal_64.c
index 97c1e4b683fc..2cb0c94cafa5 100644
--- a/arch/powerpc/kernel/signal_64.c
+++ b/arch/powerpc/kernel/signal_64.c
@@ -708,20 +708,19 @@ badframe:
         return 0;
 }
 
-int handle_rt_signal64(int signr, struct k_sigaction *ka, siginfo_t *info,
-                sigset_t *set, struct pt_regs *regs)
+int handle_rt_signal64(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
         unsigned long newsp = 0;
         long err = 0;
 
-        frame = get_sigframe(ka, get_tm_stackpointer(regs), sizeof(*frame), 0);
+        frame = get_sigframe(ksig, get_tm_stackpointer(regs), sizeof(*frame), 0);
         if (unlikely(frame == NULL))
                 goto badframe;
 
         err |= __put_user(&frame->info, &frame->pinfo);
         err |= __put_user(&frame->uc, &frame->puc);
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
         if (err)
                 goto badframe;
 
@@ -736,15 +735,15 @@ int handle_rt_signal64(int signr, struct k_sigaction *ka, siginfo_t *info,
                 err |= __put_user(&frame->uc_transact, &frame->uc.uc_link);
                 err |= setup_tm_sigcontexts(&frame->uc.uc_mcontext,
                                             &frame->uc_transact.uc_mcontext,
-                                            regs, signr,
+                                            regs, ksig->sig,
                                             NULL,
-                                            (unsigned long)ka->sa.sa_handler);
+                                            (unsigned long)ksig->ka.sa.sa_handler);
         } else
 #endif
         {
                 err |= __put_user(0, &frame->uc.uc_link);
-                err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, signr,
-                                        NULL, (unsigned long)ka->sa.sa_handler,
+                err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, ksig->sig,
+                                        NULL, (unsigned long)ksig->ka.sa.sa_handler,
                                         1);
         }
         err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
@@ -770,7 +769,7 @@ int handle_rt_signal64(int signr, struct k_sigaction *ka, siginfo_t *info,
 
         /* Set up "regs" so we "return" to the signal handler. */
         if (is_elf2_task()) {
-                regs->nip = (unsigned long) ka->sa.sa_handler;
+                regs->nip = (unsigned long) ksig->ka.sa.sa_handler;
                 regs->gpr[12] = regs->nip;
         } else {
                 /* Handler is *really* a pointer to the function descriptor for
@@ -779,7 +778,7 @@ int handle_rt_signal64(int signr, struct k_sigaction *ka, siginfo_t *info,
                  * entry is the TOC value we need to use.
                  */
                 func_descr_t __user *funct_desc_ptr =
-                        (func_descr_t __user *) ka->sa.sa_handler;
+                        (func_descr_t __user *) ksig->ka.sa.sa_handler;
 
                 err |= get_user(regs->nip, &funct_desc_ptr->entry);
                 err |= get_user(regs->gpr[2], &funct_desc_ptr->toc);
@@ -789,9 +788,9 @@ int handle_rt_signal64(int signr, struct k_sigaction *ka, siginfo_t *info,
         regs->msr &= ~MSR_LE;
         regs->msr |= (MSR_KERNEL & MSR_LE);
         regs->gpr[1] = newsp;
-        regs->gpr[3] = signr;
+        regs->gpr[3] = ksig->sig;
         regs->result = 0;
-        if (ka->sa.sa_flags & SA_SIGINFO) {
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO) {
                 err |= get_user(regs->gpr[4], (unsigned long __user *)&frame->pinfo);
                 err |= get_user(regs->gpr[5], (unsigned long __user *)&frame->puc);
                 regs->gpr[6] = (unsigned long) frame;
@@ -801,7 +800,7 @@ int handle_rt_signal64(int signr, struct k_sigaction *ka, siginfo_t *info,
         if (err)
                 goto badframe;
 
-        return 1;
+        return 0;
 
 badframe:
         if (show_unhandled_signals)
@@ -809,6 +808,5 @@ badframe:
                                    current->comm, current->pid, "setup_rt_frame",
                                    (long)frame, regs->nip, regs->link);
 
-        force_sigsegv(signr, current);
-        return 0;
+        return 1;
 }
diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c
index f204d6920368..598b0b42668b 100644
--- a/arch/s390/kernel/compat_signal.c
+++ b/arch/s390/kernel/compat_signal.c
@@ -320,38 +320,39 @@ static inline int map_signal(int sig)
                 return sig;
 }
 
-static int setup_frame32(int sig, struct k_sigaction *ka,
-                        sigset_t *set, struct pt_regs * regs)
+static int setup_frame32(struct ksignal *ksig, sigset_t *set,
+                         struct pt_regs *regs)
 {
-        sigframe32 __user *frame = get_sigframe(ka, regs, sizeof(sigframe32));
+        int sig = ksig->sig;
+        sigframe32 __user *frame = get_sigframe(&ksig->ka, regs, sizeof(sigframe32));
 
         if (frame == (void __user *) -1UL)
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (__copy_to_user(&frame->sc.oldmask, &set->sig, _SIGMASK_COPY_SIZE32))
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (save_sigregs32(regs, &frame->sregs))
-                goto give_sigsegv;
+                return -EFAULT;
         if (save_sigregs_gprs_high(regs, frame->gprs_high))
-                goto give_sigsegv;
+                return -EFAULT;
         if (__put_user((unsigned long) &frame->sregs, &frame->sc.sregs))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up to return from userspace.  If provided, use a stub
            already in userspace.  */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                regs->gprs[14] = (__u64 __force) ka->sa.sa_restorer | PSW32_ADDR_AMODE;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                regs->gprs[14] = (__u64 __force) ksig->ka.sa.sa_restorer | PSW32_ADDR_AMODE;
         } else {
                 regs->gprs[14] = (__u64 __force) frame->retcode | PSW32_ADDR_AMODE;
                 if (__put_user(S390_SYSCALL_OPCODE | __NR_sigreturn,
                                (u16 __force __user *)(frame->retcode)))
-                        goto give_sigsegv;
+                        return -EFAULT;
         }
 
         /* Set up backchain. */
         if (__put_user(regs->gprs[15], (unsigned int __user *) frame))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up registers for signal handler */
         regs->gprs[15] = (__force __u64) frame;
@@ -359,7 +360,7 @@ static int setup_frame32(int sig, struct k_sigaction *ka,
         regs->psw.mask = PSW_MASK_BA |
                 (PSW_USER_BITS & PSW_MASK_ASC) |
                 (regs->psw.mask & ~PSW_MASK_ASC);
-        regs->psw.addr = (__force __u64) ka->sa.sa_handler;
+        regs->psw.addr = (__force __u64) ksig->ka.sa.sa_handler;
 
         regs->gprs[2] = map_signal(sig);
         regs->gprs[3] = (__force __u64) &frame->sc;
@@ -376,25 +377,21 @@ static int setup_frame32(int sig, struct k_sigaction *ka,
 
         /* Place signal number on stack to allow backtrace from handler.  */
         if (__put_user(regs->gprs[2], (int __force __user *) &frame->signo))
-                goto give_sigsegv;
+                return -EFAULT;
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
-static int setup_rt_frame32(int sig, struct k_sigaction *ka, siginfo_t *info,
-                           sigset_t *set, struct pt_regs * regs)
+static int setup_rt_frame32(struct ksignal *ksig, sigset_t *set,
+                            struct pt_regs *regs)
 {
         int err = 0;
-        rt_sigframe32 __user *frame = get_sigframe(ka, regs, sizeof(rt_sigframe32));
+        rt_sigframe32 __user *frame = get_sigframe(&ksig->ka, regs, sizeof(rt_sigframe32));
 
         if (frame == (void __user *) -1UL)
-                goto give_sigsegv;
+                return -EFAULT;
 
-        if (copy_siginfo_to_user32(&frame->info, info))
-                goto give_sigsegv;
+        if (copy_siginfo_to_user32(&frame->info, &ksig->info))
+                return -EFAULT;
 
         /* Create the ucontext.  */
         err |= __put_user(UC_EXTENDED, &frame->uc.uc_flags);
@@ -404,22 +401,22 @@ static int setup_rt_frame32(int sig, struct k_sigaction *ka, siginfo_t *info,
         err |= save_sigregs_gprs_high(regs, frame->gprs_high);
         err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up to return from userspace.  If provided, use a stub
            already in userspace.  */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                regs->gprs[14] = (__u64 __force) ka->sa.sa_restorer | PSW32_ADDR_AMODE;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                regs->gprs[14] = (__u64 __force) ksig->ka.sa.sa_restorer | PSW32_ADDR_AMODE;
         } else {
                 regs->gprs[14] = (__u64 __force) frame->retcode | PSW32_ADDR_AMODE;
                 if (__put_user(S390_SYSCALL_OPCODE | __NR_rt_sigreturn,
                                (u16 __force __user *)(frame->retcode)))
-                        goto give_sigsegv;
+                        return -EFAULT;
         }
 
         /* Set up backchain. */
         if (__put_user(regs->gprs[15], (unsigned int __force __user *) frame))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up registers for signal handler */
         regs->gprs[15] = (__force __u64) frame;
@@ -427,36 +424,30 @@ static int setup_rt_frame32(int sig, struct k_sigaction *ka, siginfo_t *info,
         regs->psw.mask = PSW_MASK_BA |
                 (PSW_USER_BITS & PSW_MASK_ASC) |
                 (regs->psw.mask & ~PSW_MASK_ASC);
-        regs->psw.addr = (__u64 __force) ka->sa.sa_handler;
+        regs->psw.addr = (__u64 __force) ksig->ka.sa.sa_handler;
 
-        regs->gprs[2] = map_signal(sig);
+        regs->gprs[2] = map_signal(ksig->sig);
         regs->gprs[3] = (__force __u64) &frame->info;
         regs->gprs[4] = (__force __u64) &frame->uc;
         regs->gprs[5] = task_thread_info(current)->last_break;
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
 /*
  * OK, we're invoking a handler
  */     
 
-void handle_signal32(unsigned long sig, struct k_sigaction *ka,
-                    siginfo_t *info, sigset_t *oldset, struct pt_regs *regs)
+void handle_signal32(struct ksignal *ksig, sigset_t *oldset,
+                     struct pt_regs *regs)
 {
         int ret;
 
         /* Set up the stack frame */
-        if (ka->sa.sa_flags & SA_SIGINFO)
-                ret = setup_rt_frame32(sig, ka, info, oldset, regs);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                ret = setup_rt_frame32(ksig, oldset, regs);
         else
-                ret = setup_frame32(sig, ka, oldset, regs);
-        if (ret)
-                return;
-        signal_delivered(sig, info, ka, regs,
-                                 test_thread_flag(TIF_SINGLE_STEP));
+                ret = setup_frame32(ksig, oldset, regs);
+
+        signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLE_STEP));
 }
 
diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h
index 6ac78192455f..1aad48398d06 100644
--- a/arch/s390/kernel/entry.h
+++ b/arch/s390/kernel/entry.h
@@ -48,8 +48,8 @@ void do_per_trap(struct pt_regs *regs);
 void syscall_trace(struct pt_regs *regs, int entryexit);
 void kernel_stack_overflow(struct pt_regs * regs);
 void do_signal(struct pt_regs *regs);
-void handle_signal32(unsigned long sig, struct k_sigaction *ka,
-                    siginfo_t *info, sigset_t *oldset, struct pt_regs *regs);
+void handle_signal32(struct ksignal *ksig, sigset_t *oldset,
+                     struct pt_regs *regs);
 void do_notify_resume(struct pt_regs *regs);
 
 void __init init_IRQ(void);
diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c
index 42b49f9e19bf..469c4c6d9182 100644
--- a/arch/s390/kernel/signal.c
+++ b/arch/s390/kernel/signal.c
@@ -200,15 +200,15 @@ static int setup_frame(int sig, struct k_sigaction *ka,
         frame = get_sigframe(ka, regs, sizeof(sigframe));
 
         if (frame == (void __user *) -1UL)
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (__copy_to_user(&frame->sc.oldmask, &set->sig, _SIGMASK_COPY_SIZE))
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (save_sigregs(regs, &frame->sregs))
-                goto give_sigsegv;
+                return -EFAULT;
         if (__put_user(&frame->sregs, &frame->sc.sregs))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up to return from userspace.  If provided, use a stub
            already in userspace.  */
@@ -220,12 +220,12 @@ static int setup_frame(int sig, struct k_sigaction *ka,
                         frame->retcode | PSW_ADDR_AMODE;
                 if (__put_user(S390_SYSCALL_OPCODE | __NR_sigreturn,
                                (u16 __user *)(frame->retcode)))
-                        goto give_sigsegv;
+                        return -EFAULT;
         }
 
         /* Set up backchain. */
         if (__put_user(regs->gprs[15], (addr_t __user *) frame))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up registers for signal handler */
         regs->gprs[15] = (unsigned long) frame;
@@ -250,27 +250,23 @@ static int setup_frame(int sig, struct k_sigaction *ka,
 
         /* Place signal number on stack to allow backtrace from handler.  */
         if (__put_user(regs->gprs[2], (int __user *) &frame->signo))
-                goto give_sigsegv;
+                return -EFAULT;
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
-static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                           sigset_t *set, struct pt_regs * regs)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         int err = 0;
         rt_sigframe __user *frame;
 
-        frame = get_sigframe(ka, regs, sizeof(rt_sigframe));
+        frame = get_sigframe(&ksig->ka, regs, sizeof(rt_sigframe));
 
         if (frame == (void __user *) -1UL)
-                goto give_sigsegv;
+                return -EFAULT;
 
-        if (copy_siginfo_to_user(&frame->info, info))
-                goto give_sigsegv;
+        if (copy_siginfo_to_user(&frame->info, &ksig->info))
+                return -EFAULT;
 
         /* Create the ucontext.  */
         err |= __put_user(0, &frame->uc.uc_flags);
@@ -279,24 +275,24 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         err |= save_sigregs(regs, &frame->uc.uc_mcontext);
         err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up to return from userspace.  If provided, use a stub
            already in userspace.  */
-        if (ka->sa.sa_flags & SA_RESTORER) {
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
                 regs->gprs[14] = (unsigned long)
-                        ka->sa.sa_restorer | PSW_ADDR_AMODE;
+                        ksig->ka.sa.sa_restorer | PSW_ADDR_AMODE;
         } else {
                 regs->gprs[14] = (unsigned long)
                         frame->retcode | PSW_ADDR_AMODE;
                 if (__put_user(S390_SYSCALL_OPCODE | __NR_rt_sigreturn,
                                (u16 __user *)(frame->retcode)))
-                        goto give_sigsegv;
+                        return -EFAULT;
         }
 
         /* Set up backchain. */
         if (__put_user(regs->gprs[15], (addr_t __user *) frame))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up registers for signal handler */
         regs->gprs[15] = (unsigned long) frame;
@@ -304,34 +300,27 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         regs->psw.mask = PSW_MASK_EA | PSW_MASK_BA |
                 (PSW_USER_BITS & PSW_MASK_ASC) |
                 (regs->psw.mask & ~PSW_MASK_ASC);
-        regs->psw.addr = (unsigned long) ka->sa.sa_handler | PSW_ADDR_AMODE;
+        regs->psw.addr = (unsigned long) ksig->ka.sa.sa_handler | PSW_ADDR_AMODE;
 
-        regs->gprs[2] = map_signal(sig);
+        regs->gprs[2] = map_signal(ksig->sig);
         regs->gprs[3] = (unsigned long) &frame->info;
         regs->gprs[4] = (unsigned long) &frame->uc;
         regs->gprs[5] = task_thread_info(current)->last_break;
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
-static void handle_signal(unsigned long sig, struct k_sigaction *ka,
-                         siginfo_t *info, sigset_t *oldset,
-                         struct pt_regs *regs)
+static void handle_signal(struct ksignal *ksig, sigset_t *oldset,
+                          struct pt_regs *regs)
 {
         int ret;
 
         /* Set up the stack frame */
-        if (ka->sa.sa_flags & SA_SIGINFO)
-                ret = setup_rt_frame(sig, ka, info, oldset, regs);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                ret = setup_rt_frame(ksig, oldset, regs);
         else
-                ret = setup_frame(sig, ka, oldset, regs);
-        if (ret)
-                return;
-        signal_delivered(sig, info, ka, regs,
-                                 test_thread_flag(TIF_SINGLE_STEP));
+                ret = setup_frame(ksig->sig, &ksig->ka, oldset, regs);
+
+        signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLE_STEP));
 }
 
 /*
@@ -345,9 +334,7 @@ static void handle_signal(unsigned long sig, struct k_sigaction *ka,
  */
 void do_signal(struct pt_regs *regs)
 {
-        siginfo_t info;
-        int signr;
-        struct k_sigaction ka;
+        struct ksignal ksig;
         sigset_t *oldset = sigmask_to_save();
 
         /*
@@ -357,9 +344,8 @@ void do_signal(struct pt_regs *regs)
          */
         current_thread_info()->system_call =
                 test_pt_regs_flag(regs, PIF_SYSCALL) ? regs->int_code : 0;
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
 
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /* Whee!  Actually deliver the signal.  */
                 if (current_thread_info()->system_call) {
                         regs->int_code = current_thread_info()->system_call;
@@ -370,7 +356,7 @@ void do_signal(struct pt_regs *regs)
                                 regs->gprs[2] = -EINTR;
                                 break;
                         case -ERESTARTSYS:
-                                if (!(ka.sa.sa_flags & SA_RESTART)) {
+                                if (!(ksig.ka.sa.sa_flags & SA_RESTART)) {
                                         regs->gprs[2] = -EINTR;
                                         break;
                                 }
@@ -387,9 +373,9 @@ void do_signal(struct pt_regs *regs)
                 clear_pt_regs_flag(regs, PIF_SYSCALL);
 
                 if (is_compat_task())
-                        handle_signal32(signr, &ka, &info, oldset, regs);
+                        handle_signal32(&ksig, oldset, regs);
                 else
-                        handle_signal(signr, &ka, &info, oldset, regs);
+                        handle_signal(&ksig, oldset, regs);
                 return;
         }
 
diff --git a/arch/score/kernel/signal.c b/arch/score/kernel/signal.c
index a00fba32b0eb..1651807774ad 100644
--- a/arch/score/kernel/signal.c
+++ b/arch/score/kernel/signal.c
@@ -173,15 +173,15 @@ badframe:
         return 0;
 }
 
-static int setup_rt_frame(struct k_sigaction *ka, struct pt_regs *regs,
-                int signr, sigset_t *set, siginfo_t *info)
+static int setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs,
+                          sigset_t *set)
 {
         struct rt_sigframe __user *frame;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(&ksig->ka, regs, sizeof(*frame));
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         /*
          * Set up the return code ...
@@ -194,7 +194,7 @@ static int setup_rt_frame(struct k_sigaction *ka, struct pt_regs *regs,
         err |= __put_user(0x80008002, frame->rs_code + 1);
         flush_cache_sigtramp((unsigned long) frame->rs_code);
 
-        err |= copy_siginfo_to_user(&frame->rs_info, info);
+        err |= copy_siginfo_to_user(&frame->rs_info, &ksig->info);
         err |= __put_user(0, &frame->rs_uc.uc_flags);
         err |= __put_user(NULL, &frame->rs_uc.uc_link);
         err |= __save_altstack(&frame->rs_uc.uc_stack, regs->regs[0]);
@@ -202,26 +202,23 @@ static int setup_rt_frame(struct k_sigaction *ka, struct pt_regs *regs,
         err |= __copy_to_user(&frame->rs_uc.uc_sigmask, set, sizeof(*set));
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         regs->regs[0] = (unsigned long) frame;
         regs->regs[3] = (unsigned long) frame->rs_code;
-        regs->regs[4] = signr;
+        regs->regs[4] = ksig->sig;
         regs->regs[5] = (unsigned long) &frame->rs_info;
         regs->regs[6] = (unsigned long) &frame->rs_uc;
-        regs->regs[29] = (unsigned long) ka->sa.sa_handler;
-        regs->cp0_epc = (unsigned long) ka->sa.sa_handler;
+        regs->regs[29] = (unsigned long) ksig->ka.sa.sa_handler;
+        regs->cp0_epc = (unsigned long) ksig->ka.sa.sa_handler;
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(signr, current);
-        return -EFAULT;
 }
 
-static void handle_signal(unsigned long sig, siginfo_t *info,
-        struct k_sigaction *ka, struct pt_regs *regs)
+static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
+        int ret;
+
         if (regs->is_syscall) {
                 switch (regs->regs[4]) {
                 case ERESTART_RESTARTBLOCK:
@@ -229,7 +226,7 @@ static void handle_signal(unsigned long sig, siginfo_t *info,
                         regs->regs[4] = EINTR;
                         break;
                 case ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
+                        if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                 regs->regs[4] = EINTR;
                                 break;
                         }
@@ -245,17 +242,14 @@ static void handle_signal(unsigned long sig, siginfo_t *info,
         /*
          * Set up the stack frame
          */
-        if (setup_rt_frame(ka, regs, sig, sigmask_to_save(), info) < 0)
-                return;
+        ret = setup_rt_frame(ksig, regs, sigmask_to_save());
 
-        signal_delivered(sig, info, ka, regs, 0);
+        signal_setup_done(ret, ksig, 0);
 }
 
 static void do_signal(struct pt_regs *regs)
 {
-        struct k_sigaction ka;
-        siginfo_t info;
-        int signr;
+        struct ksignal ksig;
 
         /*
          * We want the common case to go fast, which is why we may in certain
@@ -265,10 +259,9 @@ static void do_signal(struct pt_regs *regs)
         if (!user_mode(regs))
                 return;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /* Actually deliver the signal.  */
-                handle_signal(signr, &info, &ka, regs);
+                handle_signal(&ksig, regs);
                 return;
         }
 
diff --git a/arch/sh/kernel/signal_32.c b/arch/sh/kernel/signal_32.c
index 594cd371aa28..2f002b24fb92 100644
--- a/arch/sh/kernel/signal_32.c
+++ b/arch/sh/kernel/signal_32.c
@@ -262,17 +262,17 @@ get_sigframe(struct k_sigaction *ka, unsigned long sp, size_t frame_size)
 extern void __kernel_sigreturn(void);
 extern void __kernel_rt_sigreturn(void);
 
-static int setup_frame(int sig, struct k_sigaction *ka,
-                        sigset_t *set, struct pt_regs *regs)
+static int setup_frame(struct ksignal *ksig, sigset_t *set,
+                       struct pt_regs *regs)
 {
         struct sigframe __user *frame;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
         int signal;
 
-        frame = get_sigframe(ka, regs->regs[15], sizeof(*frame));
+        frame = get_sigframe(&ksig->ka, regs->regs[15], sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         signal = current_thread_info()->exec_domain
                 && current_thread_info()->exec_domain->signal_invmap
@@ -288,8 +288,8 @@ static int setup_frame(int sig, struct k_sigaction *ka,
 
         /* Set up to return from userspace.  If provided, use a stub
            already in userspace.  */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                regs->pr = (unsigned long) ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                regs->pr = (unsigned long) ksig->ka.sa.sa_restorer;
 #ifdef CONFIG_VSYSCALL
         } else if (likely(current->mm->context.vdso)) {
                 regs->pr = VDSO_SYM(&__kernel_sigreturn);
@@ -309,7 +309,7 @@ static int setup_frame(int sig, struct k_sigaction *ka,
         }
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up registers for signal handler */
         regs->regs[15] = (unsigned long) frame;
@@ -319,15 +319,15 @@ static int setup_frame(int sig, struct k_sigaction *ka,
 
         if (current->personality & FDPIC_FUNCPTRS) {
                 struct fdpic_func_descriptor __user *funcptr =
-                        (struct fdpic_func_descriptor __user *)ka->sa.sa_handler;
+                        (struct fdpic_func_descriptor __user *)ksig->ka.sa.sa_handler;
 
                 err |= __get_user(regs->pc, &funcptr->text);
                 err |= __get_user(regs->regs[12], &funcptr->GOT);
         } else
-                regs->pc = (unsigned long)ka->sa.sa_handler;
+                regs->pc = (unsigned long)ksig->ka.sa.sa_handler;
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         set_fs(USER_DS);
 
@@ -335,23 +335,19 @@ static int setup_frame(int sig, struct k_sigaction *ka,
                  current->comm, task_pid_nr(current), frame, regs->pc, regs->pr);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
-static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                           sigset_t *set, struct pt_regs *regs)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
         int signal;
 
-        frame = get_sigframe(ka, regs->regs[15], sizeof(*frame));
+        frame = get_sigframe(&ksig->ka, regs->regs[15], sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         signal = current_thread_info()->exec_domain
                 && current_thread_info()->exec_domain->signal_invmap
@@ -359,7 +355,7 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 ? current_thread_info()->exec_domain->signal_invmap[sig]
                 : sig;
 
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
 
         /* Create the ucontext.  */
         err |= __put_user(0, &frame->uc.uc_flags);
@@ -371,8 +367,8 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
         /* Set up to return from userspace.  If provided, use a stub
            already in userspace.  */
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                regs->pr = (unsigned long) ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                regs->pr = (unsigned long) ksig->ka.sa.sa_restorer;
 #ifdef CONFIG_VSYSCALL
         } else if (likely(current->mm->context.vdso)) {
                 regs->pr = VDSO_SYM(&__kernel_rt_sigreturn);
@@ -392,7 +388,7 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         }
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up registers for signal handler */
         regs->regs[15] = (unsigned long) frame;
@@ -402,15 +398,15 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
         if (current->personality & FDPIC_FUNCPTRS) {
                 struct fdpic_func_descriptor __user *funcptr =
-                        (struct fdpic_func_descriptor __user *)ka->sa.sa_handler;
+                        (struct fdpic_func_descriptor __user *)ksig->ka.sa.sa_handler;
 
                 err |= __get_user(regs->pc, &funcptr->text);
                 err |= __get_user(regs->regs[12], &funcptr->GOT);
         } else
-                regs->pc = (unsigned long)ka->sa.sa_handler;
+                regs->pc = (unsigned long)ksig->ka.sa.sa_handler;
 
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         set_fs(USER_DS);
 
@@ -418,10 +414,6 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                  current->comm, task_pid_nr(current), frame, regs->pc, regs->pr);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
 static inline void
@@ -455,22 +447,18 @@ handle_syscall_restart(unsigned long save_r0, struct pt_regs *regs,
  * OK, we're invoking a handler
  */
 static void
-handle_signal(unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
-              struct pt_regs *regs, unsigned int save_r0)
+handle_signal(struct ksignal *ksig, struct pt_regs *regs, unsigned int save_r0)
 {
         sigset_t *oldset = sigmask_to_save();
         int ret;
 
         /* Set up the stack frame */
-        if (ka->sa.sa_flags & SA_SIGINFO)
-                ret = setup_rt_frame(sig, ka, info, oldset, regs);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                ret = setup_rt_frame(ksig, oldset, regs);
         else
-                ret = setup_frame(sig, ka, oldset, regs);
+                ret = setup_frame(ksig, oldset, regs);
 
-        if (ret)
-                return;
-        signal_delivered(sig, info, ka, regs,
-                        test_thread_flag(TIF_SINGLESTEP));
+        signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLESTEP));
 }
 
 /*
@@ -484,9 +472,7 @@ handle_signal(unsigned long sig, struct k_sigaction *ka, siginfo_t *info,
  */
 static void do_signal(struct pt_regs *regs, unsigned int save_r0)
 {
-        siginfo_t info;
-        int signr;
-        struct k_sigaction ka;
+        struct ksignal ksig;
 
         /*
          * We want the common case to go fast, which
@@ -497,12 +483,11 @@ static void do_signal(struct pt_regs *regs, unsigned int save_r0)
         if (!user_mode(regs))
                 return;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
-                handle_syscall_restart(save_r0, regs, &ka.sa);
+        if (get_signal(&ksig)) {
+                handle_syscall_restart(save_r0, regs, &ksig.ka.sa);
 
                 /* Whee!  Actually deliver the signal.  */
-                handle_signal(signr, &ka, &info, regs, save_r0);
+                handle_signal(&ksig, regs, save_r0);
                 return;
         }
 
diff --git a/arch/sh/kernel/signal_64.c b/arch/sh/kernel/signal_64.c
index 23d4c71c91af..897abe7b871e 100644
--- a/arch/sh/kernel/signal_64.c
+++ b/arch/sh/kernel/signal_64.c
@@ -41,8 +41,7 @@
 #define DEBUG_SIG 0
 
 static void
-handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
-                struct pt_regs * regs);
+handle_signal(struct ksignal *ksig, struct pt_regs *regs);
 
 static inline void
 handle_syscall_restart(struct pt_regs *regs, struct sigaction *sa)
@@ -82,9 +81,7 @@ handle_syscall_restart(struct pt_regs *regs, struct sigaction *sa)
  */
 static void do_signal(struct pt_regs *regs)
 {
-        siginfo_t info;
-        int signr;
-        struct k_sigaction ka;
+        struct ksignal ksig;
 
         /*
          * We want the common case to go fast, which
@@ -95,12 +92,11 @@ static void do_signal(struct pt_regs *regs)
         if (!user_mode(regs))
                 return;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, 0);
-        if (signr > 0) {
-                handle_syscall_restart(regs, &ka.sa);
+        if (get_signal(&ksig)) {
+                handle_syscall_restart(regs, &ksig.ka.sa);
 
                 /* Whee!  Actually deliver the signal.  */
-                handle_signal(signr, &info, &ka, regs);
+                handle_signal(&ksig, regs);
                 return;
         }
 
@@ -378,17 +374,16 @@ get_sigframe(struct k_sigaction *ka, unsigned long sp, size_t frame_size)
 void sa_default_restorer(void);         /* See comments below */
 void sa_default_rt_restorer(void);      /* See comments below */
 
-static int setup_frame(int sig, struct k_sigaction *ka,
-                       sigset_t *set, struct pt_regs *regs)
+static int setup_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs)
 {
         struct sigframe __user *frame;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
         int signal;
 
-        frame = get_sigframe(ka, regs->regs[REG_SP], sizeof(*frame));
+        frame = get_sigframe(&ksig->ka, regs->regs[REG_SP], sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         signal = current_thread_info()->exec_domain
                 && current_thread_info()->exec_domain->signal_invmap
@@ -400,7 +395,7 @@ static int setup_frame(int sig, struct k_sigaction *ka,
 
         /* Give up earlier as i386, in case */
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         if (_NSIG_WORDS > 1) {
                 err |= __copy_to_user(frame->extramask, &set->sig[1],
@@ -408,16 +403,16 @@ static int setup_frame(int sig, struct k_sigaction *ka,
 
         /* Give up earlier as i386, in case */
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up to return from userspace.  If provided, use a stub
            already in userspace.  */
-        if (ka->sa.sa_flags & SA_RESTORER) {
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
                 /*
                  * On SH5 all edited pointers are subject to NEFF
                  */
                 DEREF_REG_PR = neff_sign_extend((unsigned long)
-                        ka->sa.sa_restorer | 0x1);
+                        ksig->ka->sa.sa_restorer | 0x1);
         } else {
                 /*
                  * Different approach on SH5.
@@ -435,7 +430,7 @@ static int setup_frame(int sig, struct k_sigaction *ka,
 
                 if (__copy_to_user(frame->retcode,
                         (void *)((unsigned long)sa_default_restorer & (~1)), 16) != 0)
-                        goto give_sigsegv;
+                        return -EFAULT;
 
                 /* Cohere the trampoline with the I-cache. */
                 flush_cache_sigtramp(DEREF_REG_PR-1);
@@ -460,7 +455,7 @@ static int setup_frame(int sig, struct k_sigaction *ka,
         regs->regs[REG_ARG2] = (unsigned long long)(unsigned long)(signed long)&frame->sc;
         regs->regs[REG_ARG3] = (unsigned long long)(unsigned long)(signed long)&frame->sc;
 
-        regs->pc = neff_sign_extend((unsigned long)ka->sa.sa_handler);
+        regs->pc = neff_sign_extend((unsigned long)ksig->ka.sa.sa_handler);
 
         set_fs(USER_DS);
 
@@ -471,23 +466,19 @@ static int setup_frame(int sig, struct k_sigaction *ka,
                  DEREF_REG_PR >> 32, DEREF_REG_PR & 0xffffffff);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
-static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                          sigset_t *set, struct pt_regs *regs)
+static int setup_rt_frame(struct ksignal *kig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
         int signal;
 
-        frame = get_sigframe(ka, regs->regs[REG_SP], sizeof(*frame));
+        frame = get_sigframe(&ksig->ka, regs->regs[REG_SP], sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                return -EFAULT;
 
         signal = current_thread_info()->exec_domain
                 && current_thread_info()->exec_domain->signal_invmap
@@ -497,11 +488,11 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
         err |= __put_user(&frame->info, &frame->pinfo);
         err |= __put_user(&frame->uc, &frame->puc);
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
 
         /* Give up earlier as i386, in case */
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Create the ucontext.  */
         err |= __put_user(0, &frame->uc.uc_flags);
@@ -513,16 +504,16 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
         /* Give up earlier as i386, in case */
         if (err)
-                goto give_sigsegv;
+                return -EFAULT;
 
         /* Set up to return from userspace.  If provided, use a stub
            already in userspace.  */
-        if (ka->sa.sa_flags & SA_RESTORER) {
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
                 /*
                  * On SH5 all edited pointers are subject to NEFF
                  */
                 DEREF_REG_PR = neff_sign_extend((unsigned long)
-                        ka->sa.sa_restorer | 0x1);
+                        ksig->ka.sa.sa_restorer | 0x1);
         } else {
                 /*
                  * Different approach on SH5.
@@ -540,7 +531,7 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
                 if (__copy_to_user(frame->retcode,
                         (void *)((unsigned long)sa_default_rt_restorer & (~1)), 16) != 0)
-                        goto give_sigsegv;
+                        return -EFAULT;
 
                 /* Cohere the trampoline with the I-cache. */
                 flush_icache_range(DEREF_REG_PR-1, DEREF_REG_PR-1+15);
@@ -554,7 +545,7 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         regs->regs[REG_ARG1] = signal; /* Arg for signal handler */
         regs->regs[REG_ARG2] = (unsigned long long)(unsigned long)(signed long)&frame->info;
         regs->regs[REG_ARG3] = (unsigned long long)(unsigned long)(signed long)&frame->uc.uc_mcontext;
-        regs->pc = neff_sign_extend((unsigned long)ka->sa.sa_handler);
+        regs->pc = neff_sign_extend((unsigned long)ksig->ka.sa.sa_handler);
 
         set_fs(USER_DS);
 
@@ -564,33 +555,24 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                  DEREF_REG_PR >> 32, DEREF_REG_PR & 0xffffffff);
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
 /*
  * OK, we're invoking a handler
  */
 static void
-handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
-                struct pt_regs * regs)
+handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
         sigset_t *oldset = sigmask_to_save();
         int ret;
 
         /* Set up the stack frame */
-        if (ka->sa.sa_flags & SA_SIGINFO)
-                ret = setup_rt_frame(sig, ka, info, oldset, regs);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                ret = setup_rt_frame(ksig, oldset, regs);
         else
-                ret = setup_frame(sig, ka, oldset, regs);
-
-        if (ret)
-                return;
+                ret = setup_frame(ksig, oldset, regs);
 
-        signal_delivered(sig, info, ka, regs,
-                        test_thread_flag(TIF_SINGLESTEP));
+        signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLESTEP));
 }
 
 asmlinkage void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags)
diff --git a/arch/tile/include/asm/compat.h b/arch/tile/include/asm/compat.h
index ffd4493efc78..c14e36f008c8 100644
--- a/arch/tile/include/asm/compat.h
+++ b/arch/tile/include/asm/compat.h
@@ -267,8 +267,7 @@ static inline int is_compat_task(void)
         return current_thread_info()->status & TS_COMPAT;
 }
 
-extern int compat_setup_rt_frame(int sig, struct k_sigaction *ka,
-                                 siginfo_t *info, sigset_t *set,
+extern int compat_setup_rt_frame(struct ksignal *ksig, sigset_t *set,
                                  struct pt_regs *regs);
 
 /* Compat syscalls. */
diff --git a/arch/tile/kernel/compat_signal.c b/arch/tile/kernel/compat_signal.c
index 19c04b5ce408..8c5abf2e4794 100644
--- a/arch/tile/kernel/compat_signal.c
+++ b/arch/tile/kernel/compat_signal.c
@@ -190,18 +190,18 @@ static inline void __user *compat_get_sigframe(struct k_sigaction *ka,
         return (void __user *) sp;
 }
 
-int compat_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                          sigset_t *set, struct pt_regs *regs)
+int compat_setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         unsigned long restorer;
         struct compat_rt_sigframe __user *frame;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
         int usig;
 
-        frame = compat_get_sigframe(ka, regs, sizeof(*frame));
+        frame = compat_get_sigframe(&ksig->ka, regs, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                goto err;
 
         usig = current_thread_info()->exec_domain
                 && current_thread_info()->exec_domain->signal_invmap
@@ -210,12 +210,12 @@ int compat_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 : sig;
 
         /* Always write at least the signal number for the stack backtracer. */
-        if (ka->sa.sa_flags & SA_SIGINFO) {
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO) {
                 /* At sigreturn time, restore the callee-save registers too. */
-                err |= copy_siginfo_to_user32(&frame->info, info);
+                err |= copy_siginfo_to_user32(&frame->info, &ksig->info);
                 regs->flags |= PT_FLAGS_RESTORE_REGS;
         } else {
-                err |= __put_user(info->si_signo, &frame->info.si_signo);
+                err |= __put_user(ksig->info.si_signo, &frame->info.si_signo);
         }
 
         /* Create the ucontext.  */
@@ -226,11 +226,11 @@ int compat_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         err |= setup_sigcontext(&frame->uc.uc_mcontext, regs);
         err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
         if (err)
-                goto give_sigsegv;
+                goto err;
 
         restorer = VDSO_SYM(&__vdso_rt_sigreturn);
-        if (ka->sa.sa_flags & SA_RESTORER)
-                restorer = ptr_to_compat_reg(ka->sa.sa_restorer);
+        if (ksig->ka.sa.sa_flags & SA_RESTORER)
+                restorer = ptr_to_compat_reg(ksig->ka.sa.sa_restorer);
 
         /*
          * Set up registers for signal handler.
@@ -239,7 +239,7 @@ int compat_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
          * We always pass siginfo and mcontext, regardless of SA_SIGINFO,
          * since some things rely on this (e.g. glibc's debug/segfault.c).
          */
-        regs->pc = ptr_to_compat_reg(ka->sa.sa_handler);
+        regs->pc = ptr_to_compat_reg(ksig->ka.sa.sa_handler);
         regs->ex1 = PL_ICS_EX1(USER_PL, 1); /* set crit sec in handler */
         regs->sp = ptr_to_compat_reg(frame);
         regs->lr = restorer;
@@ -249,7 +249,8 @@ int compat_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         regs->flags |= PT_FLAGS_CALLER_SAVES;
         return 0;
 
-give_sigsegv:
-        signal_fault("bad setup frame", regs, frame, sig);
+err:
+        trace_unhandled_signal("bad sigreturn frame", regs,
+                              (unsigned long)frame, SIGSEGV);
         return -EFAULT;
 }
diff --git a/arch/tile/kernel/signal.c b/arch/tile/kernel/signal.c
index d1d026f01267..7c2fecc52177 100644
--- a/arch/tile/kernel/signal.c
+++ b/arch/tile/kernel/signal.c
@@ -153,18 +153,18 @@ static inline void __user *get_sigframe(struct k_sigaction *ka,
         return (void __user *) sp;
 }
 
-static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                           sigset_t *set, struct pt_regs *regs)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         unsigned long restorer;
         struct rt_sigframe __user *frame;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
         int usig;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame));
+        frame = get_sigframe(&ksig->ka, regs, sizeof(*frame));
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
-                goto give_sigsegv;
+                goto err;
 
         usig = current_thread_info()->exec_domain
                 && current_thread_info()->exec_domain->signal_invmap
@@ -173,12 +173,12 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 : sig;
 
         /* Always write at least the signal number for the stack backtracer. */
-        if (ka->sa.sa_flags & SA_SIGINFO) {
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO) {
                 /* At sigreturn time, restore the callee-save registers too. */
-                err |= copy_siginfo_to_user(&frame->info, info);
+                err |= copy_siginfo_to_user(&frame->info, &ksig->info);
                 regs->flags |= PT_FLAGS_RESTORE_REGS;
         } else {
-                err |= __put_user(info->si_signo, &frame->info.si_signo);
+                err |= __put_user(ksig->info.si_signo, &frame->info.si_signo);
         }
 
         /* Create the ucontext.  */
@@ -189,11 +189,11 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         err |= setup_sigcontext(&frame->uc.uc_mcontext, regs);
         err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
         if (err)
-                goto give_sigsegv;
+                goto err;
 
         restorer = VDSO_SYM(&__vdso_rt_sigreturn);
-        if (ka->sa.sa_flags & SA_RESTORER)
-                restorer = (unsigned long) ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER)
+                restorer = (unsigned long) ksig->ka.sa.sa_restorer;
 
         /*
          * Set up registers for signal handler.
@@ -202,7 +202,7 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
          * We always pass siginfo and mcontext, regardless of SA_SIGINFO,
          * since some things rely on this (e.g. glibc's debug/segfault.c).
          */
-        regs->pc = (unsigned long) ka->sa.sa_handler;
+        regs->pc = (unsigned long) ksig->ka.sa.sa_handler;
         regs->ex1 = PL_ICS_EX1(USER_PL, 1); /* set crit sec in handler */
         regs->sp = (unsigned long) frame;
         regs->lr = restorer;
@@ -212,8 +212,9 @@ static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         regs->flags |= PT_FLAGS_CALLER_SAVES;
         return 0;
 
-give_sigsegv:
-        signal_fault("bad setup frame", regs, frame, sig);
+err:
+        trace_unhandled_signal("bad sigreturn frame", regs,
+                              (unsigned long)frame, SIGSEGV);
         return -EFAULT;
 }
 
@@ -221,9 +222,7 @@ give_sigsegv:
  * OK, we're invoking a handler
  */
 
-static void handle_signal(unsigned long sig, siginfo_t *info,
-                         struct k_sigaction *ka,
-                         struct pt_regs *regs)
+static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
         sigset_t *oldset = sigmask_to_save();
         int ret;
@@ -238,7 +237,7 @@ static void handle_signal(unsigned long sig, siginfo_t *info,
                         break;
 
                 case -ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
+                        if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                 regs->regs[0] = -EINTR;
                                 break;
                         }
@@ -254,14 +253,12 @@ static void handle_signal(unsigned long sig, siginfo_t *info,
         /* Set up the stack frame */
 #ifdef CONFIG_COMPAT
         if (is_compat_task())
-                ret = compat_setup_rt_frame(sig, ka, info, oldset, regs);
+                ret = compat_setup_rt_frame(ksig, oldset, regs);
         else
 #endif
-                ret = setup_rt_frame(sig, ka, info, oldset, regs);
-        if (ret)
-                return;
-        signal_delivered(sig, info, ka, regs,
-                        test_thread_flag(TIF_SINGLESTEP));
+                ret = setup_rt_frame(ksig, oldset, regs);
+
+        signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLESTEP));
 }
 
 /*
@@ -271,9 +268,7 @@ static void handle_signal(unsigned long sig, siginfo_t *info,
  */
 void do_signal(struct pt_regs *regs)
 {
-        siginfo_t info;
-        int signr;
-        struct k_sigaction ka;
+        struct ksignal ksig;
 
         /*
          * i386 will check if we're coming from kernel mode and bail out
@@ -282,10 +277,9 @@ void do_signal(struct pt_regs *regs)
          * helpful, we can reinstate the check on "!user_mode(regs)".
          */
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /* Whee! Actually deliver the signal.  */
-                handle_signal(signr, &info, &ka, regs);
+                handle_signal(&ksig, regs);
                 goto done;
         }
 
diff --git a/arch/um/include/shared/frame_kern.h b/arch/um/include/shared/frame_kern.h
index f2ca5702a4e2..a5cde5c433b4 100644
--- a/arch/um/include/shared/frame_kern.h
+++ b/arch/um/include/shared/frame_kern.h
@@ -6,14 +6,10 @@
 #ifndef __FRAME_KERN_H_
 #define __FRAME_KERN_H_
 
-extern int setup_signal_stack_sc(unsigned long stack_top, int sig,
-                                 struct k_sigaction *ka,
-                                 struct pt_regs *regs,
-                                 sigset_t *mask);
-extern int setup_signal_stack_si(unsigned long stack_top, int sig,
-                                 struct k_sigaction *ka,
-                                 struct pt_regs *regs, struct siginfo *info,
-                                 sigset_t *mask);
+extern int setup_signal_stack_sc(unsigned long stack_top, struct ksignal *ksig,
+                                 struct pt_regs *regs, sigset_t *mask);
+extern int setup_signal_stack_si(unsigned long stack_top, struct ksignal *ksig,
+                                 struct pt_regs *regs, sigset_t *mask);
 
 #endif
 
diff --git a/arch/um/kernel/signal.c b/arch/um/kernel/signal.c
index f57e02e7910f..4f60e4aad790 100644
--- a/arch/um/kernel/signal.c
+++ b/arch/um/kernel/signal.c
@@ -18,8 +18,7 @@ EXPORT_SYMBOL(unblock_signals);
 /*
  * OK, we're invoking a handler
  */
-static void handle_signal(struct pt_regs *regs, unsigned long signr,
-                         struct k_sigaction *ka, struct siginfo *info)
+static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
         sigset_t *oldset = sigmask_to_save();
         int singlestep = 0;
@@ -39,7 +38,7 @@ static void handle_signal(struct pt_regs *regs, unsigned long signr,
                         break;
 
                 case -ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
+                        if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                 PT_REGS_SYSCALL_RET(regs) = -EINTR;
                                 break;
                         }
@@ -52,32 +51,28 @@ static void handle_signal(struct pt_regs *regs, unsigned long signr,
         }
 
         sp = PT_REGS_SP(regs);
-        if ((ka->sa.sa_flags & SA_ONSTACK) && (sas_ss_flags(sp) == 0))
+        if ((ksig->ka.sa.sa_flags & SA_ONSTACK) && (sas_ss_flags(sp) == 0))
                 sp = current->sas_ss_sp + current->sas_ss_size;
 
 #ifdef CONFIG_ARCH_HAS_SC_SIGNALS
-        if (!(ka->sa.sa_flags & SA_SIGINFO))
-                err = setup_signal_stack_sc(sp, signr, ka, regs, oldset);
+        if (!(ksig->ka.sa.sa_flags & SA_SIGINFO))
+                err = setup_signal_stack_sc(sp, ksig, regs, oldset);
         else
 #endif
-                err = setup_signal_stack_si(sp, signr, ka, regs, info, oldset);
+                err = setup_signal_stack_si(sp, ksig, regs, oldset);
 
-        if (err)
-                force_sigsegv(signr, current);
-        else
-                signal_delivered(signr, info, ka, regs, singlestep);
+        signal_setup_done(err, ksig, singlestep);
 }
 
 static int kern_do_signal(struct pt_regs *regs)
 {
-        struct k_sigaction ka_copy;
-        struct siginfo info;
-        int sig, handled_sig = 0;
+        struct ksignal ksig;
+        int handled_sig = 0;
 
-        while ((sig = get_signal_to_deliver(&info, &ka_copy, regs, NULL)) > 0) {
+        while (get_signal(&ksig)) {
                 handled_sig = 1;
                 /* Whee!  Actually deliver the signal.  */
-                handle_signal(regs, sig, &ka_copy, &info);
+                handle_signal(&ksig, regs);
         }
 
         /* Did we come from a system call? */
diff --git a/arch/unicore32/kernel/signal.c b/arch/unicore32/kernel/signal.c
index 6905f0ebdc77..780d77388dec 100644
--- a/arch/unicore32/kernel/signal.c
+++ b/arch/unicore32/kernel/signal.c
@@ -238,10 +238,10 @@ static int setup_return(struct pt_regs *regs, struct k_sigaction *ka,
         return 0;
 }
 
-static int setup_frame(int usig, struct k_sigaction *ka,
-                sigset_t *set, struct pt_regs *regs)
+static int setup_frame(struct ksignal *ksig, sigset_t *set,
+                       struct pt_regs *regs)
 {
-        struct sigframe __user *frame = get_sigframe(ka, regs, sizeof(*frame));
+        struct sigframe __user *frame = get_sigframe(&ksig->ka, regs, sizeof(*frame));
         int err = 0;
 
         if (!frame)
@@ -254,29 +254,29 @@ static int setup_frame(int usig, struct k_sigaction *ka,
 
         err |= setup_sigframe(frame, regs, set);
         if (err == 0)
-                err |= setup_return(regs, ka, frame->retcode, frame, usig);
+                err |= setup_return(regs, &ksig->ka, frame->retcode, frame, usig);
 
         return err;
 }
 
-static int setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
-               sigset_t *set, struct pt_regs *regs)
+static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
+                          struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame =
-                        get_sigframe(ka, regs, sizeof(*frame));
+                        get_sigframe(&ksig->ka, regs, sizeof(*frame));
         int err = 0;
 
         if (!frame)
                 return 1;
 
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
 
         err |= __put_user(0, &frame->sig.uc.uc_flags);
         err |= __put_user(NULL, &frame->sig.uc.uc_link);
         err |= __save_altstack(&frame->sig.uc.uc_stack, regs->UCreg_sp);
         err |= setup_sigframe(&frame->sig, regs, set);
         if (err == 0)
-                err |= setup_return(regs, ka, frame->sig.retcode, frame, usig);
+                err |= setup_return(regs, &ksig->ka, frame->sig.retcode, frame, usig);
 
         if (err == 0) {
                 /*
@@ -299,13 +299,13 @@ static inline void setup_syscall_restart(struct pt_regs *regs)
 /*
  * OK, we're invoking a handler
  */
-static void handle_signal(unsigned long sig, struct k_sigaction *ka,
-              siginfo_t *info, struct pt_regs *regs, int syscall)
+static void handle_signal(struct ksignal *ksig, struct pt_regs *regs,
+                          int syscall)
 {
         struct thread_info *thread = current_thread_info();
         struct task_struct *tsk = current;
         sigset_t *oldset = sigmask_to_save();
-        int usig = sig;
+        int usig = ksig->sig;
         int ret;
 
         /*
@@ -318,7 +318,7 @@ static void handle_signal(unsigned long sig, struct k_sigaction *ka,
                         regs->UCreg_00 = -EINTR;
                         break;
                 case -ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
+                        if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                 regs->UCreg_00 = -EINTR;
                                 break;
                         }
@@ -338,22 +338,17 @@ static void handle_signal(unsigned long sig, struct k_sigaction *ka,
         /*
          * Set up the stack frame
          */
-        if (ka->sa.sa_flags & SA_SIGINFO)
-                ret = setup_rt_frame(usig, ka, info, oldset, regs);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                ret = setup_rt_frame(ksig, oldset, regs);
         else
-                ret = setup_frame(usig, ka, oldset, regs);
+                ret = setup_frame(ksig, oldset, regs);
 
         /*
          * Check that the resulting registers are actually sane.
          */
         ret |= !valid_user_regs(regs);
 
-        if (ret != 0) {
-                force_sigsegv(sig, tsk);
-                return;
-        }
-
-        signal_delivered(sig, info, ka, regs, 0);
+        signal_setup_done(ret, ksig, 0);
 }
 
 /*
@@ -367,9 +362,7 @@ static void handle_signal(unsigned long sig, struct k_sigaction *ka,
  */
 static void do_signal(struct pt_regs *regs, int syscall)
 {
-        struct k_sigaction ka;
-        siginfo_t info;
-        int signr;
+        struct ksignal ksig;
 
         /*
          * We want the common case to go fast, which
@@ -380,9 +373,8 @@ static void do_signal(struct pt_regs *regs, int syscall)
         if (!user_mode(regs))
                 return;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
-                handle_signal(signr, &ka, &info, regs, syscall);
+        if (get_signsl(&ksig)) {
+                handle_signal(&ksig, regs, syscall);
                 return;
         }
 
diff --git a/arch/x86/um/signal.c b/arch/x86/um/signal.c
index 5e04a1c899fa..79d824551c1a 100644
--- a/arch/x86/um/signal.c
+++ b/arch/x86/um/signal.c
@@ -370,13 +370,12 @@ struct rt_sigframe
         char retcode[8];
 };
 
-int setup_signal_stack_sc(unsigned long stack_top, int sig,
-                          struct k_sigaction *ka, struct pt_regs *regs,
-                          sigset_t *mask)
+int setup_signal_stack_sc(unsigned long stack_top, struct ksignal *ksig,
+                          struct pt_regs *regs, sigset_t *mask)
 {
         struct sigframe __user *frame;
         void __user *restorer;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
 
         /* This is the same calculation as i386 - ((sp + 4) & 15) == 0 */
         stack_top = ((stack_top + 4) & -16UL) - 4;
@@ -385,8 +384,8 @@ int setup_signal_stack_sc(unsigned long stack_top, int sig,
                 return 1;
 
         restorer = frame->retcode;
-        if (ka->sa.sa_flags & SA_RESTORER)
-                restorer = ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER)
+                restorer = ksig->ka.sa.sa_restorer;
 
         err |= __put_user(restorer, &frame->pretcode);
         err |= __put_user(sig, &frame->sig);
@@ -410,20 +409,19 @@ int setup_signal_stack_sc(unsigned long stack_top, int sig,
                 return err;
 
         PT_REGS_SP(regs) = (unsigned long) frame;
-        PT_REGS_IP(regs) = (unsigned long) ka->sa.sa_handler;
+        PT_REGS_IP(regs) = (unsigned long) ksig->ka.sa.sa_handler;
         PT_REGS_AX(regs) = (unsigned long) sig;
         PT_REGS_DX(regs) = (unsigned long) 0;
         PT_REGS_CX(regs) = (unsigned long) 0;
         return 0;
 }
 
-int setup_signal_stack_si(unsigned long stack_top, int sig,
-                          struct k_sigaction *ka, struct pt_regs *regs,
-                          siginfo_t *info, sigset_t *mask)
+int setup_signal_stack_si(unsigned long stack_top, struct ksignal *ksig,
+                          struct pt_regs *regs, sigset_t *mask)
 {
         struct rt_sigframe __user *frame;
         void __user *restorer;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
 
         stack_top &= -8UL;
         frame = (struct rt_sigframe __user *) stack_top - 1;
@@ -431,14 +429,14 @@ int setup_signal_stack_si(unsigned long stack_top, int sig,
                 return 1;
 
         restorer = frame->retcode;
-        if (ka->sa.sa_flags & SA_RESTORER)
-                restorer = ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER)
+                restorer = ksig->ka.sa.sa_restorer;
 
         err |= __put_user(restorer, &frame->pretcode);
         err |= __put_user(sig, &frame->sig);
         err |= __put_user(&frame->info, &frame->pinfo);
         err |= __put_user(&frame->uc, &frame->puc);
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
         err |= copy_ucontext_to_user(&frame->uc, &frame->fpstate, mask,
                                         PT_REGS_SP(regs));
 
@@ -457,7 +455,7 @@ int setup_signal_stack_si(unsigned long stack_top, int sig,
                 return err;
 
         PT_REGS_SP(regs) = (unsigned long) frame;
-        PT_REGS_IP(regs) = (unsigned long) ka->sa.sa_handler;
+        PT_REGS_IP(regs) = (unsigned long) ksig->ka.sa.sa_handler;
         PT_REGS_AX(regs) = (unsigned long) sig;
         PT_REGS_DX(regs) = (unsigned long) &frame->info;
         PT_REGS_CX(regs) = (unsigned long) &frame->uc;
@@ -502,12 +500,11 @@ struct rt_sigframe
         struct _fpstate fpstate;
 };
 
-int setup_signal_stack_si(unsigned long stack_top, int sig,
-                          struct k_sigaction *ka, struct pt_regs * regs,
-                          siginfo_t *info, sigset_t *set)
+int setup_signal_stack_si(unsigned long stack_top, struct ksignal *ksig,
+                          struct pt_regs *regs, sigset_t *set)
 {
         struct rt_sigframe __user *frame;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
 
         frame = (struct rt_sigframe __user *)
                 round_down(stack_top - sizeof(struct rt_sigframe), 16);
@@ -517,8 +514,8 @@ int setup_signal_stack_si(unsigned long stack_top, int sig,
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
                 goto out;
 
-        if (ka->sa.sa_flags & SA_SIGINFO) {
-                err |= copy_siginfo_to_user(&frame->info, info);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO) {
+                err |= copy_siginfo_to_user(&frame->info, &ksig->info);
                 if (err)
                         goto out;
         }
@@ -543,8 +540,8 @@ int setup_signal_stack_si(unsigned long stack_top, int sig,
          * already in userspace.
          */
         /* x86-64 should always use SA_RESTORER. */
-        if (ka->sa.sa_flags & SA_RESTORER)
-                err |= __put_user(ka->sa.sa_restorer, &frame->pretcode);
+        if (ksig->ka.sa.sa_flags & SA_RESTORER)
+                err |= __put_user(ksig->ka.sa.sa_restorer, &frame->pretcode);
         else
                 /* could use a vstub here */
                 return err;
@@ -570,7 +567,7 @@ int setup_signal_stack_si(unsigned long stack_top, int sig,
          */
         PT_REGS_SI(regs) = (unsigned long) &frame->info;
         PT_REGS_DX(regs) = (unsigned long) &frame->uc;
-        PT_REGS_IP(regs) = (unsigned long) ka->sa.sa_handler;
+        PT_REGS_IP(regs) = (unsigned long) ksig->ka.sa.sa_handler;
  out:
         return err;
 }
diff --git a/arch/xtensa/kernel/signal.c b/arch/xtensa/kernel/signal.c
index 98b67d5f1514..4612321c73cc 100644
--- a/arch/xtensa/kernel/signal.c
+++ b/arch/xtensa/kernel/signal.c
@@ -331,17 +331,17 @@ gen_return_code(unsigned char *codemem)
 }
 
 
-static int setup_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                       sigset_t *set, struct pt_regs *regs)
+static int setup_frame(struct ksignal *ksig, sigset_t *set,
+                       struct pt_regs *regs)
 {
         struct rt_sigframe *frame;
-        int err = 0;
+        int err = 0, sig = ksig->sig;
         int signal;
         unsigned long sp, ra, tp;
 
         sp = regs->areg[1];
 
-        if ((ka->sa.sa_flags & SA_ONSTACK) != 0 && sas_ss_flags(sp) == 0) {
+        if ((ksig->ka.sa.sa_flags & SA_ONSTACK) != 0 && sas_ss_flags(sp) == 0) {
                 sp = current->sas_ss_sp + current->sas_ss_size;
         }
 
@@ -351,7 +351,7 @@ static int setup_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 panic ("Double exception sys_sigreturn\n");
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) {
-                goto give_sigsegv;
+                return -EFAULT;
         }
 
         signal = current_thread_info()->exec_domain
@@ -360,8 +360,8 @@ static int setup_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 ? current_thread_info()->exec_domain->signal_invmap[sig]
                 : sig;
 
-        if (ka->sa.sa_flags & SA_SIGINFO) {
-                err |= copy_siginfo_to_user(&frame->info, info);
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO) {
+                err |= copy_siginfo_to_user(&frame->info, &ksig->info);
         }
 
         /* Create the user context.  */
@@ -372,8 +372,8 @@ static int setup_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         err |= setup_sigcontext(frame, regs);
         err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
 
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                ra = (unsigned long)ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                ra = (unsigned long)ksig->ka.sa.sa_restorer;
         } else {
 
                 /* Create sys_rt_sigreturn syscall in stack frame */
@@ -381,7 +381,7 @@ static int setup_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 err |= gen_return_code(frame->retcode);
 
                 if (err) {
-                        goto give_sigsegv;
+                        return -EFAULT;
                 }
                 ra = (unsigned long) frame->retcode;
         }
@@ -393,7 +393,7 @@ static int setup_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
         /* Set up registers for signal handler; preserve the threadptr */
         tp = regs->threadptr;
-        start_thread(regs, (unsigned long) ka->sa.sa_handler,
+        start_thread(regs, (unsigned long) ksig->ka.sa.sa_handler,
                      (unsigned long) frame);
 
         /* Set up a stack frame for a call4
@@ -416,10 +416,6 @@ static int setup_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 #endif
 
         return 0;
-
-give_sigsegv:
-        force_sigsegv(sig, current);
-        return -EFAULT;
 }
 
 /*
@@ -433,15 +429,11 @@ give_sigsegv:
  */
 static void do_signal(struct pt_regs *regs)
 {
-        siginfo_t info;
-        int signr;
-        struct k_sigaction ka;
+        struct ksignal ksig;
 
         task_pt_regs(current)->icountlevel = 0;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 int ret;
 
                 /* Are we from a system call? */
@@ -457,7 +449,7 @@ static void do_signal(struct pt_regs *regs)
                                         break;
 
                                 case -ERESTARTSYS:
-                                        if (!(ka.sa.sa_flags & SA_RESTART)) {
+                                        if (!(ksig.ka.sa.sa_flags & SA_RESTART)) {
                                                 regs->areg[2] = -EINTR;
                                                 break;
                                         }
@@ -476,11 +468,8 @@ static void do_signal(struct pt_regs *regs)
 
                 /* Whee!  Actually deliver the signal.  */
                 /* Set up the stack frame */
-                ret = setup_frame(signr, &ka, &info, sigmask_to_save(), regs);
-                if (ret)
-                        return;
-
-                signal_delivered(signr, &info, &ka, regs, 0);
+                ret = setup_frame(&ksig, sigmask_to_save(), regs);
+                signal_setup_done(ret, &ksig, 0);
                 if (current->ptrace & PT_SINGLESTEP)
                         task_pt_regs(current)->icountlevel = 1;
 
diff --git a/drivers/gpu/drm/nouveau/Kconfig b/drivers/gpu/drm/nouveau/Kconfig
index 637c29a33127..40afc69a3778 100644
--- a/drivers/gpu/drm/nouveau/Kconfig
+++ b/drivers/gpu/drm/nouveau/Kconfig
@@ -1,5 +1,5 @@
 config DRM_NOUVEAU
-        tristate "Nouveau (nVidia) cards"
+        tristate "Nouveau (NVIDIA) cards"
         depends on DRM && PCI
         select FW_LOADER
         select DRM_KMS_HELPER
@@ -23,7 +23,15 @@ config DRM_NOUVEAU
         select THERMAL if ACPI && X86
         select ACPI_VIDEO if ACPI && X86
         help
-          Choose this option for open-source nVidia support.
+          Choose this option for open-source NVIDIA support.
+
+config NOUVEAU_PLATFORM_DRIVER
+        tristate "Nouveau (NVIDIA) SoC GPUs"
+        depends on DRM_NOUVEAU && ARCH_TEGRA
+        default y
+        help
+          Support for Nouveau platform driver, used for SoC GPUs as found
+          on NVIDIA Tegra K1.
 
 config NOUVEAU_DEBUG
         int "Maximum debug level"
diff --git a/drivers/gpu/drm/nouveau/Makefile b/drivers/gpu/drm/nouveau/Makefile
index 8b307e143632..f5d7f7ce4bc6 100644
--- a/drivers/gpu/drm/nouveau/Makefile
+++ b/drivers/gpu/drm/nouveau/Makefile
@@ -14,8 +14,10 @@ nouveau-y += core/core/enum.o
 nouveau-y += core/core/event.o
 nouveau-y += core/core/gpuobj.o
 nouveau-y += core/core/handle.o
+nouveau-y += core/core/ioctl.o
 nouveau-y += core/core/mm.o
 nouveau-y += core/core/namedb.o
+nouveau-y += core/core/notify.o
 nouveau-y += core/core/object.o
 nouveau-y += core/core/option.o
 nouveau-y += core/core/parent.o
@@ -26,6 +28,7 @@ nouveau-y += core/core/subdev.o
 nouveau-y += core/subdev/bar/base.o
 nouveau-y += core/subdev/bar/nv50.o
 nouveau-y += core/subdev/bar/nvc0.o
+nouveau-y += core/subdev/bar/gk20a.o
 nouveau-y += core/subdev/bios/base.o
 nouveau-y += core/subdev/bios/bit.o
 nouveau-y += core/subdev/bios/boost.o
@@ -64,6 +67,7 @@ nouveau-y += core/subdev/clock/nva3.o
 nouveau-y += core/subdev/clock/nvaa.o
 nouveau-y += core/subdev/clock/nvc0.o
 nouveau-y += core/subdev/clock/nve0.o
+nouveau-y += core/subdev/clock/gk20a.o
 nouveau-y += core/subdev/clock/pllnv04.o
 nouveau-y += core/subdev/clock/pllnva3.o
 nouveau-y += core/subdev/devinit/base.o
@@ -149,8 +153,10 @@ nouveau-y += core/subdev/instmem/base.o
 nouveau-y += core/subdev/instmem/nv04.o
 nouveau-y += core/subdev/instmem/nv40.o
 nouveau-y += core/subdev/instmem/nv50.o
-nouveau-y += core/subdev/ltcg/gf100.o
-nouveau-y += core/subdev/ltcg/gm107.o
+nouveau-y += core/subdev/ltc/base.o
+nouveau-y += core/subdev/ltc/gf100.o
+nouveau-y += core/subdev/ltc/gk104.o
+nouveau-y += core/subdev/ltc/gm107.o
 nouveau-y += core/subdev/mc/base.o
 nouveau-y += core/subdev/mc/nv04.o
 nouveau-y += core/subdev/mc/nv40.o
@@ -161,6 +167,7 @@ nouveau-y += core/subdev/mc/nv94.o
 nouveau-y += core/subdev/mc/nv98.o
 nouveau-y += core/subdev/mc/nvc0.o
 nouveau-y += core/subdev/mc/nvc3.o
+nouveau-y += core/subdev/mc/gk20a.o
 nouveau-y += core/subdev/mxm/base.o
 nouveau-y += core/subdev/mxm/mxms.o
 nouveau-y += core/subdev/mxm/nv50.o
@@ -169,6 +176,7 @@ nouveau-y += core/subdev/pwr/memx.o
 nouveau-y += core/subdev/pwr/nva3.o
 nouveau-y += core/subdev/pwr/nvc0.o
 nouveau-y += core/subdev/pwr/nvd0.o
+nouveau-y += core/subdev/pwr/gk104.o
 nouveau-y += core/subdev/pwr/nv108.o
 nouveau-y += core/subdev/therm/base.o
 nouveau-y += core/subdev/therm/fan.o
@@ -211,6 +219,7 @@ nouveau-y += core/engine/copy/nvc0.o
 nouveau-y += core/engine/copy/nve0.o
 nouveau-y += core/engine/crypt/nv84.o
 nouveau-y += core/engine/crypt/nv98.o
+nouveau-y += core/engine/device/acpi.o
 nouveau-y += core/engine/device/base.o
 nouveau-y += core/engine/device/ctrl.o
 nouveau-y += core/engine/device/nv04.o
@@ -270,6 +279,7 @@ nouveau-y += core/engine/graph/ctxnvd9.o
 nouveau-y += core/engine/graph/ctxnve4.o
 nouveau-y += core/engine/graph/ctxgk20a.o
 nouveau-y += core/engine/graph/ctxnvf0.o
+nouveau-y += core/engine/graph/ctxgk110b.o
 nouveau-y += core/engine/graph/ctxnv108.o
 nouveau-y += core/engine/graph/ctxgm107.o
 nouveau-y += core/engine/graph/nv04.o
@@ -291,6 +301,7 @@ nouveau-y += core/engine/graph/nvd9.o
 nouveau-y += core/engine/graph/nve4.o
 nouveau-y += core/engine/graph/gk20a.o
 nouveau-y += core/engine/graph/nvf0.o
+nouveau-y += core/engine/graph/gk110b.o
 nouveau-y += core/engine/graph/nv108.o
 nouveau-y += core/engine/graph/gm107.o
 nouveau-y += core/engine/mpeg/nv31.o
@@ -318,11 +329,18 @@ nouveau-y += core/engine/vp/nv98.o
 nouveau-y += core/engine/vp/nvc0.o
 nouveau-y += core/engine/vp/nve0.o
 
+# nvif
+nouveau-y += nvif/object.o
+nouveau-y += nvif/client.o
+nouveau-y += nvif/device.o
+nouveau-y += nvif/notify.o
+
 # drm/core
 nouveau-y += nouveau_drm.o nouveau_chan.o nouveau_dma.o nouveau_fence.o
 nouveau-y += nouveau_vga.o nouveau_agp.o
 nouveau-y += nouveau_ttm.o nouveau_sgdma.o nouveau_bo.o nouveau_gem.o
 nouveau-y += nouveau_prime.o nouveau_abi16.o
+nouveau-y += nouveau_nvif.o nouveau_usif.o
 nouveau-y += nv04_fence.o nv10_fence.o nv17_fence.o
 nouveau-y += nv50_fence.o nv84_fence.o nvc0_fence.o
 
@@ -349,3 +367,6 @@ nouveau-$(CONFIG_DRM_NOUVEAU_BACKLIGHT) += nouveau_backlight.o
 nouveau-$(CONFIG_DEBUG_FS) += nouveau_debugfs.o
 
 obj-$(CONFIG_DRM_NOUVEAU)+= nouveau.o
+
+# platform driver
+obj-$(CONFIG_NOUVEAU_PLATFORM_DRIVER) += nouveau_platform.o
diff --git a/drivers/gpu/drm/nouveau/core/core/client.c b/drivers/gpu/drm/nouveau/core/core/client.c
index 9079c0ac58e6..10598dede9e9 100644
--- a/drivers/gpu/drm/nouveau/core/core/client.c
+++ b/drivers/gpu/drm/nouveau/core/core/client.c
@@ -26,13 +26,167 @@
 #include <core/client.h>
 #include <core/handle.h>
 #include <core/option.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
+
+#include <nvif/unpack.h>
+#include <nvif/event.h>
 
 #include <engine/device.h>
 
+struct nvkm_client_notify {
+        struct nouveau_client *client;
+        struct nvkm_notify n;
+        u8 version;
+        u8 size;
+        union {
+                struct nvif_notify_rep_v0 v0;
+        } rep;
+};
+
+static int
+nvkm_client_notify(struct nvkm_notify *n)
+{
+        struct nvkm_client_notify *notify = container_of(n, typeof(*notify), n);
+        struct nouveau_client *client = notify->client;
+        return client->ntfy(&notify->rep, notify->size, n->data, n->size);
+}
+
+int
+nvkm_client_notify_put(struct nouveau_client *client, int index)
+{
+        if (index < ARRAY_SIZE(client->notify)) {
+                if (client->notify[index]) {
+                        nvkm_notify_put(&client->notify[index]->n);
+                        return 0;
+                }
+        }
+        return -ENOENT;
+}
+
+int
+nvkm_client_notify_get(struct nouveau_client *client, int index)
+{
+        if (index < ARRAY_SIZE(client->notify)) {
+                if (client->notify[index]) {
+                        nvkm_notify_get(&client->notify[index]->n);
+                        return 0;
+                }
+        }
+        return -ENOENT;
+}
+
+int
+nvkm_client_notify_del(struct nouveau_client *client, int index)
+{
+        if (index < ARRAY_SIZE(client->notify)) {
+                if (client->notify[index]) {
+                        nvkm_notify_fini(&client->notify[index]->n);
+                        kfree(client->notify[index]);
+                        client->notify[index] = NULL;
+                        return 0;
+                }
+        }
+        return -ENOENT;
+}
+
+int
+nvkm_client_notify_new(struct nouveau_client *client,
+                       struct nvkm_event *event, void *data, u32 size)
+{
+        struct nvkm_client_notify *notify;
+        union {
+                struct nvif_notify_req_v0 v0;
+        } *req = data;
+        u8  index, reply;
+        int ret;
+
+        for (index = 0; index < ARRAY_SIZE(client->notify); index++) {
+                if (!client->notify[index])
+                        break;
+        }
+
+        if (index == ARRAY_SIZE(client->notify))
+                return -ENOSPC;
+
+        notify = kzalloc(sizeof(*notify), GFP_KERNEL);
+        if (!notify)
+                return -ENOMEM;
+
+        nv_ioctl(client, "notify new size %d\n", size);
+        if (nvif_unpack(req->v0, 0, 0, true)) {
+                nv_ioctl(client, "notify new vers %d reply %d route %02x "
+                                 "token %llx\n", req->v0.version,
+                         req->v0.reply, req->v0.route, req->v0.token);
+                notify->version = req->v0.version;
+                notify->size = sizeof(notify->rep.v0);
+                notify->rep.v0.version = req->v0.version;
+                notify->rep.v0.route = req->v0.route;
+                notify->rep.v0.token = req->v0.token;
+                reply = req->v0.reply;
+        }
+
+        if (ret == 0) {
+                ret = nvkm_notify_init(event, nvkm_client_notify, false,
+                                       data, size, reply, &notify->n);
+                if (ret == 0) {
+                        client->notify[index] = notify;
+                        notify->client = client;
+                        return 0;
+                }
+        }
+
+        kfree(notify);
+        return 0;
+}
+
+static int
+nouveau_client_devlist(struct nouveau_object *object, void *data, u32 size)
+{
+        union {
+                struct nv_client_devlist_v0 v0;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "client devlist size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                nv_ioctl(object, "client devlist vers %d count %d\n",
+                         args->v0.version, args->v0.count);
+                if (size == sizeof(args->v0.device[0]) * args->v0.count) {
+                        ret = nouveau_device_list(args->v0.device,
+                                                  args->v0.count);
+                        if (ret >= 0) {
+                                args->v0.count = ret;
+                                ret = 0;
+                        }
+                } else {
+                        ret = -EINVAL;
+                }
+        }
+
+        return ret;
+}
+
+static int
+nouveau_client_mthd(struct nouveau_object *object, u32 mthd,
+                    void *data, u32 size)
+{
+        switch (mthd) {
+        case NV_CLIENT_DEVLIST:
+                return nouveau_client_devlist(object, data, size);
+        default:
+                break;
+        }
+        return -EINVAL;
+}
+
 static void
 nouveau_client_dtor(struct nouveau_object *object)
 {
         struct nouveau_client *client = (void *)object;
+        int i;
+        for (i = 0; i < ARRAY_SIZE(client->notify); i++)
+                nvkm_client_notify_del(client, i);
         nouveau_object_ref(NULL, &client->device);
         nouveau_handle_destroy(client->root);
         nouveau_namedb_destroy(&client->base);
@@ -42,6 +196,7 @@ static struct nouveau_oclass
 nouveau_client_oclass = {
         .ofuncs = &(struct nouveau_ofuncs) {
                 .dtor = nouveau_client_dtor,
+                .mthd = nouveau_client_mthd,
         },
 };
 
@@ -93,9 +248,12 @@ int
 nouveau_client_fini(struct nouveau_client *client, bool suspend)
 {
         const char *name[2] = { "fini", "suspend" };
-        int ret;
-
+        int ret, i;
         nv_debug(client, "%s running\n", name[suspend]);
+        nv_debug(client, "%s notify\n", name[suspend]);
+        for (i = 0; i < ARRAY_SIZE(client->notify); i++)
+                nvkm_client_notify_put(client, i);
+        nv_debug(client, "%s object\n", name[suspend]);
         ret = nouveau_handle_fini(client->root, suspend);
         nv_debug(client, "%s completed with %d\n", name[suspend], ret);
         return ret;
diff --git a/drivers/gpu/drm/nouveau/core/core/event.c b/drivers/gpu/drm/nouveau/core/core/event.c
index ae81d3b5d8b7..0540a48c5678 100644
--- a/drivers/gpu/drm/nouveau/core/core/event.c
+++ b/drivers/gpu/drm/nouveau/core/core/event.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013 Red Hat Inc.
+ * Copyright 2013-2014 Red Hat Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -24,173 +24,77 @@
 #include <core/event.h>
 
 void
-nouveau_event_put(struct nouveau_eventh *handler)
+nvkm_event_put(struct nvkm_event *event, u32 types, int index)
 {
-        struct nouveau_event *event = handler->event;
-        unsigned long flags;
-        u32 m, t;
-
-        if (!__test_and_clear_bit(NVKM_EVENT_ENABLE, &handler->flags))
-                return;
-
-        spin_lock_irqsave(&event->refs_lock, flags);
-        for (m = handler->types; t = __ffs(m), m; m &= ~(1 << t)) {
-                if (!--event->refs[handler->index * event->types_nr + t]) {
-                        if (event->disable)
-                                event->disable(event, 1 << t, handler->index);
+        BUG_ON(!spin_is_locked(&event->refs_lock));
+        while (types) {
+                int type = __ffs(types); types &= ~(1 << type);
+                if (--event->refs[index * event->types_nr + type] == 0) {
+                        if (event->func->fini)
+                                event->func->fini(event, 1 << type, index);
                 }
-
         }
-        spin_unlock_irqrestore(&event->refs_lock, flags);
 }
 
 void
-nouveau_event_get(struct nouveau_eventh *handler)
+nvkm_event_get(struct nvkm_event *event, u32 types, int index)
 {
-        struct nouveau_event *event = handler->event;
-        unsigned long flags;
-        u32 m, t;
-
-        if (__test_and_set_bit(NVKM_EVENT_ENABLE, &handler->flags))
-                return;
-
-        spin_lock_irqsave(&event->refs_lock, flags);
-        for (m = handler->types; t = __ffs(m), m; m &= ~(1 << t)) {
-                if (!event->refs[handler->index * event->types_nr + t]++) {
-                        if (event->enable)
-                                event->enable(event, 1 << t, handler->index);
+        BUG_ON(!spin_is_locked(&event->refs_lock));
+        while (types) {
+                int type = __ffs(types); types &= ~(1 << type);
+                if (++event->refs[index * event->types_nr + type] == 1) {
+                        if (event->func->init)
+                                event->func->init(event, 1 << type, index);
                 }
-
         }
-        spin_unlock_irqrestore(&event->refs_lock, flags);
-}
-
-static void
-nouveau_event_fini(struct nouveau_eventh *handler)
-{
-        struct nouveau_event *event = handler->event;
-        unsigned long flags;
-        nouveau_event_put(handler);
-        spin_lock_irqsave(&event->list_lock, flags);
-        list_del(&handler->head);
-        spin_unlock_irqrestore(&event->list_lock, flags);
-}
-
-static int
-nouveau_event_init(struct nouveau_event *event, u32 types, int index,
-                   int (*func)(void *, u32, int), void *priv,
-                   struct nouveau_eventh *handler)
-{
-        unsigned long flags;
-
-        if (types & ~((1 << event->types_nr) - 1))
-                return -EINVAL;
-        if (index >= event->index_nr)
-                return -EINVAL;
-
-        handler->event = event;
-        handler->flags = 0;
-        handler->types = types;
-        handler->index = index;
-        handler->func = func;
-        handler->priv = priv;
-
-        spin_lock_irqsave(&event->list_lock, flags);
-        list_add_tail(&handler->head, &event->list[index]);
-        spin_unlock_irqrestore(&event->list_lock, flags);
-        return 0;
-}
-
-int
-nouveau_event_new(struct nouveau_event *event, u32 types, int index,
-                  int (*func)(void *, u32, int), void *priv,
-                  struct nouveau_eventh **phandler)
-{
-        struct nouveau_eventh *handler;
-        int ret = -ENOMEM;
-
-        if (event->check) {
-                ret = event->check(event, types, index);
-                if (ret)
-                        return ret;
-        }
-
-        handler = *phandler = kmalloc(sizeof(*handler), GFP_KERNEL);
-        if (handler) {
-                ret = nouveau_event_init(event, types, index, func, priv, handler);
-                if (ret)
-                        kfree(handler);
-        }
-
-        return ret;
-}
-
-void
-nouveau_event_ref(struct nouveau_eventh *handler, struct nouveau_eventh **ref)
-{
-        BUG_ON(handler != NULL);
-        if (*ref) {
-                nouveau_event_fini(*ref);
-                kfree(*ref);
-        }
-        *ref = handler;
 }
 
 void
-nouveau_event_trigger(struct nouveau_event *event, u32 types, int index)
+nvkm_event_send(struct nvkm_event *event, u32 types, int index,
+                void *data, u32 size)
 {
-        struct nouveau_eventh *handler;
+        struct nvkm_notify *notify;
         unsigned long flags;
 
-        if (WARN_ON(index >= event->index_nr))
+        if (!event->refs || WARN_ON(index >= event->index_nr))
                 return;
 
         spin_lock_irqsave(&event->list_lock, flags);
-        list_for_each_entry(handler, &event->list[index], head) {
-                if (!test_bit(NVKM_EVENT_ENABLE, &handler->flags))
-                        continue;
-                if (!(handler->types & types))
-                        continue;
-                if (handler->func(handler->priv, handler->types & types, index)
-                                != NVKM_EVENT_DROP)
-                        continue;
-                nouveau_event_put(handler);
+        list_for_each_entry(notify, &event->list, head) {
+                if (notify->index == index && (notify->types & types)) {
+                        if (event->func->send) {
+                                event->func->send(data, size, notify);
+                                continue;
+                        }
+                        nvkm_notify_send(notify, data, size);
+                }
         }
         spin_unlock_irqrestore(&event->list_lock, flags);
 }
 
 void
-nouveau_event_destroy(struct nouveau_event **pevent)
+nvkm_event_fini(struct nvkm_event *event)
 {
-        struct nouveau_event *event = *pevent;
-        if (event) {
-                kfree(event);
-                *pevent = NULL;
+        if (event->refs) {
+                kfree(event->refs);
+                event->refs = NULL;
         }
 }
 
 int
-nouveau_event_create(int types_nr, int index_nr, struct nouveau_event **pevent)
+nvkm_event_init(const struct nvkm_event_func *func, int types_nr, int index_nr,
+                struct nvkm_event *event)
 {
-        struct nouveau_event *event;
-        int i;
-
-        event = *pevent = kzalloc(sizeof(*event) + (index_nr * types_nr) *
-                                  sizeof(event->refs[0]), GFP_KERNEL);
-        if (!event)
-                return -ENOMEM;
-
-        event->list = kmalloc(sizeof(*event->list) * index_nr, GFP_KERNEL);
-        if (!event->list) {
-                kfree(event);
+        event->refs = kzalloc(sizeof(*event->refs) * index_nr * types_nr,
+                              GFP_KERNEL);
+        if (!event->refs)
                 return -ENOMEM;
-        }
 
-        spin_lock_init(&event->list_lock);
-        spin_lock_init(&event->refs_lock);
-        for (i = 0; i < index_nr; i++)
-                INIT_LIST_HEAD(&event->list[i]);
+        event->func = func;
         event->types_nr = types_nr;
         event->index_nr = index_nr;
+        spin_lock_init(&event->refs_lock);
+        spin_lock_init(&event->list_lock);
+        INIT_LIST_HEAD(&event->list);
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/core/handle.c b/drivers/gpu/drm/nouveau/core/core/handle.c
index 264c2b338ac3..a490b805d7e3 100644
--- a/drivers/gpu/drm/nouveau/core/core/handle.c
+++ b/drivers/gpu/drm/nouveau/core/core/handle.c
@@ -146,9 +146,7 @@ nouveau_handle_create(struct nouveau_object *parent, u32 _parent, u32 _handle,
         }
 
         hprintk(handle, TRACE, "created\n");
-
         *phandle = handle;
-
         return 0;
 }
 
@@ -224,3 +222,116 @@ nouveau_handle_put(struct nouveau_handle *handle)
         if (handle)
                 nouveau_namedb_put(handle);
 }
+
+int
+nouveau_handle_new(struct nouveau_object *client, u32 _parent, u32 _handle,
+                   u16 _oclass, void *data, u32 size,
+                   struct nouveau_object **pobject)
+{
+        struct nouveau_object *parent = NULL;
+        struct nouveau_object *engctx = NULL;
+        struct nouveau_object *object = NULL;
+        struct nouveau_object *engine;
+        struct nouveau_oclass *oclass;
+        struct nouveau_handle *handle;
+        int ret;
+
+        /* lookup parent object and ensure it *is* a parent */
+        parent = nouveau_handle_ref(client, _parent);
+        if (!parent) {
+                nv_error(client, "parent 0x%08x not found\n", _parent);
+                return -ENOENT;
+        }
+
+        if (!nv_iclass(parent, NV_PARENT_CLASS)) {
+                nv_error(parent, "cannot have children\n");
+                ret = -EINVAL;
+                goto fail_class;
+        }
+
+        /* check that parent supports the requested subclass */
+        ret = nouveau_parent_sclass(parent, _oclass, &engine, &oclass);
+        if (ret) {
+                nv_debug(parent, "illegal class 0x%04x\n", _oclass);
+                goto fail_class;
+        }
+
+        /* make sure engine init has been completed *before* any objects
+         * it controls are created - the constructors may depend on
+         * state calculated at init (ie. default context construction)
+         */
+        if (engine) {
+                ret = nouveau_object_inc(engine);
+                if (ret)
+                        goto fail_class;
+        }
+
+        /* if engine requires it, create a context object to insert
+         * between the parent and its children (eg. PGRAPH context)
+         */
+        if (engine && nv_engine(engine)->cclass) {
+                ret = nouveau_object_ctor(parent, engine,
+                                          nv_engine(engine)->cclass,
+                                          data, size, &engctx);
+                if (ret)
+                        goto fail_engctx;
+        } else {
+                nouveau_object_ref(parent, &engctx);
+        }
+
+        /* finally, create new object and bind it to its handle */
+        ret = nouveau_object_ctor(engctx, engine, oclass, data, size, &object);
+        *pobject = object;
+        if (ret)
+                goto fail_ctor;
+
+        ret = nouveau_object_inc(object);
+        if (ret)
+                goto fail_init;
+
+        ret = nouveau_handle_create(parent, _parent, _handle, object, &handle);
+        if (ret)
+                goto fail_handle;
+
+        ret = nouveau_handle_init(handle);
+        if (ret)
+                nouveau_handle_destroy(handle);
+
+fail_handle:
+        nouveau_object_dec(object, false);
+fail_init:
+        nouveau_object_ref(NULL, &object);
+fail_ctor:
+        nouveau_object_ref(NULL, &engctx);
+fail_engctx:
+        if (engine)
+                nouveau_object_dec(engine, false);
+fail_class:
+        nouveau_object_ref(NULL, &parent);
+        return ret;
+}
+
+int
+nouveau_handle_del(struct nouveau_object *client, u32 _parent, u32 _handle)
+{
+        struct nouveau_object *parent = NULL;
+        struct nouveau_object *namedb = NULL;
+        struct nouveau_handle *handle = NULL;
+
+        parent = nouveau_handle_ref(client, _parent);
+        if (!parent)
+                return -ENOENT;
+
+        namedb = nv_pclass(parent, NV_NAMEDB_CLASS);
+        if (namedb) {
+                handle = nouveau_namedb_get(nv_namedb(namedb), _handle);
+                if (handle) {
+                        nouveau_namedb_put(handle);
+                        nouveau_handle_fini(handle, false);
+                        nouveau_handle_destroy(handle);
+                }
+        }
+
+        nouveau_object_ref(NULL, &parent);
+        return handle ? 0 : -EINVAL;
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/ioctl.c b/drivers/gpu/drm/nouveau/core/core/ioctl.c
new file mode 100644
index 000000000000..f7e19bfb489c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/ioctl.c
@@ -0,0 +1,531 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <core/object.h>
+#include <core/parent.h>
+#include <core/handle.h>
+#include <core/namedb.h>
+#include <core/client.h>
+#include <core/device.h>
+#include <core/ioctl.h>
+#include <core/event.h>
+
+#include <nvif/unpack.h>
+#include <nvif/ioctl.h>
+
+static int
+nvkm_ioctl_nop(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_object *object = handle->object;
+        union {
+                struct nvif_ioctl_nop none;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "nop size %d\n", size);
+        if (nvif_unvers(args->none)) {
+                nv_ioctl(object, "nop\n");
+        }
+
+        return ret;
+}
+
+static int
+nvkm_ioctl_sclass(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_object *object = handle->object;
+        union {
+                struct nvif_ioctl_sclass_v0 v0;
+        } *args = data;
+        int ret;
+
+        if (!nv_iclass(object, NV_PARENT_CLASS)) {
+                nv_debug(object, "cannot have children (sclass)\n");
+                return -ENODEV;
+        }
+
+        nv_ioctl(object, "sclass size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                nv_ioctl(object, "sclass vers %d count %d\n",
+                         args->v0.version, args->v0.count);
+                if (size == args->v0.count * sizeof(args->v0.oclass[0])) {
+                        ret = nouveau_parent_lclass(object, args->v0.oclass,
+                                                            args->v0.count);
+                        if (ret >= 0) {
+                                args->v0.count = ret;
+                                ret = 0;
+                        }
+                } else {
+                        ret = -EINVAL;
+                }
+        }
+
+        return ret;
+}
+
+static int
+nvkm_ioctl_new(struct nouveau_handle *parent, void *data, u32 size)
+{
+        union {
+                struct nvif_ioctl_new_v0 v0;
+        } *args = data;
+        struct nouveau_client *client = nouveau_client(parent->object);
+        struct nouveau_object *engctx = NULL;
+        struct nouveau_object *object = NULL;
+        struct nouveau_object *engine;
+        struct nouveau_oclass *oclass;
+        struct nouveau_handle *handle;
+        u32 _handle, _oclass;
+        int ret;
+
+        nv_ioctl(client, "new size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                _handle = args->v0.handle;
+                _oclass = args->v0.oclass;
+        } else
+                return ret;
+
+        nv_ioctl(client, "new vers %d handle %08x class %08x "
+                         "route %02x token %llx\n",
+                args->v0.version, _handle, _oclass,
+                args->v0.route, args->v0.token);
+
+        if (!nv_iclass(parent->object, NV_PARENT_CLASS)) {
+                nv_debug(parent->object, "cannot have children (ctor)\n");
+                ret = -ENODEV;
+                goto fail_class;
+        }
+
+        /* check that parent supports the requested subclass */
+        ret = nouveau_parent_sclass(parent->object, _oclass, &engine, &oclass);
+        if (ret) {
+                nv_debug(parent->object, "illegal class 0x%04x\n", _oclass);
+                goto fail_class;
+        }
+
+        /* make sure engine init has been completed *before* any objects
+         * it controls are created - the constructors may depend on
+         * state calculated at init (ie. default context construction)
+         */
+        if (engine) {
+                ret = nouveau_object_inc(engine);
+                if (ret)
+                        goto fail_class;
+        }
+
+        /* if engine requires it, create a context object to insert
+         * between the parent and its children (eg. PGRAPH context)
+         */
+        if (engine && nv_engine(engine)->cclass) {
+                ret = nouveau_object_ctor(parent->object, engine,
+                                          nv_engine(engine)->cclass,
+                                          data, size, &engctx);
+                if (ret)
+                        goto fail_engctx;
+        } else {
+                nouveau_object_ref(parent->object, &engctx);
+        }
+
+        /* finally, create new object and bind it to its handle */
+        ret = nouveau_object_ctor(engctx, engine, oclass, data, size, &object);
+        client->data = object;
+        if (ret)
+                goto fail_ctor;
+
+        ret = nouveau_object_inc(object);
+        if (ret)
+                goto fail_init;
+
+        ret = nouveau_handle_create(parent->object, parent->name,
+                                    _handle, object, &handle);
+        if (ret)
+                goto fail_handle;
+
+        ret = nouveau_handle_init(handle);
+        handle->route = args->v0.route;
+        handle->token = args->v0.token;
+        if (ret)
+                nouveau_handle_destroy(handle);
+
+fail_handle:
+        nouveau_object_dec(object, false);
+fail_init:
+        nouveau_object_ref(NULL, &object);
+fail_ctor:
+        nouveau_object_ref(NULL, &engctx);
+fail_engctx:
+        if (engine)
+                nouveau_object_dec(engine, false);
+fail_class:
+        return ret;
+}
+
+static int
+nvkm_ioctl_del(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_object *object = handle->object;
+        union {
+                struct nvif_ioctl_del none;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "delete size %d\n", size);
+        if (nvif_unvers(args->none)) {
+                nv_ioctl(object, "delete\n");
+                nouveau_handle_fini(handle, false);
+                nouveau_handle_destroy(handle);
+        }
+
+        return ret;
+}
+
+static int
+nvkm_ioctl_mthd(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_object *object = handle->object;
+        struct nouveau_ofuncs *ofuncs = object->oclass->ofuncs;
+        union {
+                struct nvif_ioctl_mthd_v0 v0;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "mthd size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                nv_ioctl(object, "mthd vers %d mthd %02x\n",
+                         args->v0.version, args->v0.method);
+                if (ret = -ENODEV, ofuncs->mthd)
+                        ret = ofuncs->mthd(object, args->v0.method, data, size);
+        }
+
+        return ret;
+}
+
+
+static int
+nvkm_ioctl_rd(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_object *object = handle->object;
+        struct nouveau_ofuncs *ofuncs = object->oclass->ofuncs;
+        union {
+                struct nvif_ioctl_rd_v0 v0;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "rd size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "rd vers %d size %d addr %016llx\n",
+                        args->v0.version, args->v0.size, args->v0.addr);
+                switch (args->v0.size) {
+                case 1:
+                        if (ret = -ENODEV, ofuncs->rd08) {
+                                args->v0.data = nv_ro08(object, args->v0.addr);
+                                ret = 0;
+                        }
+                        break;
+                case 2:
+                        if (ret = -ENODEV, ofuncs->rd16) {
+                                args->v0.data = nv_ro16(object, args->v0.addr);
+                                ret = 0;
+                        }
+                        break;
+                case 4:
+                        if (ret = -ENODEV, ofuncs->rd32) {
+                                args->v0.data = nv_ro32(object, args->v0.addr);
+                                ret = 0;
+                        }
+                        break;
+                default:
+                        ret = -EINVAL;
+                        break;
+                }
+        }
+
+        return ret;
+}
+
+static int
+nvkm_ioctl_wr(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_object *object = handle->object;
+        struct nouveau_ofuncs *ofuncs = object->oclass->ofuncs;
+        union {
+                struct nvif_ioctl_wr_v0 v0;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "wr size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "wr vers %d size %d addr %016llx data %08x\n",
+                         args->v0.version, args->v0.size, args->v0.addr,
+                         args->v0.data);
+                switch (args->v0.size) {
+                case 1:
+                        if (ret = -ENODEV, ofuncs->wr08) {
+                                nv_wo08(object, args->v0.addr, args->v0.data);
+                                ret = 0;
+                        }
+                        break;
+                case 2:
+                        if (ret = -ENODEV, ofuncs->wr16) {
+                                nv_wo16(object, args->v0.addr, args->v0.data);
+                                ret = 0;
+                        }
+                        break;
+                case 4:
+                        if (ret = -ENODEV, ofuncs->wr32) {
+                                nv_wo32(object, args->v0.addr, args->v0.data);
+                                ret = 0;
+                        }
+                        break;
+                default:
+                        ret = -EINVAL;
+                        break;
+                }
+        }
+
+        return ret;
+}
+
+static int
+nvkm_ioctl_map(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_object *object = handle->object;
+        struct nouveau_ofuncs *ofuncs = object->oclass->ofuncs;
+        union {
+                struct nvif_ioctl_map_v0 v0;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "map size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "map vers %d\n", args->v0.version);
+                if (ret = -ENODEV, ofuncs->map) {
+                        ret = ofuncs->map(object, &args->v0.handle,
+                                                  &args->v0.length);
+                }
+        }
+
+        return ret;
+}
+
+static int
+nvkm_ioctl_unmap(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_object *object = handle->object;
+        union {
+                struct nvif_ioctl_unmap none;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "unmap size %d\n", size);
+        if (nvif_unvers(args->none)) {
+                nv_ioctl(object, "unmap\n");
+        }
+
+        return ret;
+}
+
+static int
+nvkm_ioctl_ntfy_new(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_client *client = nouveau_client(handle->object);
+        struct nouveau_object *object = handle->object;
+        struct nouveau_ofuncs *ofuncs = object->oclass->ofuncs;
+        union {
+                struct nvif_ioctl_ntfy_new_v0 v0;
+        } *args = data;
+        struct nvkm_event *event;
+        int ret;
+
+        nv_ioctl(object, "ntfy new size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                nv_ioctl(object, "ntfy new vers %d event %02x\n",
+                         args->v0.version, args->v0.event);
+                if (ret = -ENODEV, ofuncs->ntfy)
+                        ret = ofuncs->ntfy(object, args->v0.event, &event);
+                if (ret == 0) {
+                        ret = nvkm_client_notify_new(client, event, data, size);
+                        if (ret >= 0) {
+                                args->v0.index = ret;
+                                ret = 0;
+                        }
+                }
+        }
+
+        return ret;
+}
+
+static int
+nvkm_ioctl_ntfy_del(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_client *client = nouveau_client(handle->object);
+        struct nouveau_object *object = handle->object;
+        union {
+                struct nvif_ioctl_ntfy_del_v0 v0;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "ntfy del size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "ntfy del vers %d index %d\n",
+                         args->v0.version, args->v0.index);
+                ret = nvkm_client_notify_del(client, args->v0.index);
+        }
+
+        return ret;
+}
+
+static int
+nvkm_ioctl_ntfy_get(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_client *client = nouveau_client(handle->object);
+        struct nouveau_object *object = handle->object;
+        union {
+                struct nvif_ioctl_ntfy_get_v0 v0;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "ntfy get size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "ntfy get vers %d index %d\n",
+                         args->v0.version, args->v0.index);
+                ret = nvkm_client_notify_get(client, args->v0.index);
+        }
+
+        return ret;
+}
+
+static int
+nvkm_ioctl_ntfy_put(struct nouveau_handle *handle, void *data, u32 size)
+{
+        struct nouveau_client *client = nouveau_client(handle->object);
+        struct nouveau_object *object = handle->object;
+        union {
+                struct nvif_ioctl_ntfy_put_v0 v0;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "ntfy put size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "ntfy put vers %d index %d\n",
+                         args->v0.version, args->v0.index);
+                ret = nvkm_client_notify_put(client, args->v0.index);
+        }
+
+        return ret;
+}
+
+static struct {
+        int version;
+        int (*func)(struct nouveau_handle *, void *, u32);
+}
+nvkm_ioctl_v0[] = {
+        { 0x00, nvkm_ioctl_nop },
+        { 0x00, nvkm_ioctl_sclass },
+        { 0x00, nvkm_ioctl_new },
+        { 0x00, nvkm_ioctl_del },
+        { 0x00, nvkm_ioctl_mthd },
+        { 0x00, nvkm_ioctl_rd },
+        { 0x00, nvkm_ioctl_wr },
+        { 0x00, nvkm_ioctl_map },
+        { 0x00, nvkm_ioctl_unmap },
+        { 0x00, nvkm_ioctl_ntfy_new },
+        { 0x00, nvkm_ioctl_ntfy_del },
+        { 0x00, nvkm_ioctl_ntfy_get },
+        { 0x00, nvkm_ioctl_ntfy_put },
+};
+
+static int
+nvkm_ioctl_path(struct nouveau_handle *parent, u32 type, u32 nr,
+                  u32 *path, void *data, u32 size,
+                  u8 owner, u8 *route, u64 *token)
+{
+        struct nouveau_handle *handle = parent;
+        struct nouveau_namedb *namedb;
+        struct nouveau_object *object;
+        int ret;
+
+        while ((object = parent->object), nr--) {
+                nv_ioctl(object, "path 0x%08x\n", path[nr]);
+                if (!nv_iclass(object, NV_PARENT_CLASS)) {
+                        nv_debug(object, "cannot have children (path)\n");
+                        return -EINVAL;
+                }
+
+                if (!(namedb = (void *)nv_pclass(object, NV_NAMEDB_CLASS)) ||
+                    !(handle = nouveau_namedb_get(namedb, path[nr]))) {
+                        nv_debug(object, "handle 0x%08x not found\n", path[nr]);
+                        return -ENOENT;
+                }
+                nouveau_namedb_put(handle);
+                parent = handle;
+        }
+
+        if (owner != NVIF_IOCTL_V0_OWNER_ANY &&
+            owner != handle->route) {
+                nv_ioctl(object, "object route != owner\n");
+                return -EACCES;
+        }
+        *route = handle->route;
+        *token = handle->token;
+
+        if (ret = -EINVAL, type < ARRAY_SIZE(nvkm_ioctl_v0)) {
+                if (nvkm_ioctl_v0[type].version == 0) {
+                        ret = nvkm_ioctl_v0[type].func(handle, data, size);
+                }
+        }
+
+        return ret;
+}
+
+int
+nvkm_ioctl(struct nouveau_client *client, bool supervisor,
+           void *data, u32 size, void **hack)
+{
+        union {
+                struct nvif_ioctl_v0 v0;
+        } *args = data;
+        int ret;
+
+        client->super = supervisor;
+        nv_ioctl(client, "size %d\n", size);
+
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                nv_ioctl(client, "vers %d type %02x path %d owner %02x\n",
+                         args->v0.version, args->v0.type, args->v0.path_nr,
+                         args->v0.owner);
+                ret = nvkm_ioctl_path(client->root, args->v0.type,
+                                      args->v0.path_nr, args->v0.path,
+                                      data, size, args->v0.owner,
+                                     &args->v0.route, &args->v0.token);
+        }
+
+        nv_ioctl(client, "return %d\n", ret);
+        if (hack) {
+                *hack = client->data;
+                client->data = NULL;
+        }
+        client->super = false;
+        return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/notify.c b/drivers/gpu/drm/nouveau/core/core/notify.c
new file mode 100644
index 000000000000..76adb81bdea2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/notify.c
@@ -0,0 +1,167 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <core/client.h>
+#include <core/event.h>
+#include <core/notify.h>
+
+#include <nvif/unpack.h>
+#include <nvif/event.h>
+
+static inline void
+nvkm_notify_put_locked(struct nvkm_notify *notify)
+{
+        if (notify->block++ == 0)
+                nvkm_event_put(notify->event, notify->types, notify->index);
+}
+
+void
+nvkm_notify_put(struct nvkm_notify *notify)
+{
+        struct nvkm_event *event = notify->event;
+        unsigned long flags;
+        if (likely(event) &&
+            test_and_clear_bit(NVKM_NOTIFY_USER, &notify->flags)) {
+                spin_lock_irqsave(&event->refs_lock, flags);
+                nvkm_notify_put_locked(notify);
+                spin_unlock_irqrestore(&event->refs_lock, flags);
+                if (test_bit(NVKM_NOTIFY_WORK, &notify->flags))
+                        flush_work(&notify->work);
+        }
+}
+
+static inline void
+nvkm_notify_get_locked(struct nvkm_notify *notify)
+{
+        if (--notify->block == 0)
+                nvkm_event_get(notify->event, notify->types, notify->index);
+}
+
+void
+nvkm_notify_get(struct nvkm_notify *notify)
+{
+        struct nvkm_event *event = notify->event;
+        unsigned long flags;
+        if (likely(event) &&
+            !test_and_set_bit(NVKM_NOTIFY_USER, &notify->flags)) {
+                spin_lock_irqsave(&event->refs_lock, flags);
+                nvkm_notify_get_locked(notify);
+                spin_unlock_irqrestore(&event->refs_lock, flags);
+        }
+}
+
+static inline void
+nvkm_notify_func(struct nvkm_notify *notify)
+{
+        struct nvkm_event *event = notify->event;
+        int ret = notify->func(notify);
+        unsigned long flags;
+        if ((ret == NVKM_NOTIFY_KEEP) ||
+            !test_and_clear_bit(NVKM_NOTIFY_USER, &notify->flags)) {
+                spin_lock_irqsave(&event->refs_lock, flags);
+                nvkm_notify_get_locked(notify);
+                spin_unlock_irqrestore(&event->refs_lock, flags);
+        }
+}
+
+static void
+nvkm_notify_work(struct work_struct *work)
+{
+        struct nvkm_notify *notify = container_of(work, typeof(*notify), work);
+        nvkm_notify_func(notify);
+}
+
+void
+nvkm_notify_send(struct nvkm_notify *notify, void *data, u32 size)
+{
+        struct nvkm_event *event = notify->event;
+        unsigned long flags;
+
+        BUG_ON(!spin_is_locked(&event->list_lock));
+        BUG_ON(size != notify->size);
+
+        spin_lock_irqsave(&event->refs_lock, flags);
+        if (notify->block) {
+                spin_unlock_irqrestore(&event->refs_lock, flags);
+                return;
+        }
+        nvkm_notify_put_locked(notify);
+        spin_unlock_irqrestore(&event->refs_lock, flags);
+
+        if (test_bit(NVKM_NOTIFY_WORK, &notify->flags)) {
+                memcpy((void *)notify->data, data, size);
+                schedule_work(&notify->work);
+        } else {
+                notify->data = data;
+                nvkm_notify_func(notify);
+                notify->data = NULL;
+        }
+}
+
+void
+nvkm_notify_fini(struct nvkm_notify *notify)
+{
+        unsigned long flags;
+        if (notify->event) {
+                nvkm_notify_put(notify);
+                spin_lock_irqsave(&notify->event->list_lock, flags);
+                list_del(&notify->head);
+                spin_unlock_irqrestore(&notify->event->list_lock, flags);
+                kfree((void *)notify->data);
+                notify->event = NULL;
+        }
+}
+
+int
+nvkm_notify_init(struct nvkm_event *event, int (*func)(struct nvkm_notify *),
+                 bool work, void *data, u32 size, u32 reply,
+                 struct nvkm_notify *notify)
+{
+        unsigned long flags;
+        int ret = -ENODEV;
+        if ((notify->event = event), event->refs) {
+                ret = event->func->ctor(data, size, notify);
+                if (ret == 0 && (ret = -EINVAL, notify->size == reply)) {
+                        notify->flags = 0;
+                        notify->block = 1;
+                        notify->func = func;
+                        notify->data = NULL;
+                        if (ret = 0, work) {
+                                INIT_WORK(&notify->work, nvkm_notify_work);
+                                set_bit(NVKM_NOTIFY_WORK, &notify->flags);
+                                notify->data = kmalloc(reply, GFP_KERNEL);
+                                if (!notify->data)
+                                        ret = -ENOMEM;
+                        }
+                }
+                if (ret == 0) {
+                        spin_lock_irqsave(&event->list_lock, flags);
+                        list_add_tail(&notify->head, &event->list);
+                        spin_unlock_irqrestore(&event->list_lock, flags);
+                }
+        }
+        if (ret)
+                notify->event = NULL;
+        return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/object.c b/drivers/gpu/drm/nouveau/core/core/object.c
index 124538555904..b08630577c82 100644
--- a/drivers/gpu/drm/nouveau/core/core/object.c
+++ b/drivers/gpu/drm/nouveau/core/core/object.c
@@ -23,9 +23,6 @@
  */
 
 #include <core/object.h>
-#include <core/parent.h>
-#include <core/namedb.h>
-#include <core/handle.h>
 #include <core/engine.h>
 
 #ifdef NOUVEAU_OBJECT_MAGIC
@@ -61,21 +58,15 @@ nouveau_object_create_(struct nouveau_object *parent,
         return 0;
 }
 
-static int
+int
 _nouveau_object_ctor(struct nouveau_object *parent,
                      struct nouveau_object *engine,
                      struct nouveau_oclass *oclass, void *data, u32 size,
                      struct nouveau_object **pobject)
 {
-        struct nouveau_object *object;
-        int ret;
-
-        ret = nouveau_object_create(parent, engine, oclass, 0, &object);
-        *pobject = nv_object(object);
-        if (ret)
-                return ret;
-
-        return 0;
+        if (size != 0)
+                return -ENOSYS;
+        return nouveau_object_create(parent, engine, oclass, 0, pobject);
 }
 
 void
@@ -91,42 +82,24 @@ nouveau_object_destroy(struct nouveau_object *object)
         kfree(object);
 }
 
-static void
-_nouveau_object_dtor(struct nouveau_object *object)
-{
-        nouveau_object_destroy(object);
-}
-
 int
 nouveau_object_init(struct nouveau_object *object)
 {
         return 0;
 }
 
-static int
-_nouveau_object_init(struct nouveau_object *object)
-{
-        return nouveau_object_init(object);
-}
-
 int
 nouveau_object_fini(struct nouveau_object *object, bool suspend)
 {
         return 0;
 }
 
-static int
-_nouveau_object_fini(struct nouveau_object *object, bool suspend)
-{
-        return nouveau_object_fini(object, suspend);
-}
-
 struct nouveau_ofuncs
 nouveau_object_ofuncs = {
         .ctor = _nouveau_object_ctor,
-        .dtor = _nouveau_object_dtor,
-        .init = _nouveau_object_init,
-        .fini = _nouveau_object_fini,
+        .dtor = nouveau_object_destroy,
+        .init = nouveau_object_init,
+        .fini = nouveau_object_fini,
 };
 
 int
@@ -189,119 +162,6 @@ nouveau_object_ref(struct nouveau_object *obj, struct nouveau_object **ref)
 }
 
 int
-nouveau_object_new(struct nouveau_object *client, u32 _parent, u32 _handle,
-                   u16 _oclass, void *data, u32 size,
-                   struct nouveau_object **pobject)
-{
-        struct nouveau_object *parent = NULL;
-        struct nouveau_object *engctx = NULL;
-        struct nouveau_object *object = NULL;
-        struct nouveau_object *engine;
-        struct nouveau_oclass *oclass;
-        struct nouveau_handle *handle;
-        int ret;
-
-        /* lookup parent object and ensure it *is* a parent */
-        parent = nouveau_handle_ref(client, _parent);
-        if (!parent) {
-                nv_error(client, "parent 0x%08x not found\n", _parent);
-                return -ENOENT;
-        }
-
-        if (!nv_iclass(parent, NV_PARENT_CLASS)) {
-                nv_error(parent, "cannot have children\n");
-                ret = -EINVAL;
-                goto fail_class;
-        }
-
-        /* check that parent supports the requested subclass */
-        ret = nouveau_parent_sclass(parent, _oclass, &engine, &oclass);
-        if (ret) {
-                nv_debug(parent, "illegal class 0x%04x\n", _oclass);
-                goto fail_class;
-        }
-
-        /* make sure engine init has been completed *before* any objects
-         * it controls are created - the constructors may depend on
-         * state calculated at init (ie. default context construction)
-         */
-        if (engine) {
-                ret = nouveau_object_inc(engine);
-                if (ret)
-                        goto fail_class;
-        }
-
-        /* if engine requires it, create a context object to insert
-         * between the parent and its children (eg. PGRAPH context)
-         */
-        if (engine && nv_engine(engine)->cclass) {
-                ret = nouveau_object_ctor(parent, engine,
-                                          nv_engine(engine)->cclass,
-                                          data, size, &engctx);
-                if (ret)
-                        goto fail_engctx;
-        } else {
-                nouveau_object_ref(parent, &engctx);
-        }
-
-        /* finally, create new object and bind it to its handle */
-        ret = nouveau_object_ctor(engctx, engine, oclass, data, size, &object);
-        *pobject = object;
-        if (ret)
-                goto fail_ctor;
-
-        ret = nouveau_object_inc(object);
-        if (ret)
-                goto fail_init;
-
-        ret = nouveau_handle_create(parent, _parent, _handle, object, &handle);
-        if (ret)
-                goto fail_handle;
-
-        ret = nouveau_handle_init(handle);
-        if (ret)
-                nouveau_handle_destroy(handle);
-
-fail_handle:
-        nouveau_object_dec(object, false);
-fail_init:
-        nouveau_object_ref(NULL, &object);
-fail_ctor:
-        nouveau_object_ref(NULL, &engctx);
-fail_engctx:
-        if (engine)
-                nouveau_object_dec(engine, false);
-fail_class:
-        nouveau_object_ref(NULL, &parent);
-        return ret;
-}
-
-int
-nouveau_object_del(struct nouveau_object *client, u32 _parent, u32 _handle)
-{
-        struct nouveau_object *parent = NULL;
-        struct nouveau_object *namedb = NULL;
-        struct nouveau_handle *handle = NULL;
-
-        parent = nouveau_handle_ref(client, _parent);
-        if (!parent)
-                return -ENOENT;
-
-        namedb = nv_pclass(parent, NV_NAMEDB_CLASS);
-        if (namedb) {
-                handle = nouveau_namedb_get(nv_namedb(namedb), _handle);
-                if (handle) {
-                        nouveau_namedb_put(handle);
-                        nouveau_handle_fini(handle, false);
-                        nouveau_handle_destroy(handle);
-                }
-        }
-
-        nouveau_object_ref(NULL, &parent);
-        return handle ? 0 : -EINVAL;
-}
-
-int
 nouveau_object_inc(struct nouveau_object *object)
 {
         int ref = atomic_add_return(1, &object->usecount);
diff --git a/drivers/gpu/drm/nouveau/core/core/parent.c b/drivers/gpu/drm/nouveau/core/core/parent.c
index dee5d1235e9b..8701968a9743 100644
--- a/drivers/gpu/drm/nouveau/core/core/parent.c
+++ b/drivers/gpu/drm/nouveau/core/core/parent.c
@@ -75,6 +75,39 @@ nouveau_parent_sclass(struct nouveau_object *parent, u16 handle,
 }
 
 int
+nouveau_parent_lclass(struct nouveau_object *parent, u32 *lclass, int size)
+{
+        struct nouveau_sclass *sclass;
+        struct nouveau_engine *engine;
+        struct nouveau_oclass *oclass;
+        int nr = -1, i;
+        u64 mask;
+
+        sclass = nv_parent(parent)->sclass;
+        while (sclass) {
+                if (++nr < size)
+                        lclass[nr] = sclass->oclass->handle;
+                sclass = sclass->sclass;
+        }
+
+        mask = nv_parent(parent)->engine;
+        while (i = __ffs64(mask), mask) {
+                engine = nouveau_engine(parent, i);
+                if (engine && (oclass = engine->sclass)) {
+                        while (oclass->ofuncs) {
+                                if (++nr < size)
+                                        lclass[nr] = oclass->handle;
+                                oclass++;
+                        }
+                }
+
+                mask &= ~(1ULL << i);
+        }
+
+        return nr + 1;
+}
+
+int
 nouveau_parent_create_(struct nouveau_object *parent,
                        struct nouveau_object *engine,
                        struct nouveau_oclass *oclass, u32 pclass,
diff --git a/drivers/gpu/drm/nouveau/core/engine/copy/nva3.c b/drivers/gpu/drm/nouveau/core/engine/copy/nva3.c
index f31527733e00..abb410ef09ea 100644
--- a/drivers/gpu/drm/nouveau/core/engine/copy/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/engine/copy/nva3.c
@@ -30,7 +30,6 @@
 #include <subdev/vm.h>
 
 #include <core/client.h>
-#include <core/class.h>
 #include <core/enum.h>
 
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/copy/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/copy/nvc0.c
index ac3291f781f6..9261694d0d35 100644
--- a/drivers/gpu/drm/nouveau/core/engine/copy/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/copy/nvc0.c
@@ -26,9 +26,7 @@
 #include <engine/fifo.h>
 #include <engine/copy.h>
 
-#include <core/class.h>
 #include <core/enum.h>
-#include <core/class.h>
 #include <core/enum.h>
 
 #include "fuc/nvc0.fuc.h"
diff --git a/drivers/gpu/drm/nouveau/core/engine/copy/nve0.c b/drivers/gpu/drm/nouveau/core/engine/copy/nve0.c
index 748a61eb3c6f..c7194b354605 100644
--- a/drivers/gpu/drm/nouveau/core/engine/copy/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/copy/nve0.c
@@ -24,7 +24,6 @@
 
 #include <core/os.h>
 #include <core/enum.h>
-#include <core/class.h>
 #include <core/engctx.h>
 
 #include <engine/copy.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/crypt/nv84.c b/drivers/gpu/drm/nouveau/core/engine/crypt/nv84.c
index 2551dafbec73..ea5c42f31791 100644
--- a/drivers/gpu/drm/nouveau/core/engine/crypt/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/engine/crypt/nv84.c
@@ -25,7 +25,6 @@
 #include <core/client.h>
 #include <core/os.h>
 #include <core/enum.h>
-#include <core/class.h>
 #include <core/engctx.h>
 #include <core/gpuobj.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/crypt/nv98.c b/drivers/gpu/drm/nouveau/core/engine/crypt/nv98.c
index c7082377ec76..5571c09534cb 100644
--- a/drivers/gpu/drm/nouveau/core/engine/crypt/nv98.c
+++ b/drivers/gpu/drm/nouveau/core/engine/crypt/nv98.c
@@ -25,7 +25,6 @@
 #include <core/client.h>
 #include <core/os.h>
 #include <core/enum.h>
-#include <core/class.h>
 #include <core/engctx.h>
 
 #include <subdev/timer.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/acpi.c b/drivers/gpu/drm/nouveau/core/engine/device/acpi.c
new file mode 100644
index 000000000000..4dbf0ba89e5c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/device/acpi.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "acpi.h"
+
+#ifdef CONFIG_ACPI
+static int
+nvkm_acpi_ntfy(struct notifier_block *nb, unsigned long val, void *data)
+{
+        struct nouveau_device *device =
+                container_of(nb, typeof(*device), acpi.nb);
+        struct acpi_bus_event *info = data;
+
+        if (!strcmp(info->device_class, "ac_adapter"))
+                nvkm_event_send(&device->event, 1, 0, NULL, 0);
+
+        return NOTIFY_DONE;
+}
+#endif
+
+int
+nvkm_acpi_fini(struct nouveau_device *device, bool suspend)
+{
+#ifdef CONFIG_ACPI
+        unregister_acpi_notifier(&device->acpi.nb);
+#endif
+        return 0;
+}
+
+int
+nvkm_acpi_init(struct nouveau_device *device)
+{
+#ifdef CONFIG_ACPI
+        device->acpi.nb.notifier_call = nvkm_acpi_ntfy;
+        register_acpi_notifier(&device->acpi.nb);
+#endif
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/acpi.h b/drivers/gpu/drm/nouveau/core/engine/device/acpi.h
new file mode 100644
index 000000000000..cc49f4f568cd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/device/acpi.h
@@ -0,0 +1,9 @@
+#ifndef __NVKM_DEVICE_ACPI_H__
+#define __NVKM_DEVICE_ACPI_H__
+
+#include <engine/device.h>
+
+int nvkm_acpi_init(struct nouveau_device *);
+int nvkm_acpi_fini(struct nouveau_device *, bool);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/base.c b/drivers/gpu/drm/nouveau/core/engine/device/base.c
index 18c8c7245b73..8928f7981d4a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/base.c
@@ -26,10 +26,14 @@
 #include <core/device.h>
 #include <core/client.h>
 #include <core/option.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
-#include <core/class.h>
+#include <subdev/fb.h>
+#include <subdev/instmem.h>
 
 #include "priv.h"
+#include "acpi.h"
 
 static DEFINE_MUTEX(nv_devices_mutex);
 static LIST_HEAD(nv_devices);
@@ -49,74 +53,258 @@ nouveau_device_find(u64 name)
         return match;
 }
 
+int
+nouveau_device_list(u64 *name, int size)
+{
+        struct nouveau_device *device;
+        int nr = 0;
+        mutex_lock(&nv_devices_mutex);
+        list_for_each_entry(device, &nv_devices, head) {
+                if (nr++ < size)
+                        name[nr - 1] = device->handle;
+        }
+        mutex_unlock(&nv_devices_mutex);
+        return nr;
+}
+
 /******************************************************************************
  * nouveau_devobj (0x0080): class implementation
  *****************************************************************************/
+
 struct nouveau_devobj {
         struct nouveau_parent base;
         struct nouveau_object *subdev[NVDEV_SUBDEV_NR];
 };
 
+static int
+nouveau_devobj_info(struct nouveau_object *object, void *data, u32 size)
+{
+        struct nouveau_device *device = nv_device(object);
+        struct nouveau_fb *pfb = nouveau_fb(device);
+        struct nouveau_instmem *imem = nouveau_instmem(device);
+        union {
+                struct nv_device_info_v0 v0;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "device info size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "device info vers %d\n", args->v0.version);
+        } else
+                return ret;
+
+        switch (device->chipset) {
+        case 0x01a:
+        case 0x01f:
+        case 0x04c:
+        case 0x04e:
+        case 0x063:
+        case 0x067:
+        case 0x068:
+        case 0x0aa:
+        case 0x0ac:
+        case 0x0af:
+                args->v0.platform = NV_DEVICE_INFO_V0_IGP;
+                break;
+        default:
+                if (device->pdev) {
+                        if (pci_find_capability(device->pdev, PCI_CAP_ID_AGP))
+                                args->v0.platform = NV_DEVICE_INFO_V0_AGP;
+                        else
+                        if (pci_is_pcie(device->pdev))
+                                args->v0.platform = NV_DEVICE_INFO_V0_PCIE;
+                        else
+                                args->v0.platform = NV_DEVICE_INFO_V0_PCI;
+                } else {
+                        args->v0.platform = NV_DEVICE_INFO_V0_SOC;
+                }
+                break;
+        }
+
+        switch (device->card_type) {
+        case NV_04: args->v0.family = NV_DEVICE_INFO_V0_TNT; break;
+        case NV_10:
+        case NV_11: args->v0.family = NV_DEVICE_INFO_V0_CELSIUS; break;
+        case NV_20: args->v0.family = NV_DEVICE_INFO_V0_KELVIN; break;
+        case NV_30: args->v0.family = NV_DEVICE_INFO_V0_RANKINE; break;
+        case NV_40: args->v0.family = NV_DEVICE_INFO_V0_CURIE; break;
+        case NV_50: args->v0.family = NV_DEVICE_INFO_V0_TESLA; break;
+        case NV_C0: args->v0.family = NV_DEVICE_INFO_V0_FERMI; break;
+        case NV_E0: args->v0.family = NV_DEVICE_INFO_V0_KEPLER; break;
+        case GM100: args->v0.family = NV_DEVICE_INFO_V0_MAXWELL; break;
+        default:
+                args->v0.family = 0;
+                break;
+        }
+
+        args->v0.chipset  = device->chipset;
+        args->v0.revision = device->chipset >= 0x10 ? nv_rd32(device, 0) : 0x00;
+        if (pfb)  args->v0.ram_size = args->v0.ram_user = pfb->ram->size;
+        else      args->v0.ram_size = args->v0.ram_user = 0;
+        if (imem) args->v0.ram_user = args->v0.ram_user - imem->reserved;
+        return 0;
+}
+
+static int
+nouveau_devobj_mthd(struct nouveau_object *object, u32 mthd,
+                    void *data, u32 size)
+{
+        switch (mthd) {
+        case NV_DEVICE_V0_INFO:
+                return nouveau_devobj_info(object, data, size);
+        default:
+                break;
+        }
+        return -EINVAL;
+}
+
+static u8
+nouveau_devobj_rd08(struct nouveau_object *object, u64 addr)
+{
+        return nv_rd08(object->engine, addr);
+}
+
+static u16
+nouveau_devobj_rd16(struct nouveau_object *object, u64 addr)
+{
+        return nv_rd16(object->engine, addr);
+}
+
+static u32
+nouveau_devobj_rd32(struct nouveau_object *object, u64 addr)
+{
+        return nv_rd32(object->engine, addr);
+}
+
+static void
+nouveau_devobj_wr08(struct nouveau_object *object, u64 addr, u8 data)
+{
+        nv_wr08(object->engine, addr, data);
+}
+
+static void
+nouveau_devobj_wr16(struct nouveau_object *object, u64 addr, u16 data)
+{
+        nv_wr16(object->engine, addr, data);
+}
+
+static void
+nouveau_devobj_wr32(struct nouveau_object *object, u64 addr, u32 data)
+{
+        nv_wr32(object->engine, addr, data);
+}
+
+static int
+nouveau_devobj_map(struct nouveau_object *object, u64 *addr, u32 *size)
+{
+        struct nouveau_device *device = nv_device(object);
+        *addr = nv_device_resource_start(device, 0);
+        *size = nv_device_resource_len(device, 0);
+        return 0;
+}
+
 static const u64 disable_map[] = {
-        [NVDEV_SUBDEV_VBIOS]    = NV_DEVICE_DISABLE_VBIOS,
-        [NVDEV_SUBDEV_DEVINIT]  = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_GPIO]     = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_I2C]      = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_CLOCK]    = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_MXM]      = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_MC]       = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_BUS]      = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_TIMER]    = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_FB]       = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_LTCG]     = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_IBUS]     = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_INSTMEM]  = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_VM]       = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_BAR]      = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_VOLT]     = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_THERM]    = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_SUBDEV_PWR]      = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_ENGINE_DMAOBJ]   = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_ENGINE_PERFMON]  = NV_DEVICE_DISABLE_CORE,
-        [NVDEV_ENGINE_FIFO]     = NV_DEVICE_DISABLE_FIFO,
-        [NVDEV_ENGINE_SW]       = NV_DEVICE_DISABLE_FIFO,
-        [NVDEV_ENGINE_GR]       = NV_DEVICE_DISABLE_GRAPH,
-        [NVDEV_ENGINE_MPEG]     = NV_DEVICE_DISABLE_MPEG,
-        [NVDEV_ENGINE_ME]       = NV_DEVICE_DISABLE_ME,
-        [NVDEV_ENGINE_VP]       = NV_DEVICE_DISABLE_VP,
-        [NVDEV_ENGINE_CRYPT]    = NV_DEVICE_DISABLE_CRYPT,
-        [NVDEV_ENGINE_BSP]      = NV_DEVICE_DISABLE_BSP,
-        [NVDEV_ENGINE_PPP]      = NV_DEVICE_DISABLE_PPP,
-        [NVDEV_ENGINE_COPY0]    = NV_DEVICE_DISABLE_COPY0,
-        [NVDEV_ENGINE_COPY1]    = NV_DEVICE_DISABLE_COPY1,
-        [NVDEV_ENGINE_VIC]      = NV_DEVICE_DISABLE_VIC,
-        [NVDEV_ENGINE_VENC]     = NV_DEVICE_DISABLE_VENC,
-        [NVDEV_ENGINE_DISP]     = NV_DEVICE_DISABLE_DISP,
+        [NVDEV_SUBDEV_VBIOS]    = NV_DEVICE_V0_DISABLE_VBIOS,
+        [NVDEV_SUBDEV_DEVINIT]  = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_GPIO]     = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_I2C]      = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_CLOCK]    = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_MXM]      = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_MC]       = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_BUS]      = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_TIMER]    = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_FB]       = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_LTC]      = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_IBUS]     = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_INSTMEM]  = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_VM]       = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_BAR]      = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_VOLT]     = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_THERM]    = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_PWR]      = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_ENGINE_DMAOBJ]   = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_ENGINE_PERFMON]  = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_ENGINE_FIFO]     = NV_DEVICE_V0_DISABLE_FIFO,
+        [NVDEV_ENGINE_SW]       = NV_DEVICE_V0_DISABLE_FIFO,
+        [NVDEV_ENGINE_GR]       = NV_DEVICE_V0_DISABLE_GRAPH,
+        [NVDEV_ENGINE_MPEG]     = NV_DEVICE_V0_DISABLE_MPEG,
+        [NVDEV_ENGINE_ME]       = NV_DEVICE_V0_DISABLE_ME,
+        [NVDEV_ENGINE_VP]       = NV_DEVICE_V0_DISABLE_VP,
+        [NVDEV_ENGINE_CRYPT]    = NV_DEVICE_V0_DISABLE_CRYPT,
+        [NVDEV_ENGINE_BSP]      = NV_DEVICE_V0_DISABLE_BSP,
+        [NVDEV_ENGINE_PPP]      = NV_DEVICE_V0_DISABLE_PPP,
+        [NVDEV_ENGINE_COPY0]    = NV_DEVICE_V0_DISABLE_COPY0,
+        [NVDEV_ENGINE_COPY1]    = NV_DEVICE_V0_DISABLE_COPY1,
+        [NVDEV_ENGINE_VIC]      = NV_DEVICE_V0_DISABLE_VIC,
+        [NVDEV_ENGINE_VENC]     = NV_DEVICE_V0_DISABLE_VENC,
+        [NVDEV_ENGINE_DISP]     = NV_DEVICE_V0_DISABLE_DISP,
         [NVDEV_SUBDEV_NR]       = 0,
 };
 
+static void
+nouveau_devobj_dtor(struct nouveau_object *object)
+{
+        struct nouveau_devobj *devobj = (void *)object;
+        int i;
+
+        for (i = NVDEV_SUBDEV_NR - 1; i >= 0; i--)
+                nouveau_object_ref(NULL, &devobj->subdev[i]);
+
+        nouveau_parent_destroy(&devobj->base);
+}
+
+static struct nouveau_oclass
+nouveau_devobj_oclass_super = {
+        .handle = NV_DEVICE,
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .dtor = nouveau_devobj_dtor,
+                .init = _nouveau_parent_init,
+                .fini = _nouveau_parent_fini,
+                .mthd = nouveau_devobj_mthd,
+                .map  = nouveau_devobj_map,
+                .rd08 = nouveau_devobj_rd08,
+                .rd16 = nouveau_devobj_rd16,
+                .rd32 = nouveau_devobj_rd32,
+                .wr08 = nouveau_devobj_wr08,
+                .wr16 = nouveau_devobj_wr16,
+                .wr32 = nouveau_devobj_wr32,
+        }
+};
+
 static int
 nouveau_devobj_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
+        union {
+                struct nv_device_v0 v0;
+        } *args = data;
         struct nouveau_client *client = nv_client(parent);
         struct nouveau_device *device;
         struct nouveau_devobj *devobj;
-        struct nv_device_class *args = data;
         u32 boot0, strap;
         u64 disable, mmio_base, mmio_size;
         void __iomem *map;
         int ret, i, c;
 
-        if (size < sizeof(struct nv_device_class))
-                return -EINVAL;
+        nv_ioctl(parent, "create device size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create device v%d device %016llx "
+                                 "disable %016llx debug0 %016llx\n",
+                         args->v0.version, args->v0.device,
+                         args->v0.disable, args->v0.debug0);
+        } else
+                return ret;
+
+        /* give priviledged clients register access */
+        if (client->super)
+                oclass = &nouveau_devobj_oclass_super;
 
         /* find the device subdev that matches what the client requested */
         device = nv_device(client->device);
-        if (args->device != ~0) {
-                device = nouveau_device_find(args->device);
+        if (args->v0.device != ~0) {
+                device = nouveau_device_find(args->v0.device);
                 if (!device)
                         return -ENODEV;
         }
@@ -135,14 +323,14 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
         mmio_size = nv_device_resource_len(device, 0);
 
         /* translate api disable mask into internal mapping */
-        disable = args->debug0;
+        disable = args->v0.debug0;
         for (i = 0; i < NVDEV_SUBDEV_NR; i++) {
-                if (args->disable & disable_map[i])
+                if (args->v0.disable & disable_map[i])
                         disable |= (1ULL << i);
         }
 
         /* identify the chipset, and determine classes of subdev/engines */
-        if (!(args->disable & NV_DEVICE_DISABLE_IDENTIFY) &&
+        if (!(args->v0.disable & NV_DEVICE_V0_DISABLE_IDENTIFY) &&
             !device->card_type) {
                 map = ioremap(mmio_base, 0x102000);
                 if (map == NULL)
@@ -180,8 +368,8 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                         case 0x080:
                         case 0x090:
                         case 0x0a0: device->card_type = NV_50; break;
-                        case 0x0c0: device->card_type = NV_C0; break;
-                        case 0x0d0: device->card_type = NV_D0; break;
+                        case 0x0c0:
+                        case 0x0d0: device->card_type = NV_C0; break;
                         case 0x0e0:
                         case 0x0f0:
                         case 0x100: device->card_type = NV_E0; break;
@@ -206,8 +394,7 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                 case NV_30: ret = nv30_identify(device); break;
                 case NV_40: ret = nv40_identify(device); break;
                 case NV_50: ret = nv50_identify(device); break;
-                case NV_C0:
-                case NV_D0: ret = nvc0_identify(device); break;
+                case NV_C0: ret = nvc0_identify(device); break;
                 case NV_E0: ret = nve0_identify(device); break;
                 case GM100: ret = gm100_identify(device); break;
                 default:
@@ -242,7 +429,7 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                 nv_debug(device, "crystal freq: %dKHz\n", device->crystal);
         }
 
-        if (!(args->disable & NV_DEVICE_DISABLE_MMIO) &&
+        if (!(args->v0.disable & NV_DEVICE_V0_DISABLE_MMIO) &&
             !nv_subdev(device)->mmio) {
                 nv_subdev(device)->mmio  = ioremap(mmio_base, mmio_size);
                 if (!nv_subdev(device)->mmio) {
@@ -298,71 +485,19 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
         return 0;
 }
 
-static void
-nouveau_devobj_dtor(struct nouveau_object *object)
-{
-        struct nouveau_devobj *devobj = (void *)object;
-        int i;
-
-        for (i = NVDEV_SUBDEV_NR - 1; i >= 0; i--)
-                nouveau_object_ref(NULL, &devobj->subdev[i]);
-
-        nouveau_parent_destroy(&devobj->base);
-}
-
-static u8
-nouveau_devobj_rd08(struct nouveau_object *object, u64 addr)
-{
-        return nv_rd08(object->engine, addr);
-}
-
-static u16
-nouveau_devobj_rd16(struct nouveau_object *object, u64 addr)
-{
-        return nv_rd16(object->engine, addr);
-}
-
-static u32
-nouveau_devobj_rd32(struct nouveau_object *object, u64 addr)
-{
-        return nv_rd32(object->engine, addr);
-}
-
-static void
-nouveau_devobj_wr08(struct nouveau_object *object, u64 addr, u8 data)
-{
-        nv_wr08(object->engine, addr, data);
-}
-
-static void
-nouveau_devobj_wr16(struct nouveau_object *object, u64 addr, u16 data)
-{
-        nv_wr16(object->engine, addr, data);
-}
-
-static void
-nouveau_devobj_wr32(struct nouveau_object *object, u64 addr, u32 data)
-{
-        nv_wr32(object->engine, addr, data);
-}
-
 static struct nouveau_ofuncs
 nouveau_devobj_ofuncs = {
         .ctor = nouveau_devobj_ctor,
         .dtor = nouveau_devobj_dtor,
         .init = _nouveau_parent_init,
         .fini = _nouveau_parent_fini,
-        .rd08 = nouveau_devobj_rd08,
-        .rd16 = nouveau_devobj_rd16,
-        .rd32 = nouveau_devobj_rd32,
-        .wr08 = nouveau_devobj_wr08,
-        .wr16 = nouveau_devobj_wr16,
-        .wr32 = nouveau_devobj_wr32,
+        .mthd = nouveau_devobj_mthd,
 };
 
 /******************************************************************************
  * nouveau_device: engine functions
  *****************************************************************************/
+
 static struct nouveau_oclass
 nouveau_device_sclass[] = {
         { 0x0080, &nouveau_devobj_ofuncs },
@@ -370,6 +505,23 @@ nouveau_device_sclass[] = {
 };
 
 static int
+nouveau_device_event_ctor(void *data, u32 size, struct nvkm_notify *notify)
+{
+        if (!WARN_ON(size != 0)) {
+                notify->size  = 0;
+                notify->types = 1;
+                notify->index = 0;
+                return 0;
+        }
+        return -EINVAL;
+}
+
+static const struct nvkm_event_func
+nouveau_device_event_func = {
+        .ctor = nouveau_device_event_ctor,
+};
+
+static int
 nouveau_device_fini(struct nouveau_object *object, bool suspend)
 {
         struct nouveau_device *device = (void *)object;
@@ -386,7 +538,7 @@ nouveau_device_fini(struct nouveau_object *object, bool suspend)
                 }
         }
 
-        ret = 0;
+        ret = nvkm_acpi_fini(device, suspend);
 fail:
         for (; ret && i < NVDEV_SUBDEV_NR; i++) {
                 if ((subdev = device->subdev[i])) {
@@ -407,7 +559,11 @@ nouveau_device_init(struct nouveau_object *object)
 {
         struct nouveau_device *device = (void *)object;
         struct nouveau_object *subdev;
-        int ret, i;
+        int ret, i = 0;
+
+        ret = nvkm_acpi_init(device);
+        if (ret)
+                goto fail;
 
         for (i = 0; i < NVDEV_SUBDEV_NR; i++) {
                 if ((subdev = device->subdev[i])) {
@@ -430,6 +586,8 @@ fail:
                 }
         }
 
+        if (ret)
+                nvkm_acpi_fini(device, false);
         return ret;
 }
 
@@ -438,6 +596,8 @@ nouveau_device_dtor(struct nouveau_object *object)
 {
         struct nouveau_device *device = (void *)object;
 
+        nvkm_event_fini(&device->event);
+
         mutex_lock(&nv_devices_mutex);
         list_del(&device->head);
         mutex_unlock(&nv_devices_mutex);
@@ -478,31 +638,6 @@ nv_device_resource_len(struct nouveau_device *device, unsigned int bar)
         }
 }
 
-dma_addr_t
-nv_device_map_page(struct nouveau_device *device, struct page *page)
-{
-        dma_addr_t ret;
-
-        if (nv_device_is_pci(device)) {
-                ret = pci_map_page(device->pdev, page, 0, PAGE_SIZE,
-                                   PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(device->pdev, ret))
-                        ret = 0;
-        } else {
-                ret = page_to_phys(page);
-        }
-
-        return ret;
-}
-
-void
-nv_device_unmap_page(struct nouveau_device *device, dma_addr_t addr)
-{
-        if (nv_device_is_pci(device))
-                pci_unmap_page(device->pdev, addr, PAGE_SIZE,
-                               PCI_DMA_BIDIRECTIONAL);
-}
-
 int
 nv_device_get_irq(struct nouveau_device *device, bool stall)
 {
@@ -560,6 +695,9 @@ nouveau_device_create_(void *dev, enum nv_bus_type type, u64 name,
         nv_subdev(device)->debug = nouveau_dbgopt(device->dbgopt, "DEVICE");
         nv_engine(device)->sclass = nouveau_device_sclass;
         list_add(&device->head, &nv_devices);
+
+        ret = nvkm_event_init(&nouveau_device_event_func, 1, 1,
+                              &device->event);
 done:
         mutex_unlock(&nv_devices_mutex);
         return ret;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/ctrl.c b/drivers/gpu/drm/nouveau/core/engine/device/ctrl.c
index 4b69bf56ed01..e34101a3490e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/ctrl.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/ctrl.c
@@ -22,55 +22,82 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
+#include <core/client.h>
 #include <core/object.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
+#include <nvif/ioctl.h>
 
 #include <subdev/clock.h>
 
 #include "priv.h"
 
 static int
-nouveau_control_mthd_pstate_info(struct nouveau_object *object, u32 mthd,
-                                void *data, u32 size)
+nouveau_control_mthd_pstate_info(struct nouveau_object *object,
+                                 void *data, u32 size)
 {
+        union {
+                struct nvif_control_pstate_info_v0 v0;
+        } *args = data;
         struct nouveau_clock *clk = nouveau_clock(object);
-        struct nv_control_pstate_info *args = data;
+        int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(object, "control pstate info size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "control pstate info vers %d\n",
+                         args->v0.version);
+        } else
+                return ret;
 
         if (clk) {
-                args->count  = clk->state_nr;
-                args->ustate = clk->ustate;
-                args->pstate = clk->pstate;
+                args->v0.count = clk->state_nr;
+                args->v0.ustate_ac = clk->ustate_ac;
+                args->v0.ustate_dc = clk->ustate_dc;
+                args->v0.pwrsrc = clk->pwrsrc;
+                args->v0.pstate = clk->pstate;
         } else {
-                args->count  = 0;
-                args->ustate = NV_CONTROL_PSTATE_INFO_USTATE_DISABLE;
-                args->pstate = NV_CONTROL_PSTATE_INFO_PSTATE_UNKNOWN;
+                args->v0.count = 0;
+                args->v0.ustate_ac = NVIF_CONTROL_PSTATE_INFO_V0_USTATE_DISABLE;
+                args->v0.ustate_dc = NVIF_CONTROL_PSTATE_INFO_V0_USTATE_DISABLE;
+                args->v0.pwrsrc = -ENOSYS;
+                args->v0.pstate = NVIF_CONTROL_PSTATE_INFO_V0_PSTATE_UNKNOWN;
         }
 
         return 0;
 }
 
 static int
-nouveau_control_mthd_pstate_attr(struct nouveau_object *object, u32 mthd,
-                                void *data, u32 size)
+nouveau_control_mthd_pstate_attr(struct nouveau_object *object,
+                                 void *data, u32 size)
 {
+        union {
+                struct nvif_control_pstate_attr_v0 v0;
+        } *args = data;
         struct nouveau_clock *clk = nouveau_clock(object);
-        struct nv_control_pstate_attr *args = data;
         struct nouveau_clocks *domain;
         struct nouveau_pstate *pstate;
         struct nouveau_cstate *cstate;
         int i = 0, j = -1;
         u32 lo, hi;
-
-        if ((size < sizeof(*args)) || !clk ||
-            (args->state >= 0 && args->state >= clk->state_nr))
-                return -EINVAL;
+        int ret;
+
+        nv_ioctl(object, "control pstate attr size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "control pstate attr vers %d state %d "
+                                 "index %d\n",
+                         args->v0.version, args->v0.state, args->v0.index);
+                if (!clk)
+                        return -ENODEV;
+                if (args->v0.state < NVIF_CONTROL_PSTATE_ATTR_V0_STATE_CURRENT)
+                        return -EINVAL;
+                if (args->v0.state >= clk->state_nr)
+                        return -EINVAL;
+        } else
+                return ret;
         domain = clk->domains;
 
         while (domain->name != nv_clk_src_max) {
-                if (domain->mname && ++j == args->index)
+                if (domain->mname && ++j == args->v0.index)
                         break;
                 domain++;
         }
@@ -78,9 +105,9 @@ nouveau_control_mthd_pstate_attr(struct nouveau_object *object, u32 mthd,
         if (domain->name == nv_clk_src_max)
                 return -EINVAL;
 
-        if (args->state != NV_CONTROL_PSTATE_ATTR_STATE_CURRENT) {
+        if (args->v0.state != NVIF_CONTROL_PSTATE_ATTR_V0_STATE_CURRENT) {
                 list_for_each_entry(pstate, &clk->states, head) {
-                        if (i++ == args->state)
+                        if (i++ == args->v0.state)
                                 break;
                 }
 
@@ -91,21 +118,21 @@ nouveau_control_mthd_pstate_attr(struct nouveau_object *object, u32 mthd,
                         hi = max(hi, cstate->domain[domain->name]);
                 }
 
-                args->state = pstate->pstate;
+                args->v0.state = pstate->pstate;
         } else {
                 lo = max(clk->read(clk, domain->name), 0);
                 hi = lo;
         }
 
-        snprintf(args->name, sizeof(args->name), "%s", domain->mname);
-        snprintf(args->unit, sizeof(args->unit), "MHz");
-        args->min = lo / domain->mdiv;
-        args->max = hi / domain->mdiv;
+        snprintf(args->v0.name, sizeof(args->v0.name), "%s", domain->mname);
+        snprintf(args->v0.unit, sizeof(args->v0.unit), "MHz");
+        args->v0.min = lo / domain->mdiv;
+        args->v0.max = hi / domain->mdiv;
 
-        args->index = 0;
+        args->v0.index = 0;
         while ((++domain)->name != nv_clk_src_max) {
                 if (domain->mname) {
-                        args->index = ++j;
+                        args->v0.index = ++j;
                         break;
                 }
         }
@@ -114,31 +141,65 @@ nouveau_control_mthd_pstate_attr(struct nouveau_object *object, u32 mthd,
 }
 
 static int
-nouveau_control_mthd_pstate_user(struct nouveau_object *object, u32 mthd,
-                                void *data, u32 size)
+nouveau_control_mthd_pstate_user(struct nouveau_object *object,
+                                 void *data, u32 size)
 {
+        union {
+                struct nvif_control_pstate_user_v0 v0;
+        } *args = data;
         struct nouveau_clock *clk = nouveau_clock(object);
-        struct nv_control_pstate_user *args = data;
+        int ret;
+
+        nv_ioctl(object, "control pstate user size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "control pstate user vers %d ustate %d "
+                                 "pwrsrc %d\n", args->v0.version,
+                         args->v0.ustate, args->v0.pwrsrc);
+                if (!clk)
+                        return -ENODEV;
+        } else
+                return ret;
+
+        if (args->v0.pwrsrc >= 0) {
+                ret |= nouveau_clock_ustate(clk, args->v0.ustate, args->v0.pwrsrc);
+        } else {
+                ret |= nouveau_clock_ustate(clk, args->v0.ustate, 0);
+                ret |= nouveau_clock_ustate(clk, args->v0.ustate, 1);
+        }
 
-        if (size < sizeof(*args) || !clk)
-                return -EINVAL;
+        return ret;
+}
 
-        return nouveau_clock_ustate(clk, args->state);
+static int
+nouveau_control_mthd(struct nouveau_object *object, u32 mthd,
+                     void *data, u32 size)
+{
+        switch (mthd) {
+        case NVIF_CONTROL_PSTATE_INFO:
+                return nouveau_control_mthd_pstate_info(object, data, size);
+        case NVIF_CONTROL_PSTATE_ATTR:
+                return nouveau_control_mthd_pstate_attr(object, data, size);
+        case NVIF_CONTROL_PSTATE_USER:
+                return nouveau_control_mthd_pstate_user(object, data, size);
+        default:
+                break;
+        }
+        return -EINVAL;
 }
 
+static struct nouveau_ofuncs
+nouveau_control_ofuncs = {
+        .ctor = _nouveau_object_ctor,
+        .dtor = nouveau_object_destroy,
+        .init = nouveau_object_init,
+        .fini = nouveau_object_fini,
+        .mthd = nouveau_control_mthd,
+};
+
 struct nouveau_oclass
 nouveau_control_oclass[] = {
-        { .handle = NV_CONTROL_CLASS,
-          .ofuncs = &nouveau_object_ofuncs,
-          .omthds = (struct nouveau_omthds[]) {
-                  { NV_CONTROL_PSTATE_INFO,
-                    NV_CONTROL_PSTATE_INFO, nouveau_control_mthd_pstate_info },
-                  { NV_CONTROL_PSTATE_ATTR,
-                    NV_CONTROL_PSTATE_ATTR, nouveau_control_mthd_pstate_attr },
-                  { NV_CONTROL_PSTATE_USER,
-                    NV_CONTROL_PSTATE_USER, nouveau_control_mthd_pstate_user },
-                  {},
-          },
+        { .handle = NVIF_IOCTL_NEW_V0_CONTROL,
+          .ofuncs = &nouveau_control_ofuncs
         },
         {}
 };
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/gm100.c b/drivers/gpu/drm/nouveau/core/engine/device/gm100.c
index a520029e25d9..377ec0b8851e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/gm100.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/gm100.c
@@ -33,7 +33,7 @@
 #include <subdev/mc.h>
 #include <subdev/timer.h>
 #include <subdev/fb.h>
-#include <subdev/ltcg.h>
+#include <subdev/ltc.h>
 #include <subdev/ibus.h>
 #include <subdev/instmem.h>
 #include <subdev/vm.h>
@@ -68,20 +68,20 @@ gm100_identify(struct nouveau_device *device)
 #endif
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] =  gm107_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  gk20a_mc_oclass;
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &gk20a_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  gm107_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gm107_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gm107_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
 #if 0
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nv108_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nv108_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
 #endif
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv108_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  gm107_graph_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv04.c b/drivers/gpu/drm/nouveau/core/engine/device/nv04.c
index 40b29d0214cb..573b55f5c2f9 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv04.c
@@ -56,7 +56,7 @@ nv04_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv04_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv04_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv04_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv04_graph_oclass;
@@ -74,7 +74,7 @@ nv04_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv04_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv04_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv04_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv04_graph_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv10.c b/drivers/gpu/drm/nouveau/core/engine/device/nv10.c
index 5f7c25ff523d..183a85a6204e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv10.c
@@ -58,7 +58,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
@@ -75,7 +75,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
@@ -94,7 +94,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
@@ -113,7 +113,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv1a_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
@@ -132,7 +132,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
@@ -151,7 +151,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
@@ -170,7 +170,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv1a_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
@@ -189,7 +189,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv20.c b/drivers/gpu/drm/nouveau/core/engine/device/nv20.c
index 75fed11bba0a..aa564c68a920 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv20.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv20.c
@@ -59,7 +59,7 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv20_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv20_graph_oclass;
@@ -78,7 +78,7 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv25_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv25_graph_oclass;
@@ -97,7 +97,7 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv25_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv25_graph_oclass;
@@ -116,7 +116,7 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv25_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv2a_graph_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv30.c b/drivers/gpu/drm/nouveau/core/engine/device/nv30.c
index 36919d7db7cc..11bd31da82ab 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv30.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv30.c
@@ -59,7 +59,7 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv30_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv30_graph_oclass;
@@ -78,7 +78,7 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv35_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv35_graph_oclass;
@@ -97,7 +97,7 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv30_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv30_graph_oclass;
@@ -117,7 +117,7 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv36_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv35_graph_oclass;
@@ -137,7 +137,7 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv34_graph_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv40.c b/drivers/gpu/drm/nouveau/core/engine/device/nv40.c
index 1130a62be2c7..e96c223cb797 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv40.c
@@ -65,7 +65,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -88,7 +88,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -111,7 +111,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -134,7 +134,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -157,7 +157,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -180,7 +180,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -203,7 +203,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -226,7 +226,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -249,7 +249,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -272,7 +272,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -295,7 +295,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -318,7 +318,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -341,7 +341,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -364,7 +364,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -387,7 +387,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
@@ -410,7 +410,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv50.c b/drivers/gpu/drm/nouveau/core/engine/device/nv50.c
index ef0b0bde1a91..932f84fae459 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv50.c
@@ -74,7 +74,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv50_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -99,7 +99,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -127,7 +127,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -155,7 +155,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -183,7 +183,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -211,7 +211,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -239,7 +239,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -267,7 +267,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -295,7 +295,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -323,7 +323,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -350,9 +350,9 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nva3_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nva3_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -380,9 +380,9 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nva3_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nva3_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -409,9 +409,9 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nva3_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nva3_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
@@ -438,9 +438,9 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nva3_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nva3_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nv50_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c
index 8d55ed633b19..b4a2917ce555 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c
@@ -33,7 +33,7 @@
 #include <subdev/mc.h>
 #include <subdev/timer.h>
 #include <subdev/fb.h>
-#include <subdev/ltcg.h>
+#include <subdev/ltc.h>
 #include <subdev/ibus.h>
 #include <subdev/instmem.h>
 #include <subdev/vm.h>
@@ -70,14 +70,14 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gf100_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nvc0_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvc0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc0_graph_oclass;
@@ -102,14 +102,14 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gf100_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nvc0_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvc0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc4_graph_oclass;
@@ -134,14 +134,14 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gf100_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nvc0_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvc0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc4_graph_oclass;
@@ -165,14 +165,14 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gf100_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nvc0_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvc0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc4_graph_oclass;
@@ -197,14 +197,14 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gf100_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nvc0_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvc0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc4_graph_oclass;
@@ -229,14 +229,14 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gf100_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nvc0_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvc0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc1_graph_oclass;
@@ -260,14 +260,14 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gf100_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nvc0_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvc0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc8_graph_oclass;
@@ -292,14 +292,14 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gf100_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvd0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nvd0_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvd9_graph_oclass;
@@ -323,12 +323,12 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gf100_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvd7_graph_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nve0.c b/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
index 2d1e97d4264f..54ec53bc6252 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
@@ -33,7 +33,7 @@
 #include <subdev/mc.h>
 #include <subdev/timer.h>
 #include <subdev/fb.h>
-#include <subdev/ltcg.h>
+#include <subdev/ltc.h>
 #include <subdev/ibus.h>
 #include <subdev/instmem.h>
 #include <subdev/vm.h>
@@ -70,14 +70,14 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gk104_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvd0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  gk104_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nve4_graph_oclass;
@@ -103,14 +103,14 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gk104_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvd0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nvd0_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nve4_graph_oclass;
@@ -136,14 +136,14 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gk104_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvd0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  gk104_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nve4_graph_oclass;
@@ -158,15 +158,16 @@ nve0_identify(struct nouveau_device *device)
                 break;
         case 0xea:
                 device->cname = "GK20A";
-                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &gk20a_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  gk20a_mc_oclass;
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &gk20a_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  gk20a_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &gk20a_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
-                device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+                device->oclass[NVDEV_SUBDEV_BAR    ] = &gk20a_bar_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  gk20a_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  gk20a_graph_oclass;
@@ -186,14 +187,14 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gk104_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvd0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nvd0_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvf0_graph_oclass;
@@ -219,17 +220,17 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gk104_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvd0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nvd0_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
-                device->oclass[NVDEV_ENGINE_GR     ] =  nvf0_graph_oclass;
+                device->oclass[NVDEV_ENGINE_GR     ] =  gk110b_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] =  nvf0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nve0_copy1_oclass;
@@ -248,18 +249,18 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nvd0_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] =  nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  gk20a_mc_oclass;
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gk104_ltc_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
-                device->oclass[NVDEV_SUBDEV_PWR    ] = &nv108_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nv108_pwr_oclass;
                 device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
-                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv108_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nv108_graph_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/base.c b/drivers/gpu/drm/nouveau/core/engine/disp/base.c
index 9c38c5e40500..22d55f6cde50 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/base.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/base.c
@@ -22,23 +22,93 @@
  * Authors: Ben Skeggs
  */
 
+#include <core/os.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
+#include <nvif/event.h>
+
 #include "priv.h"
 #include "outp.h"
 #include "conn.h"
 
+int
+nouveau_disp_vblank_ctor(void *data, u32 size, struct nvkm_notify *notify)
+{
+        struct nouveau_disp *disp =
+                container_of(notify->event, typeof(*disp), vblank);
+        union {
+                struct nvif_notify_head_req_v0 v0;
+        } *req = data;
+        int ret;
+
+        if (nvif_unpack(req->v0, 0, 0, false)) {
+                notify->size = sizeof(struct nvif_notify_head_rep_v0);
+                if (ret = -ENXIO, req->v0.head <= disp->vblank.index_nr) {
+                        notify->types = 1;
+                        notify->index = req->v0.head;
+                        return 0;
+                }
+        }
+
+        return ret;
+}
+
+void
+nouveau_disp_vblank(struct nouveau_disp *disp, int head)
+{
+        struct nvif_notify_head_rep_v0 rep = {};
+        nvkm_event_send(&disp->vblank, 1, head, &rep, sizeof(rep));
+}
+
 static int
-nouveau_disp_hpd_check(struct nouveau_event *event, u32 types, int index)
+nouveau_disp_hpd_ctor(void *data, u32 size, struct nvkm_notify *notify)
 {
-        struct nouveau_disp *disp = event->priv;
+        struct nouveau_disp *disp =
+                container_of(notify->event, typeof(*disp), hpd);
+        union {
+                struct nvif_notify_conn_req_v0 v0;
+        } *req = data;
         struct nvkm_output *outp;
-        list_for_each_entry(outp, &disp->outp, head) {
-                if (outp->conn->index == index) {
-                        if (outp->conn->hpd.event)
-                                return 0;
-                        break;
+        int ret;
+
+        if (nvif_unpack(req->v0, 0, 0, false)) {
+                notify->size = sizeof(struct nvif_notify_conn_rep_v0);
+                list_for_each_entry(outp, &disp->outp, head) {
+                        if (ret = -ENXIO, outp->conn->index == req->v0.conn) {
+                                if (ret = -ENODEV, outp->conn->hpd.event) {
+                                        notify->types = req->v0.mask;
+                                        notify->index = req->v0.conn;
+                                        ret = 0;
+                                }
+                                break;
+                        }
                 }
         }
-        return -ENOSYS;
+
+        return ret;
+}
+
+static const struct nvkm_event_func
+nouveau_disp_hpd_func = {
+        .ctor = nouveau_disp_hpd_ctor
+};
+
+int
+nouveau_disp_ntfy(struct nouveau_object *object, u32 type,
+                  struct nvkm_event **event)
+{
+        struct nouveau_disp *disp = (void *)object->engine;
+        switch (type) {
+        case NV04_DISP_NTFY_VBLANK:
+                *event = &disp->vblank;
+                return 0;
+        case NV04_DISP_NTFY_CONN:
+                *event = &disp->hpd;
+                return 0;
+        default:
+                break;
+        }
+        return -EINVAL;
 }
 
 int
@@ -97,7 +167,8 @@ _nouveau_disp_dtor(struct nouveau_object *object)
         struct nouveau_disp *disp = (void *)object;
         struct nvkm_output *outp, *outt;
 
-        nouveau_event_destroy(&disp->vblank);
+        nvkm_event_fini(&disp->vblank);
+        nvkm_event_fini(&disp->hpd);
 
         if (disp->outp.next) {
                 list_for_each_entry_safe(outp, outt, &disp->outp, head) {
@@ -157,14 +228,11 @@ nouveau_disp_create_(struct nouveau_object *parent,
                 hpd = max(hpd, (u8)(dcbE.connector + 1));
         }
 
-        ret = nouveau_event_create(3, hpd, &disp->hpd);
+        ret = nvkm_event_init(&nouveau_disp_hpd_func, 3, hpd, &disp->hpd);
         if (ret)
                 return ret;
 
-        disp->hpd->priv = disp;
-        disp->hpd->check = nouveau_disp_hpd_check;
-
-        ret = nouveau_event_create(1, heads, &disp->vblank);
+        ret = nvkm_event_init(impl->vblank, 1, heads, &disp->vblank);
         if (ret)
                 return ret;
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/conn.c b/drivers/gpu/drm/nouveau/core/engine/disp/conn.c
index 4ffbc70ecf5a..3d1070228977 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/conn.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/conn.c
@@ -22,39 +22,41 @@
  * Authors: Ben Skeggs
  */
 
+#include <core/os.h>
+#include <nvif/event.h>
+
 #include <subdev/gpio.h>
 
 #include "conn.h"
 #include "outp.h"
 
-static void
-nvkm_connector_hpd_work(struct work_struct *w)
+static int
+nvkm_connector_hpd(struct nvkm_notify *notify)
 {
-        struct nvkm_connector *conn = container_of(w, typeof(*conn), hpd.work);
+        struct nvkm_connector *conn = container_of(notify, typeof(*conn), hpd);
         struct nouveau_disp *disp = nouveau_disp(conn);
         struct nouveau_gpio *gpio = nouveau_gpio(conn);
-        u32 send = NVKM_HPD_UNPLUG;
-        if (gpio->get(gpio, 0, DCB_GPIO_UNUSED, conn->hpd.event->index))
-                send = NVKM_HPD_PLUG;
-        nouveau_event_trigger(disp->hpd, send, conn->index);
-        nouveau_event_get(conn->hpd.event);
-}
+        const struct nvkm_gpio_ntfy_rep *line = notify->data;
+        struct nvif_notify_conn_rep_v0 rep;
+        int index = conn->index;
 
-static int
-nvkm_connector_hpd(void *data, u32 type, int index)
-{
-        struct nvkm_connector *conn = data;
-        DBG("HPD: %d\n", type);
-        schedule_work(&conn->hpd.work);
-        return NVKM_EVENT_DROP;
+        DBG("HPD: %d\n", line->mask);
+
+        if (!gpio->get(gpio, 0, DCB_GPIO_UNUSED, conn->hpd.index))
+                rep.mask = NVIF_NOTIFY_CONN_V0_UNPLUG;
+        else
+                rep.mask = NVIF_NOTIFY_CONN_V0_PLUG;
+        rep.version = 0;
+
+        nvkm_event_send(&disp->hpd, rep.mask, index, &rep, sizeof(rep));
+        return NVKM_NOTIFY_KEEP;
 }
 
 int
 _nvkm_connector_fini(struct nouveau_object *object, bool suspend)
 {
         struct nvkm_connector *conn = (void *)object;
-        if (conn->hpd.event)
-                nouveau_event_put(conn->hpd.event);
+        nvkm_notify_put(&conn->hpd);
         return nouveau_object_fini(&conn->base, suspend);
 }
 
@@ -63,10 +65,8 @@ _nvkm_connector_init(struct nouveau_object *object)
 {
         struct nvkm_connector *conn = (void *)object;
         int ret = nouveau_object_init(&conn->base);
-        if (ret == 0) {
-                if (conn->hpd.event)
-                        nouveau_event_get(conn->hpd.event);
-        }
+        if (ret == 0)
+                nvkm_notify_get(&conn->hpd);
         return ret;
 }
 
@@ -74,7 +74,7 @@ void
 _nvkm_connector_dtor(struct nouveau_object *object)
 {
         struct nvkm_connector *conn = (void *)object;
-        nouveau_event_ref(NULL, &conn->hpd.event);
+        nvkm_notify_fini(&conn->hpd);
         nouveau_object_destroy(&conn->base);
 }
 
@@ -116,19 +116,24 @@ nvkm_connector_create_(struct nouveau_object *parent,
         if ((info->hpd = ffs(info->hpd))) {
                 if (--info->hpd >= ARRAY_SIZE(hpd)) {
                         ERR("hpd %02x unknown\n", info->hpd);
-                        goto done;
+                        return 0;
                 }
                 info->hpd = hpd[info->hpd];
 
                 ret = gpio->find(gpio, 0, info->hpd, DCB_GPIO_UNUSED, &func);
                 if (ret) {
                         ERR("func %02x lookup failed, %d\n", info->hpd, ret);
-                        goto done;
+                        return 0;
                 }
 
-                ret = nouveau_event_new(gpio->events, NVKM_GPIO_TOGGLED,
-                                        func.line, nvkm_connector_hpd,
-                                        conn, &conn->hpd.event);
+                ret = nvkm_notify_init(&gpio->event, nvkm_connector_hpd, true,
+                                       &(struct nvkm_gpio_ntfy_req) {
+                                        .mask = NVKM_GPIO_TOGGLED,
+                                        .line = func.line,
+                                       },
+                                       sizeof(struct nvkm_gpio_ntfy_req),
+                                       sizeof(struct nvkm_gpio_ntfy_rep),
+                                       &conn->hpd);
                 if (ret) {
                         ERR("func %02x failed, %d\n", info->hpd, ret);
                 } else {
@@ -136,8 +141,6 @@ nvkm_connector_create_(struct nouveau_object *parent,
                 }
         }
 
-done:
-        INIT_WORK(&conn->hpd.work, nvkm_connector_hpd_work);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/conn.h b/drivers/gpu/drm/nouveau/core/engine/disp/conn.h
index 035ebeacbb1c..55e5f5c82c14 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/conn.h
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/conn.h
@@ -10,10 +10,7 @@ struct nvkm_connector {
         struct nvbios_connE info;
         int index;
 
-        struct {
-                struct nouveau_eventh *event;
-                struct work_struct work;
-        } hpd;
+        struct nvkm_notify hpd;
 };
 
 #define nvkm_connector_create(p,e,c,b,i,d)                                     \
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/dacnv50.c b/drivers/gpu/drm/nouveau/core/engine/disp/dacnv50.c
index a66b27c0fcab..b36addff06a9 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/dacnv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/dacnv50.c
@@ -22,8 +22,9 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <subdev/bios.h>
 #include <subdev/bios/dcb.h>
@@ -32,13 +33,28 @@
 #include "nv50.h"
 
 int
-nv50_dac_power(struct nv50_disp_priv *priv, int or, u32 data)
+nv50_dac_power(NV50_DISP_MTHD_V1)
 {
-        const u32 stat = (data & NV50_DISP_DAC_PWR_HSYNC) |
-                         (data & NV50_DISP_DAC_PWR_VSYNC) |
-                         (data & NV50_DISP_DAC_PWR_DATA) |
-                         (data & NV50_DISP_DAC_PWR_STATE);
-        const u32 doff = (or * 0x800);
+        const u32 doff = outp->or * 0x800;
+        union {
+                struct nv50_disp_dac_pwr_v0 v0;
+        } *args = data;
+        u32 stat;
+        int ret;
+
+        nv_ioctl(object, "disp dac pwr size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "disp dac pwr vers %d state %d data %d "
+                                 "vsync %d hsync %d\n",
+                         args->v0.version, args->v0.state, args->v0.data,
+                         args->v0.vsync, args->v0.hsync);
+                stat  = 0x00000040 * !args->v0.state;
+                stat |= 0x00000010 * !args->v0.data;
+                stat |= 0x00000004 * !args->v0.vsync;
+                stat |= 0x00000001 * !args->v0.hsync;
+        } else
+                return ret;
+
         nv_wait(priv, 0x61a004 + doff, 0x80000000, 0x00000000);
         nv_mask(priv, 0x61a004 + doff, 0xc000007f, 0x80000000 | stat);
         nv_wait(priv, 0x61a004 + doff, 0x80000000, 0x00000000);
@@ -46,9 +62,24 @@ nv50_dac_power(struct nv50_disp_priv *priv, int or, u32 data)
 }
 
 int
-nv50_dac_sense(struct nv50_disp_priv *priv, int or, u32 loadval)
+nv50_dac_sense(NV50_DISP_MTHD_V1)
 {
-        const u32 doff = (or * 0x800);
+        union {
+                struct nv50_disp_dac_load_v0 v0;
+        } *args = data;
+        const u32 doff = outp->or * 0x800;
+        u32 loadval;
+        int ret;
+
+        nv_ioctl(object, "disp dac load size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "disp dac load vers %d data %08x\n",
+                         args->v0.version, args->v0.data);
+                if (args->v0.data & 0xfff00000)
+                        return -EINVAL;
+                loadval = args->v0.data;
+        } else
+                return ret;
 
         nv_mask(priv, 0x61a004 + doff, 0x807f0000, 0x80150000);
         nv_wait(priv, 0x61a004 + doff, 0x80000000, 0x00000000);
@@ -61,38 +92,10 @@ nv50_dac_sense(struct nv50_disp_priv *priv, int or, u32 loadval)
         nv_mask(priv, 0x61a004 + doff, 0x807f0000, 0x80550000);
         nv_wait(priv, 0x61a004 + doff, 0x80000000, 0x00000000);
 
-        nv_debug(priv, "DAC%d sense: 0x%08x\n", or, loadval);
+        nv_debug(priv, "DAC%d sense: 0x%08x\n", outp->or, loadval);
         if (!(loadval & 0x80000000))
                 return -ETIMEDOUT;
 
-        return (loadval & 0x38000000) >> 27;
-}
-
-int
-nv50_dac_mthd(struct nouveau_object *object, u32 mthd, void *args, u32 size)
-{
-        struct nv50_disp_priv *priv = (void *)object->engine;
-        const u8 or = (mthd & NV50_DISP_DAC_MTHD_OR);
-        u32 *data = args;
-        int ret;
-
-        if (size < sizeof(u32))
-                return -EINVAL;
-
-        switch (mthd & ~0x3f) {
-        case NV50_DISP_DAC_PWR:
-                ret = priv->dac.power(priv, or, data[0]);
-                break;
-        case NV50_DISP_DAC_LOAD:
-                ret = priv->dac.sense(priv, or, data[0]);
-                if (ret >= 0) {
-                        data[0] = ret;
-                        ret = 0;
-                }
-                break;
-        default:
-                BUG_ON(1);
-        }
-
-        return ret;
+        args->v0.load = (loadval & 0x38000000) >> 27;
+        return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/dport.c b/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
index 5a5b59b21130..39890221b91c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
@@ -30,7 +30,7 @@
 
 #include <engine/disp.h>
 
-#include <core/class.h>
+#include <nvif/class.h>
 
 #include "dport.h"
 #include "outpdp.h"
@@ -335,7 +335,7 @@ nouveau_dp_train(struct work_struct *w)
         int ret;
 
         /* bring capabilities within encoder limits */
-        if (nv_mclass(disp) < NVD0_DISP_CLASS)
+        if (nv_mclass(disp) < GF110_DISP)
                 outp->dpcd[2] &= ~DPCD_RC02_TPS3_SUPPORTED;
         if ((outp->dpcd[2] & 0x1f) > outp->base.info.dpconf.link_nr) {
                 outp->dpcd[2] &= ~DPCD_RC02_MAX_LANE_COUNT;
@@ -354,7 +354,7 @@ nouveau_dp_train(struct work_struct *w)
         cfg--;
 
         /* disable link interrupt handling during link training */
-        nouveau_event_put(outp->irq);
+        nvkm_notify_put(&outp->irq);
 
         /* enable down-spreading and execute pre-train script from vbios */
         dp_link_train_init(dp, outp->dpcd[3] & 0x01);
@@ -395,5 +395,5 @@ nouveau_dp_train(struct work_struct *w)
         DBG("training complete\n");
         atomic_set(&outp->lt.done, 1);
         wake_up(&outp->lt.wait);
-        nouveau_event_get(outp->irq);
+        nvkm_notify_get(&outp->irq);
 }
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c b/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c
index 9fc7447fec90..d54da8b5f87e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c
@@ -25,7 +25,7 @@
 #include <engine/software.h>
 #include <engine/disp.h>
 
-#include <core/class.h>
+#include <nvif/class.h>
 
 #include "nv50.h"
 
@@ -35,17 +35,17 @@
 
 static struct nouveau_oclass
 gm107_disp_sclass[] = {
-        { GM107_DISP_MAST_CLASS, &nvd0_disp_mast_ofuncs },
-        { GM107_DISP_SYNC_CLASS, &nvd0_disp_sync_ofuncs },
-        { GM107_DISP_OVLY_CLASS, &nvd0_disp_ovly_ofuncs },
-        { GM107_DISP_OIMM_CLASS, &nvd0_disp_oimm_ofuncs },
-        { GM107_DISP_CURS_CLASS, &nvd0_disp_curs_ofuncs },
+        { GM107_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
+        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GK104_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
+        { GK104_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
+        { GK104_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
         {}
 };
 
 static struct nouveau_oclass
 gm107_disp_base_oclass[] = {
-        { GM107_DISP_CLASS, &nvd0_disp_base_ofuncs, nvd0_disp_base_omthds },
+        { GM107_DISP, &nvd0_disp_base_ofuncs },
         {}
 };
 
@@ -93,9 +93,11 @@ gm107_disp_oclass = &(struct nv50_disp_impl) {
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
+        .base.vblank = &nvd0_disp_vblank_func,
         .base.outp =  nvd0_disp_outp_sclass,
         .mthd.core = &nve0_disp_mast_mthd_chan,
         .mthd.base = &nvd0_disp_sync_mthd_chan,
         .mthd.ovly = &nve0_disp_ovly_mthd_chan,
         .mthd.prev = -0x020000,
+        .head.scanoutpos = nvd0_disp_base_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/hdanva3.c b/drivers/gpu/drm/nouveau/core/engine/disp/hdanva3.c
index a19e7d79b847..8b4e06abe533 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/hdanva3.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/hdanva3.c
@@ -22,25 +22,37 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include "nv50.h"
 
 int
-nva3_hda_eld(struct nv50_disp_priv *priv, int or, u8 *data, u32 size)
+nva3_hda_eld(NV50_DISP_MTHD_V1)
 {
-        const u32 soff = (or * 0x800);
-        int i;
+        union {
+                struct nv50_disp_sor_hda_eld_v0 v0;
+        } *args = data;
+        const u32 soff = outp->or * 0x800;
+        int ret, i;
 
-        if (data && data[0]) {
+        nv_ioctl(object, "disp sor hda eld size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                nv_ioctl(object, "disp sor hda eld vers %d\n", args->v0.version);
+                if (size > 0x60)
+                        return -E2BIG;
+        } else
+                return ret;
+
+        if (size && args->v0.data[0]) {
                 for (i = 0; i < size; i++)
-                        nv_wr32(priv, 0x61c440 + soff, (i << 8) | data[i]);
+                        nv_wr32(priv, 0x61c440 + soff, (i << 8) | args->v0.data[0]);
                 for (; i < 0x60; i++)
                         nv_wr32(priv, 0x61c440 + soff, (i << 8));
                 nv_mask(priv, 0x61c448 + soff, 0x80000003, 0x80000003);
         } else
-        if (data) {
+        if (size) {
                 nv_mask(priv, 0x61c448 + soff, 0x80000003, 0x80000001);
         } else {
                 nv_mask(priv, 0x61c448 + soff, 0x80000003, 0x80000000);
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/hdanvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/hdanvd0.c
index 717639386ced..baf558fc12fb 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/hdanvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/hdanvd0.c
@@ -22,8 +22,9 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <subdev/bios.h>
 #include <subdev/bios/dcb.h>
@@ -33,19 +34,30 @@
 #include "nv50.h"
 
 int
-nvd0_hda_eld(struct nv50_disp_priv *priv, int or, u8 *data, u32 size)
+nvd0_hda_eld(NV50_DISP_MTHD_V1)
 {
-        const u32 soff = (or * 0x030);
-        int i;
+        union {
+                struct nv50_disp_sor_hda_eld_v0 v0;
+        } *args = data;
+        const u32 soff = outp->or * 0x030;
+        int ret, i;
 
-        if (data && data[0]) {
+        nv_ioctl(object, "disp sor hda eld size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                nv_ioctl(object, "disp sor hda eld vers %d\n", args->v0.version);
+                if (size > 0x60)
+                        return -E2BIG;
+        } else
+                return ret;
+
+        if (size && args->v0.data[0]) {
                 for (i = 0; i < size; i++)
-                        nv_wr32(priv, 0x10ec00 + soff, (i << 8) | data[i]);
+                        nv_wr32(priv, 0x10ec00 + soff, (i << 8) | args->v0.data[i]);
                 for (; i < 0x60; i++)
                         nv_wr32(priv, 0x10ec00 + soff, (i << 8));
                 nv_mask(priv, 0x10ec10 + soff, 0x80000003, 0x80000003);
         } else
-        if (data) {
+        if (size) {
                 nv_mask(priv, 0x10ec10 + soff, 0x80000003, 0x80000001);
         } else {
                 nv_mask(priv, 0x10ec10 + soff, 0x80000003, 0x80000000);
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/hdminv84.c b/drivers/gpu/drm/nouveau/core/engine/disp/hdminv84.c
index 7fdade6e604d..fa276dede9cd 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/hdminv84.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/hdminv84.c
@@ -22,17 +22,38 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include "nv50.h"
 
 int
-nv84_hdmi_ctrl(struct nv50_disp_priv *priv, int head, int or, u32 data)
+nv84_hdmi_ctrl(NV50_DISP_MTHD_V1)
 {
         const u32 hoff = (head * 0x800);
+        union {
+                struct nv50_disp_sor_hdmi_pwr_v0 v0;
+        } *args = data;
+        u32 ctrl;
+        int ret;
 
-        if (!(data & NV84_DISP_SOR_HDMI_PWR_STATE_ON)) {
+        nv_ioctl(object, "disp sor hdmi ctrl size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "disp sor hdmi ctrl vers %d state %d "
+                                 "max_ac_packet %d rekey %d\n",
+                         args->v0.version, args->v0.state,
+                         args->v0.max_ac_packet, args->v0.rekey);
+                if (args->v0.max_ac_packet > 0x1f || args->v0.rekey > 0x7f)
+                        return -EINVAL;
+                ctrl  = 0x40000000 * !!args->v0.state;
+                ctrl |= args->v0.max_ac_packet << 16;
+                ctrl |= args->v0.rekey;
+                ctrl |= 0x1f000000; /* ??? */
+        } else
+                return ret;
+
+        if (!(ctrl & 0x40000000)) {
                 nv_mask(priv, 0x6165a4 + hoff, 0x40000000, 0x00000000);
                 nv_mask(priv, 0x616520 + hoff, 0x00000001, 0x00000000);
                 nv_mask(priv, 0x616500 + hoff, 0x00000001, 0x00000000);
@@ -65,6 +86,6 @@ nv84_hdmi_ctrl(struct nv50_disp_priv *priv, int head, int or, u32 data)
         nv_mask(priv, 0x61733c, 0x00100000, 0x00000000); /* !RESETF */
 
         /* HDMI_CTRL */
-        nv_mask(priv, 0x6165a4 + hoff, 0x5f1f007f, data | 0x1f000000 /* ??? */);
+        nv_mask(priv, 0x6165a4 + hoff, 0x5f1f007f, ctrl);
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/hdminva3.c b/drivers/gpu/drm/nouveau/core/engine/disp/hdminva3.c
index db8c6fd46278..57eeed1d1942 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/hdminva3.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/hdminva3.c
@@ -22,17 +22,38 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include "nv50.h"
 
 int
-nva3_hdmi_ctrl(struct nv50_disp_priv *priv, int head, int or, u32 data)
+nva3_hdmi_ctrl(NV50_DISP_MTHD_V1)
 {
-        const u32 soff = (or * 0x800);
+        const u32 soff = outp->or * 0x800;
+        union {
+                struct nv50_disp_sor_hdmi_pwr_v0 v0;
+        } *args = data;
+        u32 ctrl;
+        int ret;
 
-        if (!(data & NV84_DISP_SOR_HDMI_PWR_STATE_ON)) {
+        nv_ioctl(object, "disp sor hdmi ctrl size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "disp sor hdmi ctrl vers %d state %d "
+                                 "max_ac_packet %d rekey %d\n",
+                         args->v0.version, args->v0.state,
+                         args->v0.max_ac_packet, args->v0.rekey);
+                if (args->v0.max_ac_packet > 0x1f || args->v0.rekey > 0x7f)
+                        return -EINVAL;
+                ctrl  = 0x40000000 * !!args->v0.state;
+                ctrl |= args->v0.max_ac_packet << 16;
+                ctrl |= args->v0.rekey;
+                ctrl |= 0x1f000000; /* ??? */
+        } else
+                return ret;
+
+        if (!(ctrl & 0x40000000)) {
                 nv_mask(priv, 0x61c5a4 + soff, 0x40000000, 0x00000000);
                 nv_mask(priv, 0x61c520 + soff, 0x00000001, 0x00000000);
                 nv_mask(priv, 0x61c500 + soff, 0x00000001, 0x00000000);
@@ -65,6 +86,6 @@ nva3_hdmi_ctrl(struct nv50_disp_priv *priv, int head, int or, u32 data)
         nv_mask(priv, 0x61733c, 0x00100000, 0x00000000); /* !RESETF */
 
         /* HDMI_CTRL */
-        nv_mask(priv, 0x61c5a4 + soff, 0x5f1f007f, data | 0x1f000000 /* ??? */);
+        nv_mask(priv, 0x61c5a4 + soff, 0x5f1f007f, ctrl);
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/hdminvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/hdminvd0.c
index 5151bb261832..3106d295b48d 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/hdminvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/hdminvd0.c
@@ -22,17 +22,37 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include "nv50.h"
 
 int
-nvd0_hdmi_ctrl(struct nv50_disp_priv *priv, int head, int or, u32 data)
+nvd0_hdmi_ctrl(NV50_DISP_MTHD_V1)
 {
         const u32 hoff = (head * 0x800);
+        union {
+                struct nv50_disp_sor_hdmi_pwr_v0 v0;
+        } *args = data;
+        u32 ctrl;
+        int ret;
 
-        if (!(data & NV84_DISP_SOR_HDMI_PWR_STATE_ON)) {
+        nv_ioctl(object, "disp sor hdmi ctrl size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "disp sor hdmi ctrl vers %d state %d "
+                                 "max_ac_packet %d rekey %d\n",
+                         args->v0.version, args->v0.state,
+                         args->v0.max_ac_packet, args->v0.rekey);
+                if (args->v0.max_ac_packet > 0x1f || args->v0.rekey > 0x7f)
+                        return -EINVAL;
+                ctrl  = 0x40000000 * !!args->v0.state;
+                ctrl |= args->v0.max_ac_packet << 16;
+                ctrl |= args->v0.rekey;
+        } else
+                return ret;
+
+        if (!(ctrl & 0x40000000)) {
                 nv_mask(priv, 0x616798 + hoff, 0x40000000, 0x00000000);
                 nv_mask(priv, 0x6167a4 + hoff, 0x00000001, 0x00000000);
                 nv_mask(priv, 0x616714 + hoff, 0x00000001, 0x00000000);
@@ -54,7 +74,7 @@ nvd0_hdmi_ctrl(struct nv50_disp_priv *priv, int head, int or, u32 data)
         nv_mask(priv, 0x6167a4 + hoff, 0x00000001, 0x00000001);
 
         /* HDMI_CTRL */
-        nv_mask(priv, 0x616798 + hoff, 0x401f007f, data);
+        nv_mask(priv, 0x616798 + hoff, 0x401f007f, ctrl);
 
         /* NFI, audio doesn't work without it though.. */
         nv_mask(priv, 0x616548 + hoff, 0x00000070, 0x00000000);
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c
index a32666ed0c47..366f315fc9a5 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c
@@ -24,60 +24,100 @@
 
 #include "priv.h"
 
+#include <core/client.h>
 #include <core/event.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 struct nv04_disp_priv {
         struct nouveau_disp base;
 };
 
 static int
-nv04_disp_scanoutpos(struct nouveau_object *object, u32 mthd,
-                     void *data, u32 size)
+nv04_disp_scanoutpos(struct nouveau_object *object, struct nv04_disp_priv *priv,
+                     void *data, u32 size, int head)
 {
-        struct nv04_disp_priv *priv = (void *)object->engine;
-        struct nv04_display_scanoutpos *args = data;
-        const int head = (mthd & NV04_DISP_MTHD_HEAD);
+        const u32 hoff = head * 0x2000;
+        union {
+                struct nv04_disp_scanoutpos_v0 v0;
+        } *args = data;
         u32 line;
+        int ret;
+
+        nv_ioctl(object, "disp scanoutpos size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "disp scanoutpos vers %d\n", args->v0.version);
+                args->v0.vblanks = nv_rd32(priv, 0x680800 + hoff) & 0xffff;
+                args->v0.vtotal  = nv_rd32(priv, 0x680804 + hoff) & 0xffff;
+                args->v0.vblanke = args->v0.vtotal - 1;
+
+                args->v0.hblanks = nv_rd32(priv, 0x680820 + hoff) & 0xffff;
+                args->v0.htotal  = nv_rd32(priv, 0x680824 + hoff) & 0xffff;
+                args->v0.hblanke = args->v0.htotal - 1;
+
+                /*
+                 * If output is vga instead of digital then vtotal/htotal is
+                 * invalid so we have to give up and trigger the timestamping
+                 * fallback in the drm core.
+                 */
+                if (!args->v0.vtotal || !args->v0.htotal)
+                        return -ENOTSUPP;
+
+                args->v0.time[0] = ktime_to_ns(ktime_get());
+                line = nv_rd32(priv, 0x600868 + hoff);
+                args->v0.time[1] = ktime_to_ns(ktime_get());
+                args->v0.hline = (line & 0xffff0000) >> 16;
+                args->v0.vline = (line & 0x0000ffff);
+        } else
+                return ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
-
-        args->vblanks = nv_rd32(priv, 0x680800 + (head * 0x2000)) & 0xffff;
-        args->vtotal  = nv_rd32(priv, 0x680804 + (head * 0x2000)) & 0xffff;
-        args->vblanke = args->vtotal - 1;
-
-        args->hblanks = nv_rd32(priv, 0x680820 + (head * 0x2000)) & 0xffff;
-        args->htotal  = nv_rd32(priv, 0x680824 + (head * 0x2000)) & 0xffff;
-        args->hblanke = args->htotal - 1;
-
-        /*
-         * If output is vga instead of digital then vtotal/htotal is invalid
-         * so we have to give up and trigger the timestamping fallback in the
-         * drm core.
-         */
-        if (!args->vtotal || !args->htotal)
-                return -ENOTSUPP;
-
-        args->time[0] = ktime_to_ns(ktime_get());
-        line = nv_rd32(priv, 0x600868 + (head * 0x2000));
-        args->time[1] = ktime_to_ns(ktime_get());
-        args->hline = (line & 0xffff0000) >> 16;
-        args->vline = (line & 0x0000ffff);
         return 0;
 }
 
-#define HEAD_MTHD(n) (n), (n) + 0x01
+static int
+nv04_disp_mthd(struct nouveau_object *object, u32 mthd, void *data, u32 size)
+{
+        union {
+                struct nv04_disp_mthd_v0 v0;
+        } *args = data;
+        struct nv04_disp_priv *priv = (void *)object->engine;
+        int head, ret;
+
+        nv_ioctl(object, "disp mthd size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                nv_ioctl(object, "disp mthd vers %d mthd %02x head %d\n",
+                         args->v0.version, args->v0.method, args->v0.head);
+                mthd = args->v0.method;
+                head = args->v0.head;
+        } else
+                return ret;
 
-static struct nouveau_omthds
-nv04_disp_omthds[] = {
-        { HEAD_MTHD(NV04_DISP_SCANOUTPOS), nv04_disp_scanoutpos },
-        {}
+        if (head < 0 || head >= 2)
+                return -ENXIO;
+
+        switch (mthd) {
+        case NV04_DISP_SCANOUTPOS:
+                return nv04_disp_scanoutpos(object, priv, data, size, head);
+        default:
+                break;
+        }
+
+        return -EINVAL;
+}
+
+static struct nouveau_ofuncs
+nv04_disp_ofuncs = {
+        .ctor = _nouveau_object_ctor,
+        .dtor = nouveau_object_destroy,
+        .init = nouveau_object_init,
+        .fini = nouveau_object_fini,
+        .mthd = nv04_disp_mthd,
+        .ntfy = nouveau_disp_ntfy,
 };
 
 static struct nouveau_oclass
 nv04_disp_sclass[] = {
-        { NV04_DISP_CLASS, &nouveau_object_ofuncs, nv04_disp_omthds },
+        { NV04_DISP, &nv04_disp_ofuncs },
         {},
 };
 
@@ -86,17 +126,26 @@ nv04_disp_sclass[] = {
  ******************************************************************************/
 
 static void
-nv04_disp_vblank_enable(struct nouveau_event *event, int type, int head)
+nv04_disp_vblank_init(struct nvkm_event *event, int type, int head)
 {
-        nv_wr32(event->priv, 0x600140 + (head * 0x2000) , 0x00000001);
+        struct nouveau_disp *disp = container_of(event, typeof(*disp), vblank);
+        nv_wr32(disp, 0x600140 + (head * 0x2000) , 0x00000001);
 }
 
 static void
-nv04_disp_vblank_disable(struct nouveau_event *event, int type, int head)
+nv04_disp_vblank_fini(struct nvkm_event *event, int type, int head)
 {
-        nv_wr32(event->priv, 0x600140 + (head * 0x2000) , 0x00000000);
+        struct nouveau_disp *disp = container_of(event, typeof(*disp), vblank);
+        nv_wr32(disp, 0x600140 + (head * 0x2000) , 0x00000000);
 }
 
+static const struct nvkm_event_func
+nv04_disp_vblank_func = {
+        .ctor = nouveau_disp_vblank_ctor,
+        .init = nv04_disp_vblank_init,
+        .fini = nv04_disp_vblank_fini,
+};
+
 static void
 nv04_disp_intr(struct nouveau_subdev *subdev)
 {
@@ -106,12 +155,12 @@ nv04_disp_intr(struct nouveau_subdev *subdev)
         u32 pvideo;
 
         if (crtc0 & 0x00000001) {
-                nouveau_event_trigger(priv->base.vblank, 1, 0);
+                nouveau_disp_vblank(&priv->base, 0);
                 nv_wr32(priv, 0x600100, 0x00000001);
         }
 
         if (crtc1 & 0x00000001) {
-                nouveau_event_trigger(priv->base.vblank, 1, 1);
+                nouveau_disp_vblank(&priv->base, 1);
                 nv_wr32(priv, 0x602100, 0x00000001);
         }
 
@@ -140,9 +189,6 @@ nv04_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         nv_engine(priv)->sclass = nv04_disp_sclass;
         nv_subdev(priv)->intr = nv04_disp_intr;
-        priv->base.vblank->priv = priv;
-        priv->base.vblank->enable = nv04_disp_vblank_enable;
-        priv->base.vblank->disable = nv04_disp_vblank_disable;
         return 0;
 }
 
@@ -155,4 +201,5 @@ nv04_disp_oclass = &(struct nouveau_disp_impl) {
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
+        .vblank = &nv04_disp_vblank_func,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
index 2283c442a10d..4b5bb5d58a54 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
@@ -23,10 +23,12 @@
  */
 
 #include <core/object.h>
+#include <core/client.h>
 #include <core/parent.h>
 #include <core/handle.h>
-#include <core/class.h>
 #include <core/enum.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <subdev/bios.h>
 #include <subdev/bios/dcb.h>
@@ -43,14 +45,16 @@
  * EVO channel base class
  ******************************************************************************/
 
-int
+static int
 nv50_disp_chan_create_(struct nouveau_object *parent,
                        struct nouveau_object *engine,
-                       struct nouveau_oclass *oclass, int chid,
+                       struct nouveau_oclass *oclass, int head,
                        int length, void **pobject)
 {
+        const struct nv50_disp_chan_impl *impl = (void *)oclass->ofuncs;
         struct nv50_disp_base *base = (void *)parent;
         struct nv50_disp_chan *chan;
+        int chid = impl->chid + head;
         int ret;
 
         if (base->chan & (1 << chid))
@@ -63,12 +67,14 @@ nv50_disp_chan_create_(struct nouveau_object *parent,
         chan = *pobject;
         if (ret)
                 return ret;
-
         chan->chid = chid;
+
+        nv_parent(chan)->object_attach = impl->attach;
+        nv_parent(chan)->object_detach = impl->detach;
         return 0;
 }
 
-void
+static void
 nv50_disp_chan_destroy(struct nv50_disp_chan *chan)
 {
         struct nv50_disp_base *base = (void *)nv_object(chan)->parent;
@@ -76,6 +82,16 @@ nv50_disp_chan_destroy(struct nv50_disp_chan *chan)
         nouveau_namedb_destroy(&chan->base);
 }
 
+int
+nv50_disp_chan_map(struct nouveau_object *object, u64 *addr, u32 *size)
+{
+        struct nv50_disp_chan *chan = (void *)object;
+        *addr = nv_device_resource_start(nv_device(object), 0) +
+                0x640000 + (chan->chid * 0x1000);
+        *size = 0x001000;
+        return 0;
+}
+
 u32
 nv50_disp_chan_rd32(struct nouveau_object *object, u64 addr)
 {
@@ -115,16 +131,16 @@ nv50_disp_dmac_object_detach(struct nouveau_object *parent, int cookie)
         nouveau_ramht_remove(base->ramht, cookie);
 }
 
-int
+static int
 nv50_disp_dmac_create_(struct nouveau_object *parent,
                        struct nouveau_object *engine,
-                       struct nouveau_oclass *oclass, u32 pushbuf, int chid,
+                       struct nouveau_oclass *oclass, u32 pushbuf, int head,
                        int length, void **pobject)
 {
         struct nv50_disp_dmac *dmac;
         int ret;
 
-        ret = nv50_disp_chan_create_(parent, engine, oclass, chid,
+        ret = nv50_disp_chan_create_(parent, engine, oclass, head,
                                      length, pobject);
         dmac = *pobject;
         if (ret)
@@ -397,27 +413,32 @@ nv50_disp_mast_mthd_chan = {
         }
 };
 
-static int
+int
 nv50_disp_mast_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
-        struct nv50_display_mast_class *args = data;
+        union {
+                struct nv50_disp_core_channel_dma_v0 v0;
+        } *args = data;
         struct nv50_disp_dmac *mast;
         int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create disp core channel dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create disp core channel dma vers %d "
+                                 "pushbuf %08x\n",
+                         args->v0.version, args->v0.pushbuf);
+        } else
+                return ret;
 
-        ret = nv50_disp_dmac_create_(parent, engine, oclass, args->pushbuf,
+        ret = nv50_disp_dmac_create_(parent, engine, oclass, args->v0.pushbuf,
                                      0, sizeof(*mast), (void **)&mast);
         *pobject = nv_object(mast);
         if (ret)
                 return ret;
 
-        nv_parent(mast)->object_attach = nv50_disp_dmac_object_attach;
-        nv_parent(mast)->object_detach = nv50_disp_dmac_object_detach;
         return 0;
 }
 
@@ -479,14 +500,18 @@ nv50_disp_mast_fini(struct nouveau_object *object, bool suspend)
         return nv50_disp_chan_fini(&mast->base, suspend);
 }
 
-struct nouveau_ofuncs
+struct nv50_disp_chan_impl
 nv50_disp_mast_ofuncs = {
-        .ctor = nv50_disp_mast_ctor,
-        .dtor = nv50_disp_dmac_dtor,
-        .init = nv50_disp_mast_init,
-        .fini = nv50_disp_mast_fini,
-        .rd32 = nv50_disp_chan_rd32,
-        .wr32 = nv50_disp_chan_wr32,
+        .base.ctor = nv50_disp_mast_ctor,
+        .base.dtor = nv50_disp_dmac_dtor,
+        .base.init = nv50_disp_mast_init,
+        .base.fini = nv50_disp_mast_fini,
+        .base.map  = nv50_disp_chan_map,
+        .base.rd32 = nv50_disp_chan_rd32,
+        .base.wr32 = nv50_disp_chan_wr32,
+        .chid = 0,
+        .attach = nv50_disp_dmac_object_attach,
+        .detach = nv50_disp_dmac_object_detach,
 };
 
 /*******************************************************************************
@@ -543,39 +568,51 @@ nv50_disp_sync_mthd_chan = {
         }
 };
 
-static int
+int
 nv50_disp_sync_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
-        struct nv50_display_sync_class *args = data;
+        union {
+                struct nv50_disp_base_channel_dma_v0 v0;
+        } *args = data;
+        struct nv50_disp_priv *priv = (void *)engine;
         struct nv50_disp_dmac *dmac;
         int ret;
 
-        if (size < sizeof(*args) || args->head > 1)
-                return -EINVAL;
+        nv_ioctl(parent, "create disp base channel dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create disp base channel dma vers %d "
+                                 "pushbuf %08x head %d\n",
+                         args->v0.version, args->v0.pushbuf, args->v0.head);
+                if (args->v0.head > priv->head.nr)
+                        return -EINVAL;
+        } else
+                return ret;
 
-        ret = nv50_disp_dmac_create_(parent, engine, oclass, args->pushbuf,
-                                     1 + args->head, sizeof(*dmac),
+        ret = nv50_disp_dmac_create_(parent, engine, oclass, args->v0.pushbuf,
+                                     args->v0.head, sizeof(*dmac),
                                      (void **)&dmac);
         *pobject = nv_object(dmac);
         if (ret)
                 return ret;
 
-        nv_parent(dmac)->object_attach = nv50_disp_dmac_object_attach;
-        nv_parent(dmac)->object_detach = nv50_disp_dmac_object_detach;
         return 0;
 }
 
-struct nouveau_ofuncs
+struct nv50_disp_chan_impl
 nv50_disp_sync_ofuncs = {
-        .ctor = nv50_disp_sync_ctor,
-        .dtor = nv50_disp_dmac_dtor,
-        .init = nv50_disp_dmac_init,
-        .fini = nv50_disp_dmac_fini,
-        .rd32 = nv50_disp_chan_rd32,
-        .wr32 = nv50_disp_chan_wr32,
+        .base.ctor = nv50_disp_sync_ctor,
+        .base.dtor = nv50_disp_dmac_dtor,
+        .base.init = nv50_disp_dmac_init,
+        .base.fini = nv50_disp_dmac_fini,
+        .base.map  = nv50_disp_chan_map,
+        .base.rd32 = nv50_disp_chan_rd32,
+        .base.wr32 = nv50_disp_chan_wr32,
+        .chid = 1,
+        .attach = nv50_disp_dmac_object_attach,
+        .detach = nv50_disp_dmac_object_detach,
 };
 
 /*******************************************************************************
@@ -620,39 +657,51 @@ nv50_disp_ovly_mthd_chan = {
         }
 };
 
-static int
+int
 nv50_disp_ovly_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
-        struct nv50_display_ovly_class *args = data;
+        union {
+                struct nv50_disp_overlay_channel_dma_v0 v0;
+        } *args = data;
+        struct nv50_disp_priv *priv = (void *)engine;
         struct nv50_disp_dmac *dmac;
         int ret;
 
-        if (size < sizeof(*args) || args->head > 1)
-                return -EINVAL;
+        nv_ioctl(parent, "create disp overlay channel dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create disp overlay channel dma vers %d "
+                                 "pushbuf %08x head %d\n",
+                         args->v0.version, args->v0.pushbuf, args->v0.head);
+                if (args->v0.head > priv->head.nr)
+                        return -EINVAL;
+        } else
+                return ret;
 
-        ret = nv50_disp_dmac_create_(parent, engine, oclass, args->pushbuf,
-                                     3 + args->head, sizeof(*dmac),
+        ret = nv50_disp_dmac_create_(parent, engine, oclass, args->v0.pushbuf,
+                                     args->v0.head, sizeof(*dmac),
                                      (void **)&dmac);
         *pobject = nv_object(dmac);
         if (ret)
                 return ret;
 
-        nv_parent(dmac)->object_attach = nv50_disp_dmac_object_attach;
-        nv_parent(dmac)->object_detach = nv50_disp_dmac_object_detach;
         return 0;
 }
 
-struct nouveau_ofuncs
+struct nv50_disp_chan_impl
 nv50_disp_ovly_ofuncs = {
-        .ctor = nv50_disp_ovly_ctor,
-        .dtor = nv50_disp_dmac_dtor,
-        .init = nv50_disp_dmac_init,
-        .fini = nv50_disp_dmac_fini,
-        .rd32 = nv50_disp_chan_rd32,
-        .wr32 = nv50_disp_chan_wr32,
+        .base.ctor = nv50_disp_ovly_ctor,
+        .base.dtor = nv50_disp_dmac_dtor,
+        .base.init = nv50_disp_dmac_init,
+        .base.fini = nv50_disp_dmac_fini,
+        .base.map  = nv50_disp_chan_map,
+        .base.rd32 = nv50_disp_chan_rd32,
+        .base.wr32 = nv50_disp_chan_wr32,
+        .chid = 3,
+        .attach = nv50_disp_dmac_object_attach,
+        .detach = nv50_disp_dmac_object_detach,
 };
 
 /*******************************************************************************
@@ -662,14 +711,14 @@ nv50_disp_ovly_ofuncs = {
 static int
 nv50_disp_pioc_create_(struct nouveau_object *parent,
                        struct nouveau_object *engine,
-                       struct nouveau_oclass *oclass, int chid,
+                       struct nouveau_oclass *oclass, int head,
                        int length, void **pobject)
 {
-        return nv50_disp_chan_create_(parent, engine, oclass, chid,
+        return nv50_disp_chan_create_(parent, engine, oclass, head,
                                       length, pobject);
 }
 
-static void
+void
 nv50_disp_pioc_dtor(struct nouveau_object *object)
 {
         struct nv50_disp_pioc *pioc = (void *)object;
@@ -727,20 +776,29 @@ nv50_disp_pioc_fini(struct nouveau_object *object, bool suspend)
  * EVO immediate overlay channel objects
  ******************************************************************************/
 
-static int
+int
 nv50_disp_oimm_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
-        struct nv50_display_oimm_class *args = data;
+        union {
+                struct nv50_disp_overlay_v0 v0;
+        } *args = data;
+        struct nv50_disp_priv *priv = (void *)engine;
         struct nv50_disp_pioc *pioc;
         int ret;
 
-        if (size < sizeof(*args) || args->head > 1)
-                return -EINVAL;
+        nv_ioctl(parent, "create disp overlay size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create disp overlay vers %d head %d\n",
+                         args->v0.version, args->v0.head);
+                if (args->v0.head > priv->head.nr)
+                        return -EINVAL;
+        } else
+                return ret;
 
-        ret = nv50_disp_pioc_create_(parent, engine, oclass, 5 + args->head,
+        ret = nv50_disp_pioc_create_(parent, engine, oclass, args->v0.head,
                                      sizeof(*pioc), (void **)&pioc);
         *pobject = nv_object(pioc);
         if (ret)
@@ -749,34 +807,45 @@ nv50_disp_oimm_ctor(struct nouveau_object *parent,
         return 0;
 }
 
-struct nouveau_ofuncs
+struct nv50_disp_chan_impl
 nv50_disp_oimm_ofuncs = {
-        .ctor = nv50_disp_oimm_ctor,
-        .dtor = nv50_disp_pioc_dtor,
-        .init = nv50_disp_pioc_init,
-        .fini = nv50_disp_pioc_fini,
-        .rd32 = nv50_disp_chan_rd32,
-        .wr32 = nv50_disp_chan_wr32,
+        .base.ctor = nv50_disp_oimm_ctor,
+        .base.dtor = nv50_disp_pioc_dtor,
+        .base.init = nv50_disp_pioc_init,
+        .base.fini = nv50_disp_pioc_fini,
+        .base.map  = nv50_disp_chan_map,
+        .base.rd32 = nv50_disp_chan_rd32,
+        .base.wr32 = nv50_disp_chan_wr32,
+        .chid = 5,
 };
 
 /*******************************************************************************
  * EVO cursor channel objects
  ******************************************************************************/
 
-static int
+int
 nv50_disp_curs_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
-        struct nv50_display_curs_class *args = data;
+        union {
+                struct nv50_disp_cursor_v0 v0;
+        } *args = data;
+        struct nv50_disp_priv *priv = (void *)engine;
         struct nv50_disp_pioc *pioc;
         int ret;
 
-        if (size < sizeof(*args) || args->head > 1)
-                return -EINVAL;
+        nv_ioctl(parent, "create disp cursor size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create disp cursor vers %d head %d\n",
+                         args->v0.version, args->v0.head);
+                if (args->v0.head > priv->head.nr)
+                        return -EINVAL;
+        } else
+                return ret;
 
-        ret = nv50_disp_pioc_create_(parent, engine, oclass, 7 + args->head,
+        ret = nv50_disp_pioc_create_(parent, engine, oclass, args->v0.head,
                                      sizeof(*pioc), (void **)&pioc);
         *pobject = nv_object(pioc);
         if (ret)
@@ -785,14 +854,16 @@ nv50_disp_curs_ctor(struct nouveau_object *parent,
         return 0;
 }
 
-struct nouveau_ofuncs
+struct nv50_disp_chan_impl
 nv50_disp_curs_ofuncs = {
-        .ctor = nv50_disp_curs_ctor,
-        .dtor = nv50_disp_pioc_dtor,
-        .init = nv50_disp_pioc_init,
-        .fini = nv50_disp_pioc_fini,
-        .rd32 = nv50_disp_chan_rd32,
-        .wr32 = nv50_disp_chan_wr32,
+        .base.ctor = nv50_disp_curs_ctor,
+        .base.dtor = nv50_disp_pioc_dtor,
+        .base.init = nv50_disp_pioc_init,
+        .base.fini = nv50_disp_pioc_fini,
+        .base.map  = nv50_disp_chan_map,
+        .base.rd32 = nv50_disp_chan_rd32,
+        .base.wr32 = nv50_disp_chan_wr32,
+        .chid = 7,
 };
 
 /*******************************************************************************
@@ -800,47 +871,162 @@ nv50_disp_curs_ofuncs = {
  ******************************************************************************/
 
 int
-nv50_disp_base_scanoutpos(struct nouveau_object *object, u32 mthd,
-                          void *data, u32 size)
+nv50_disp_base_scanoutpos(NV50_DISP_MTHD_V0)
 {
-        struct nv50_disp_priv *priv = (void *)object->engine;
-        struct nv04_display_scanoutpos *args = data;
-        const int head = (mthd & NV50_DISP_MTHD_HEAD);
-        u32 blanke, blanks, total;
+        const u32 blanke = nv_rd32(priv, 0x610aec + (head * 0x540));
+        const u32 blanks = nv_rd32(priv, 0x610af4 + (head * 0x540));
+        const u32 total  = nv_rd32(priv, 0x610afc + (head * 0x540));
+        union {
+                struct nv04_disp_scanoutpos_v0 v0;
+        } *args = data;
+        int ret;
+
+        nv_ioctl(object, "disp scanoutpos size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "disp scanoutpos vers %d\n", args->v0.version);
+                args->v0.vblanke = (blanke & 0xffff0000) >> 16;
+                args->v0.hblanke = (blanke & 0x0000ffff);
+                args->v0.vblanks = (blanks & 0xffff0000) >> 16;
+                args->v0.hblanks = (blanks & 0x0000ffff);
+                args->v0.vtotal  = ( total & 0xffff0000) >> 16;
+                args->v0.htotal  = ( total & 0x0000ffff);
+                args->v0.time[0] = ktime_to_ns(ktime_get());
+                args->v0.vline = /* vline read locks hline */
+                        nv_rd32(priv, 0x616340 + (head * 0x800)) & 0xffff;
+                args->v0.time[1] = ktime_to_ns(ktime_get());
+                args->v0.hline =
+                        nv_rd32(priv, 0x616344 + (head * 0x800)) & 0xffff;
+        } else
+                return ret;
 
-        if (size < sizeof(*args) || head >= priv->head.nr)
-                return -EINVAL;
-        blanke = nv_rd32(priv, 0x610aec + (head * 0x540));
-        blanks = nv_rd32(priv, 0x610af4 + (head * 0x540));
-        total  = nv_rd32(priv, 0x610afc + (head * 0x540));
-
-        args->vblanke = (blanke & 0xffff0000) >> 16;
-        args->hblanke = (blanke & 0x0000ffff);
-        args->vblanks = (blanks & 0xffff0000) >> 16;
-        args->hblanks = (blanks & 0x0000ffff);
-        args->vtotal  = ( total & 0xffff0000) >> 16;
-        args->htotal  = ( total & 0x0000ffff);
-
-        args->time[0] = ktime_to_ns(ktime_get());
-        args->vline   = nv_rd32(priv, 0x616340 + (head * 0x800)) & 0xffff;
-        args->time[1] = ktime_to_ns(ktime_get()); /* vline read locks hline */
-        args->hline   = nv_rd32(priv, 0x616344 + (head * 0x800)) & 0xffff;
         return 0;
 }
 
-static void
-nv50_disp_base_vblank_enable(struct nouveau_event *event, int type, int head)
+int
+nv50_disp_base_mthd(struct nouveau_object *object, u32 mthd,
+                    void *data, u32 size)
 {
-        nv_mask(event->priv, 0x61002c, (4 << head), (4 << head));
-}
+        const struct nv50_disp_impl *impl = (void *)nv_oclass(object->engine);
+        union {
+                struct nv50_disp_mthd_v0 v0;
+                struct nv50_disp_mthd_v1 v1;
+        } *args = data;
+        struct nv50_disp_priv *priv = (void *)object->engine;
+        struct nvkm_output *outp = NULL;
+        struct nvkm_output *temp;
+        u16 type, mask = 0;
+        int head, ret;
 
-static void
-nv50_disp_base_vblank_disable(struct nouveau_event *event, int type, int head)
-{
-        nv_mask(event->priv, 0x61002c, (4 << head), 0);
+        if (mthd != NV50_DISP_MTHD)
+                return -EINVAL;
+
+        nv_ioctl(object, "disp mthd size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                nv_ioctl(object, "disp mthd vers %d mthd %02x head %d\n",
+                         args->v0.version, args->v0.method, args->v0.head);
+                mthd = args->v0.method;
+                head = args->v0.head;
+        } else
+        if (nvif_unpack(args->v1, 1, 1, true)) {
+                nv_ioctl(object, "disp mthd vers %d mthd %02x "
+                                 "type %04x mask %04x\n",
+                         args->v1.version, args->v1.method,
+                         args->v1.hasht, args->v1.hashm);
+                mthd = args->v1.method;
+                type = args->v1.hasht;
+                mask = args->v1.hashm;
+                head = ffs((mask >> 8) & 0x0f) - 1;
+        } else
+                return ret;
+
+        if (head < 0 || head >= priv->head.nr)
+                return -ENXIO;
+
+        if (mask) {
+                list_for_each_entry(temp, &priv->base.outp, head) {
+                        if ((temp->info.hasht         == type) &&
+                            (temp->info.hashm & mask) == mask) {
+                                outp = temp;
+                                break;
+                        }
+                }
+                if (outp == NULL)
+                        return -ENXIO;
+        }
+
+        switch (mthd) {
+        case NV50_DISP_SCANOUTPOS:
+                return impl->head.scanoutpos(object, priv, data, size, head);
+        default:
+                break;
+        }
+
+        switch (mthd * !!outp) {
+        case NV50_DISP_MTHD_V1_DAC_PWR:
+                return priv->dac.power(object, priv, data, size, head, outp);
+        case NV50_DISP_MTHD_V1_DAC_LOAD:
+                return priv->dac.sense(object, priv, data, size, head, outp);
+        case NV50_DISP_MTHD_V1_SOR_PWR:
+                return priv->sor.power(object, priv, data, size, head, outp);
+        case NV50_DISP_MTHD_V1_SOR_HDA_ELD:
+                if (!priv->sor.hda_eld)
+                        return -ENODEV;
+                return priv->sor.hda_eld(object, priv, data, size, head, outp);
+        case NV50_DISP_MTHD_V1_SOR_HDMI_PWR:
+                if (!priv->sor.hdmi)
+                        return -ENODEV;
+                return priv->sor.hdmi(object, priv, data, size, head, outp);
+        case NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT: {
+                union {
+                        struct nv50_disp_sor_lvds_script_v0 v0;
+                } *args = data;
+                nv_ioctl(object, "disp sor lvds script size %d\n", size);
+                if (nvif_unpack(args->v0, 0, 0, false)) {
+                        nv_ioctl(object, "disp sor lvds script "
+                                         "vers %d name %04x\n",
+                                 args->v0.version, args->v0.script);
+                        priv->sor.lvdsconf = args->v0.script;
+                        return 0;
+                } else
+                        return ret;
+        }
+                break;
+        case NV50_DISP_MTHD_V1_SOR_DP_PWR: {
+                struct nvkm_output_dp *outpdp = (void *)outp;
+                union {
+                        struct nv50_disp_sor_dp_pwr_v0 v0;
+                } *args = data;
+                nv_ioctl(object, "disp sor dp pwr size %d\n", size);
+                if (nvif_unpack(args->v0, 0, 0, false)) {
+                        nv_ioctl(object, "disp sor dp pwr vers %d state %d\n",
+                                 args->v0.version, args->v0.state);
+                        if (args->v0.state == 0) {
+                                nvkm_notify_put(&outpdp->irq);
+                                ((struct nvkm_output_dp_impl *)nv_oclass(outp))
+                                        ->lnk_pwr(outpdp, 0);
+                                atomic_set(&outpdp->lt.done, 0);
+                                return 0;
+                        } else
+                        if (args->v0.state != 0) {
+                                nvkm_output_dp_train(&outpdp->base, 0, true);
+                                return 0;
+                        }
+                } else
+                        return ret;
+        }
+                break;
+        case NV50_DISP_MTHD_V1_PIOR_PWR:
+                if (!priv->pior.power)
+                        return -ENODEV;
+                return priv->pior.power(object, priv, data, size, head, outp);
+        default:
+                break;
+        }
+
+        return -EINVAL;
 }
 
-static int
+int
 nv50_disp_base_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
                     struct nouveau_oclass *oclass, void *data, u32 size,
@@ -856,14 +1042,11 @@ nv50_disp_base_ctor(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
-        priv->base.vblank->priv = priv;
-        priv->base.vblank->enable = nv50_disp_base_vblank_enable;
-        priv->base.vblank->disable = nv50_disp_base_vblank_disable;
         return nouveau_ramht_new(nv_object(base), nv_object(base), 0x1000, 0,
                                 &base->ramht);
 }
 
-static void
+void
 nv50_disp_base_dtor(struct nouveau_object *object)
 {
         struct nv50_disp_base *base = (void *)object;
@@ -958,34 +1141,23 @@ nv50_disp_base_ofuncs = {
         .dtor = nv50_disp_base_dtor,
         .init = nv50_disp_base_init,
         .fini = nv50_disp_base_fini,
-};
-
-static struct nouveau_omthds
-nv50_disp_base_omthds[] = {
-        { HEAD_MTHD(NV50_DISP_SCANOUTPOS)     , nv50_disp_base_scanoutpos },
-        { SOR_MTHD(NV50_DISP_SOR_PWR)         , nv50_sor_mthd },
-        { SOR_MTHD(NV50_DISP_SOR_LVDS_SCRIPT) , nv50_sor_mthd },
-        { DAC_MTHD(NV50_DISP_DAC_PWR)         , nv50_dac_mthd },
-        { DAC_MTHD(NV50_DISP_DAC_LOAD)        , nv50_dac_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_PWR)       , nv50_pior_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_TMDS_PWR)  , nv50_pior_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_DP_PWR)    , nv50_pior_mthd },
-        {},
+        .mthd = nv50_disp_base_mthd,
+        .ntfy = nouveau_disp_ntfy,
 };
 
 static struct nouveau_oclass
 nv50_disp_base_oclass[] = {
-        { NV50_DISP_CLASS, &nv50_disp_base_ofuncs, nv50_disp_base_omthds },
+        { NV50_DISP, &nv50_disp_base_ofuncs },
         {}
 };
 
 static struct nouveau_oclass
 nv50_disp_sclass[] = {
-        { NV50_DISP_MAST_CLASS, &nv50_disp_mast_ofuncs },
-        { NV50_DISP_SYNC_CLASS, &nv50_disp_sync_ofuncs },
-        { NV50_DISP_OVLY_CLASS, &nv50_disp_ovly_ofuncs },
-        { NV50_DISP_OIMM_CLASS, &nv50_disp_oimm_ofuncs },
-        { NV50_DISP_CURS_CLASS, &nv50_disp_curs_ofuncs },
+        { NV50_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
+        { NV50_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { NV50_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
+        { NV50_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
+        { NV50_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
         {}
 };
 
@@ -1005,7 +1177,7 @@ nv50_disp_data_ctor(struct nouveau_object *parent,
         int ret = -EBUSY;
 
         /* no context needed for channel objects... */
-        if (nv_mclass(parent) != NV_DEVICE_CLASS) {
+        if (nv_mclass(parent) != NV_DEVICE) {
                 atomic_inc(&parent->refcount);
                 *pobject = parent;
                 return 1;
@@ -1040,6 +1212,27 @@ nv50_disp_cclass = {
  * Display engine implementation
  ******************************************************************************/
 
+static void
+nv50_disp_vblank_fini(struct nvkm_event *event, int type, int head)
+{
+        struct nouveau_disp *disp = container_of(event, typeof(*disp), vblank);
+        nv_mask(disp, 0x61002c, (4 << head), 0);
+}
+
+static void
+nv50_disp_vblank_init(struct nvkm_event *event, int type, int head)
+{
+        struct nouveau_disp *disp = container_of(event, typeof(*disp), vblank);
+        nv_mask(disp, 0x61002c, (4 << head), (4 << head));
+}
+
+const struct nvkm_event_func
+nv50_disp_vblank_func = {
+        .ctor = nouveau_disp_vblank_ctor,
+        .init = nv50_disp_vblank_init,
+        .fini = nv50_disp_vblank_fini,
+};
+
 static const struct nouveau_enum
 nv50_disp_intr_error_type[] = {
         { 3, "ILLEGAL_MTHD" },
@@ -1381,7 +1574,7 @@ nv50_disp_intr_unk20_2_dp(struct nv50_disp_priv *priv,
         int TU, VTUi, VTUf, VTUa;
         u64 link_data_rate, link_ratio, unk;
         u32 best_diff = 64 * symbol;
-        u32 link_nr, link_bw, bits, r;
+        u32 link_nr, link_bw, bits;
 
         /* calculate packed data rate for each lane */
         if      (dpctrl > 0x00030000) link_nr = 4;
@@ -1401,7 +1594,7 @@ nv50_disp_intr_unk20_2_dp(struct nv50_disp_priv *priv,
 
         /* calculate ratio of packed data rate to link symbol rate */
         link_ratio = link_data_rate * symbol;
-        r = do_div(link_ratio, link_bw);
+        do_div(link_ratio, link_bw);
 
         for (TU = 64; TU >= 32; TU--) {
                 /* calculate average number of valid symbols in each TU */
@@ -1462,8 +1655,8 @@ nv50_disp_intr_unk20_2_dp(struct nv50_disp_priv *priv,
         /* XXX close to vbios numbers, but not right */
         unk  = (symbol - link_ratio) * bestTU;
         unk *= link_ratio;
-        r = do_div(unk, symbol);
-        r = do_div(unk, symbol);
+        do_div(unk, symbol);
+        do_div(unk, symbol);
         unk += 6;
 
         nv_mask(priv, 0x61c10c + loff, 0x000001fc, bestTU << 2);
@@ -1654,13 +1847,13 @@ nv50_disp_intr(struct nouveau_subdev *subdev)
         }
 
         if (intr1 & 0x00000004) {
-                nouveau_event_trigger(priv->base.vblank, 1, 0);
+                nouveau_disp_vblank(&priv->base, 0);
                 nv_wr32(priv, 0x610024, 0x00000004);
                 intr1 &= ~0x00000004;
         }
 
         if (intr1 & 0x00000008) {
-                nouveau_event_trigger(priv->base.vblank, 1, 1);
+                nouveau_disp_vblank(&priv->base, 1);
                 nv_wr32(priv, 0x610024, 0x00000008);
                 intr1 &= ~0x00000008;
         }
@@ -1718,9 +1911,11 @@ nv50_disp_oclass = &(struct nv50_disp_impl) {
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
+        .base.vblank = &nv50_disp_vblank_func,
         .base.outp =  nv50_disp_outp_sclass,
         .mthd.core = &nv50_disp_mast_mthd_chan,
         .mthd.base = &nv50_disp_sync_mthd_chan,
         .mthd.ovly = &nv50_disp_ovly_mthd_chan,
         .mthd.prev = 0x000004,
+        .head.scanoutpos = nv50_disp_base_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
index 1a886472b6f5..8ab14461f70c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
@@ -14,15 +14,10 @@
 #include "outp.h"
 #include "outpdp.h"
 
-struct nv50_disp_impl {
-        struct nouveau_disp_impl base;
-        struct {
-                const struct nv50_disp_mthd_chan *core;
-                const struct nv50_disp_mthd_chan *base;
-                const struct nv50_disp_mthd_chan *ovly;
-                int prev;
-        } mthd;
-};
+#define NV50_DISP_MTHD_ struct nouveau_object *object,                         \
+        struct nv50_disp_priv *priv, void *data, u32 size
+#define NV50_DISP_MTHD_V0 NV50_DISP_MTHD_, int head
+#define NV50_DISP_MTHD_V1 NV50_DISP_MTHD_, int head, struct nvkm_output *outp
 
 struct nv50_disp_priv {
         struct nouveau_disp base;
@@ -36,44 +31,52 @@ struct nv50_disp_priv {
         } head;
         struct {
                 int nr;
-                int (*power)(struct nv50_disp_priv *, int dac, u32 data);
-                int (*sense)(struct nv50_disp_priv *, int dac, u32 load);
+                int (*power)(NV50_DISP_MTHD_V1);
+                int (*sense)(NV50_DISP_MTHD_V1);
         } dac;
         struct {
                 int nr;
-                int (*power)(struct nv50_disp_priv *, int sor, u32 data);
-                int (*hda_eld)(struct nv50_disp_priv *, int sor, u8 *, u32);
-                int (*hdmi)(struct nv50_disp_priv *, int head, int sor, u32);
+                int (*power)(NV50_DISP_MTHD_V1);
+                int (*hda_eld)(NV50_DISP_MTHD_V1);
+                int (*hdmi)(NV50_DISP_MTHD_V1);
                 u32 lvdsconf;
         } sor;
         struct {
                 int nr;
-                int (*power)(struct nv50_disp_priv *, int ext, u32 data);
+                int (*power)(NV50_DISP_MTHD_V1);
                 u8 type[3];
         } pior;
 };
 
-#define HEAD_MTHD(n) (n), (n) + 0x03
-
-int nv50_disp_base_scanoutpos(struct nouveau_object *, u32, void *, u32);
+struct nv50_disp_impl {
+        struct nouveau_disp_impl base;
+        struct {
+                const struct nv50_disp_mthd_chan *core;
+                const struct nv50_disp_mthd_chan *base;
+                const struct nv50_disp_mthd_chan *ovly;
+                int prev;
+        } mthd;
+        struct {
+                int (*scanoutpos)(NV50_DISP_MTHD_V0);
+        } head;
+};
 
-#define DAC_MTHD(n) (n), (n) + 0x03
+int nv50_disp_base_scanoutpos(NV50_DISP_MTHD_V0);
+int nv50_disp_base_mthd(struct nouveau_object *, u32, void *, u32);
 
-int nv50_dac_mthd(struct nouveau_object *, u32, void *, u32);
-int nv50_dac_power(struct nv50_disp_priv *, int, u32);
-int nv50_dac_sense(struct nv50_disp_priv *, int, u32);
+int nvd0_disp_base_scanoutpos(NV50_DISP_MTHD_V0);
 
-#define SOR_MTHD(n) (n), (n) + 0x3f
+int nv50_dac_power(NV50_DISP_MTHD_V1);
+int nv50_dac_sense(NV50_DISP_MTHD_V1);
 
-int nva3_hda_eld(struct nv50_disp_priv *, int, u8 *, u32);
-int nvd0_hda_eld(struct nv50_disp_priv *, int, u8 *, u32);
+int nva3_hda_eld(NV50_DISP_MTHD_V1);
+int nvd0_hda_eld(NV50_DISP_MTHD_V1);
 
-int nv84_hdmi_ctrl(struct nv50_disp_priv *, int, int, u32);
-int nva3_hdmi_ctrl(struct nv50_disp_priv *, int, int, u32);
-int nvd0_hdmi_ctrl(struct nv50_disp_priv *, int, int, u32);
+int nv84_hdmi_ctrl(NV50_DISP_MTHD_V1);
+int nva3_hdmi_ctrl(NV50_DISP_MTHD_V1);
+int nvd0_hdmi_ctrl(NV50_DISP_MTHD_V1);
 
-int nv50_sor_mthd(struct nouveau_object *, u32, void *, u32);
-int nv50_sor_power(struct nv50_disp_priv *, int, u32);
+int nv50_sor_power(NV50_DISP_MTHD_V1);
 
 int nv94_sor_dp_train_init(struct nv50_disp_priv *, int, int, int, u16, u16,
                            u32, struct dcb_output *);
@@ -93,10 +96,7 @@ int nvd0_sor_dp_lnkctl(struct nv50_disp_priv *, int, int, int, u16, u16, u32,
 int nvd0_sor_dp_drvctl(struct nv50_disp_priv *, int, int, int, u16, u16, u32,
                        struct dcb_output *);
 
-#define PIOR_MTHD(n) (n), (n) + 0x03
-
-int nv50_pior_mthd(struct nouveau_object *, u32, void *, u32);
-int nv50_pior_power(struct nv50_disp_priv *, int, u32);
+int nv50_pior_power(NV50_DISP_MTHD_V1);
 
 struct nv50_disp_base {
         struct nouveau_parent base;
@@ -104,14 +104,19 @@ struct nv50_disp_base {
         u32 chan;
 };
 
+struct nv50_disp_chan_impl {
+        struct nouveau_ofuncs base;
+        int chid;
+        int  (*attach)(struct nouveau_object *, struct nouveau_object *, u32);
+        void (*detach)(struct nouveau_object *, int);
+};
+
 struct nv50_disp_chan {
         struct nouveau_namedb base;
         int chid;
 };
 
-int  nv50_disp_chan_create_(struct nouveau_object *, struct nouveau_object *,
-                            struct nouveau_oclass *, int, int, void **);
-void nv50_disp_chan_destroy(struct nv50_disp_chan *);
+int  nv50_disp_chan_map(struct nouveau_object *, u64 *, u32 *);
 u32  nv50_disp_chan_rd32(struct nouveau_object *, u64);
 void nv50_disp_chan_wr32(struct nouveau_object *, u64, u32);
 
@@ -120,20 +125,20 @@ void nv50_disp_chan_wr32(struct nouveau_object *, u64, u32);
 #define nv50_disp_chan_fini(a,b)                                               \
         nouveau_namedb_fini(&(a)->base, (b))
 
-int  nv50_disp_dmac_create_(struct nouveau_object *, struct nouveau_object *,
-                            struct nouveau_oclass *, u32, int, int, void **);
-void nv50_disp_dmac_dtor(struct nouveau_object *);
-
 struct nv50_disp_dmac {
         struct nv50_disp_chan base;
         struct nouveau_dmaobj *pushdma;
         u32 push;
 };
 
+void nv50_disp_dmac_dtor(struct nouveau_object *);
+
 struct nv50_disp_pioc {
         struct nv50_disp_chan base;
 };
 
+void nv50_disp_pioc_dtor(struct nouveau_object *);
+
 struct nv50_disp_mthd_list {
         u32 mthd;
         u32 addr;
@@ -154,47 +159,67 @@ struct nv50_disp_mthd_chan {
         } data[];
 };
 
-extern struct nouveau_ofuncs nv50_disp_mast_ofuncs;
+extern struct nv50_disp_chan_impl nv50_disp_mast_ofuncs;
+int nv50_disp_mast_ctor(struct nouveau_object *, struct nouveau_object *,
+                        struct nouveau_oclass *, void *, u32,
+                        struct nouveau_object **);
 extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_base;
 extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_sor;
 extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_pior;
-extern struct nouveau_ofuncs nv50_disp_sync_ofuncs;
+extern struct nv50_disp_chan_impl nv50_disp_sync_ofuncs;
+int nv50_disp_sync_ctor(struct nouveau_object *, struct nouveau_object *,
+                        struct nouveau_oclass *, void *, u32,
+                        struct nouveau_object **);
 extern const struct nv50_disp_mthd_list nv50_disp_sync_mthd_image;
-extern struct nouveau_ofuncs nv50_disp_ovly_ofuncs;
+extern struct nv50_disp_chan_impl nv50_disp_ovly_ofuncs;
+int nv50_disp_ovly_ctor(struct nouveau_object *, struct nouveau_object *,
+                        struct nouveau_oclass *, void *, u32,
+                        struct nouveau_object **);
 extern const struct nv50_disp_mthd_list nv50_disp_ovly_mthd_base;
-extern struct nouveau_ofuncs nv50_disp_oimm_ofuncs;
-extern struct nouveau_ofuncs nv50_disp_curs_ofuncs;
+extern struct nv50_disp_chan_impl nv50_disp_oimm_ofuncs;
+int nv50_disp_oimm_ctor(struct nouveau_object *, struct nouveau_object *,
+                        struct nouveau_oclass *, void *, u32,
+                        struct nouveau_object **);
+extern struct nv50_disp_chan_impl nv50_disp_curs_ofuncs;
+int nv50_disp_curs_ctor(struct nouveau_object *, struct nouveau_object *,
+                        struct nouveau_oclass *, void *, u32,
+                        struct nouveau_object **);
 extern struct nouveau_ofuncs nv50_disp_base_ofuncs;
+int  nv50_disp_base_ctor(struct nouveau_object *, struct nouveau_object *,
+                         struct nouveau_oclass *, void *, u32,
+                         struct nouveau_object **);
+void nv50_disp_base_dtor(struct nouveau_object *);
+extern struct nouveau_omthds nv50_disp_base_omthds[];
 extern struct nouveau_oclass nv50_disp_cclass;
 void nv50_disp_mthd_chan(struct nv50_disp_priv *, int debug, int head,
                          const struct nv50_disp_mthd_chan *);
 void nv50_disp_intr_supervisor(struct work_struct *);
 void nv50_disp_intr(struct nouveau_subdev *);
+extern const struct nvkm_event_func nv50_disp_vblank_func;
 
 extern const struct nv50_disp_mthd_chan nv84_disp_mast_mthd_chan;
 extern const struct nv50_disp_mthd_list nv84_disp_mast_mthd_dac;
 extern const struct nv50_disp_mthd_list nv84_disp_mast_mthd_head;
 extern const struct nv50_disp_mthd_chan nv84_disp_sync_mthd_chan;
 extern const struct nv50_disp_mthd_chan nv84_disp_ovly_mthd_chan;
-extern struct nouveau_omthds nv84_disp_base_omthds[];
 
 extern const struct nv50_disp_mthd_chan nv94_disp_mast_mthd_chan;
 
-extern struct nouveau_ofuncs nvd0_disp_mast_ofuncs;
+extern struct nv50_disp_chan_impl nvd0_disp_mast_ofuncs;
 extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_base;
 extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_dac;
 extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_sor;
 extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_pior;
-extern struct nouveau_ofuncs nvd0_disp_sync_ofuncs;
-extern struct nouveau_ofuncs nvd0_disp_ovly_ofuncs;
+extern struct nv50_disp_chan_impl nvd0_disp_sync_ofuncs;
+extern struct nv50_disp_chan_impl nvd0_disp_ovly_ofuncs;
 extern const struct nv50_disp_mthd_chan nvd0_disp_sync_mthd_chan;
-extern struct nouveau_ofuncs nvd0_disp_oimm_ofuncs;
-extern struct nouveau_ofuncs nvd0_disp_curs_ofuncs;
-extern struct nouveau_omthds nvd0_disp_base_omthds[];
+extern struct nv50_disp_chan_impl nvd0_disp_oimm_ofuncs;
+extern struct nv50_disp_chan_impl nvd0_disp_curs_ofuncs;
 extern struct nouveau_ofuncs nvd0_disp_base_ofuncs;
 extern struct nouveau_oclass nvd0_disp_cclass;
 void nvd0_disp_intr_supervisor(struct work_struct *);
 void nvd0_disp_intr(struct nouveau_subdev *);
+extern const struct nvkm_event_func nvd0_disp_vblank_func;
 
 extern const struct nv50_disp_mthd_chan nve0_disp_mast_mthd_chan;
 extern const struct nv50_disp_mthd_chan nve0_disp_ovly_mthd_chan;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
index 1cc62e434683..788ced1b6182 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
@@ -25,7 +25,7 @@
 #include <engine/software.h>
 #include <engine/disp.h>
 
-#include <core/class.h>
+#include <nvif/class.h>
 
 #include "nv50.h"
 
@@ -204,31 +204,17 @@ nv84_disp_ovly_mthd_chan = {
 
 static struct nouveau_oclass
 nv84_disp_sclass[] = {
-        { NV84_DISP_MAST_CLASS, &nv50_disp_mast_ofuncs },
-        { NV84_DISP_SYNC_CLASS, &nv50_disp_sync_ofuncs },
-        { NV84_DISP_OVLY_CLASS, &nv50_disp_ovly_ofuncs },
-        { NV84_DISP_OIMM_CLASS, &nv50_disp_oimm_ofuncs },
-        { NV84_DISP_CURS_CLASS, &nv50_disp_curs_ofuncs },
+        { G82_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
+        { G82_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { G82_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
+        { G82_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
+        { G82_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
         {}
 };
 
-struct nouveau_omthds
-nv84_disp_base_omthds[] = {
-        { HEAD_MTHD(NV50_DISP_SCANOUTPOS)     , nv50_disp_base_scanoutpos },
-        { SOR_MTHD(NV50_DISP_SOR_PWR)         , nv50_sor_mthd },
-        { SOR_MTHD(NV84_DISP_SOR_HDMI_PWR)    , nv50_sor_mthd },
-        { SOR_MTHD(NV50_DISP_SOR_LVDS_SCRIPT) , nv50_sor_mthd },
-        { DAC_MTHD(NV50_DISP_DAC_PWR)         , nv50_dac_mthd },
-        { DAC_MTHD(NV50_DISP_DAC_LOAD)        , nv50_dac_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_PWR)       , nv50_pior_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_TMDS_PWR)  , nv50_pior_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_DP_PWR)    , nv50_pior_mthd },
-        {},
-};
-
 static struct nouveau_oclass
 nv84_disp_base_oclass[] = {
-        { NV84_DISP_CLASS, &nv50_disp_base_ofuncs, nv84_disp_base_omthds },
+        { G82_DISP, &nv50_disp_base_ofuncs },
         {}
 };
 
@@ -276,9 +262,11 @@ nv84_disp_oclass = &(struct nv50_disp_impl) {
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
+        .base.vblank = &nv50_disp_vblank_func,
         .base.outp =  nv50_disp_outp_sclass,
         .mthd.core = &nv84_disp_mast_mthd_chan,
         .mthd.base = &nv84_disp_sync_mthd_chan,
         .mthd.ovly = &nv84_disp_ovly_mthd_chan,
         .mthd.prev = 0x000004,
+        .head.scanoutpos = nv50_disp_base_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
index 4f718a9f5aef..fa79de906eae 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
@@ -25,7 +25,7 @@
 #include <engine/software.h>
 #include <engine/disp.h>
 
-#include <core/class.h>
+#include <nvif/class.h>
 
 #include "nv50.h"
 
@@ -63,32 +63,17 @@ nv94_disp_mast_mthd_chan = {
 
 static struct nouveau_oclass
 nv94_disp_sclass[] = {
-        { NV94_DISP_MAST_CLASS, &nv50_disp_mast_ofuncs },
-        { NV94_DISP_SYNC_CLASS, &nv50_disp_sync_ofuncs },
-        { NV94_DISP_OVLY_CLASS, &nv50_disp_ovly_ofuncs },
-        { NV94_DISP_OIMM_CLASS, &nv50_disp_oimm_ofuncs },
-        { NV94_DISP_CURS_CLASS, &nv50_disp_curs_ofuncs },
+        { GT206_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
+        { GT200_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { GT200_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
+        { G82_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
+        { G82_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
         {}
 };
 
-static struct nouveau_omthds
-nv94_disp_base_omthds[] = {
-        { HEAD_MTHD(NV50_DISP_SCANOUTPOS)     , nv50_disp_base_scanoutpos },
-        { SOR_MTHD(NV50_DISP_SOR_PWR)         , nv50_sor_mthd },
-        { SOR_MTHD(NV84_DISP_SOR_HDMI_PWR)    , nv50_sor_mthd },
-        { SOR_MTHD(NV50_DISP_SOR_LVDS_SCRIPT) , nv50_sor_mthd },
-        { SOR_MTHD(NV94_DISP_SOR_DP_PWR)      , nv50_sor_mthd },
-        { DAC_MTHD(NV50_DISP_DAC_PWR)         , nv50_dac_mthd },
-        { DAC_MTHD(NV50_DISP_DAC_LOAD)        , nv50_dac_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_PWR)       , nv50_pior_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_TMDS_PWR)  , nv50_pior_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_DP_PWR)    , nv50_pior_mthd },
-        {},
-};
-
 static struct nouveau_oclass
 nv94_disp_base_oclass[] = {
-        { NV94_DISP_CLASS, &nv50_disp_base_ofuncs, nv94_disp_base_omthds },
+        { GT206_DISP, &nv50_disp_base_ofuncs },
         {}
 };
 
@@ -143,9 +128,11 @@ nv94_disp_oclass = &(struct nv50_disp_impl) {
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
+        .base.vblank = &nv50_disp_vblank_func,
         .base.outp =  nv94_disp_outp_sclass,
         .mthd.core = &nv94_disp_mast_mthd_chan,
         .mthd.base = &nv84_disp_sync_mthd_chan,
         .mthd.ovly = &nv84_disp_ovly_mthd_chan,
         .mthd.prev = 0x000004,
+        .head.scanoutpos = nv50_disp_base_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
index 6237a9a36f70..7af15f5d48dc 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
@@ -25,7 +25,7 @@
 #include <engine/software.h>
 #include <engine/disp.h>
 
-#include <core/class.h>
+#include <nvif/class.h>
 
 #include "nv50.h"
 
@@ -80,17 +80,17 @@ nva0_disp_ovly_mthd_chan = {
 
 static struct nouveau_oclass
 nva0_disp_sclass[] = {
-        { NVA0_DISP_MAST_CLASS, &nv50_disp_mast_ofuncs },
-        { NVA0_DISP_SYNC_CLASS, &nv50_disp_sync_ofuncs },
-        { NVA0_DISP_OVLY_CLASS, &nv50_disp_ovly_ofuncs },
-        { NVA0_DISP_OIMM_CLASS, &nv50_disp_oimm_ofuncs },
-        { NVA0_DISP_CURS_CLASS, &nv50_disp_curs_ofuncs },
+        { GT200_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
+        { GT200_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { GT200_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
+        { G82_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
+        { G82_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
         {}
 };
 
 static struct nouveau_oclass
 nva0_disp_base_oclass[] = {
-        { NVA0_DISP_CLASS, &nv50_disp_base_ofuncs, nv84_disp_base_omthds },
+        { GT200_DISP, &nv50_disp_base_ofuncs },
         {}
 };
 
@@ -138,9 +138,11 @@ nva0_disp_oclass = &(struct nv50_disp_impl) {
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
+        .base.vblank = &nv50_disp_vblank_func,
         .base.outp =  nv50_disp_outp_sclass,
         .mthd.core = &nv84_disp_mast_mthd_chan,
         .mthd.base = &nv84_disp_sync_mthd_chan,
         .mthd.ovly = &nva0_disp_ovly_mthd_chan,
         .mthd.prev = 0x000004,
+        .head.scanoutpos = nv50_disp_base_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c b/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
index 019124d4782b..6bd39448f8da 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
@@ -25,7 +25,7 @@
 #include <engine/software.h>
 #include <engine/disp.h>
 
-#include <core/class.h>
+#include <nvif/class.h>
 
 #include "nv50.h"
 
@@ -35,33 +35,17 @@
 
 static struct nouveau_oclass
 nva3_disp_sclass[] = {
-        { NVA3_DISP_MAST_CLASS, &nv50_disp_mast_ofuncs },
-        { NVA3_DISP_SYNC_CLASS, &nv50_disp_sync_ofuncs },
-        { NVA3_DISP_OVLY_CLASS, &nv50_disp_ovly_ofuncs },
-        { NVA3_DISP_OIMM_CLASS, &nv50_disp_oimm_ofuncs },
-        { NVA3_DISP_CURS_CLASS, &nv50_disp_curs_ofuncs },
+        { GT214_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
+        { GT214_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { GT214_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
+        { GT214_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
+        { GT214_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
         {}
 };
 
-static struct nouveau_omthds
-nva3_disp_base_omthds[] = {
-        { HEAD_MTHD(NV50_DISP_SCANOUTPOS)     , nv50_disp_base_scanoutpos },
-        { SOR_MTHD(NV50_DISP_SOR_PWR)         , nv50_sor_mthd },
-        { SOR_MTHD(NVA3_DISP_SOR_HDA_ELD)     , nv50_sor_mthd },
-        { SOR_MTHD(NV84_DISP_SOR_HDMI_PWR)    , nv50_sor_mthd },
-        { SOR_MTHD(NV50_DISP_SOR_LVDS_SCRIPT) , nv50_sor_mthd },
-        { SOR_MTHD(NV94_DISP_SOR_DP_PWR)      , nv50_sor_mthd },
-        { DAC_MTHD(NV50_DISP_DAC_PWR)         , nv50_dac_mthd },
-        { DAC_MTHD(NV50_DISP_DAC_LOAD)        , nv50_dac_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_PWR)       , nv50_pior_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_TMDS_PWR)  , nv50_pior_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_DP_PWR)    , nv50_pior_mthd },
-        {},
-};
-
 static struct nouveau_oclass
 nva3_disp_base_oclass[] = {
-        { NVA3_DISP_CLASS, &nv50_disp_base_ofuncs, nva3_disp_base_omthds },
+        { GT214_DISP, &nv50_disp_base_ofuncs },
         {}
 };
 
@@ -110,9 +94,11 @@ nva3_disp_oclass = &(struct nv50_disp_impl) {
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
+        .base.vblank = &nv50_disp_vblank_func,
         .base.outp =  nv94_disp_outp_sclass,
         .mthd.core = &nv94_disp_mast_mthd_chan,
         .mthd.base = &nv84_disp_sync_mthd_chan,
         .mthd.ovly = &nv84_disp_ovly_mthd_chan,
         .mthd.prev = 0x000004,
+        .head.scanoutpos = nv50_disp_base_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
index fa30d8196f35..a4bb3c774ee1 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
@@ -23,9 +23,11 @@
  */
 
 #include <core/object.h>
+#include <core/client.h>
 #include <core/parent.h>
 #include <core/handle.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <engine/disp.h>
 
@@ -265,30 +267,6 @@ nvd0_disp_mast_mthd_chan = {
 };
 
 static int
-nvd0_disp_mast_ctor(struct nouveau_object *parent,
-                    struct nouveau_object *engine,
-                    struct nouveau_oclass *oclass, void *data, u32 size,
-                    struct nouveau_object **pobject)
-{
-        struct nv50_display_mast_class *args = data;
-        struct nv50_disp_dmac *mast;
-        int ret;
-
-        if (size < sizeof(*args))
-                return -EINVAL;
-
-        ret = nv50_disp_dmac_create_(parent, engine, oclass, args->pushbuf,
-                                     0, sizeof(*mast), (void **)&mast);
-        *pobject = nv_object(mast);
-        if (ret)
-                return ret;
-
-        nv_parent(mast)->object_attach = nvd0_disp_dmac_object_attach;
-        nv_parent(mast)->object_detach = nvd0_disp_dmac_object_detach;
-        return 0;
-}
-
-static int
 nvd0_disp_mast_init(struct nouveau_object *object)
 {
         struct nv50_disp_priv *priv = (void *)object->engine;
@@ -342,14 +320,18 @@ nvd0_disp_mast_fini(struct nouveau_object *object, bool suspend)
         return nv50_disp_chan_fini(&mast->base, suspend);
 }
 
-struct nouveau_ofuncs
+struct nv50_disp_chan_impl
 nvd0_disp_mast_ofuncs = {
-        .ctor = nvd0_disp_mast_ctor,
-        .dtor = nv50_disp_dmac_dtor,
-        .init = nvd0_disp_mast_init,
-        .fini = nvd0_disp_mast_fini,
-        .rd32 = nv50_disp_chan_rd32,
-        .wr32 = nv50_disp_chan_wr32,
+        .base.ctor = nv50_disp_mast_ctor,
+        .base.dtor = nv50_disp_dmac_dtor,
+        .base.init = nvd0_disp_mast_init,
+        .base.fini = nvd0_disp_mast_fini,
+        .base.map  = nv50_disp_chan_map,
+        .base.rd32 = nv50_disp_chan_rd32,
+        .base.wr32 = nv50_disp_chan_wr32,
+        .chid = 0,
+        .attach = nvd0_disp_dmac_object_attach,
+        .detach = nvd0_disp_dmac_object_detach,
 };
 
 /*******************************************************************************
@@ -431,40 +413,18 @@ nvd0_disp_sync_mthd_chan = {
         }
 };
 
-static int
-nvd0_disp_sync_ctor(struct nouveau_object *parent,
-                    struct nouveau_object *engine,
-                    struct nouveau_oclass *oclass, void *data, u32 size,
-                    struct nouveau_object **pobject)
-{
-        struct nv50_display_sync_class *args = data;
-        struct nv50_disp_priv *priv = (void *)engine;
-        struct nv50_disp_dmac *dmac;
-        int ret;
-
-        if (size < sizeof(*args) || args->head >= priv->head.nr)
-                return -EINVAL;
-
-        ret = nv50_disp_dmac_create_(parent, engine, oclass, args->pushbuf,
-                                     1 + args->head, sizeof(*dmac),
-                                     (void **)&dmac);
-        *pobject = nv_object(dmac);
-        if (ret)
-                return ret;
-
-        nv_parent(dmac)->object_attach = nvd0_disp_dmac_object_attach;
-        nv_parent(dmac)->object_detach = nvd0_disp_dmac_object_detach;
-        return 0;
-}
-
-struct nouveau_ofuncs
+struct nv50_disp_chan_impl
 nvd0_disp_sync_ofuncs = {
-        .ctor = nvd0_disp_sync_ctor,
-        .dtor = nv50_disp_dmac_dtor,
-        .init = nvd0_disp_dmac_init,
-        .fini = nvd0_disp_dmac_fini,
-        .rd32 = nv50_disp_chan_rd32,
-        .wr32 = nv50_disp_chan_wr32,
+        .base.ctor = nv50_disp_sync_ctor,
+        .base.dtor = nv50_disp_dmac_dtor,
+        .base.init = nvd0_disp_dmac_init,
+        .base.fini = nvd0_disp_dmac_fini,
+        .base.map  = nv50_disp_chan_map,
+        .base.rd32 = nv50_disp_chan_rd32,
+        .base.wr32 = nv50_disp_chan_wr32,
+        .chid = 1,
+        .attach = nvd0_disp_dmac_object_attach,
+        .detach = nvd0_disp_dmac_object_detach,
 };
 
 /*******************************************************************************
@@ -533,40 +493,18 @@ nvd0_disp_ovly_mthd_chan = {
         }
 };
 
-static int
-nvd0_disp_ovly_ctor(struct nouveau_object *parent,
-                    struct nouveau_object *engine,
-                    struct nouveau_oclass *oclass, void *data, u32 size,
-                    struct nouveau_object **pobject)
-{
-        struct nv50_display_ovly_class *args = data;
-        struct nv50_disp_priv *priv = (void *)engine;
-        struct nv50_disp_dmac *dmac;
-        int ret;
-
-        if (size < sizeof(*args) || args->head >= priv->head.nr)
-                return -EINVAL;
-
-        ret = nv50_disp_dmac_create_(parent, engine, oclass, args->pushbuf,
-                                     5 + args->head, sizeof(*dmac),
-                                     (void **)&dmac);
-        *pobject = nv_object(dmac);
-        if (ret)
-                return ret;
-
-        nv_parent(dmac)->object_attach = nvd0_disp_dmac_object_attach;
-        nv_parent(dmac)->object_detach = nvd0_disp_dmac_object_detach;
-        return 0;
-}
-
-struct nouveau_ofuncs
+struct nv50_disp_chan_impl
 nvd0_disp_ovly_ofuncs = {
-        .ctor = nvd0_disp_ovly_ctor,
-        .dtor = nv50_disp_dmac_dtor,
-        .init = nvd0_disp_dmac_init,
-        .fini = nvd0_disp_dmac_fini,
-        .rd32 = nv50_disp_chan_rd32,
-        .wr32 = nv50_disp_chan_wr32,
+        .base.ctor = nv50_disp_ovly_ctor,
+        .base.dtor = nv50_disp_dmac_dtor,
+        .base.init = nvd0_disp_dmac_init,
+        .base.fini = nvd0_disp_dmac_fini,
+        .base.map  = nv50_disp_chan_map,
+        .base.rd32 = nv50_disp_chan_rd32,
+        .base.wr32 = nv50_disp_chan_wr32,
+        .chid = 5,
+        .attach = nvd0_disp_dmac_object_attach,
+        .detach = nvd0_disp_dmac_object_detach,
 };
 
 /*******************************************************************************
@@ -574,23 +512,6 @@ nvd0_disp_ovly_ofuncs = {
  ******************************************************************************/
 
 static int
-nvd0_disp_pioc_create_(struct nouveau_object *parent,
-                       struct nouveau_object *engine,
-                       struct nouveau_oclass *oclass, int chid,
-                       int length, void **pobject)
-{
-        return nv50_disp_chan_create_(parent, engine, oclass, chid,
-                                      length, pobject);
-}
-
-static void
-nvd0_disp_pioc_dtor(struct nouveau_object *object)
-{
-        struct nv50_disp_pioc *pioc = (void *)object;
-        nv50_disp_chan_destroy(&pioc->base);
-}
-
-static int
 nvd0_disp_pioc_init(struct nouveau_object *object)
 {
         struct nv50_disp_priv *priv = (void *)object->engine;
@@ -643,152 +564,68 @@ nvd0_disp_pioc_fini(struct nouveau_object *object, bool suspend)
  * EVO immediate overlay channel objects
  ******************************************************************************/
 
-static int
-nvd0_disp_oimm_ctor(struct nouveau_object *parent,
-                    struct nouveau_object *engine,
-                    struct nouveau_oclass *oclass, void *data, u32 size,
-                    struct nouveau_object **pobject)
-{
-        struct nv50_display_oimm_class *args = data;
-        struct nv50_disp_priv *priv = (void *)engine;
-        struct nv50_disp_pioc *pioc;
-        int ret;
-
-        if (size < sizeof(*args) || args->head >= priv->head.nr)
-                return -EINVAL;
-
-        ret = nvd0_disp_pioc_create_(parent, engine, oclass, 9 + args->head,
-                                     sizeof(*pioc), (void **)&pioc);
-        *pobject = nv_object(pioc);
-        if (ret)
-                return ret;
-
-        return 0;
-}
-
-struct nouveau_ofuncs
+struct nv50_disp_chan_impl
 nvd0_disp_oimm_ofuncs = {
-        .ctor = nvd0_disp_oimm_ctor,
-        .dtor = nvd0_disp_pioc_dtor,
-        .init = nvd0_disp_pioc_init,
-        .fini = nvd0_disp_pioc_fini,
-        .rd32 = nv50_disp_chan_rd32,
-        .wr32 = nv50_disp_chan_wr32,
+        .base.ctor = nv50_disp_oimm_ctor,
+        .base.dtor = nv50_disp_pioc_dtor,
+        .base.init = nvd0_disp_pioc_init,
+        .base.fini = nvd0_disp_pioc_fini,
+        .base.map  = nv50_disp_chan_map,
+        .base.rd32 = nv50_disp_chan_rd32,
+        .base.wr32 = nv50_disp_chan_wr32,
+        .chid = 9,
 };
 
 /*******************************************************************************
  * EVO cursor channel objects
  ******************************************************************************/
 
-static int
-nvd0_disp_curs_ctor(struct nouveau_object *parent,
-                    struct nouveau_object *engine,
-                    struct nouveau_oclass *oclass, void *data, u32 size,
-                    struct nouveau_object **pobject)
-{
-        struct nv50_display_curs_class *args = data;
-        struct nv50_disp_priv *priv = (void *)engine;
-        struct nv50_disp_pioc *pioc;
-        int ret;
-
-        if (size < sizeof(*args) || args->head >= priv->head.nr)
-                return -EINVAL;
-
-        ret = nvd0_disp_pioc_create_(parent, engine, oclass, 13 + args->head,
-                                     sizeof(*pioc), (void **)&pioc);
-        *pobject = nv_object(pioc);
-        if (ret)
-                return ret;
-
-        return 0;
-}
-
-struct nouveau_ofuncs
+struct nv50_disp_chan_impl
 nvd0_disp_curs_ofuncs = {
-        .ctor = nvd0_disp_curs_ctor,
-        .dtor = nvd0_disp_pioc_dtor,
-        .init = nvd0_disp_pioc_init,
-        .fini = nvd0_disp_pioc_fini,
-        .rd32 = nv50_disp_chan_rd32,
-        .wr32 = nv50_disp_chan_wr32,
+        .base.ctor = nv50_disp_curs_ctor,
+        .base.dtor = nv50_disp_pioc_dtor,
+        .base.init = nvd0_disp_pioc_init,
+        .base.fini = nvd0_disp_pioc_fini,
+        .base.map  = nv50_disp_chan_map,
+        .base.rd32 = nv50_disp_chan_rd32,
+        .base.wr32 = nv50_disp_chan_wr32,
+        .chid = 13,
 };
 
 /*******************************************************************************
  * Base display object
  ******************************************************************************/
 
-static int
-nvd0_disp_base_scanoutpos(struct nouveau_object *object, u32 mthd,
-                          void *data, u32 size)
-{
-        struct nv50_disp_priv *priv = (void *)object->engine;
-        struct nv04_display_scanoutpos *args = data;
-        const int head = (mthd & NV50_DISP_MTHD_HEAD);
-        u32 blanke, blanks, total;
-
-        if (size < sizeof(*args) || head >= priv->head.nr)
-                return -EINVAL;
-
-        total  = nv_rd32(priv, 0x640414 + (head * 0x300));
-        blanke = nv_rd32(priv, 0x64041c + (head * 0x300));
-        blanks = nv_rd32(priv, 0x640420 + (head * 0x300));
-
-        args->vblanke = (blanke & 0xffff0000) >> 16;
-        args->hblanke = (blanke & 0x0000ffff);
-        args->vblanks = (blanks & 0xffff0000) >> 16;
-        args->hblanks = (blanks & 0x0000ffff);
-        args->vtotal  = ( total & 0xffff0000) >> 16;
-        args->htotal  = ( total & 0x0000ffff);
-
-        args->time[0] = ktime_to_ns(ktime_get());
-        args->vline   = nv_rd32(priv, 0x616340 + (head * 0x800)) & 0xffff;
-        args->time[1] = ktime_to_ns(ktime_get()); /* vline read locks hline */
-        args->hline   = nv_rd32(priv, 0x616344 + (head * 0x800)) & 0xffff;
-        return 0;
-}
-
-static void
-nvd0_disp_base_vblank_enable(struct nouveau_event *event, int type, int head)
-{
-        nv_mask(event->priv, 0x6100c0 + (head * 0x800), 0x00000001, 0x00000001);
-}
-
-static void
-nvd0_disp_base_vblank_disable(struct nouveau_event *event, int type, int head)
+int
+nvd0_disp_base_scanoutpos(NV50_DISP_MTHD_V0)
 {
-        nv_mask(event->priv, 0x6100c0 + (head * 0x800), 0x00000001, 0x00000000);
-}
-
-static int
-nvd0_disp_base_ctor(struct nouveau_object *parent,
-                    struct nouveau_object *engine,
-                    struct nouveau_oclass *oclass, void *data, u32 size,
-                    struct nouveau_object **pobject)
-{
-        struct nv50_disp_priv *priv = (void *)engine;
-        struct nv50_disp_base *base;
+        const u32 total  = nv_rd32(priv, 0x640414 + (head * 0x300));
+        const u32 blanke = nv_rd32(priv, 0x64041c + (head * 0x300));
+        const u32 blanks = nv_rd32(priv, 0x640420 + (head * 0x300));
+        union {
+                struct nv04_disp_scanoutpos_v0 v0;
+        } *args = data;
         int ret;
 
-        ret = nouveau_parent_create(parent, engine, oclass, 0,
-                                    priv->sclass, 0, &base);
-        *pobject = nv_object(base);
-        if (ret)
+        nv_ioctl(object, "disp scanoutpos size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "disp scanoutpos vers %d\n", args->v0.version);
+                args->v0.vblanke = (blanke & 0xffff0000) >> 16;
+                args->v0.hblanke = (blanke & 0x0000ffff);
+                args->v0.vblanks = (blanks & 0xffff0000) >> 16;
+                args->v0.hblanks = (blanks & 0x0000ffff);
+                args->v0.vtotal  = ( total & 0xffff0000) >> 16;
+                args->v0.htotal  = ( total & 0x0000ffff);
+                args->v0.time[0] = ktime_to_ns(ktime_get());
+                args->v0.vline = /* vline read locks hline */
+                        nv_rd32(priv, 0x616340 + (head * 0x800)) & 0xffff;
+                args->v0.time[1] = ktime_to_ns(ktime_get());
+                args->v0.hline =
+                        nv_rd32(priv, 0x616344 + (head * 0x800)) & 0xffff;
+        } else
                 return ret;
 
-        priv->base.vblank->priv = priv;
-        priv->base.vblank->enable = nvd0_disp_base_vblank_enable;
-        priv->base.vblank->disable = nvd0_disp_base_vblank_disable;
-
-        return nouveau_ramht_new(nv_object(base), nv_object(base), 0x1000, 0,
-                                &base->ramht);
-}
-
-static void
-nvd0_disp_base_dtor(struct nouveau_object *object)
-{
-        struct nv50_disp_base *base = (void *)object;
-        nouveau_ramht_ref(NULL, &base->ramht);
-        nouveau_parent_destroy(&base->base);
+        return 0;
 }
 
 static int
@@ -874,41 +711,27 @@ nvd0_disp_base_fini(struct nouveau_object *object, bool suspend)
 
 struct nouveau_ofuncs
 nvd0_disp_base_ofuncs = {
-        .ctor = nvd0_disp_base_ctor,
-        .dtor = nvd0_disp_base_dtor,
+        .ctor = nv50_disp_base_ctor,
+        .dtor = nv50_disp_base_dtor,
         .init = nvd0_disp_base_init,
         .fini = nvd0_disp_base_fini,
-};
-
-struct nouveau_omthds
-nvd0_disp_base_omthds[] = {
-        { HEAD_MTHD(NV50_DISP_SCANOUTPOS)     , nvd0_disp_base_scanoutpos },
-        { SOR_MTHD(NV50_DISP_SOR_PWR)         , nv50_sor_mthd },
-        { SOR_MTHD(NVA3_DISP_SOR_HDA_ELD)     , nv50_sor_mthd },
-        { SOR_MTHD(NV84_DISP_SOR_HDMI_PWR)    , nv50_sor_mthd },
-        { SOR_MTHD(NV50_DISP_SOR_LVDS_SCRIPT) , nv50_sor_mthd },
-        { SOR_MTHD(NV94_DISP_SOR_DP_PWR)      , nv50_sor_mthd },
-        { DAC_MTHD(NV50_DISP_DAC_PWR)         , nv50_dac_mthd },
-        { DAC_MTHD(NV50_DISP_DAC_LOAD)        , nv50_dac_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_PWR)       , nv50_pior_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_TMDS_PWR)  , nv50_pior_mthd },
-        { PIOR_MTHD(NV50_DISP_PIOR_DP_PWR)    , nv50_pior_mthd },
-        {},
+        .mthd = nv50_disp_base_mthd,
+        .ntfy = nouveau_disp_ntfy,
 };
 
 static struct nouveau_oclass
 nvd0_disp_base_oclass[] = {
-        { NVD0_DISP_CLASS, &nvd0_disp_base_ofuncs, nvd0_disp_base_omthds },
+        { GF110_DISP, &nvd0_disp_base_ofuncs },
         {}
 };
 
 static struct nouveau_oclass
 nvd0_disp_sclass[] = {
-        { NVD0_DISP_MAST_CLASS, &nvd0_disp_mast_ofuncs },
-        { NVD0_DISP_SYNC_CLASS, &nvd0_disp_sync_ofuncs },
-        { NVD0_DISP_OVLY_CLASS, &nvd0_disp_ovly_ofuncs },
-        { NVD0_DISP_OIMM_CLASS, &nvd0_disp_oimm_ofuncs },
-        { NVD0_DISP_CURS_CLASS, &nvd0_disp_curs_ofuncs },
+        { GF110_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
+        { GF110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GF110_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
+        { GF110_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
+        { GF110_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
         {}
 };
 
@@ -916,6 +739,27 @@ nvd0_disp_sclass[] = {
  * Display engine implementation
  ******************************************************************************/
 
+static void
+nvd0_disp_vblank_init(struct nvkm_event *event, int type, int head)
+{
+        struct nouveau_disp *disp = container_of(event, typeof(*disp), vblank);
+        nv_mask(disp, 0x6100c0 + (head * 0x800), 0x00000001, 0x00000001);
+}
+
+static void
+nvd0_disp_vblank_fini(struct nvkm_event *event, int type, int head)
+{
+        struct nouveau_disp *disp = container_of(event, typeof(*disp), vblank);
+        nv_mask(disp, 0x6100c0 + (head * 0x800), 0x00000001, 0x00000000);
+}
+
+const struct nvkm_event_func
+nvd0_disp_vblank_func = {
+        .ctor = nouveau_disp_vblank_ctor,
+        .init = nvd0_disp_vblank_init,
+        .fini = nvd0_disp_vblank_fini,
+};
+
 static struct nvkm_output *
 exec_lookup(struct nv50_disp_priv *priv, int head, int or, u32 ctrl,
             u32 *data, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
@@ -1343,7 +1187,7 @@ nvd0_disp_intr(struct nouveau_subdev *subdev)
                 if (mask & intr) {
                         u32 stat = nv_rd32(priv, 0x6100bc + (i * 0x800));
                         if (stat & 0x00000001)
-                                nouveau_event_trigger(priv->base.vblank, 1, i);
+                                nouveau_disp_vblank(&priv->base, i);
                         nv_mask(priv, 0x6100bc + (i * 0x800), 0, 0);
                         nv_rd32(priv, 0x6100c0 + (i * 0x800));
                 }
@@ -1396,9 +1240,11 @@ nvd0_disp_oclass = &(struct nv50_disp_impl) {
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
+        .base.vblank = &nvd0_disp_vblank_func,
         .base.outp =  nvd0_disp_outp_sclass,
         .mthd.core = &nvd0_disp_mast_mthd_chan,
         .mthd.base = &nvd0_disp_sync_mthd_chan,
         .mthd.ovly = &nvd0_disp_ovly_mthd_chan,
         .mthd.prev = -0x020000,
+        .head.scanoutpos = nvd0_disp_base_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
index 11328e3f5df1..47fef1e398c4 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
@@ -25,7 +25,7 @@
 #include <engine/software.h>
 #include <engine/disp.h>
 
-#include <core/class.h>
+#include <nvif/class.h>
 
 #include "nv50.h"
 
@@ -200,17 +200,17 @@ nve0_disp_ovly_mthd_chan = {
 
 static struct nouveau_oclass
 nve0_disp_sclass[] = {
-        { NVE0_DISP_MAST_CLASS, &nvd0_disp_mast_ofuncs },
-        { NVE0_DISP_SYNC_CLASS, &nvd0_disp_sync_ofuncs },
-        { NVE0_DISP_OVLY_CLASS, &nvd0_disp_ovly_ofuncs },
-        { NVE0_DISP_OIMM_CLASS, &nvd0_disp_oimm_ofuncs },
-        { NVE0_DISP_CURS_CLASS, &nvd0_disp_curs_ofuncs },
+        { GK104_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
+        { GK104_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GK104_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
+        { GK104_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
+        { GK104_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
         {}
 };
 
 static struct nouveau_oclass
 nve0_disp_base_oclass[] = {
-        { NVE0_DISP_CLASS, &nvd0_disp_base_ofuncs, nvd0_disp_base_omthds },
+        { GK104_DISP, &nvd0_disp_base_ofuncs },
         {}
 };
 
@@ -258,9 +258,11 @@ nve0_disp_oclass = &(struct nv50_disp_impl) {
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
+        .base.vblank = &nvd0_disp_vblank_func,
         .base.outp =  nvd0_disp_outp_sclass,
         .mthd.core = &nve0_disp_mast_mthd_chan,
         .mthd.base = &nvd0_disp_sync_mthd_chan,
         .mthd.ovly = &nve0_disp_ovly_mthd_chan,
         .mthd.prev = -0x020000,
+        .head.scanoutpos = nvd0_disp_base_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
index 104388081d73..04bda4ac4ed3 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
@@ -25,7 +25,7 @@
 #include <engine/software.h>
 #include <engine/disp.h>
 
-#include <core/class.h>
+#include <nvif/class.h>
 
 #include "nv50.h"
 
@@ -35,17 +35,17 @@
 
 static struct nouveau_oclass
 nvf0_disp_sclass[] = {
-        { NVF0_DISP_MAST_CLASS, &nvd0_disp_mast_ofuncs },
-        { NVF0_DISP_SYNC_CLASS, &nvd0_disp_sync_ofuncs },
-        { NVF0_DISP_OVLY_CLASS, &nvd0_disp_ovly_ofuncs },
-        { NVF0_DISP_OIMM_CLASS, &nvd0_disp_oimm_ofuncs },
-        { NVF0_DISP_CURS_CLASS, &nvd0_disp_curs_ofuncs },
+        { GK110_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
+        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GK104_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
+        { GK104_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
+        { GK104_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
         {}
 };
 
 static struct nouveau_oclass
 nvf0_disp_base_oclass[] = {
-        { NVF0_DISP_CLASS, &nvd0_disp_base_ofuncs, nvd0_disp_base_omthds },
+        { GK110_DISP, &nvd0_disp_base_ofuncs },
         {}
 };
 
@@ -93,9 +93,11 @@ nvf0_disp_oclass = &(struct nv50_disp_impl) {
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
+        .base.vblank = &nvd0_disp_vblank_func,
         .base.outp =  nvd0_disp_outp_sclass,
         .mthd.core = &nve0_disp_mast_mthd_chan,
         .mthd.base = &nvd0_disp_sync_mthd_chan,
         .mthd.ovly = &nve0_disp_ovly_mthd_chan,
         .mthd.prev = -0x020000,
+        .head.scanoutpos = nvd0_disp_base_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/outp.c b/drivers/gpu/drm/nouveau/core/engine/disp/outp.c
index ad9ba7ccec7f..a5ff00a9cedc 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/outp.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/outp.c
@@ -78,6 +78,7 @@ nvkm_output_create_(struct nouveau_object *parent,
 
         outp->info = *dcbE;
         outp->index = index;
+        outp->or = ffs(outp->info.or) - 1;
 
         DBG("type %02x loc %d or %d link %d con %x edid %x bus %d head %x\n",
             dcbE->type, dcbE->location, dcbE->or, dcbE->type >= 2 ?
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/outp.h b/drivers/gpu/drm/nouveau/core/engine/disp/outp.h
index bc76fbf85710..187f435ad0e2 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/outp.h
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/outp.h
@@ -9,6 +9,7 @@ struct nvkm_output {
 
         struct dcb_output info;
         int index;
+        int or;
 
         struct nouveau_i2c_port *port;
         struct nouveau_i2c_port *edid;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/outpdp.c b/drivers/gpu/drm/nouveau/core/engine/disp/outpdp.c
index eb2d7789555d..6f6e2a898270 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/outpdp.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/outpdp.c
@@ -22,6 +22,9 @@
  * Authors: Ben Skeggs
  */
 
+#include <core/os.h>
+#include <nvif/event.h>
+
 #include <subdev/i2c.h>
 
 #include "outpdp.h"
@@ -86,7 +89,7 @@ done:
                 atomic_set(&outp->lt.done, 0);
                 schedule_work(&outp->lt.work);
         } else {
-                nouveau_event_get(outp->irq);
+                nvkm_notify_get(&outp->irq);
         }
 
         if (wait) {
@@ -133,46 +136,59 @@ nvkm_output_dp_detect(struct nvkm_output_dp *outp)
         }
 }
 
-static void
-nvkm_output_dp_service_work(struct work_struct *work)
+static int
+nvkm_output_dp_hpd(struct nvkm_notify *notify)
 {
-        struct nvkm_output_dp *outp = container_of(work, typeof(*outp), work);
-        struct nouveau_disp *disp = nouveau_disp(outp);
-        int type = atomic_xchg(&outp->pending, 0);
-        u32 send = 0;
-
-        if (type & (NVKM_I2C_PLUG | NVKM_I2C_UNPLUG)) {
-                nvkm_output_dp_detect(outp);
-                if (type & NVKM_I2C_UNPLUG)
-                        send |= NVKM_HPD_UNPLUG;
-                if (type & NVKM_I2C_PLUG)
-                        send |= NVKM_HPD_PLUG;
-                nouveau_event_get(outp->base.conn->hpd.event);
-        }
-
-        if (type & NVKM_I2C_IRQ) {
-                nvkm_output_dp_train(&outp->base, 0, true);
-                send |= NVKM_HPD_IRQ;
+        struct nvkm_connector *conn = container_of(notify, typeof(*conn), hpd);
+        struct nvkm_output_dp *outp;
+        struct nouveau_disp *disp = nouveau_disp(conn);
+        const struct nvkm_i2c_ntfy_rep *line = notify->data;
+        struct nvif_notify_conn_rep_v0 rep = {};
+
+        list_for_each_entry(outp, &disp->outp, base.head) {
+                if (outp->base.conn == conn &&
+                    outp->info.type == DCB_OUTPUT_DP) {
+                        DBG("HPD: %d\n", line->mask);
+                        nvkm_output_dp_detect(outp);
+
+                        if (line->mask & NVKM_I2C_UNPLUG)
+                                rep.mask |= NVIF_NOTIFY_CONN_V0_UNPLUG;
+                        if (line->mask & NVKM_I2C_PLUG)
+                                rep.mask |= NVIF_NOTIFY_CONN_V0_PLUG;
+
+                        nvkm_event_send(&disp->hpd, rep.mask, conn->index,
+                                        &rep, sizeof(rep));
+                        return NVKM_NOTIFY_KEEP;
+                }
         }
 
-        nouveau_event_trigger(disp->hpd, send, outp->base.info.connector);
+        WARN_ON(1);
+        return NVKM_NOTIFY_DROP;
 }
 
 static int
-nvkm_output_dp_service(void *data, u32 type, int index)
+nvkm_output_dp_irq(struct nvkm_notify *notify)
 {
-        struct nvkm_output_dp *outp = data;
-        DBG("HPD: %d\n", type);
-        atomic_or(type, &outp->pending);
-        schedule_work(&outp->work);
-        return NVKM_EVENT_DROP;
+        struct nvkm_output_dp *outp = container_of(notify, typeof(*outp), irq);
+        struct nouveau_disp *disp = nouveau_disp(outp);
+        const struct nvkm_i2c_ntfy_rep *line = notify->data;
+        struct nvif_notify_conn_rep_v0 rep = {
+                .mask = NVIF_NOTIFY_CONN_V0_IRQ,
+        };
+        int index = outp->base.info.connector;
+
+        DBG("IRQ: %d\n", line->mask);
+        nvkm_output_dp_train(&outp->base, 0, true);
+
+        nvkm_event_send(&disp->hpd, rep.mask, index, &rep, sizeof(rep));
+        return NVKM_NOTIFY_DROP;
 }
 
 int
 _nvkm_output_dp_fini(struct nouveau_object *object, bool suspend)
 {
         struct nvkm_output_dp *outp = (void *)object;
-        nouveau_event_put(outp->irq);
+        nvkm_notify_put(&outp->irq);
         nvkm_output_dp_enable(outp, false);
         return nvkm_output_fini(&outp->base, suspend);
 }
@@ -189,7 +205,7 @@ void
 _nvkm_output_dp_dtor(struct nouveau_object *object)
 {
         struct nvkm_output_dp *outp = (void *)object;
-        nouveau_event_ref(NULL, &outp->irq);
+        nvkm_notify_fini(&outp->irq);
         nvkm_output_destroy(&outp->base);
 }
 
@@ -213,7 +229,7 @@ nvkm_output_dp_create_(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
-        nouveau_event_ref(NULL, &outp->base.conn->hpd.event);
+        nvkm_notify_fini(&outp->base.conn->hpd);
 
         /* access to the aux channel is not optional... */
         if (!outp->base.edid) {
@@ -238,20 +254,28 @@ nvkm_output_dp_create_(struct nouveau_object *parent,
         atomic_set(&outp->lt.done, 0);
 
         /* link maintenance */
-        ret = nouveau_event_new(i2c->ntfy, NVKM_I2C_IRQ, outp->base.edid->index,
-                                nvkm_output_dp_service, outp, &outp->irq);
+        ret = nvkm_notify_init(&i2c->event, nvkm_output_dp_irq, true,
+                               &(struct nvkm_i2c_ntfy_req) {
+                                .mask = NVKM_I2C_IRQ,
+                                .port = outp->base.edid->index,
+                               },
+                               sizeof(struct nvkm_i2c_ntfy_req),
+                               sizeof(struct nvkm_i2c_ntfy_rep),
+                               &outp->irq);
         if (ret) {
                 ERR("error monitoring aux irq event: %d\n", ret);
                 return ret;
         }
 
-        INIT_WORK(&outp->work, nvkm_output_dp_service_work);
-
         /* hotplug detect, replaces gpio-based mechanism with aux events */
-        ret = nouveau_event_new(i2c->ntfy, NVKM_I2C_PLUG | NVKM_I2C_UNPLUG,
-                                outp->base.edid->index,
-                                nvkm_output_dp_service, outp,
-                               &outp->base.conn->hpd.event);
+        ret = nvkm_notify_init(&i2c->event, nvkm_output_dp_hpd, true,
+                               &(struct nvkm_i2c_ntfy_req) {
+                                .mask = NVKM_I2C_PLUG | NVKM_I2C_UNPLUG,
+                                .port = outp->base.edid->index,
+                               },
+                               sizeof(struct nvkm_i2c_ntfy_req),
+                               sizeof(struct nvkm_i2c_ntfy_rep),
+                               &outp->base.conn->hpd);
         if (ret) {
                 ERR("error monitoring aux hpd events: %d\n", ret);
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/outpdp.h b/drivers/gpu/drm/nouveau/core/engine/disp/outpdp.h
index ff33ba12cb67..1fac367cc867 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/outpdp.h
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/outpdp.h
@@ -12,10 +12,7 @@ struct nvkm_output_dp {
         struct nvbios_dpout info;
         u8 version;
 
-        struct nouveau_eventh *irq;
-        struct nouveau_eventh *hpd;
-        struct work_struct work;
-        atomic_t pending;
+        struct nvkm_notify irq;
         bool present;
         u8 dpcd[16];
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/piornv50.c b/drivers/gpu/drm/nouveau/core/engine/disp/piornv50.c
index fe0f256f11bf..d00f89a468a7 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/piornv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/piornv50.c
@@ -22,8 +22,9 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <subdev/bios.h>
 #include <subdev/bios/dcb.h>
@@ -143,38 +144,29 @@ nv50_pior_dp_impl = {
  *****************************************************************************/
 
 int
-nv50_pior_power(struct nv50_disp_priv *priv, int or, u32 data)
+nv50_pior_power(NV50_DISP_MTHD_V1)
 {
-        const u32 stat = data & NV50_DISP_PIOR_PWR_STATE;
-        const u32 soff = (or * 0x800);
+        const u32 soff = outp->or * 0x800;
+        union {
+                struct nv50_disp_pior_pwr_v0 v0;
+        } *args = data;
+        u32 ctrl, type;
+        int ret;
+
+        nv_ioctl(object, "disp pior pwr size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "disp pior pwr vers %d state %d type %x\n",
+                         args->v0.version, args->v0.state, args->v0.type);
+                if (args->v0.type > 0x0f)
+                        return -EINVAL;
+                ctrl = !!args->v0.state;
+                type = args->v0.type;
+        } else
+                return ret;
+
         nv_wait(priv, 0x61e004 + soff, 0x80000000, 0x00000000);
-        nv_mask(priv, 0x61e004 + soff, 0x80000101, 0x80000000 | stat);
+        nv_mask(priv, 0x61e004 + soff, 0x80000101, 0x80000000 | ctrl);
         nv_wait(priv, 0x61e004 + soff, 0x80000000, 0x00000000);
+        priv->pior.type[outp->or] = type;
         return 0;
 }
-
-int
-nv50_pior_mthd(struct nouveau_object *object, u32 mthd, void *args, u32 size)
-{
-        struct nv50_disp_priv *priv = (void *)object->engine;
-        const u8 type = (mthd & NV50_DISP_PIOR_MTHD_TYPE) >> 12;
-        const u8 or   = (mthd & NV50_DISP_PIOR_MTHD_OR);
-        u32 *data = args;
-        int ret;
-
-        if (size < sizeof(u32))
-                return -EINVAL;
-
-        mthd &= ~NV50_DISP_PIOR_MTHD_TYPE;
-        mthd &= ~NV50_DISP_PIOR_MTHD_OR;
-        switch (mthd) {
-        case NV50_DISP_PIOR_PWR:
-                ret = priv->pior.power(priv, or, data[0]);
-                priv->pior.type[or] = type;
-                break;
-        default:
-                return -EINVAL;
-        }
-
-        return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/priv.h b/drivers/gpu/drm/nouveau/core/engine/disp/priv.h
index 26e9a42569c7..dbd43ae9df81 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/priv.h
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/priv.h
@@ -11,6 +11,7 @@ struct nouveau_disp_impl {
         struct nouveau_oclass base;
         struct nouveau_oclass **outp;
         struct nouveau_oclass **conn;
+        const struct nvkm_event_func *vblank;
 };
 
 #define nouveau_disp_create(p,e,c,h,i,x,d)                                     \
@@ -39,4 +40,8 @@ int  _nouveau_disp_fini(struct nouveau_object *, bool);
 extern struct nouveau_oclass *nvkm_output_oclass;
 extern struct nouveau_oclass *nvkm_connector_oclass;
 
+int  nouveau_disp_vblank_ctor(void *data, u32 size, struct nvkm_notify *);
+void nouveau_disp_vblank(struct nouveau_disp *, int head);
+int  nouveau_disp_ntfy(struct nouveau_object *, u32, struct nvkm_event **);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/sornv50.c b/drivers/gpu/drm/nouveau/core/engine/disp/sornv50.c
index 7a1ebdfa9e1b..ddf1760c4400 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/sornv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/sornv50.c
@@ -22,8 +22,9 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <subdev/bios.h>
 #include <subdev/bios/dcb.h>
@@ -32,77 +33,26 @@
 #include "nv50.h"
 
 int
-nv50_sor_power(struct nv50_disp_priv *priv, int or, u32 data)
+nv50_sor_power(NV50_DISP_MTHD_V1)
 {
-        const u32 stat = data & NV50_DISP_SOR_PWR_STATE;
-        const u32 soff = (or * 0x800);
+        union {
+                struct nv50_disp_sor_pwr_v0 v0;
+        } *args = data;
+        const u32 soff = outp->or * 0x800;
+        u32 stat;
+        int ret;
+
+        nv_ioctl(object, "disp sor pwr size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "disp sor pwr vers %d state %d\n",
+                         args->v0.version, args->v0.state);
+                stat = !!args->v0.state;
+        } else
+                return ret;
+
         nv_wait(priv, 0x61c004 + soff, 0x80000000, 0x00000000);
         nv_mask(priv, 0x61c004 + soff, 0x80000001, 0x80000000 | stat);
         nv_wait(priv, 0x61c004 + soff, 0x80000000, 0x00000000);
         nv_wait(priv, 0x61c030 + soff, 0x10000000, 0x00000000);
         return 0;
 }
-
-int
-nv50_sor_mthd(struct nouveau_object *object, u32 mthd, void *args, u32 size)
-{
-        struct nv50_disp_priv *priv = (void *)object->engine;
-        const u8  type = (mthd & NV50_DISP_SOR_MTHD_TYPE) >> 12;
-        const u8  head = (mthd & NV50_DISP_SOR_MTHD_HEAD) >> 3;
-        const u8  link = (mthd & NV50_DISP_SOR_MTHD_LINK) >> 2;
-        const u8    or = (mthd & NV50_DISP_SOR_MTHD_OR);
-        const u16 mask = (0x0100 << head) | (0x0040 << link) | (0x0001 << or);
-        struct nvkm_output *outp = NULL, *temp;
-        u32 data;
-        int ret = -EINVAL;
-
-        if (size < sizeof(u32))
-                return -EINVAL;
-        data = *(u32 *)args;
-
-        list_for_each_entry(temp, &priv->base.outp, head) {
-                if ((temp->info.hasht & 0xff) == type &&
-                    (temp->info.hashm & mask) == mask) {
-                        outp = temp;
-                        break;
-                }
-        }
-
-        switch (mthd & ~0x3f) {
-        case NV50_DISP_SOR_PWR:
-                ret = priv->sor.power(priv, or, data);
-                break;
-        case NVA3_DISP_SOR_HDA_ELD:
-                ret = priv->sor.hda_eld(priv, or, args, size);
-                break;
-        case NV84_DISP_SOR_HDMI_PWR:
-                ret = priv->sor.hdmi(priv, head, or, data);
-                break;
-        case NV50_DISP_SOR_LVDS_SCRIPT:
-                priv->sor.lvdsconf = data & NV50_DISP_SOR_LVDS_SCRIPT_ID;
-                ret = 0;
-                break;
-        case NV94_DISP_SOR_DP_PWR:
-                if (outp) {
-                        struct nvkm_output_dp *outpdp = (void *)outp;
-                        switch (data) {
-                        case NV94_DISP_SOR_DP_PWR_STATE_OFF:
-                                nouveau_event_put(outpdp->irq);
-                                ((struct nvkm_output_dp_impl *)nv_oclass(outp))
-                                        ->lnk_pwr(outpdp, 0);
-                                atomic_set(&outpdp->lt.done, 0);
-                                break;
-                        case NV94_DISP_SOR_DP_PWR_STATE_ON:
-                                nvkm_output_dp_train(&outpdp->base, 0, true);
-                                break;
-                        default:
-                                return -EINVAL;
-                        }
-                }
-                break;
-        default:
-                BUG_ON(1);
-        }
-
-        return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/sornv94.c b/drivers/gpu/drm/nouveau/core/engine/disp/sornv94.c
index 05487cda84a8..39f85d627336 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/sornv94.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/sornv94.c
@@ -23,7 +23,6 @@
  */
 
 #include <core/os.h>
-#include <core/class.h>
 
 #include <subdev/bios.h>
 #include <subdev/bios/dcb.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
index 97f0e9cd3d40..7b7bbc3e459e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
@@ -23,7 +23,6 @@
  */
 
 #include <core/os.h>
-#include <core/class.h>
 
 #include <subdev/bios.h>
 #include <subdev/bios/dcb.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/base.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/base.c
index 5103e88d1877..e1500f77a56a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/dmaobj/base.c
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/base.c
@@ -23,98 +23,143 @@
  */
 
 #include <core/object.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <subdev/fb.h>
-#include <engine/dmaobj.h>
+#include <subdev/instmem.h>
+
+#include "priv.h"
 
 static int
-nouveau_dmaobj_ctor(struct nouveau_object *parent,
+nvkm_dmaobj_bind(struct nouveau_dmaobj *dmaobj, struct nouveau_object *parent,
+                 struct nouveau_gpuobj **pgpuobj)
+{
+        const struct nvkm_dmaeng_impl *impl = (void *)
+                nv_oclass(nv_object(dmaobj)->engine);
+        int ret = 0;
+
+        if (nv_object(dmaobj) == parent) { /* ctor bind */
+                if (nv_mclass(parent->parent) == NV_DEVICE) {
+                        /* delayed, or no, binding */
+                        return 0;
+                }
+                ret = impl->bind(dmaobj, parent, pgpuobj);
+                if (ret == 0)
+                        nouveau_object_ref(NULL, &parent);
+                return ret;
+        }
+
+        return impl->bind(dmaobj, parent, pgpuobj);
+}
+
+int
+nvkm_dmaobj_create_(struct nouveau_object *parent,
                     struct nouveau_object *engine,
-                    struct nouveau_oclass *oclass, void *data, u32 size,
-                    struct nouveau_object **pobject)
+                    struct nouveau_oclass *oclass, void **pdata, u32 *psize,
+                    int length, void **pobject)
 {
-        struct nouveau_dmaeng *dmaeng = (void *)engine;
+        union {
+                struct nv_dma_v0 v0;
+        } *args = *pdata;
+        struct nouveau_instmem *instmem = nouveau_instmem(parent);
+        struct nouveau_client *client = nouveau_client(parent);
+        struct nouveau_device *device = nv_device(parent);
+        struct nouveau_fb *pfb = nouveau_fb(parent);
         struct nouveau_dmaobj *dmaobj;
-        struct nouveau_gpuobj *gpuobj;
-        struct nv_dma_class *args = data;
+        void *data = *pdata;
+        u32 size = *psize;
         int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
-
-        ret = nouveau_object_create(parent, engine, oclass, 0, &dmaobj);
-        *pobject = nv_object(dmaobj);
+        ret = nouveau_object_create_(parent, engine, oclass, 0, length, pobject);
+        dmaobj = *pobject;
         if (ret)
                 return ret;
 
-        switch (args->flags & NV_DMA_TARGET_MASK) {
-        case NV_DMA_TARGET_VM:
+        nv_ioctl(parent, "create dma size %d\n", *psize);
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                nv_ioctl(parent, "create dma vers %d target %d access %d "
+                                 "start %016llx limit %016llx\n",
+                         args->v0.version, args->v0.target, args->v0.access,
+                         args->v0.start, args->v0.limit);
+                dmaobj->target = args->v0.target;
+                dmaobj->access = args->v0.access;
+                dmaobj->start  = args->v0.start;
+                dmaobj->limit  = args->v0.limit;
+        } else
+                return ret;
+
+        *pdata = data;
+        *psize = size;
+
+        if (dmaobj->start > dmaobj->limit)
+                return -EINVAL;
+
+        switch (dmaobj->target) {
+        case NV_DMA_V0_TARGET_VM:
                 dmaobj->target = NV_MEM_TARGET_VM;
                 break;
-        case NV_DMA_TARGET_VRAM:
+        case NV_DMA_V0_TARGET_VRAM:
+                if (!client->super) {
+                        if (dmaobj->limit >= pfb->ram->size - instmem->reserved)
+                                return -EACCES;
+                        if (device->card_type >= NV_50)
+                                return -EACCES;
+                }
                 dmaobj->target = NV_MEM_TARGET_VRAM;
                 break;
-        case NV_DMA_TARGET_PCI:
+        case NV_DMA_V0_TARGET_PCI:
+                if (!client->super)
+                        return -EACCES;
                 dmaobj->target = NV_MEM_TARGET_PCI;
                 break;
-        case NV_DMA_TARGET_PCI_US:
-        case NV_DMA_TARGET_AGP:
+        case NV_DMA_V0_TARGET_PCI_US:
+        case NV_DMA_V0_TARGET_AGP:
+                if (!client->super)
+                        return -EACCES;
                 dmaobj->target = NV_MEM_TARGET_PCI_NOSNOOP;
                 break;
         default:
                 return -EINVAL;
         }
 
-        switch (args->flags & NV_DMA_ACCESS_MASK) {
-        case NV_DMA_ACCESS_VM:
+        switch (dmaobj->access) {
+        case NV_DMA_V0_ACCESS_VM:
                 dmaobj->access = NV_MEM_ACCESS_VM;
                 break;
-        case NV_DMA_ACCESS_RD:
+        case NV_DMA_V0_ACCESS_RD:
                 dmaobj->access = NV_MEM_ACCESS_RO;
                 break;
-        case NV_DMA_ACCESS_WR:
+        case NV_DMA_V0_ACCESS_WR:
                 dmaobj->access = NV_MEM_ACCESS_WO;
                 break;
-        case NV_DMA_ACCESS_RDWR:
+        case NV_DMA_V0_ACCESS_RDWR:
                 dmaobj->access = NV_MEM_ACCESS_RW;
                 break;
         default:
                 return -EINVAL;
         }
 
-        dmaobj->start = args->start;
-        dmaobj->limit = args->limit;
-        dmaobj->conf0 = args->conf0;
-
-        switch (nv_mclass(parent)) {
-        case NV_DEVICE_CLASS:
-                /* delayed, or no, binding */
-                break;
-        default:
-                ret = dmaeng->bind(dmaeng, *pobject, dmaobj, &gpuobj);
-                if (ret == 0) {
-                        nouveau_object_ref(NULL, pobject);
-                        *pobject = nv_object(gpuobj);
-                }
-                break;
-        }
-
         return ret;
 }
 
-static struct nouveau_ofuncs
-nouveau_dmaobj_ofuncs = {
-        .ctor = nouveau_dmaobj_ctor,
-        .dtor = nouveau_object_destroy,
-        .init = nouveau_object_init,
-        .fini = nouveau_object_fini,
-};
-
-struct nouveau_oclass
-nouveau_dmaobj_sclass[] = {
-        { NV_DMA_FROM_MEMORY_CLASS, &nouveau_dmaobj_ofuncs },
-        { NV_DMA_TO_MEMORY_CLASS, &nouveau_dmaobj_ofuncs },
-        { NV_DMA_IN_MEMORY_CLASS, &nouveau_dmaobj_ofuncs },
-        {}
-};
+int
+_nvkm_dmaeng_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                  struct nouveau_oclass *oclass, void *data, u32 size,
+                  struct nouveau_object **pobject)
+{
+        const struct nvkm_dmaeng_impl *impl = (void *)oclass;
+        struct nouveau_dmaeng *dmaeng;
+        int ret;
+
+        ret = nouveau_engine_create(parent, engine, oclass, true, "DMAOBJ",
+                                    "dmaobj", &dmaeng);
+        *pobject = nv_object(dmaeng);
+        if (ret)
+                return ret;
+
+        nv_engine(dmaeng)->sclass = impl->sclass;
+        dmaeng->bind = nvkm_dmaobj_bind;
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv04.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv04.c
index 027d8217c0fa..20c9dbfe3b2e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv04.c
@@ -23,121 +23,143 @@
  */
 
 #include <core/gpuobj.h>
-#include <core/class.h>
+#include <nvif/class.h>
 
 #include <subdev/fb.h>
 #include <subdev/vm/nv04.h>
 
-#include <engine/dmaobj.h>
+#include "priv.h"
 
-struct nv04_dmaeng_priv {
-        struct nouveau_dmaeng base;
+struct nv04_dmaobj_priv {
+        struct nouveau_dmaobj base;
+        bool clone;
+        u32 flags0;
+        u32 flags2;
 };
 
 static int
-nv04_dmaobj_bind(struct nouveau_dmaeng *dmaeng,
+nv04_dmaobj_bind(struct nouveau_dmaobj *dmaobj,
                  struct nouveau_object *parent,
-                 struct nouveau_dmaobj *dmaobj,
                  struct nouveau_gpuobj **pgpuobj)
 {
-        struct nv04_vmmgr_priv *vmm = nv04_vmmgr(dmaeng);
+        struct nv04_dmaobj_priv *priv = (void *)dmaobj;
         struct nouveau_gpuobj *gpuobj;
-        u32 flags0 = nv_mclass(dmaobj);
-        u32 flags2 = 0x00000000;
-        u64 offset = dmaobj->start & 0xfffff000;
-        u64 adjust = dmaobj->start & 0x00000fff;
-        u32 length = dmaobj->limit - dmaobj->start;
+        u64 offset = priv->base.start & 0xfffff000;
+        u64 adjust = priv->base.start & 0x00000fff;
+        u32 length = priv->base.limit - priv->base.start;
         int ret;
 
         if (!nv_iclass(parent, NV_ENGCTX_CLASS)) {
                 switch (nv_mclass(parent->parent)) {
-                case NV03_CHANNEL_DMA_CLASS:
-                case NV10_CHANNEL_DMA_CLASS:
-                case NV17_CHANNEL_DMA_CLASS:
-                case NV40_CHANNEL_DMA_CLASS:
+                case NV03_CHANNEL_DMA:
+                case NV10_CHANNEL_DMA:
+                case NV17_CHANNEL_DMA:
+                case NV40_CHANNEL_DMA:
                         break;
                 default:
                         return -EINVAL;
                 }
         }
 
-        if (dmaobj->target == NV_MEM_TARGET_VM) {
-                if (nv_object(vmm)->oclass == &nv04_vmmgr_oclass) {
-                        struct nouveau_gpuobj *pgt = vmm->vm->pgt[0].obj[0];
-                        if (!dmaobj->start)
-                                return nouveau_gpuobj_dup(parent, pgt, pgpuobj);
-                        offset  = nv_ro32(pgt, 8 + (offset >> 10));
-                        offset &= 0xfffff000;
-                }
+        if (priv->clone) {
+                struct nv04_vmmgr_priv *vmm = nv04_vmmgr(dmaobj);
+                struct nouveau_gpuobj *pgt = vmm->vm->pgt[0].obj[0];
+                if (!dmaobj->start)
+                        return nouveau_gpuobj_dup(parent, pgt, pgpuobj);
+                offset  = nv_ro32(pgt, 8 + (offset >> 10));
+                offset &= 0xfffff000;
+        }
+
+        ret = nouveau_gpuobj_new(parent, parent, 16, 16, 0, &gpuobj);
+        *pgpuobj = gpuobj;
+        if (ret == 0) {
+                nv_wo32(*pgpuobj, 0x00, priv->flags0 | (adjust << 20));
+                nv_wo32(*pgpuobj, 0x04, length);
+                nv_wo32(*pgpuobj, 0x08, priv->flags2 | offset);
+                nv_wo32(*pgpuobj, 0x0c, priv->flags2 | offset);
+        }
+
+        return ret;
+}
+
+static int
+nv04_dmaobj_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                 struct nouveau_oclass *oclass, void *data, u32 size,
+                 struct nouveau_object **pobject)
+{
+        struct nouveau_dmaeng *dmaeng = (void *)engine;
+        struct nv04_vmmgr_priv *vmm = nv04_vmmgr(engine);
+        struct nv04_dmaobj_priv *priv;
+        int ret;
+
+        ret = nvkm_dmaobj_create(parent, engine, oclass, &data, &size, &priv);
+        *pobject = nv_object(priv);
+        if (ret || (ret = -ENOSYS, size))
+                return ret;
 
-                dmaobj->target = NV_MEM_TARGET_PCI;
-                dmaobj->access = NV_MEM_ACCESS_RW;
+        if (priv->base.target == NV_MEM_TARGET_VM) {
+                if (nv_object(vmm)->oclass == &nv04_vmmgr_oclass)
+                        priv->clone = true;
+                priv->base.target = NV_MEM_TARGET_PCI;
+                priv->base.access = NV_MEM_ACCESS_RW;
         }
 
-        switch (dmaobj->target) {
+        priv->flags0 = nv_mclass(priv);
+        switch (priv->base.target) {
         case NV_MEM_TARGET_VRAM:
-                flags0 |= 0x00003000;
+                priv->flags0 |= 0x00003000;
                 break;
         case NV_MEM_TARGET_PCI:
-                flags0 |= 0x00023000;
+                priv->flags0 |= 0x00023000;
                 break;
         case NV_MEM_TARGET_PCI_NOSNOOP:
-                flags0 |= 0x00033000;
+                priv->flags0 |= 0x00033000;
                 break;
         default:
                 return -EINVAL;
         }
 
-        switch (dmaobj->access) {
+        switch (priv->base.access) {
         case NV_MEM_ACCESS_RO:
-                flags0 |= 0x00004000;
+                priv->flags0 |= 0x00004000;
                 break;
         case NV_MEM_ACCESS_WO:
-                flags0 |= 0x00008000;
+                priv->flags0 |= 0x00008000;
         case NV_MEM_ACCESS_RW:
-                flags2 |= 0x00000002;
+                priv->flags2 |= 0x00000002;
                 break;
         default:
                 return -EINVAL;
         }
 
-        ret = nouveau_gpuobj_new(parent, parent, 16, 16, 0, &gpuobj);
-        *pgpuobj = gpuobj;
-        if (ret == 0) {
-                nv_wo32(*pgpuobj, 0x00, flags0 | (adjust << 20));
-                nv_wo32(*pgpuobj, 0x04, length);
-                nv_wo32(*pgpuobj, 0x08, flags2 | offset);
-                nv_wo32(*pgpuobj, 0x0c, flags2 | offset);
-        }
-
-        return ret;
+        return dmaeng->bind(&priv->base, nv_object(priv), (void *)pobject);
 }
 
-static int
-nv04_dmaeng_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-                 struct nouveau_oclass *oclass, void *data, u32 size,
-                 struct nouveau_object **pobject)
-{
-        struct nv04_dmaeng_priv *priv;
-        int ret;
-
-        ret = nouveau_dmaeng_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
+static struct nouveau_ofuncs
+nv04_dmaobj_ofuncs = {
+        .ctor =  nv04_dmaobj_ctor,
+        .dtor = _nvkm_dmaobj_dtor,
+        .init = _nvkm_dmaobj_init,
+        .fini = _nvkm_dmaobj_fini,
+};
 
-        nv_engine(priv)->sclass = nouveau_dmaobj_sclass;
-        priv->base.bind = nv04_dmaobj_bind;
-        return 0;
-}
+static struct nouveau_oclass
+nv04_dmaeng_sclass[] = {
+        { NV_DMA_FROM_MEMORY, &nv04_dmaobj_ofuncs },
+        { NV_DMA_TO_MEMORY, &nv04_dmaobj_ofuncs },
+        { NV_DMA_IN_MEMORY, &nv04_dmaobj_ofuncs },
+        {}
+};
 
-struct nouveau_oclass
-nv04_dmaeng_oclass = {
-        .handle = NV_ENGINE(DMAOBJ, 0x04),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv04_dmaeng_ctor,
-                .dtor = _nouveau_dmaeng_dtor,
-                .init = _nouveau_dmaeng_init,
-                .fini = _nouveau_dmaeng_fini,
+struct nouveau_oclass *
+nv04_dmaeng_oclass = &(struct nvkm_dmaeng_impl) {
+        .base.handle = NV_ENGINE(DMAOBJ, 0x04),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nvkm_dmaeng_ctor,
+                .dtor = _nvkm_dmaeng_dtor,
+                .init = _nvkm_dmaeng_init,
+                .fini = _nvkm_dmaeng_fini,
         },
-};
+        .sclass = nv04_dmaeng_sclass,
+        .bind = nv04_dmaobj_bind,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv50.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv50.c
index 750183f7c057..a740ddba2ee2 100644
--- a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv50.c
@@ -22,140 +22,176 @@
  * Authors: Ben Skeggs
  */
 
+#include <core/client.h>
 #include <core/gpuobj.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <subdev/fb.h>
-#include <engine/dmaobj.h>
 
-struct nv50_dmaeng_priv {
-        struct nouveau_dmaeng base;
+#include "priv.h"
+
+struct nv50_dmaobj_priv {
+        struct nouveau_dmaobj base;
+        u32 flags0;
+        u32 flags5;
 };
 
 static int
-nv50_dmaobj_bind(struct nouveau_dmaeng *dmaeng,
+nv50_dmaobj_bind(struct nouveau_dmaobj *dmaobj,
                  struct nouveau_object *parent,
-                 struct nouveau_dmaobj *dmaobj,
                  struct nouveau_gpuobj **pgpuobj)
 {
-        u32 flags0 = nv_mclass(dmaobj);
-        u32 flags5 = 0x00000000;
+        struct nv50_dmaobj_priv *priv = (void *)dmaobj;
         int ret;
 
         if (!nv_iclass(parent, NV_ENGCTX_CLASS)) {
                 switch (nv_mclass(parent->parent)) {
-                case NV50_CHANNEL_DMA_CLASS:
-                case NV84_CHANNEL_DMA_CLASS:
-                case NV50_CHANNEL_IND_CLASS:
-                case NV84_CHANNEL_IND_CLASS:
-                case NV50_DISP_MAST_CLASS:
-                case NV84_DISP_MAST_CLASS:
-                case NV94_DISP_MAST_CLASS:
-                case NVA0_DISP_MAST_CLASS:
-                case NVA3_DISP_MAST_CLASS:
-                case NV50_DISP_SYNC_CLASS:
-                case NV84_DISP_SYNC_CLASS:
-                case NV94_DISP_SYNC_CLASS:
-                case NVA0_DISP_SYNC_CLASS:
-                case NVA3_DISP_SYNC_CLASS:
-                case NV50_DISP_OVLY_CLASS:
-                case NV84_DISP_OVLY_CLASS:
-                case NV94_DISP_OVLY_CLASS:
-                case NVA0_DISP_OVLY_CLASS:
-                case NVA3_DISP_OVLY_CLASS:
+                case NV40_CHANNEL_DMA:
+                case NV50_CHANNEL_GPFIFO:
+                case G82_CHANNEL_GPFIFO:
+                case NV50_DISP_CORE_CHANNEL_DMA:
+                case G82_DISP_CORE_CHANNEL_DMA:
+                case GT206_DISP_CORE_CHANNEL_DMA:
+                case GT200_DISP_CORE_CHANNEL_DMA:
+                case GT214_DISP_CORE_CHANNEL_DMA:
+                case NV50_DISP_BASE_CHANNEL_DMA:
+                case G82_DISP_BASE_CHANNEL_DMA:
+                case GT200_DISP_BASE_CHANNEL_DMA:
+                case GT214_DISP_BASE_CHANNEL_DMA:
+                case NV50_DISP_OVERLAY_CHANNEL_DMA:
+                case G82_DISP_OVERLAY_CHANNEL_DMA:
+                case GT200_DISP_OVERLAY_CHANNEL_DMA:
+                case GT214_DISP_OVERLAY_CHANNEL_DMA:
                         break;
                 default:
                         return -EINVAL;
                 }
         }
 
-        if (!(dmaobj->conf0 & NV50_DMA_CONF0_ENABLE)) {
-                if (dmaobj->target == NV_MEM_TARGET_VM) {
-                        dmaobj->conf0  = NV50_DMA_CONF0_PRIV_VM;
-                        dmaobj->conf0 |= NV50_DMA_CONF0_PART_VM;
-                        dmaobj->conf0 |= NV50_DMA_CONF0_COMP_VM;
-                        dmaobj->conf0 |= NV50_DMA_CONF0_TYPE_VM;
+        ret = nouveau_gpuobj_new(parent, parent, 24, 32, 0, pgpuobj);
+        if (ret == 0) {
+                nv_wo32(*pgpuobj, 0x00, priv->flags0 | nv_mclass(dmaobj));
+                nv_wo32(*pgpuobj, 0x04, lower_32_bits(priv->base.limit));
+                nv_wo32(*pgpuobj, 0x08, lower_32_bits(priv->base.start));
+                nv_wo32(*pgpuobj, 0x0c, upper_32_bits(priv->base.limit) << 24 |
+                                        upper_32_bits(priv->base.start));
+                nv_wo32(*pgpuobj, 0x10, 0x00000000);
+                nv_wo32(*pgpuobj, 0x14, priv->flags5);
+        }
+
+        return ret;
+}
+
+static int
+nv50_dmaobj_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                 struct nouveau_oclass *oclass, void *data, u32 size,
+                 struct nouveau_object **pobject)
+{
+        struct nouveau_dmaeng *dmaeng = (void *)engine;
+        union {
+                struct nv50_dma_v0 v0;
+        } *args;
+        struct nv50_dmaobj_priv *priv;
+        u32 user, part, comp, kind;
+        int ret;
+
+        ret = nvkm_dmaobj_create(parent, engine, oclass, &data, &size, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+        args = data;
+
+        nv_ioctl(parent, "create nv50 dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create nv50 dma vers %d priv %d part %d "
+                                 "comp %d kind %02x\n", args->v0.version,
+                         args->v0.priv, args->v0.part, args->v0.comp,
+                         args->v0.kind);
+                user = args->v0.priv;
+                part = args->v0.part;
+                comp = args->v0.comp;
+                kind = args->v0.kind;
+        } else
+        if (size == 0) {
+                if (priv->base.target != NV_MEM_TARGET_VM) {
+                        user = NV50_DMA_V0_PRIV_US;
+                        part = NV50_DMA_V0_PART_256;
+                        comp = NV50_DMA_V0_COMP_NONE;
+                        kind = NV50_DMA_V0_KIND_PITCH;
                 } else {
-                        dmaobj->conf0  = NV50_DMA_CONF0_PRIV_US;
-                        dmaobj->conf0 |= NV50_DMA_CONF0_PART_256;
-                        dmaobj->conf0 |= NV50_DMA_CONF0_COMP_NONE;
-                        dmaobj->conf0 |= NV50_DMA_CONF0_TYPE_LINEAR;
+                        user = NV50_DMA_V0_PRIV_VM;
+                        part = NV50_DMA_V0_PART_VM;
+                        comp = NV50_DMA_V0_COMP_VM;
+                        kind = NV50_DMA_V0_KIND_VM;
                 }
-        }
+        } else
+                return ret;
 
-        flags0 |= (dmaobj->conf0 & NV50_DMA_CONF0_COMP) << 22;
-        flags0 |= (dmaobj->conf0 & NV50_DMA_CONF0_TYPE) << 22;
-        flags0 |= (dmaobj->conf0 & NV50_DMA_CONF0_PRIV);
-        flags5 |= (dmaobj->conf0 & NV50_DMA_CONF0_PART);
+        if (user > 2 || part > 2 || comp > 3 || kind > 0x7f)
+                return -EINVAL;
+        priv->flags0 = (comp << 29) | (kind << 22) | (user << 20);
+        priv->flags5 = (part << 16);
 
-        switch (dmaobj->target) {
+        switch (priv->base.target) {
         case NV_MEM_TARGET_VM:
-                flags0 |= 0x00000000;
+                priv->flags0 |= 0x00000000;
                 break;
         case NV_MEM_TARGET_VRAM:
-                flags0 |= 0x00010000;
+                priv->flags0 |= 0x00010000;
                 break;
         case NV_MEM_TARGET_PCI:
-                flags0 |= 0x00020000;
+                priv->flags0 |= 0x00020000;
                 break;
         case NV_MEM_TARGET_PCI_NOSNOOP:
-                flags0 |= 0x00030000;
+                priv->flags0 |= 0x00030000;
                 break;
         default:
                 return -EINVAL;
         }
 
-        switch (dmaobj->access) {
+        switch (priv->base.access) {
         case NV_MEM_ACCESS_VM:
                 break;
         case NV_MEM_ACCESS_RO:
-                flags0 |= 0x00040000;
+                priv->flags0 |= 0x00040000;
                 break;
         case NV_MEM_ACCESS_WO:
         case NV_MEM_ACCESS_RW:
-                flags0 |= 0x00080000;
+                priv->flags0 |= 0x00080000;
                 break;
+        default:
+                return -EINVAL;
         }
 
-        ret = nouveau_gpuobj_new(parent, parent, 24, 32, 0, pgpuobj);
-        if (ret == 0) {
-                nv_wo32(*pgpuobj, 0x00, flags0);
-                nv_wo32(*pgpuobj, 0x04, lower_32_bits(dmaobj->limit));
-                nv_wo32(*pgpuobj, 0x08, lower_32_bits(dmaobj->start));
-                nv_wo32(*pgpuobj, 0x0c, upper_32_bits(dmaobj->limit) << 24 |
-                                        upper_32_bits(dmaobj->start));
-                nv_wo32(*pgpuobj, 0x10, 0x00000000);
-                nv_wo32(*pgpuobj, 0x14, flags5);
-        }
-
-        return ret;
+        return dmaeng->bind(&priv->base, nv_object(priv), (void *)pobject);
 }
 
-static int
-nv50_dmaeng_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-                 struct nouveau_oclass *oclass, void *data, u32 size,
-                 struct nouveau_object **pobject)
-{
-        struct nv50_dmaeng_priv *priv;
-        int ret;
-
-        ret = nouveau_dmaeng_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
+static struct nouveau_ofuncs
+nv50_dmaobj_ofuncs = {
+        .ctor =  nv50_dmaobj_ctor,
+        .dtor = _nvkm_dmaobj_dtor,
+        .init = _nvkm_dmaobj_init,
+        .fini = _nvkm_dmaobj_fini,
+};
 
-        nv_engine(priv)->sclass = nouveau_dmaobj_sclass;
-        priv->base.bind = nv50_dmaobj_bind;
-        return 0;
-}
+static struct nouveau_oclass
+nv50_dmaeng_sclass[] = {
+        { NV_DMA_FROM_MEMORY, &nv50_dmaobj_ofuncs },
+        { NV_DMA_TO_MEMORY, &nv50_dmaobj_ofuncs },
+        { NV_DMA_IN_MEMORY, &nv50_dmaobj_ofuncs },
+        {}
+};
 
-struct nouveau_oclass
-nv50_dmaeng_oclass = {
-        .handle = NV_ENGINE(DMAOBJ, 0x50),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv50_dmaeng_ctor,
-                .dtor = _nouveau_dmaeng_dtor,
-                .init = _nouveau_dmaeng_init,
-                .fini = _nouveau_dmaeng_fini,
+struct nouveau_oclass *
+nv50_dmaeng_oclass = &(struct nvkm_dmaeng_impl) {
+        .base.handle = NV_ENGINE(DMAOBJ, 0x50),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nvkm_dmaeng_ctor,
+                .dtor = _nvkm_dmaeng_dtor,
+                .init = _nvkm_dmaeng_init,
+                .fini = _nvkm_dmaeng_fini,
         },
-};
+        .sclass = nv50_dmaeng_sclass,
+        .bind = nv50_dmaobj_bind,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvc0.c
index cd3970d03b80..88ec33b20048 100644
--- a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvc0.c
@@ -22,32 +22,35 @@
  * Authors: Ben Skeggs
  */
 
+#include <core/client.h>
 #include <core/device.h>
 #include <core/gpuobj.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <subdev/fb.h>
-#include <engine/dmaobj.h>
 
-struct nvc0_dmaeng_priv {
-        struct nouveau_dmaeng base;
+#include "priv.h"
+
+struct nvc0_dmaobj_priv {
+        struct nouveau_dmaobj base;
+        u32 flags0;
+        u32 flags5;
 };
 
 static int
-nvc0_dmaobj_bind(struct nouveau_dmaeng *dmaeng,
+nvc0_dmaobj_bind(struct nouveau_dmaobj *dmaobj,
                  struct nouveau_object *parent,
-                 struct nouveau_dmaobj *dmaobj,
                  struct nouveau_gpuobj **pgpuobj)
 {
-        u32 flags0 = nv_mclass(dmaobj);
-        u32 flags5 = 0x00000000;
+        struct nvc0_dmaobj_priv *priv = (void *)dmaobj;
         int ret;
 
         if (!nv_iclass(parent, NV_ENGCTX_CLASS)) {
                 switch (nv_mclass(parent->parent)) {
-                case NVA3_DISP_MAST_CLASS:
-                case NVA3_DISP_SYNC_CLASS:
-                case NVA3_DISP_OVLY_CLASS:
+                case GT214_DISP_CORE_CHANNEL_DMA:
+                case GT214_DISP_BASE_CHANNEL_DMA:
+                case GT214_DISP_OVERLAY_CHANNEL_DMA:
                         break;
                 default:
                         return -EINVAL;
@@ -55,89 +58,122 @@ nvc0_dmaobj_bind(struct nouveau_dmaeng *dmaeng,
         } else
                 return 0;
 
-        if (!(dmaobj->conf0 & NVC0_DMA_CONF0_ENABLE)) {
-                if (dmaobj->target == NV_MEM_TARGET_VM) {
-                        dmaobj->conf0  = NVC0_DMA_CONF0_PRIV_VM;
-                        dmaobj->conf0 |= NVC0_DMA_CONF0_TYPE_VM;
+        ret = nouveau_gpuobj_new(parent, parent, 24, 32, 0, pgpuobj);
+        if (ret == 0) {
+                nv_wo32(*pgpuobj, 0x00, priv->flags0 | nv_mclass(dmaobj));
+                nv_wo32(*pgpuobj, 0x04, lower_32_bits(priv->base.limit));
+                nv_wo32(*pgpuobj, 0x08, lower_32_bits(priv->base.start));
+                nv_wo32(*pgpuobj, 0x0c, upper_32_bits(priv->base.limit) << 24 |
+                                        upper_32_bits(priv->base.start));
+                nv_wo32(*pgpuobj, 0x10, 0x00000000);
+                nv_wo32(*pgpuobj, 0x14, priv->flags5);
+        }
+
+        return ret;
+}
+
+static int
+nvc0_dmaobj_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                 struct nouveau_oclass *oclass, void *data, u32 size,
+                 struct nouveau_object **pobject)
+{
+        struct nouveau_dmaeng *dmaeng = (void *)engine;
+        union {
+                struct gf100_dma_v0 v0;
+        } *args;
+        struct nvc0_dmaobj_priv *priv;
+        u32 kind, user, unkn;
+        int ret;
+
+        ret = nvkm_dmaobj_create(parent, engine, oclass, &data, &size, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+        args = data;
+
+        nv_ioctl(parent, "create gf100 dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create gf100 dma vers %d priv %d kind %02x\n",
+                         args->v0.version, args->v0.priv, args->v0.kind);
+                kind = args->v0.kind;
+                user = args->v0.priv;
+                unkn = 0;
+        } else
+        if (size == 0) {
+                if (priv->base.target != NV_MEM_TARGET_VM) {
+                        kind = GF100_DMA_V0_KIND_PITCH;
+                        user = GF100_DMA_V0_PRIV_US;
+                        unkn = 2;
                 } else {
-                        dmaobj->conf0  = NVC0_DMA_CONF0_PRIV_US;
-                        dmaobj->conf0 |= NVC0_DMA_CONF0_TYPE_LINEAR;
-                        dmaobj->conf0 |= 0x00020000;
+                        kind = GF100_DMA_V0_KIND_VM;
+                        user = GF100_DMA_V0_PRIV_VM;
+                        unkn = 0;
                 }
-        }
+        } else
+                return ret;
 
-        flags0 |= (dmaobj->conf0 & NVC0_DMA_CONF0_TYPE) << 22;
-        flags0 |= (dmaobj->conf0 & NVC0_DMA_CONF0_PRIV);
-        flags5 |= (dmaobj->conf0 & NVC0_DMA_CONF0_UNKN);
+        if (user > 2)
+                return -EINVAL;
+        priv->flags0 |= (kind << 22) | (user << 20);
+        priv->flags5 |= (unkn << 16);
 
-        switch (dmaobj->target) {
+        switch (priv->base.target) {
         case NV_MEM_TARGET_VM:
-                flags0 |= 0x00000000;
+                priv->flags0 |= 0x00000000;
                 break;
         case NV_MEM_TARGET_VRAM:
-                flags0 |= 0x00010000;
+                priv->flags0 |= 0x00010000;
                 break;
         case NV_MEM_TARGET_PCI:
-                flags0 |= 0x00020000;
+                priv->flags0 |= 0x00020000;
                 break;
         case NV_MEM_TARGET_PCI_NOSNOOP:
-                flags0 |= 0x00030000;
+                priv->flags0 |= 0x00030000;
                 break;
         default:
                 return -EINVAL;
         }
 
-        switch (dmaobj->access) {
+        switch (priv->base.access) {
         case NV_MEM_ACCESS_VM:
                 break;
         case NV_MEM_ACCESS_RO:
-                flags0 |= 0x00040000;
+                priv->flags0 |= 0x00040000;
                 break;
         case NV_MEM_ACCESS_WO:
         case NV_MEM_ACCESS_RW:
-                flags0 |= 0x00080000;
+                priv->flags0 |= 0x00080000;
                 break;
         }
 
-        ret = nouveau_gpuobj_new(parent, parent, 24, 32, 0, pgpuobj);
-        if (ret == 0) {
-                nv_wo32(*pgpuobj, 0x00, flags0);
-                nv_wo32(*pgpuobj, 0x04, lower_32_bits(dmaobj->limit));
-                nv_wo32(*pgpuobj, 0x08, lower_32_bits(dmaobj->start));
-                nv_wo32(*pgpuobj, 0x0c, upper_32_bits(dmaobj->limit) << 24 |
-                                        upper_32_bits(dmaobj->start));
-                nv_wo32(*pgpuobj, 0x10, 0x00000000);
-                nv_wo32(*pgpuobj, 0x14, flags5);
-        }
-
-        return ret;
+        return dmaeng->bind(&priv->base, nv_object(priv), (void *)pobject);
 }
 
-static int
-nvc0_dmaeng_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-                 struct nouveau_oclass *oclass, void *data, u32 size,
-                 struct nouveau_object **pobject)
-{
-        struct nvc0_dmaeng_priv *priv;
-        int ret;
-
-        ret = nouveau_dmaeng_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
+static struct nouveau_ofuncs
+nvc0_dmaobj_ofuncs = {
+        .ctor =  nvc0_dmaobj_ctor,
+        .dtor = _nvkm_dmaobj_dtor,
+        .init = _nvkm_dmaobj_init,
+        .fini = _nvkm_dmaobj_fini,
+};
 
-        nv_engine(priv)->sclass = nouveau_dmaobj_sclass;
-        priv->base.bind = nvc0_dmaobj_bind;
-        return 0;
-}
+static struct nouveau_oclass
+nvc0_dmaeng_sclass[] = {
+        { NV_DMA_FROM_MEMORY, &nvc0_dmaobj_ofuncs },
+        { NV_DMA_TO_MEMORY, &nvc0_dmaobj_ofuncs },
+        { NV_DMA_IN_MEMORY, &nvc0_dmaobj_ofuncs },
+        {}
+};
 
-struct nouveau_oclass
-nvc0_dmaeng_oclass = {
-        .handle = NV_ENGINE(DMAOBJ, 0xc0),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nvc0_dmaeng_ctor,
-                .dtor = _nouveau_dmaeng_dtor,
-                .init = _nouveau_dmaeng_init,
-                .fini = _nouveau_dmaeng_fini,
+struct nouveau_oclass *
+nvc0_dmaeng_oclass = &(struct nvkm_dmaeng_impl) {
+        .base.handle = NV_ENGINE(DMAOBJ, 0xc0),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nvkm_dmaeng_ctor,
+                .dtor = _nvkm_dmaeng_dtor,
+                .init = _nvkm_dmaeng_init,
+                .fini = _nvkm_dmaeng_fini,
         },
-};
+        .sclass = nvc0_dmaeng_sclass,
+        .bind = nvc0_dmaobj_bind,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
index 1cfb3bb90131..3fc4f0b0eaca 100644
--- a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
@@ -22,40 +22,40 @@
  * Authors: Ben Skeggs
  */
 
+#include <core/client.h>
 #include <core/device.h>
 #include <core/gpuobj.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <subdev/fb.h>
-#include <engine/dmaobj.h>
 
-struct nvd0_dmaeng_priv {
-        struct nouveau_dmaeng base;
+#include "priv.h"
+
+struct nvd0_dmaobj_priv {
+        struct nouveau_dmaobj base;
+        u32 flags0;
 };
 
 static int
-nvd0_dmaobj_bind(struct nouveau_dmaeng *dmaeng,
+nvd0_dmaobj_bind(struct nouveau_dmaobj *dmaobj,
                  struct nouveau_object *parent,
-                 struct nouveau_dmaobj *dmaobj,
                  struct nouveau_gpuobj **pgpuobj)
 {
-        u32 flags0 = 0x00000000;
+        struct nvd0_dmaobj_priv *priv = (void *)dmaobj;
         int ret;
 
         if (!nv_iclass(parent, NV_ENGCTX_CLASS)) {
                 switch (nv_mclass(parent->parent)) {
-                case NVD0_DISP_MAST_CLASS:
-                case NVD0_DISP_SYNC_CLASS:
-                case NVD0_DISP_OVLY_CLASS:
-                case NVE0_DISP_MAST_CLASS:
-                case NVE0_DISP_SYNC_CLASS:
-                case NVE0_DISP_OVLY_CLASS:
-                case NVF0_DISP_MAST_CLASS:
-                case NVF0_DISP_SYNC_CLASS:
-                case NVF0_DISP_OVLY_CLASS:
-                case GM107_DISP_MAST_CLASS:
-                case GM107_DISP_SYNC_CLASS:
-                case GM107_DISP_OVLY_CLASS:
+                case GF110_DISP_CORE_CHANNEL_DMA:
+                case GK104_DISP_CORE_CHANNEL_DMA:
+                case GK110_DISP_CORE_CHANNEL_DMA:
+                case GM107_DISP_CORE_CHANNEL_DMA:
+                case GF110_DISP_BASE_CHANNEL_DMA:
+                case GK104_DISP_BASE_CHANNEL_DMA:
+                case GK110_DISP_BASE_CHANNEL_DMA:
+                case GF110_DISP_OVERLAY_CONTROL_DMA:
+                case GK104_DISP_OVERLAY_CONTROL_DMA:
                         break;
                 default:
                         return -EINVAL;
@@ -63,33 +63,11 @@ nvd0_dmaobj_bind(struct nouveau_dmaeng *dmaeng,
         } else
                 return 0;
 
-        if (!(dmaobj->conf0 & NVD0_DMA_CONF0_ENABLE)) {
-                if (dmaobj->target == NV_MEM_TARGET_VM) {
-                        dmaobj->conf0 |= NVD0_DMA_CONF0_TYPE_VM;
-                        dmaobj->conf0 |= NVD0_DMA_CONF0_PAGE_LP;
-                } else {
-                        dmaobj->conf0 |= NVD0_DMA_CONF0_TYPE_LINEAR;
-                        dmaobj->conf0 |= NVD0_DMA_CONF0_PAGE_SP;
-                }
-        }
-
-        flags0 |= (dmaobj->conf0 & NVD0_DMA_CONF0_TYPE) << 20;
-        flags0 |= (dmaobj->conf0 & NVD0_DMA_CONF0_PAGE) >> 4;
-
-        switch (dmaobj->target) {
-        case NV_MEM_TARGET_VRAM:
-                flags0 |= 0x00000009;
-                break;
-        default:
-                return -EINVAL;
-                break;
-        }
-
         ret = nouveau_gpuobj_new(parent, parent, 24, 32, 0, pgpuobj);
         if (ret == 0) {
-                nv_wo32(*pgpuobj, 0x00, flags0);
-                nv_wo32(*pgpuobj, 0x04, dmaobj->start >> 8);
-                nv_wo32(*pgpuobj, 0x08, dmaobj->limit >> 8);
+                nv_wo32(*pgpuobj, 0x00, priv->flags0);
+                nv_wo32(*pgpuobj, 0x04, priv->base.start >> 8);
+                nv_wo32(*pgpuobj, 0x08, priv->base.limit >> 8);
                 nv_wo32(*pgpuobj, 0x0c, 0x00000000);
                 nv_wo32(*pgpuobj, 0x10, 0x00000000);
                 nv_wo32(*pgpuobj, 0x14, 0x00000000);
@@ -99,30 +77,91 @@ nvd0_dmaobj_bind(struct nouveau_dmaeng *dmaeng,
 }
 
 static int
-nvd0_dmaeng_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+nvd0_dmaobj_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                  struct nouveau_oclass *oclass, void *data, u32 size,
                  struct nouveau_object **pobject)
 {
-        struct nvd0_dmaeng_priv *priv;
+        struct nouveau_dmaeng *dmaeng = (void *)engine;
+        union {
+                struct gf110_dma_v0 v0;
+        } *args;
+        struct nvd0_dmaobj_priv *priv;
+        u32 kind, page;
         int ret;
 
-        ret = nouveau_dmaeng_create(parent, engine, oclass, &priv);
+        ret = nvkm_dmaobj_create(parent, engine, oclass, &data, &size, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
+        args = data;
 
-        nv_engine(priv)->sclass = nouveau_dmaobj_sclass;
-        priv->base.bind = nvd0_dmaobj_bind;
-        return 0;
+        nv_ioctl(parent, "create gf110 dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create gf100 dma vers %d page %d kind %02x\n",
+                         args->v0.version, args->v0.page, args->v0.kind);
+                kind = args->v0.kind;
+                page = args->v0.page;
+        } else
+        if (size == 0) {
+                if (priv->base.target != NV_MEM_TARGET_VM) {
+                        kind = GF110_DMA_V0_KIND_PITCH;
+                        page = GF110_DMA_V0_PAGE_SP;
+                } else {
+                        kind = GF110_DMA_V0_KIND_VM;
+                        page = GF110_DMA_V0_PAGE_LP;
+                }
+        } else
+                return ret;
+
+        if (page > 1)
+                return -EINVAL;
+        priv->flags0 = (kind << 20) | (page << 6);
+
+        switch (priv->base.target) {
+        case NV_MEM_TARGET_VRAM:
+                priv->flags0 |= 0x00000009;
+                break;
+        case NV_MEM_TARGET_VM:
+        case NV_MEM_TARGET_PCI:
+        case NV_MEM_TARGET_PCI_NOSNOOP:
+                /* XXX: don't currently know how to construct a real one
+                 *      of these.  we only use them to represent pushbufs
+                 *      on these chipsets, and the classes that use them
+                 *      deal with the target themselves.
+                 */
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        return dmaeng->bind(&priv->base, nv_object(priv), (void *)pobject);
 }
 
-struct nouveau_oclass
-nvd0_dmaeng_oclass = {
-        .handle = NV_ENGINE(DMAOBJ, 0xd0),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nvd0_dmaeng_ctor,
-                .dtor = _nouveau_dmaeng_dtor,
-                .init = _nouveau_dmaeng_init,
-                .fini = _nouveau_dmaeng_fini,
-        },
+static struct nouveau_ofuncs
+nvd0_dmaobj_ofuncs = {
+        .ctor =  nvd0_dmaobj_ctor,
+        .dtor = _nvkm_dmaobj_dtor,
+        .init = _nvkm_dmaobj_init,
+        .fini = _nvkm_dmaobj_fini,
 };
+
+static struct nouveau_oclass
+nvd0_dmaeng_sclass[] = {
+        { NV_DMA_FROM_MEMORY, &nvd0_dmaobj_ofuncs },
+        { NV_DMA_TO_MEMORY, &nvd0_dmaobj_ofuncs },
+        { NV_DMA_IN_MEMORY, &nvd0_dmaobj_ofuncs },
+        {}
+};
+
+struct nouveau_oclass *
+nvd0_dmaeng_oclass = &(struct nvkm_dmaeng_impl) {
+        .base.handle = NV_ENGINE(DMAOBJ, 0xd0),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nvkm_dmaeng_ctor,
+                .dtor = _nvkm_dmaeng_dtor,
+                .init = _nvkm_dmaeng_init,
+                .fini = _nvkm_dmaeng_fini,
+        },
+        .sclass = nvd0_dmaeng_sclass,
+        .bind = nvd0_dmaobj_bind,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/priv.h b/drivers/gpu/drm/nouveau/core/engine/dmaobj/priv.h
new file mode 100644
index 000000000000..36f743866937
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/priv.h
@@ -0,0 +1,30 @@
+#ifndef __NVKM_DMAOBJ_PRIV_H__
+#define __NVKM_DMAOBJ_PRIV_H__
+
+#include <engine/dmaobj.h>
+
+#define nvkm_dmaobj_create(p,e,c,pa,sa,d)                                      \
+        nvkm_dmaobj_create_((p), (e), (c), (pa), (sa), sizeof(**d), (void **)d)
+
+int nvkm_dmaobj_create_(struct nouveau_object *, struct nouveau_object *,
+                        struct nouveau_oclass *, void **, u32 *,
+                        int, void **);
+#define _nvkm_dmaobj_dtor nouveau_object_destroy
+#define _nvkm_dmaobj_init nouveau_object_init
+#define _nvkm_dmaobj_fini nouveau_object_fini
+
+int _nvkm_dmaeng_ctor(struct nouveau_object *, struct nouveau_object *,
+                      struct nouveau_oclass *, void *, u32,
+                      struct nouveau_object **);
+#define _nvkm_dmaeng_dtor _nouveau_engine_dtor
+#define _nvkm_dmaeng_init _nouveau_engine_init
+#define _nvkm_dmaeng_fini _nouveau_engine_fini
+
+struct nvkm_dmaeng_impl {
+        struct nouveau_oclass base;
+        struct nouveau_oclass *sclass;
+        int (*bind)(struct nouveau_dmaobj *, struct nouveau_object *,
+                    struct nouveau_gpuobj **);
+};
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/base.c b/drivers/gpu/drm/nouveau/core/engine/fifo/base.c
index 56ed3d73bf8e..0f999fc45ab9 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/base.c
@@ -26,11 +26,30 @@
 #include <core/object.h>
 #include <core/handle.h>
 #include <core/event.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
+#include <nvif/event.h>
 
 #include <engine/dmaobj.h>
 #include <engine/fifo.h>
 
+static int
+nouveau_fifo_event_ctor(void *data, u32 size, struct nvkm_notify *notify)
+{
+        if (size == 0) {
+                notify->size  = 0;
+                notify->types = 1;
+                notify->index = 0;
+                return 0;
+        }
+        return -ENOSYS;
+}
+
+static const struct nvkm_event_func
+nouveau_fifo_event_func = {
+        .ctor = nouveau_fifo_event_ctor,
+};
+
 int
 nouveau_fifo_channel_create_(struct nouveau_object *parent,
                              struct nouveau_object *engine,
@@ -59,14 +78,14 @@ nouveau_fifo_channel_create_(struct nouveau_object *parent,
 
         dmaeng = (void *)chan->pushdma->base.engine;
         switch (chan->pushdma->base.oclass->handle) {
-        case NV_DMA_FROM_MEMORY_CLASS:
-        case NV_DMA_IN_MEMORY_CLASS:
+        case NV_DMA_FROM_MEMORY:
+        case NV_DMA_IN_MEMORY:
                 break;
         default:
                 return -EINVAL;
         }
 
-        ret = dmaeng->bind(dmaeng, parent, chan->pushdma, &chan->pushgpu);
+        ret = dmaeng->bind(chan->pushdma, parent, &chan->pushgpu);
         if (ret)
                 return ret;
 
@@ -85,15 +104,10 @@ nouveau_fifo_channel_create_(struct nouveau_object *parent,
                 return -ENOSPC;
         }
 
-        /* map fifo control registers */
-        chan->user = ioremap(nv_device_resource_start(device, bar) + addr +
-                             (chan->chid * size), size);
-        if (!chan->user)
-                return -EFAULT;
-
-        nouveau_event_trigger(priv->cevent, 1, 0);
-
+        chan->addr = nv_device_resource_start(device, bar) +
+                     addr + size * chan->chid;
         chan->size = size;
+        nvkm_event_send(&priv->cevent, 1, 0, NULL, 0);
         return 0;
 }
 
@@ -103,7 +117,8 @@ nouveau_fifo_channel_destroy(struct nouveau_fifo_chan *chan)
         struct nouveau_fifo *priv = (void *)nv_object(chan)->engine;
         unsigned long flags;
 
-        iounmap(chan->user);
+        if (chan->user)
+                iounmap(chan->user);
 
         spin_lock_irqsave(&priv->lock, flags);
         priv->channel[chan->chid] = NULL;
@@ -121,10 +136,24 @@ _nouveau_fifo_channel_dtor(struct nouveau_object *object)
         nouveau_fifo_channel_destroy(chan);
 }
 
+int
+_nouveau_fifo_channel_map(struct nouveau_object *object, u64 *addr, u32 *size)
+{
+        struct nouveau_fifo_chan *chan = (void *)object;
+        *addr = chan->addr;
+        *size = chan->size;
+        return 0;
+}
+
 u32
 _nouveau_fifo_channel_rd32(struct nouveau_object *object, u64 addr)
 {
         struct nouveau_fifo_chan *chan = (void *)object;
+        if (unlikely(!chan->user)) {
+                chan->user = ioremap(chan->addr, chan->size);
+                if (WARN_ON_ONCE(chan->user == NULL))
+                        return 0;
+        }
         return ioread32_native(chan->user + addr);
 }
 
@@ -132,9 +161,57 @@ void
 _nouveau_fifo_channel_wr32(struct nouveau_object *object, u64 addr, u32 data)
 {
         struct nouveau_fifo_chan *chan = (void *)object;
+        if (unlikely(!chan->user)) {
+                chan->user = ioremap(chan->addr, chan->size);
+                if (WARN_ON_ONCE(chan->user == NULL))
+                        return;
+        }
         iowrite32_native(data, chan->user + addr);
 }
 
+int
+nouveau_fifo_uevent_ctor(void *data, u32 size, struct nvkm_notify *notify)
+{
+        union {
+                struct nvif_notify_uevent_req none;
+        } *req = data;
+        int ret;
+
+        if (nvif_unvers(req->none)) {
+                notify->size  = sizeof(struct nvif_notify_uevent_rep);
+                notify->types = 1;
+                notify->index = 0;
+        }
+
+        return ret;
+}
+
+void
+nouveau_fifo_uevent(struct nouveau_fifo *fifo)
+{
+        struct nvif_notify_uevent_rep rep = {
+        };
+        nvkm_event_send(&fifo->uevent, 1, 0, &rep, sizeof(rep));
+}
+
+int
+_nouveau_fifo_channel_ntfy(struct nouveau_object *object, u32 type,
+                           struct nvkm_event **event)
+{
+        struct nouveau_fifo *fifo = (void *)object->engine;
+        switch (type) {
+        case G82_CHANNEL_DMA_V0_NTFY_UEVENT:
+                if (nv_mclass(object) >= G82_CHANNEL_DMA) {
+                        *event = &fifo->uevent;
+                        return 0;
+                }
+                break;
+        default:
+                break;
+        }
+        return -EINVAL;
+}
+
 static int
 nouveau_fifo_chid(struct nouveau_fifo *priv, struct nouveau_object *object)
 {
@@ -168,8 +245,8 @@ void
 nouveau_fifo_destroy(struct nouveau_fifo *priv)
 {
         kfree(priv->channel);
-        nouveau_event_destroy(&priv->uevent);
-        nouveau_event_destroy(&priv->cevent);
+        nvkm_event_fini(&priv->uevent);
+        nvkm_event_fini(&priv->cevent);
         nouveau_engine_destroy(&priv->base);
 }
 
@@ -194,11 +271,7 @@ nouveau_fifo_create_(struct nouveau_object *parent,
         if (!priv->channel)
                 return -ENOMEM;
 
-        ret = nouveau_event_create(1, 1, &priv->cevent);
-        if (ret)
-                return ret;
-
-        ret = nouveau_event_create(1, 1, &priv->uevent);
+        ret = nvkm_event_init(&nouveau_fifo_event_func, 1, 1, &priv->cevent);
         if (ret)
                 return ret;
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c
index c61b16a63884..5ae6a43893b5 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c
@@ -22,8 +22,9 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 #include <core/engctx.h>
 #include <core/namedb.h>
 #include <core/handle.h>
@@ -117,16 +118,23 @@ nv04_fifo_chan_ctor(struct nouveau_object *parent,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
+        union {
+                struct nv03_channel_dma_v0 v0;
+        } *args = data;
         struct nv04_fifo_priv *priv = (void *)engine;
         struct nv04_fifo_chan *chan;
-        struct nv03_channel_dma_class *args = data;
         int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create channel dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create channel dma vers %d pushbuf %08x "
+                                 "offset %016llx\n", args->v0.version,
+                         args->v0.pushbuf, args->v0.offset);
+        } else
+                return ret;
 
         ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0x800000,
-                                          0x10000, args->pushbuf,
+                                          0x10000, args->v0.pushbuf,
                                           (1ULL << NVDEV_ENGINE_DMAOBJ) |
                                           (1ULL << NVDEV_ENGINE_SW) |
                                           (1ULL << NVDEV_ENGINE_GR), &chan);
@@ -134,13 +142,15 @@ nv04_fifo_chan_ctor(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
+        args->v0.chid = chan->base.chid;
+
         nv_parent(chan)->object_attach = nv04_fifo_object_attach;
         nv_parent(chan)->object_detach = nv04_fifo_object_detach;
         nv_parent(chan)->context_attach = nv04_fifo_context_attach;
         chan->ramfc = chan->base.chid * 32;
 
-        nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->offset);
-        nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->offset);
+        nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->v0.offset);
+        nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->v0.offset);
         nv_wo32(priv->ramfc, chan->ramfc + 0x08, chan->base.pushgpu->addr >> 4);
         nv_wo32(priv->ramfc, chan->ramfc + 0x10,
                              NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
@@ -242,13 +252,15 @@ nv04_fifo_ofuncs = {
         .dtor = nv04_fifo_chan_dtor,
         .init = nv04_fifo_chan_init,
         .fini = nv04_fifo_chan_fini,
+        .map  = _nouveau_fifo_channel_map,
         .rd32 = _nouveau_fifo_channel_rd32,
         .wr32 = _nouveau_fifo_channel_wr32,
+        .ntfy = _nouveau_fifo_channel_ntfy
 };
 
 static struct nouveau_oclass
 nv04_fifo_sclass[] = {
-        { NV03_CHANNEL_DMA_CLASS, &nv04_fifo_ofuncs },
+        { NV03_CHANNEL_DMA, &nv04_fifo_ofuncs },
         {}
 };
 
@@ -539,7 +551,7 @@ nv04_fifo_intr(struct nouveau_subdev *subdev)
                         }
 
                         if (status & 0x40000000) {
-                                nouveau_event_trigger(priv->base.uevent, 1, 0);
+                                nouveau_fifo_uevent(&priv->base);
                                 nv_wr32(priv, 0x002100, 0x40000000);
                                 status &= ~0x40000000;
                         }
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c
index 571a22aa1ae5..2a32add51c81 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c
@@ -22,8 +22,9 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 #include <core/engctx.h>
 #include <core/ramht.h>
 
@@ -59,16 +60,23 @@ nv10_fifo_chan_ctor(struct nouveau_object *parent,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
+        union {
+                struct nv03_channel_dma_v0 v0;
+        } *args = data;
         struct nv04_fifo_priv *priv = (void *)engine;
         struct nv04_fifo_chan *chan;
-        struct nv03_channel_dma_class *args = data;
         int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create channel dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create channel dma vers %d pushbuf %08x "
+                                 "offset %016llx\n", args->v0.version,
+                         args->v0.pushbuf, args->v0.offset);
+        } else
+                return ret;
 
         ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0x800000,
-                                          0x10000, args->pushbuf,
+                                          0x10000, args->v0.pushbuf,
                                           (1ULL << NVDEV_ENGINE_DMAOBJ) |
                                           (1ULL << NVDEV_ENGINE_SW) |
                                           (1ULL << NVDEV_ENGINE_GR), &chan);
@@ -76,13 +84,15 @@ nv10_fifo_chan_ctor(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
+        args->v0.chid = chan->base.chid;
+
         nv_parent(chan)->object_attach = nv04_fifo_object_attach;
         nv_parent(chan)->object_detach = nv04_fifo_object_detach;
         nv_parent(chan)->context_attach = nv04_fifo_context_attach;
         chan->ramfc = chan->base.chid * 32;
 
-        nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->offset);
-        nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->offset);
+        nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->v0.offset);
+        nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->v0.offset);
         nv_wo32(priv->ramfc, chan->ramfc + 0x0c, chan->base.pushgpu->addr >> 4);
         nv_wo32(priv->ramfc, chan->ramfc + 0x14,
                              NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
@@ -100,13 +110,15 @@ nv10_fifo_ofuncs = {
         .dtor = nv04_fifo_chan_dtor,
         .init = nv04_fifo_chan_init,
         .fini = nv04_fifo_chan_fini,
+        .map  = _nouveau_fifo_channel_map,
         .rd32 = _nouveau_fifo_channel_rd32,
         .wr32 = _nouveau_fifo_channel_wr32,
+        .ntfy = _nouveau_fifo_channel_ntfy
 };
 
 static struct nouveau_oclass
 nv10_fifo_sclass[] = {
-        { NV10_CHANNEL_DMA_CLASS, &nv10_fifo_ofuncs },
+        { NV10_CHANNEL_DMA, &nv10_fifo_ofuncs },
         {}
 };
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv17.c
index f25760209316..12d76c8adb23 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv17.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv17.c
@@ -22,8 +22,9 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 #include <core/engctx.h>
 #include <core/ramht.h>
 
@@ -64,16 +65,23 @@ nv17_fifo_chan_ctor(struct nouveau_object *parent,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
+        union {
+                struct nv03_channel_dma_v0 v0;
+        } *args = data;
         struct nv04_fifo_priv *priv = (void *)engine;
         struct nv04_fifo_chan *chan;
-        struct nv03_channel_dma_class *args = data;
         int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create channel dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create channel dma vers %d pushbuf %08x "
+                                 "offset %016llx\n", args->v0.version,
+                         args->v0.pushbuf, args->v0.offset);
+        } else
+                return ret;
 
         ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0x800000,
-                                          0x10000, args->pushbuf,
+                                          0x10000, args->v0.pushbuf,
                                           (1ULL << NVDEV_ENGINE_DMAOBJ) |
                                           (1ULL << NVDEV_ENGINE_SW) |
                                           (1ULL << NVDEV_ENGINE_GR) |
@@ -83,13 +91,15 @@ nv17_fifo_chan_ctor(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
+        args->v0.chid = chan->base.chid;
+
         nv_parent(chan)->object_attach = nv04_fifo_object_attach;
         nv_parent(chan)->object_detach = nv04_fifo_object_detach;
         nv_parent(chan)->context_attach = nv04_fifo_context_attach;
         chan->ramfc = chan->base.chid * 64;
 
-        nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->offset);
-        nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->offset);
+        nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->v0.offset);
+        nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->v0.offset);
         nv_wo32(priv->ramfc, chan->ramfc + 0x0c, chan->base.pushgpu->addr >> 4);
         nv_wo32(priv->ramfc, chan->ramfc + 0x14,
                              NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
@@ -107,13 +117,15 @@ nv17_fifo_ofuncs = {
         .dtor = nv04_fifo_chan_dtor,
         .init = nv04_fifo_chan_init,
         .fini = nv04_fifo_chan_fini,
+        .map  = _nouveau_fifo_channel_map,
         .rd32 = _nouveau_fifo_channel_rd32,
         .wr32 = _nouveau_fifo_channel_wr32,
+        .ntfy = _nouveau_fifo_channel_ntfy
 };
 
 static struct nouveau_oclass
 nv17_fifo_sclass[] = {
-        { NV17_CHANNEL_DMA_CLASS, &nv17_fifo_ofuncs },
+        { NV17_CHANNEL_DMA, &nv17_fifo_ofuncs },
         {}
 };
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv40.c
index 343487ed2238..9f49c3a24dc6 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv40.c
@@ -22,8 +22,9 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/os.h>
-#include <core/class.h>
+#include <core/client.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 #include <core/engctx.h>
 #include <core/ramht.h>
 
@@ -182,16 +183,23 @@ nv40_fifo_chan_ctor(struct nouveau_object *parent,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
+        union {
+                struct nv03_channel_dma_v0 v0;
+        } *args = data;
         struct nv04_fifo_priv *priv = (void *)engine;
         struct nv04_fifo_chan *chan;
-        struct nv03_channel_dma_class *args = data;
         int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create channel dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create channel dma vers %d pushbuf %08x "
+                                 "offset %016llx\n", args->v0.version,
+                         args->v0.pushbuf, args->v0.offset);
+        } else
+                return ret;
 
         ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000,
-                                          0x1000, args->pushbuf,
+                                          0x1000, args->v0.pushbuf,
                                           (1ULL << NVDEV_ENGINE_DMAOBJ) |
                                           (1ULL << NVDEV_ENGINE_SW) |
                                           (1ULL << NVDEV_ENGINE_GR) |
@@ -200,14 +208,16 @@ nv40_fifo_chan_ctor(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
+        args->v0.chid = chan->base.chid;
+
         nv_parent(chan)->context_attach = nv40_fifo_context_attach;
         nv_parent(chan)->context_detach = nv40_fifo_context_detach;
         nv_parent(chan)->object_attach = nv40_fifo_object_attach;
         nv_parent(chan)->object_detach = nv04_fifo_object_detach;
         chan->ramfc = chan->base.chid * 128;
 
-        nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->offset);
-        nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->offset);
+        nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->v0.offset);
+        nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->v0.offset);
         nv_wo32(priv->ramfc, chan->ramfc + 0x0c, chan->base.pushgpu->addr >> 4);
         nv_wo32(priv->ramfc, chan->ramfc + 0x18, 0x30000000 |
                              NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
@@ -226,13 +236,15 @@ nv40_fifo_ofuncs = {
         .dtor = nv04_fifo_chan_dtor,
         .init = nv04_fifo_chan_init,
         .fini = nv04_fifo_chan_fini,
+        .map  = _nouveau_fifo_channel_map,
         .rd32 = _nouveau_fifo_channel_rd32,
         .wr32 = _nouveau_fifo_channel_wr32,
+        .ntfy = _nouveau_fifo_channel_ntfy
 };
 
 static struct nouveau_oclass
 nv40_fifo_sclass[] = {
-        { NV40_CHANNEL_DMA_CLASS, &nv40_fifo_ofuncs },
+        { NV40_CHANNEL_DMA, &nv40_fifo_ofuncs },
         {}
 };
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.c
index e6352bd5b4ff..5d1e86bc244c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.c
@@ -25,7 +25,8 @@
 #include <core/client.h>
 #include <core/engctx.h>
 #include <core/ramht.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <subdev/timer.h>
 #include <subdev/bar.h>
@@ -194,17 +195,24 @@ nv50_fifo_chan_ctor_dma(struct nouveau_object *parent,
                         struct nouveau_oclass *oclass, void *data, u32 size,
                         struct nouveau_object **pobject)
 {
+        union {
+                struct nv03_channel_dma_v0 v0;
+        } *args = data;
         struct nouveau_bar *bar = nouveau_bar(parent);
         struct nv50_fifo_base *base = (void *)parent;
         struct nv50_fifo_chan *chan;
-        struct nv03_channel_dma_class *args = data;
         int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create channel dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create channel dma vers %d pushbuf %08x "
+                                 "offset %016llx\n", args->v0.version,
+                         args->v0.pushbuf, args->v0.offset);
+        } else
+                return ret;
 
         ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000,
-                                          0x2000, args->pushbuf,
+                                          0x2000, args->v0.pushbuf,
                                           (1ULL << NVDEV_ENGINE_DMAOBJ) |
                                           (1ULL << NVDEV_ENGINE_SW) |
                                           (1ULL << NVDEV_ENGINE_GR) |
@@ -213,6 +221,8 @@ nv50_fifo_chan_ctor_dma(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
+        args->v0.chid = chan->base.chid;
+
         nv_parent(chan)->context_attach = nv50_fifo_context_attach;
         nv_parent(chan)->context_detach = nv50_fifo_context_detach;
         nv_parent(chan)->object_attach = nv50_fifo_object_attach;
@@ -223,10 +233,10 @@ nv50_fifo_chan_ctor_dma(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
-        nv_wo32(base->ramfc, 0x08, lower_32_bits(args->offset));
-        nv_wo32(base->ramfc, 0x0c, upper_32_bits(args->offset));
-        nv_wo32(base->ramfc, 0x10, lower_32_bits(args->offset));
-        nv_wo32(base->ramfc, 0x14, upper_32_bits(args->offset));
+        nv_wo32(base->ramfc, 0x08, lower_32_bits(args->v0.offset));
+        nv_wo32(base->ramfc, 0x0c, upper_32_bits(args->v0.offset));
+        nv_wo32(base->ramfc, 0x10, lower_32_bits(args->v0.offset));
+        nv_wo32(base->ramfc, 0x14, upper_32_bits(args->v0.offset));
         nv_wo32(base->ramfc, 0x3c, 0x003f6078);
         nv_wo32(base->ramfc, 0x44, 0x01003fff);
         nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4);
@@ -247,18 +257,26 @@ nv50_fifo_chan_ctor_ind(struct nouveau_object *parent,
                         struct nouveau_oclass *oclass, void *data, u32 size,
                         struct nouveau_object **pobject)
 {
-        struct nv50_channel_ind_class *args = data;
+        union {
+                struct nv50_channel_gpfifo_v0 v0;
+        } *args = data;
         struct nouveau_bar *bar = nouveau_bar(parent);
         struct nv50_fifo_base *base = (void *)parent;
         struct nv50_fifo_chan *chan;
         u64 ioffset, ilength;
         int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create channel gpfifo size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x "
+                                 "ioffset %016llx ilength %08x\n",
+                         args->v0.version, args->v0.pushbuf, args->v0.ioffset,
+                         args->v0.ilength);
+        } else
+                return ret;
 
         ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000,
-                                          0x2000, args->pushbuf,
+                                          0x2000, args->v0.pushbuf,
                                           (1ULL << NVDEV_ENGINE_DMAOBJ) |
                                           (1ULL << NVDEV_ENGINE_SW) |
                                           (1ULL << NVDEV_ENGINE_GR) |
@@ -267,6 +285,8 @@ nv50_fifo_chan_ctor_ind(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
+        args->v0.chid = chan->base.chid;
+
         nv_parent(chan)->context_attach = nv50_fifo_context_attach;
         nv_parent(chan)->context_detach = nv50_fifo_context_detach;
         nv_parent(chan)->object_attach = nv50_fifo_object_attach;
@@ -277,8 +297,8 @@ nv50_fifo_chan_ctor_ind(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
-        ioffset = args->ioffset;
-        ilength = order_base_2(args->ilength / 8);
+        ioffset = args->v0.ioffset;
+        ilength = order_base_2(args->v0.ilength / 8);
 
         nv_wo32(base->ramfc, 0x3c, 0x403f6078);
         nv_wo32(base->ramfc, 0x44, 0x01003fff);
@@ -343,8 +363,10 @@ nv50_fifo_ofuncs_dma = {
         .dtor = nv50_fifo_chan_dtor,
         .init = nv50_fifo_chan_init,
         .fini = nv50_fifo_chan_fini,
+        .map  = _nouveau_fifo_channel_map,
         .rd32 = _nouveau_fifo_channel_rd32,
         .wr32 = _nouveau_fifo_channel_wr32,
+        .ntfy = _nouveau_fifo_channel_ntfy
 };
 
 static struct nouveau_ofuncs
@@ -353,14 +375,16 @@ nv50_fifo_ofuncs_ind = {
         .dtor = nv50_fifo_chan_dtor,
         .init = nv50_fifo_chan_init,
         .fini = nv50_fifo_chan_fini,
+        .map  = _nouveau_fifo_channel_map,
         .rd32 = _nouveau_fifo_channel_rd32,
         .wr32 = _nouveau_fifo_channel_wr32,
+        .ntfy = _nouveau_fifo_channel_ntfy
 };
 
 static struct nouveau_oclass
 nv50_fifo_sclass[] = {
-        { NV50_CHANNEL_DMA_CLASS, &nv50_fifo_ofuncs_dma },
-        { NV50_CHANNEL_IND_CLASS, &nv50_fifo_ofuncs_ind },
+        { NV50_CHANNEL_DMA, &nv50_fifo_ofuncs_dma },
+        { NV50_CHANNEL_GPFIFO, &nv50_fifo_ofuncs_ind },
         {}
 };
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv84.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv84.c
index 6e5ac16e5460..1f42996b354a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv84.c
@@ -27,7 +27,8 @@
 #include <core/engctx.h>
 #include <core/ramht.h>
 #include <core/event.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 
 #include <subdev/timer.h>
 #include <subdev/bar.h>
@@ -160,17 +161,24 @@ nv84_fifo_chan_ctor_dma(struct nouveau_object *parent,
                         struct nouveau_oclass *oclass, void *data, u32 size,
                         struct nouveau_object **pobject)
 {
+        union {
+                struct nv03_channel_dma_v0 v0;
+        } *args = data;
         struct nouveau_bar *bar = nouveau_bar(parent);
         struct nv50_fifo_base *base = (void *)parent;
         struct nv50_fifo_chan *chan;
-        struct nv03_channel_dma_class *args = data;
         int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create channel dma size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create channel dma vers %d pushbuf %08x "
+                                 "offset %016llx\n", args->v0.version,
+                         args->v0.pushbuf, args->v0.offset);
+        } else
+                return ret;
 
         ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000,
-                                          0x2000, args->pushbuf,
+                                          0x2000, args->v0.pushbuf,
                                           (1ULL << NVDEV_ENGINE_DMAOBJ) |
                                           (1ULL << NVDEV_ENGINE_SW) |
                                           (1ULL << NVDEV_ENGINE_GR) |
@@ -186,6 +194,8 @@ nv84_fifo_chan_ctor_dma(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
+        args->v0.chid = chan->base.chid;
+
         ret = nouveau_ramht_new(nv_object(chan), nv_object(chan), 0x8000, 16,
                                &chan->ramht);
         if (ret)
@@ -196,10 +206,10 @@ nv84_fifo_chan_ctor_dma(struct nouveau_object *parent,
         nv_parent(chan)->object_attach = nv84_fifo_object_attach;
         nv_parent(chan)->object_detach = nv50_fifo_object_detach;
 
-        nv_wo32(base->ramfc, 0x08, lower_32_bits(args->offset));
-        nv_wo32(base->ramfc, 0x0c, upper_32_bits(args->offset));
-        nv_wo32(base->ramfc, 0x10, lower_32_bits(args->offset));
-        nv_wo32(base->ramfc, 0x14, upper_32_bits(args->offset));
+        nv_wo32(base->ramfc, 0x08, lower_32_bits(args->v0.offset));
+        nv_wo32(base->ramfc, 0x0c, upper_32_bits(args->v0.offset));
+        nv_wo32(base->ramfc, 0x10, lower_32_bits(args->v0.offset));
+        nv_wo32(base->ramfc, 0x14, upper_32_bits(args->v0.offset));
         nv_wo32(base->ramfc, 0x3c, 0x003f6078);
         nv_wo32(base->ramfc, 0x44, 0x01003fff);
         nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4);
@@ -222,18 +232,26 @@ nv84_fifo_chan_ctor_ind(struct nouveau_object *parent,
                         struct nouveau_oclass *oclass, void *data, u32 size,
                         struct nouveau_object **pobject)
 {
+        union {
+                struct nv50_channel_gpfifo_v0 v0;
+        } *args = data;
         struct nouveau_bar *bar = nouveau_bar(parent);
         struct nv50_fifo_base *base = (void *)parent;
         struct nv50_fifo_chan *chan;
-        struct nv50_channel_ind_class *args = data;
         u64 ioffset, ilength;
         int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create channel gpfifo size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x "
+                                 "ioffset %016llx ilength %08x\n",
+                         args->v0.version, args->v0.pushbuf, args->v0.ioffset,
+                         args->v0.ilength);
+        } else
+                return ret;
 
         ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000,
-                                          0x2000, args->pushbuf,
+                                          0x2000, args->v0.pushbuf,
                                           (1ULL << NVDEV_ENGINE_DMAOBJ) |
                                           (1ULL << NVDEV_ENGINE_SW) |
                                           (1ULL << NVDEV_ENGINE_GR) |
@@ -249,6 +267,8 @@ nv84_fifo_chan_ctor_ind(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
+        args->v0.chid = chan->base.chid;
+
         ret = nouveau_ramht_new(nv_object(chan), nv_object(chan), 0x8000, 16,
                                &chan->ramht);
         if (ret)
@@ -259,8 +279,8 @@ nv84_fifo_chan_ctor_ind(struct nouveau_object *parent,
         nv_parent(chan)->object_attach = nv84_fifo_object_attach;
         nv_parent(chan)->object_detach = nv50_fifo_object_detach;
 
-        ioffset = args->ioffset;
-        ilength = order_base_2(args->ilength / 8);
+        ioffset = args->v0.ioffset;
+        ilength = order_base_2(args->v0.ilength / 8);
 
         nv_wo32(base->ramfc, 0x3c, 0x403f6078);
         nv_wo32(base->ramfc, 0x44, 0x01003fff);
@@ -304,8 +324,10 @@ nv84_fifo_ofuncs_dma = {
         .dtor = nv50_fifo_chan_dtor,
         .init = nv84_fifo_chan_init,
         .fini = nv50_fifo_chan_fini,
+        .map  = _nouveau_fifo_channel_map,
         .rd32 = _nouveau_fifo_channel_rd32,
         .wr32 = _nouveau_fifo_channel_wr32,
+        .ntfy = _nouveau_fifo_channel_ntfy
 };
 
 static struct nouveau_ofuncs
@@ -314,14 +336,16 @@ nv84_fifo_ofuncs_ind = {
         .dtor = nv50_fifo_chan_dtor,
         .init = nv84_fifo_chan_init,
         .fini = nv50_fifo_chan_fini,
+        .map  = _nouveau_fifo_channel_map,
         .rd32 = _nouveau_fifo_channel_rd32,
         .wr32 = _nouveau_fifo_channel_wr32,
+        .ntfy = _nouveau_fifo_channel_ntfy
 };
 
 static struct nouveau_oclass
 nv84_fifo_sclass[] = {
-        { NV84_CHANNEL_DMA_CLASS, &nv84_fifo_ofuncs_dma },
-        { NV84_CHANNEL_IND_CLASS, &nv84_fifo_ofuncs_ind },
+        { G82_CHANNEL_DMA, &nv84_fifo_ofuncs_dma },
+        { G82_CHANNEL_GPFIFO, &nv84_fifo_ofuncs_ind },
         {}
 };
 
@@ -389,19 +413,26 @@ nv84_fifo_cclass = {
  ******************************************************************************/
 
 static void
-nv84_fifo_uevent_enable(struct nouveau_event *event, int type, int index)
+nv84_fifo_uevent_init(struct nvkm_event *event, int type, int index)
 {
-        struct nv84_fifo_priv *priv = event->priv;
-        nv_mask(priv, 0x002140, 0x40000000, 0x40000000);
+        struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent);
+        nv_mask(fifo, 0x002140, 0x40000000, 0x40000000);
 }
 
 static void
-nv84_fifo_uevent_disable(struct nouveau_event *event, int type, int index)
+nv84_fifo_uevent_fini(struct nvkm_event *event, int type, int index)
 {
-        struct nv84_fifo_priv *priv = event->priv;
-        nv_mask(priv, 0x002140, 0x40000000, 0x00000000);
+        struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent);
+        nv_mask(fifo, 0x002140, 0x40000000, 0x00000000);
 }
 
+static const struct nvkm_event_func
+nv84_fifo_uevent_func = {
+        .ctor = nouveau_fifo_uevent_ctor,
+        .init = nv84_fifo_uevent_init,
+        .fini = nv84_fifo_uevent_fini,
+};
+
 static int
 nv84_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
@@ -425,9 +456,9 @@ nv84_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        priv->base.uevent->enable = nv84_fifo_uevent_enable;
-        priv->base.uevent->disable = nv84_fifo_uevent_disable;
-        priv->base.uevent->priv = priv;
+        ret = nvkm_event_init(&nv84_fifo_uevent_func, 1, 1, &priv->base.uevent);
+        if (ret)
+                return ret;
 
         nv_subdev(priv)->unit = 0x00000100;
         nv_subdev(priv)->intr = nv04_fifo_intr;
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c
index ae4a4dc5642a..1fe1f8fbda0c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c
@@ -28,7 +28,8 @@
 #include <core/gpuobj.h>
 #include <core/engctx.h>
 #include <core/event.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 #include <core/enum.h>
 
 #include <subdev/timer.h>
@@ -187,20 +188,28 @@ nvc0_fifo_chan_ctor(struct nouveau_object *parent,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
+        union {
+                struct nv50_channel_gpfifo_v0 v0;
+        } *args = data;
         struct nouveau_bar *bar = nouveau_bar(parent);
         struct nvc0_fifo_priv *priv = (void *)engine;
         struct nvc0_fifo_base *base = (void *)parent;
         struct nvc0_fifo_chan *chan;
-        struct nv50_channel_ind_class *args = data;
         u64 usermem, ioffset, ilength;
         int ret, i;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create channel gpfifo size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x "
+                                 "ioffset %016llx ilength %08x\n",
+                         args->v0.version, args->v0.pushbuf, args->v0.ioffset,
+                         args->v0.ilength);
+        } else
+                return ret;
 
         ret = nouveau_fifo_channel_create(parent, engine, oclass, 1,
                                           priv->user.bar.offset, 0x1000,
-                                          args->pushbuf,
+                                          args->v0.pushbuf,
                                           (1ULL << NVDEV_ENGINE_SW) |
                                           (1ULL << NVDEV_ENGINE_GR) |
                                           (1ULL << NVDEV_ENGINE_COPY0) |
@@ -212,12 +221,14 @@ nvc0_fifo_chan_ctor(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
+        args->v0.chid = chan->base.chid;
+
         nv_parent(chan)->context_attach = nvc0_fifo_context_attach;
         nv_parent(chan)->context_detach = nvc0_fifo_context_detach;
 
         usermem = chan->base.chid * 0x1000;
-        ioffset = args->ioffset;
-        ilength = order_base_2(args->ilength / 8);
+        ioffset = args->v0.ioffset;
+        ilength = order_base_2(args->v0.ilength / 8);
 
         for (i = 0; i < 0x1000; i += 4)
                 nv_wo32(priv->user.mem, usermem + i, 0x00000000);
@@ -291,13 +302,15 @@ nvc0_fifo_ofuncs = {
         .dtor = _nouveau_fifo_channel_dtor,
         .init = nvc0_fifo_chan_init,
         .fini = nvc0_fifo_chan_fini,
+        .map  = _nouveau_fifo_channel_map,
         .rd32 = _nouveau_fifo_channel_rd32,
         .wr32 = _nouveau_fifo_channel_wr32,
+        .ntfy = _nouveau_fifo_channel_ntfy
 };
 
 static struct nouveau_oclass
 nvc0_fifo_sclass[] = {
-        { NVC0_CHANNEL_IND_CLASS, &nvc0_fifo_ofuncs },
+        { FERMI_CHANNEL_GPFIFO, &nvc0_fifo_ofuncs },
         {}
 };
 
@@ -654,7 +667,7 @@ nvc0_fifo_intr_fault(struct nvc0_fifo_priv *priv, int unit)
         object = engctx;
         while (object) {
                 switch (nv_mclass(object)) {
-                case NVC0_CHANNEL_IND_CLASS:
+                case FERMI_CHANNEL_GPFIFO:
                         nvc0_fifo_recover(priv, engine, (void *)object);
                         break;
                 }
@@ -730,7 +743,7 @@ nvc0_fifo_intr_engine_unit(struct nvc0_fifo_priv *priv, int engn)
         for (unkn = 0; unkn < 8; unkn++) {
                 u32 ints = (intr >> (unkn * 0x04)) & inte;
                 if (ints & 0x1) {
-                        nouveau_event_trigger(priv->base.uevent, 1, 0);
+                        nouveau_fifo_uevent(&priv->base);
                         ints &= ~1;
                 }
                 if (ints) {
@@ -827,19 +840,26 @@ nvc0_fifo_intr(struct nouveau_subdev *subdev)
 }
 
 static void
-nvc0_fifo_uevent_enable(struct nouveau_event *event, int type, int index)
+nvc0_fifo_uevent_init(struct nvkm_event *event, int type, int index)
 {
-        struct nvc0_fifo_priv *priv = event->priv;
-        nv_mask(priv, 0x002140, 0x80000000, 0x80000000);
+        struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent);
+        nv_mask(fifo, 0x002140, 0x80000000, 0x80000000);
 }
 
 static void
-nvc0_fifo_uevent_disable(struct nouveau_event *event, int type, int index)
+nvc0_fifo_uevent_fini(struct nvkm_event *event, int type, int index)
 {
-        struct nvc0_fifo_priv *priv = event->priv;
-        nv_mask(priv, 0x002140, 0x80000000, 0x00000000);
+        struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent);
+        nv_mask(fifo, 0x002140, 0x80000000, 0x00000000);
 }
 
+static const struct nvkm_event_func
+nvc0_fifo_uevent_func = {
+        .ctor = nouveau_fifo_uevent_ctor,
+        .init = nvc0_fifo_uevent_init,
+        .fini = nvc0_fifo_uevent_fini,
+};
+
 static int
 nvc0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
@@ -877,9 +897,9 @@ nvc0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        priv->base.uevent->enable = nvc0_fifo_uevent_enable;
-        priv->base.uevent->disable = nvc0_fifo_uevent_disable;
-        priv->base.uevent->priv = priv;
+        ret = nvkm_event_init(&nvc0_fifo_uevent_func, 1, 1, &priv->base.uevent);
+        if (ret)
+                return ret;
 
         nv_subdev(priv)->unit = 0x00000100;
         nv_subdev(priv)->intr = nvc0_fifo_intr;
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
index 298063edb92d..d2f0fd39c145 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
@@ -28,7 +28,8 @@
 #include <core/gpuobj.h>
 #include <core/engctx.h>
 #include <core/event.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
 #include <core/enum.h>
 
 #include <subdev/timer.h>
@@ -216,46 +217,56 @@ nve0_fifo_chan_ctor(struct nouveau_object *parent,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
 {
+        union {
+                struct kepler_channel_gpfifo_a_v0 v0;
+        } *args = data;
         struct nouveau_bar *bar = nouveau_bar(parent);
         struct nve0_fifo_priv *priv = (void *)engine;
         struct nve0_fifo_base *base = (void *)parent;
         struct nve0_fifo_chan *chan;
-        struct nve0_channel_ind_class *args = data;
         u64 usermem, ioffset, ilength;
         int ret, i;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create channel gpfifo size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x "
+                                 "ioffset %016llx ilength %08x engine %08x\n",
+                         args->v0.version, args->v0.pushbuf, args->v0.ioffset,
+                         args->v0.ilength, args->v0.engine);
+        } else
+                return ret;
 
         for (i = 0; i < FIFO_ENGINE_NR; i++) {
-                if (args->engine & (1 << i)) {
+                if (args->v0.engine & (1 << i)) {
                         if (nouveau_engine(parent, fifo_engine[i].subdev)) {
-                                args->engine = (1 << i);
+                                args->v0.engine = (1 << i);
                                 break;
                         }
                 }
         }
 
         if (i == FIFO_ENGINE_NR) {
-                nv_error(priv, "unsupported engines 0x%08x\n", args->engine);
+                nv_error(priv, "unsupported engines 0x%08x\n", args->v0.engine);
                 return -ENODEV;
         }
 
         ret = nouveau_fifo_channel_create(parent, engine, oclass, 1,
                                           priv->user.bar.offset, 0x200,
-                                          args->pushbuf,
+                                          args->v0.pushbuf,
                                           fifo_engine[i].mask, &chan);
         *pobject = nv_object(chan);
         if (ret)
                 return ret;
 
+        args->v0.chid = chan->base.chid;
+
         nv_parent(chan)->context_attach = nve0_fifo_context_attach;
         nv_parent(chan)->context_detach = nve0_fifo_context_detach;
         chan->engine = i;
 
         usermem = chan->base.chid * 0x200;
-        ioffset = args->ioffset;
-        ilength = order_base_2(args->ilength / 8);
+        ioffset = args->v0.ioffset;
+        ilength = order_base_2(args->v0.ilength / 8);
 
         for (i = 0; i < 0x200; i += 4)
                 nv_wo32(priv->user.mem, usermem + i, 0x00000000);
@@ -325,13 +336,15 @@ nve0_fifo_ofuncs = {
         .dtor = _nouveau_fifo_channel_dtor,
         .init = nve0_fifo_chan_init,
         .fini = nve0_fifo_chan_fini,
+        .map  = _nouveau_fifo_channel_map,
         .rd32 = _nouveau_fifo_channel_rd32,
         .wr32 = _nouveau_fifo_channel_wr32,
+        .ntfy = _nouveau_fifo_channel_ntfy
 };
 
 static struct nouveau_oclass
 nve0_fifo_sclass[] = {
-        { NVE0_CHANNEL_IND_CLASS, &nve0_fifo_ofuncs },
+        { KEPLER_CHANNEL_GPFIFO_A, &nve0_fifo_ofuncs },
         {}
 };
 
@@ -769,7 +782,7 @@ nve0_fifo_intr_fault(struct nve0_fifo_priv *priv, int unit)
         object = engctx;
         while (object) {
                 switch (nv_mclass(object)) {
-                case NVE0_CHANNEL_IND_CLASS:
+                case KEPLER_CHANNEL_GPFIFO_A:
                         nve0_fifo_recover(priv, engine, (void *)object);
                         break;
                 }
@@ -859,7 +872,7 @@ nve0_fifo_intr_runlist(struct nve0_fifo_priv *priv)
 static void
 nve0_fifo_intr_engine(struct nve0_fifo_priv *priv)
 {
-        nouveau_event_trigger(priv->base.uevent, 1, 0);
+        nouveau_fifo_uevent(&priv->base);
 }
 
 static void
@@ -952,19 +965,26 @@ nve0_fifo_intr(struct nouveau_subdev *subdev)
 }
 
 static void
-nve0_fifo_uevent_enable(struct nouveau_event *event, int type, int index)
+nve0_fifo_uevent_init(struct nvkm_event *event, int type, int index)
 {
-        struct nve0_fifo_priv *priv = event->priv;
-        nv_mask(priv, 0x002140, 0x80000000, 0x80000000);
+        struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent);
+        nv_mask(fifo, 0x002140, 0x80000000, 0x80000000);
 }
 
 static void
-nve0_fifo_uevent_disable(struct nouveau_event *event, int type, int index)
+nve0_fifo_uevent_fini(struct nvkm_event *event, int type, int index)
 {
-        struct nve0_fifo_priv *priv = event->priv;
-        nv_mask(priv, 0x002140, 0x80000000, 0x00000000);
+        struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent);
+        nv_mask(fifo, 0x002140, 0x80000000, 0x00000000);
 }
 
+static const struct nvkm_event_func
+nve0_fifo_uevent_func = {
+        .ctor = nouveau_fifo_uevent_ctor,
+        .init = nve0_fifo_uevent_init,
+        .fini = nve0_fifo_uevent_fini,
+};
+
 int
 nve0_fifo_fini(struct nouveau_object *object, bool suspend)
 {
@@ -1067,9 +1087,9 @@ nve0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        priv->base.uevent->enable = nve0_fifo_uevent_enable;
-        priv->base.uevent->disable = nve0_fifo_uevent_disable;
-        priv->base.uevent->priv = priv;
+        ret = nvkm_event_init(&nve0_fifo_uevent_func, 1, 1, &priv->base.uevent);
+        if (ret)
+                return ret;
 
         nv_subdev(priv)->unit = 0x00000100;
         nv_subdev(priv)->intr = nve0_fifo_intr;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxgk110b.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxgk110b.c
new file mode 100644
index 000000000000..3adb7fe91772
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxgk110b.c
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include "ctxnvc0.h"
+
+/*******************************************************************************
+ * PGRAPH context register lists
+ ******************************************************************************/
+
+static const struct nvc0_graph_init
+gk110b_grctx_init_sm_0[] = {
+        { 0x419e04,   1, 0x04, 0x00000000 },
+        { 0x419e08,   1, 0x04, 0x0000001d },
+        { 0x419e0c,   1, 0x04, 0x00000000 },
+        { 0x419e10,   1, 0x04, 0x00001c02 },
+        { 0x419e44,   1, 0x04, 0x0013eff2 },
+        { 0x419e48,   1, 0x04, 0x00000000 },
+        { 0x419e4c,   1, 0x04, 0x0000007f },
+        { 0x419e50,   2, 0x04, 0x00000000 },
+        { 0x419e58,   1, 0x04, 0x00000001 },
+        { 0x419e5c,   3, 0x04, 0x00000000 },
+        { 0x419e68,   1, 0x04, 0x00000002 },
+        { 0x419e6c,  12, 0x04, 0x00000000 },
+        { 0x419eac,   1, 0x04, 0x00001f8f },
+        { 0x419eb0,   1, 0x04, 0x0db00d2f },
+        { 0x419eb8,   1, 0x04, 0x00000000 },
+        { 0x419ec8,   1, 0x04, 0x0001304f },
+        { 0x419f30,   4, 0x04, 0x00000000 },
+        { 0x419f40,   1, 0x04, 0x00000018 },
+        { 0x419f44,   3, 0x04, 0x00000000 },
+        { 0x419f58,   1, 0x04, 0x00000000 },
+        { 0x419f70,   1, 0x04, 0x00006300 },
+        { 0x419f78,   1, 0x04, 0x000000eb },
+        { 0x419f7c,   1, 0x04, 0x00000404 },
+        {}
+};
+
+static const struct nvc0_graph_pack
+gk110b_grctx_pack_tpc[] = {
+        { nvd7_grctx_init_pe_0 },
+        { nvf0_grctx_init_tex_0 },
+        { nvf0_grctx_init_mpc_0 },
+        { nvf0_grctx_init_l1c_0 },
+        { gk110b_grctx_init_sm_0 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
+struct nouveau_oclass *
+gk110b_grctx_oclass = &(struct nvc0_grctx_oclass) {
+        .base.handle = NV_ENGCTX(GR, 0xf1),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nvc0_graph_context_ctor,
+                .dtor = nvc0_graph_context_dtor,
+                .init = _nouveau_graph_context_init,
+                .fini = _nouveau_graph_context_fini,
+                .rd32 = _nouveau_graph_context_rd32,
+                .wr32 = _nouveau_graph_context_wr32,
+        },
+        .main  = nve4_grctx_generate_main,
+        .unkn  = nve4_grctx_generate_unkn,
+        .hub   = nvf0_grctx_pack_hub,
+        .gpc   = nvf0_grctx_pack_gpc,
+        .zcull = nvc0_grctx_pack_zcull,
+        .tpc   = gk110b_grctx_pack_tpc,
+        .ppc   = nvf0_grctx_pack_ppc,
+        .icmd  = nvf0_grctx_pack_icmd,
+        .mthd  = nvf0_grctx_pack_mthd,
+        .bundle = nve4_grctx_generate_bundle,
+        .bundle_size = 0x3000,
+        .bundle_min_gpm_fifo_depth = 0x180,
+        .bundle_token_limit = 0x600,
+        .pagepool = nve4_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = nvd7_grctx_generate_attrib,
+        .attrib_nr_max = 0x324,
+        .attrib_nr = 0x218,
+        .alpha_nr_max = 0x7ff,
+        .alpha_nr = 0x648,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxgk20a.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxgk20a.c
index 224ee0287ab7..36fc9831cc93 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxgk20a.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxgk20a.c
@@ -41,7 +41,6 @@ gk20a_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .wr32 = _nouveau_graph_context_wr32,
         },
         .main  = nve4_grctx_generate_main,
-        .mods  = nve4_grctx_generate_mods,
         .unkn  = nve4_grctx_generate_unkn,
         .hub   = nve4_grctx_pack_hub,
         .gpc   = nve4_grctx_pack_gpc,
@@ -50,4 +49,15 @@ gk20a_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .ppc   = nve4_grctx_pack_ppc,
         .icmd  = nve4_grctx_pack_icmd,
         .mthd  = gk20a_grctx_pack_mthd,
+        .bundle = nve4_grctx_generate_bundle,
+        .bundle_size = 0x1800,
+        .bundle_min_gpm_fifo_depth = 0x62,
+        .bundle_token_limit = 0x100,
+        .pagepool = nve4_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = nvd7_grctx_generate_attrib,
+        .attrib_nr_max = 0x240,
+        .attrib_nr = 0x240,
+        .alpha_nr_max = 0x648 + (0x648 / 2),
+        .alpha_nr = 0x648,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxgm107.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxgm107.c
index b0d0fb2f4d08..62e918b9fa81 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxgm107.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxgm107.c
@@ -859,45 +859,74 @@ gm107_grctx_pack_ppc[] = {
  ******************************************************************************/
 
 static void
-gm107_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
+gm107_grctx_generate_bundle(struct nvc0_grctx *info)
 {
-        mmio_data(0x003000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x008000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x200000, 0x1000, NV_MEM_ACCESS_RW);
-
-        mmio_list(0x40800c, 0x00000000,  8, 1);
-        mmio_list(0x408010, 0x80000000,  0, 0);
-        mmio_list(0x419004, 0x00000000,  8, 1);
-        mmio_list(0x419008, 0x00000000,  0, 0);
-        mmio_list(0x4064cc, 0x80000000,  0, 0);
-        mmio_list(0x418e30, 0x80000000,  0, 0);
-
-        mmio_list(0x408004, 0x00000000,  8, 0);
-        mmio_list(0x408008, 0x80000030,  0, 0);
-        mmio_list(0x418e24, 0x00000000,  8, 0);
-        mmio_list(0x418e28, 0x80000030,  0, 0);
-
-        mmio_list(0x4064c8, 0x018002c0,  0, 0);
-
-        mmio_list(0x418810, 0x80000000, 12, 2);
-        mmio_list(0x419848, 0x10000000, 12, 2);
-        mmio_list(0x419c2c, 0x10000000, 12, 2);
-
-        mmio_list(0x405830, 0x0aa01000,  0, 0);
-        mmio_list(0x4064c4, 0x0400ffff,  0, 0);
-
-        /*XXX*/
-        mmio_list(0x5030c0, 0x00001540,  0, 0);
-        mmio_list(0x5030f4, 0x00000000,  0, 0);
-        mmio_list(0x5030e4, 0x00002000,  0, 0);
-        mmio_list(0x5030f8, 0x00003fc0,  0, 0);
-        mmio_list(0x418ea0, 0x07151540,  0, 0);
-
-        mmio_list(0x5032c0, 0x00001540,  0, 0);
-        mmio_list(0x5032f4, 0x00001fe0,  0, 0);
-        mmio_list(0x5032e4, 0x00002000,  0, 0);
-        mmio_list(0x5032f8, 0x00006fc0,  0, 0);
-        mmio_list(0x418ea4, 0x07151540,  0, 0);
+        const struct nvc0_grctx_oclass *impl = nvc0_grctx_impl(info->priv);
+        const u32 state_limit = min(impl->bundle_min_gpm_fifo_depth,
+                                    impl->bundle_size / 0x20);
+        const u32 token_limit = impl->bundle_token_limit;
+        const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
+        const int s = 8;
+        const int b = mmio_vram(info, impl->bundle_size, (1 << s), access);
+        mmio_refn(info, 0x408004, 0x00000000, s, b);
+        mmio_refn(info, 0x408008, 0x80000000 | (impl->bundle_size >> s), 0, b);
+        mmio_refn(info, 0x418e24, 0x00000000, s, b);
+        mmio_refn(info, 0x418e28, 0x80000000 | (impl->bundle_size >> s), 0, b);
+        mmio_wr32(info, 0x4064c8, (state_limit << 16) | token_limit);
+}
+
+static void
+gm107_grctx_generate_pagepool(struct nvc0_grctx *info)
+{
+        const struct nvc0_grctx_oclass *impl = nvc0_grctx_impl(info->priv);
+        const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
+        const int s = 8;
+        const int b = mmio_vram(info, impl->pagepool_size, (1 << s), access);
+        mmio_refn(info, 0x40800c, 0x00000000, s, b);
+        mmio_wr32(info, 0x408010, 0x80000000);
+        mmio_refn(info, 0x419004, 0x00000000, s, b);
+        mmio_wr32(info, 0x419008, 0x00000000);
+        mmio_wr32(info, 0x4064cc, 0x80000000);
+        mmio_wr32(info, 0x418e30, 0x80000000); /* guess at it being related */
+}
+
+static void
+gm107_grctx_generate_attrib(struct nvc0_grctx *info)
+{
+        struct nvc0_graph_priv *priv = info->priv;
+        const struct nvc0_grctx_oclass *impl = (void *)nvc0_grctx_impl(priv);
+        const u32  alpha = impl->alpha_nr;
+        const u32 attrib = impl->attrib_nr;
+        const u32   size = 0x20 * (impl->attrib_nr_max + impl->alpha_nr_max);
+        const u32 access = NV_MEM_ACCESS_RW;
+        const int s = 12;
+        const int b = mmio_vram(info, size * priv->tpc_total, (1 << s), access);
+        const int max_batches = 0xffff;
+        u32 bo = 0;
+        u32 ao = bo + impl->attrib_nr_max * priv->tpc_total;
+        int gpc, ppc, n = 0;
+
+        mmio_refn(info, 0x418810, 0x80000000, s, b);
+        mmio_refn(info, 0x419848, 0x10000000, s, b);
+        mmio_refn(info, 0x419c2c, 0x10000000, s, b);
+        mmio_wr32(info, 0x405830, (attrib << 16) | alpha);
+        mmio_wr32(info, 0x4064c4, ((alpha / 4) << 16) | max_batches);
+
+        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+                for (ppc = 0; ppc < priv->ppc_nr[gpc]; ppc++, n++) {
+                        const u32 as =  alpha * priv->ppc_tpc_nr[gpc][ppc];
+                        const u32 bs = attrib * priv->ppc_tpc_nr[gpc][ppc];
+                        const u32 u = 0x418ea0 + (n * 0x04);
+                        const u32 o = PPC_UNIT(gpc, ppc, 0);
+                        mmio_wr32(info, o + 0xc0, bs);
+                        mmio_wr32(info, o + 0xf4, bo);
+                        bo += impl->attrib_nr_max * priv->ppc_tpc_nr[gpc][ppc];
+                        mmio_wr32(info, o + 0xe4, as);
+                        mmio_wr32(info, o + 0xf8, ao);
+                        ao += impl->alpha_nr_max * priv->ppc_tpc_nr[gpc][ppc];
+                        mmio_wr32(info, u, (0x715 /*XXX*/ << 16) | bs);
+                }
+        }
 }
 
 static void
@@ -934,7 +963,9 @@ gm107_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 
         nv_wr32(priv, 0x404154, 0x00000000);
 
-        oclass->mods(priv, info);
+        oclass->bundle(info);
+        oclass->pagepool(info);
+        oclass->attrib(info);
         oclass->unkn(priv);
 
         gm107_grctx_generate_tpcid(priv);
@@ -979,7 +1010,6 @@ gm107_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .wr32 = _nouveau_graph_context_wr32,
         },
         .main  = gm107_grctx_generate_main,
-        .mods  = gm107_grctx_generate_mods,
         .unkn  = nve4_grctx_generate_unkn,
         .hub   = gm107_grctx_pack_hub,
         .gpc   = gm107_grctx_pack_gpc,
@@ -988,4 +1018,15 @@ gm107_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .ppc   = gm107_grctx_pack_ppc,
         .icmd  = gm107_grctx_pack_icmd,
         .mthd  = gm107_grctx_pack_mthd,
+        .bundle = gm107_grctx_generate_bundle,
+        .bundle_size = 0x3000,
+        .bundle_min_gpm_fifo_depth = 0x180,
+        .bundle_token_limit = 0x2c0,
+        .pagepool = gm107_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = gm107_grctx_generate_attrib,
+        .attrib_nr_max = 0xff0,
+        .attrib_nr = 0xaa0,
+        .alpha_nr_max = 0x1800,
+        .alpha_nr = 0x1000,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv108.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv108.c
index 8de4a4291548..ce252adbef81 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv108.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv108.c
@@ -531,50 +531,6 @@ nv108_grctx_pack_ppc[] = {
  * PGRAPH context implementation
  ******************************************************************************/
 
-static void
-nv108_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
-{
-        u32 magic[GPC_MAX][2];
-        u32 offset;
-        int gpc;
-
-        mmio_data(0x003000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x008000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x060000, 0x1000, NV_MEM_ACCESS_RW);
-        mmio_list(0x40800c, 0x00000000,  8, 1);
-        mmio_list(0x408010, 0x80000000,  0, 0);
-        mmio_list(0x419004, 0x00000000,  8, 1);
-        mmio_list(0x419008, 0x00000000,  0, 0);
-        mmio_list(0x4064cc, 0x80000000,  0, 0);
-        mmio_list(0x408004, 0x00000000,  8, 0);
-        mmio_list(0x408008, 0x80000030,  0, 0);
-        mmio_list(0x418808, 0x00000000,  8, 0);
-        mmio_list(0x41880c, 0x80000030,  0, 0);
-        mmio_list(0x4064c8, 0x00c20200,  0, 0);
-        mmio_list(0x418810, 0x80000000, 12, 2);
-        mmio_list(0x419848, 0x10000000, 12, 2);
-
-        mmio_list(0x405830, 0x02180648,  0, 0);
-        mmio_list(0x4064c4, 0x0192ffff,  0, 0);
-
-        for (gpc = 0, offset = 0; gpc < priv->gpc_nr; gpc++) {
-                u16 magic0 = 0x0218 * priv->tpc_nr[gpc];
-                u16 magic1 = 0x0648 * priv->tpc_nr[gpc];
-                magic[gpc][0]  = 0x10000000 | (magic0 << 16) | offset;
-                magic[gpc][1]  = 0x00000000 | (magic1 << 16);
-                offset += 0x0324 * priv->tpc_nr[gpc];
-        }
-
-        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-                mmio_list(GPC_UNIT(gpc, 0x30c0), magic[gpc][0], 0, 0);
-                mmio_list(GPC_UNIT(gpc, 0x30e4), magic[gpc][1] | offset, 0, 0);
-                offset += 0x07ff * priv->tpc_nr[gpc];
-        }
-
-        mmio_list(0x17e91c, 0x0b040a0b, 0, 0);
-        mmio_list(0x17e920, 0x00090d08, 0, 0);
-}
-
 struct nouveau_oclass *
 nv108_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .base.handle = NV_ENGCTX(GR, 0x08),
@@ -587,7 +543,6 @@ nv108_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .wr32 = _nouveau_graph_context_wr32,
         },
         .main  = nve4_grctx_generate_main,
-        .mods  = nv108_grctx_generate_mods,
         .unkn  = nve4_grctx_generate_unkn,
         .hub   = nv108_grctx_pack_hub,
         .gpc   = nv108_grctx_pack_gpc,
@@ -596,4 +551,15 @@ nv108_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .ppc   = nv108_grctx_pack_ppc,
         .icmd  = nv108_grctx_pack_icmd,
         .mthd  = nvf0_grctx_pack_mthd,
+        .bundle = nve4_grctx_generate_bundle,
+        .bundle_size = 0x3000,
+        .bundle_min_gpm_fifo_depth = 0xc2,
+        .bundle_token_limit = 0x200,
+        .pagepool = nve4_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = nvd7_grctx_generate_attrib,
+        .attrib_nr_max = 0x324,
+        .attrib_nr = 0x218,
+        .alpha_nr_max = 0x7ff,
+        .alpha_nr = 0x648,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.c
index 833a96508c4e..b8e5fe60a1eb 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.c
@@ -982,34 +982,93 @@ nvc0_grctx_pack_tpc[] = {
  * PGRAPH context implementation
  ******************************************************************************/
 
+int
+nvc0_grctx_mmio_data(struct nvc0_grctx *info, u32 size, u32 align, u32 access)
+{
+        if (info->data) {
+                info->buffer[info->buffer_nr] = round_up(info->addr, align);
+                info->addr = info->buffer[info->buffer_nr] + size;
+                info->data->size = size;
+                info->data->align = align;
+                info->data->access = access;
+                info->data++;
+                return info->buffer_nr++;
+        }
+        return -1;
+}
+
+void
+nvc0_grctx_mmio_item(struct nvc0_grctx *info, u32 addr, u32 data,
+                     int shift, int buffer)
+{
+        if (info->data) {
+                if (shift >= 0) {
+                        info->mmio->addr = addr;
+                        info->mmio->data = data;
+                        info->mmio->shift = shift;
+                        info->mmio->buffer = buffer;
+                        if (buffer >= 0)
+                                data |= info->buffer[buffer] >> shift;
+                        info->mmio++;
+                } else
+                        return;
+        } else {
+                if (buffer >= 0)
+                        return;
+        }
+
+        nv_wr32(info->priv, addr, data);
+}
+
+void
+nvc0_grctx_generate_bundle(struct nvc0_grctx *info)
+{
+        const struct nvc0_grctx_oclass *impl = nvc0_grctx_impl(info->priv);
+        const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
+        const int s = 8;
+        const int b = mmio_vram(info, impl->bundle_size, (1 << s), access);
+        mmio_refn(info, 0x408004, 0x00000000, s, b);
+        mmio_refn(info, 0x408008, 0x80000000 | (impl->bundle_size >> s), 0, b);
+        mmio_refn(info, 0x418808, 0x00000000, s, b);
+        mmio_refn(info, 0x41880c, 0x80000000 | (impl->bundle_size >> s), 0, b);
+}
+
 void
-nvc0_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
+nvc0_grctx_generate_pagepool(struct nvc0_grctx *info)
 {
+        const struct nvc0_grctx_oclass *impl = nvc0_grctx_impl(info->priv);
+        const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
+        const int s = 8;
+        const int b = mmio_vram(info, impl->pagepool_size, (1 << s), access);
+        mmio_refn(info, 0x40800c, 0x00000000, s, b);
+        mmio_wr32(info, 0x408010, 0x80000000);
+        mmio_refn(info, 0x419004, 0x00000000, s, b);
+        mmio_wr32(info, 0x419008, 0x00000000);
+}
+
+void
+nvc0_grctx_generate_attrib(struct nvc0_grctx *info)
+{
+        struct nvc0_graph_priv *priv = info->priv;
+        const struct nvc0_grctx_oclass *impl = nvc0_grctx_impl(priv);
+        const u32 attrib = impl->attrib_nr;
+        const u32   size = 0x20 * (impl->attrib_nr_max + impl->alpha_nr_max);
+        const u32 access = NV_MEM_ACCESS_RW;
+        const int s = 12;
+        const int b = mmio_vram(info, size * priv->tpc_total, (1 << s), access);
         int gpc, tpc;
-        u32 offset;
-
-        mmio_data(0x002000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x008000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x060000, 0x1000, NV_MEM_ACCESS_RW);
-
-        mmio_list(0x408004, 0x00000000,  8, 0);
-        mmio_list(0x408008, 0x80000018,  0, 0);
-        mmio_list(0x40800c, 0x00000000,  8, 1);
-        mmio_list(0x408010, 0x80000000,  0, 0);
-        mmio_list(0x418810, 0x80000000, 12, 2);
-        mmio_list(0x419848, 0x10000000, 12, 2);
-        mmio_list(0x419004, 0x00000000,  8, 1);
-        mmio_list(0x419008, 0x00000000,  0, 0);
-        mmio_list(0x418808, 0x00000000,  8, 0);
-        mmio_list(0x41880c, 0x80000018,  0, 0);
-
-        mmio_list(0x405830, 0x02180000, 0, 0);
-
-        for (gpc = 0, offset = 0; gpc < priv->gpc_nr; gpc++) {
+        u32 bo = 0;
+
+        mmio_refn(info, 0x418810, 0x80000000, s, b);
+        mmio_refn(info, 0x419848, 0x10000000, s, b);
+        mmio_wr32(info, 0x405830, (attrib << 16));
+
+        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
                 for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
-                        u32 addr = TPC_UNIT(gpc, tpc, 0x0520);
-                        mmio_list(addr, 0x02180000 | offset, 0, 0);
-                        offset += 0x0324;
+                        const u32 o = TPC_UNIT(gpc, tpc, 0x0520);
+                        mmio_skip(info, o, (attrib << 16) | ++bo);
+                        mmio_wr32(info, o, (attrib << 16) | --bo);
+                        bo += impl->attrib_nr_max;
                 }
         }
 }
@@ -1170,7 +1229,7 @@ nvc0_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 {
         struct nvc0_grctx_oclass *oclass = (void *)nv_engine(priv)->cclass;
 
-        nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
+        nouveau_mc(priv)->unk260(nouveau_mc(priv), 0);
 
         nvc0_graph_mmio(priv, oclass->hub);
         nvc0_graph_mmio(priv, oclass->gpc);
@@ -1180,7 +1239,9 @@ nvc0_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 
         nv_wr32(priv, 0x404154, 0x00000000);
 
-        oclass->mods(priv, info);
+        oclass->bundle(info);
+        oclass->pagepool(info);
+        oclass->attrib(info);
         oclass->unkn(priv);
 
         nvc0_grctx_generate_tpcid(priv);
@@ -1192,7 +1253,7 @@ nvc0_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
         nvc0_graph_icmd(priv, oclass->icmd);
         nv_wr32(priv, 0x404154, 0x00000400);
         nvc0_graph_mthd(priv, oclass->mthd);
-        nv_mask(priv, 0x000260, 0x00000001, 0x00000001);
+        nouveau_mc(priv)->unk260(nouveau_mc(priv), 1);
 }
 
 int
@@ -1308,7 +1369,6 @@ nvc0_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .wr32 = _nouveau_graph_context_wr32,
         },
         .main  = nvc0_grctx_generate_main,
-        .mods  = nvc0_grctx_generate_mods,
         .unkn  = nvc0_grctx_generate_unkn,
         .hub   = nvc0_grctx_pack_hub,
         .gpc   = nvc0_grctx_pack_gpc,
@@ -1316,4 +1376,11 @@ nvc0_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .tpc   = nvc0_grctx_pack_tpc,
         .icmd  = nvc0_grctx_pack_icmd,
         .mthd  = nvc0_grctx_pack_mthd,
+        .bundle = nvc0_grctx_generate_bundle,
+        .bundle_size = 0x1800,
+        .pagepool = nvc0_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = nvc0_grctx_generate_attrib,
+        .attrib_nr_max = 0x324,
+        .attrib_nr = 0x218,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.h b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.h
index 8da8b627b9d0..c776cd715e33 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.h
@@ -12,12 +12,19 @@ struct nvc0_grctx {
         u64 addr;
 };
 
+int  nvc0_grctx_mmio_data(struct nvc0_grctx *, u32 size, u32 align, u32 access);
+void nvc0_grctx_mmio_item(struct nvc0_grctx *, u32 addr, u32 data, int s, int);
+
+#define mmio_vram(a,b,c,d) nvc0_grctx_mmio_data((a), (b), (c), (d))
+#define mmio_refn(a,b,c,d,e) nvc0_grctx_mmio_item((a), (b), (c), (d), (e))
+#define mmio_skip(a,b,c) mmio_refn((a), (b), (c), -1, -1)
+#define mmio_wr32(a,b,c) mmio_refn((a), (b), (c),  0, -1)
+
 struct nvc0_grctx_oclass {
         struct nouveau_oclass base;
         /* main context generation function */
         void  (*main)(struct nvc0_graph_priv *, struct nvc0_grctx *);
         /* context-specific modify-on-first-load list generation function */
-        void  (*mods)(struct nvc0_graph_priv *, struct nvc0_grctx *);
         void  (*unkn)(struct nvc0_graph_priv *);
         /* mmio context data */
         const struct nvc0_graph_pack *hub;
@@ -28,30 +35,34 @@ struct nvc0_grctx_oclass {
         /* indirect context data, generated with icmds/mthds */
         const struct nvc0_graph_pack *icmd;
         const struct nvc0_graph_pack *mthd;
+        /* bundle circular buffer */
+        void (*bundle)(struct nvc0_grctx *);
+        u32 bundle_size;
+        u32 bundle_min_gpm_fifo_depth;
+        u32 bundle_token_limit;
+        /* pagepool */
+        void (*pagepool)(struct nvc0_grctx *);
+        u32 pagepool_size;
+        /* attribute(/alpha) circular buffer */
+        void (*attrib)(struct nvc0_grctx *);
+        u32 attrib_nr_max;
+        u32 attrib_nr;
+        u32 alpha_nr_max;
+        u32 alpha_nr;
 };
 
-#define mmio_data(s,a,p) do {                                                  \
-        info->buffer[info->buffer_nr] = round_up(info->addr, (a));             \
-        info->addr = info->buffer[info->buffer_nr++] + (s);                    \
-        info->data->size = (s);                                                \
-        info->data->align = (a);                                               \
-        info->data->access = (p);                                              \
-        info->data++;                                                          \
-} while(0)
-
-#define mmio_list(r,d,s,b) do {                                                \
-        info->mmio->addr = (r);                                                \
-        info->mmio->data = (d);                                                \
-        info->mmio->shift = (s);                                               \
-        info->mmio->buffer = (b);                                              \
-        info->mmio++;                                                          \
-        nv_wr32(priv, (r), (d) | ((s) ? (info->buffer[(b)] >> (s)) : 0));      \
-} while(0)
+static inline const struct nvc0_grctx_oclass *
+nvc0_grctx_impl(struct nvc0_graph_priv *priv)
+{
+        return (void *)nv_engine(priv)->cclass;
+}
 
 extern struct nouveau_oclass *nvc0_grctx_oclass;
 int  nvc0_grctx_generate(struct nvc0_graph_priv *);
 void nvc0_grctx_generate_main(struct nvc0_graph_priv *, struct nvc0_grctx *);
-void nvc0_grctx_generate_mods(struct nvc0_graph_priv *, struct nvc0_grctx *);
+void nvc0_grctx_generate_bundle(struct nvc0_grctx *);
+void nvc0_grctx_generate_pagepool(struct nvc0_grctx *);
+void nvc0_grctx_generate_attrib(struct nvc0_grctx *);
 void nvc0_grctx_generate_unkn(struct nvc0_graph_priv *);
 void nvc0_grctx_generate_tpcid(struct nvc0_graph_priv *);
 void nvc0_grctx_generate_r406028(struct nvc0_graph_priv *);
@@ -60,22 +71,27 @@ void nvc0_grctx_generate_r418bb8(struct nvc0_graph_priv *);
 void nvc0_grctx_generate_r406800(struct nvc0_graph_priv *);
 
 extern struct nouveau_oclass *nvc1_grctx_oclass;
-void nvc1_grctx_generate_mods(struct nvc0_graph_priv *, struct nvc0_grctx *);
+void nvc1_grctx_generate_attrib(struct nvc0_grctx *);
 void nvc1_grctx_generate_unkn(struct nvc0_graph_priv *);
 
 extern struct nouveau_oclass *nvc4_grctx_oclass;
 extern struct nouveau_oclass *nvc8_grctx_oclass;
+
 extern struct nouveau_oclass *nvd7_grctx_oclass;
+void nvd7_grctx_generate_attrib(struct nvc0_grctx *);
+
 extern struct nouveau_oclass *nvd9_grctx_oclass;
 
 extern struct nouveau_oclass *nve4_grctx_oclass;
 extern struct nouveau_oclass *gk20a_grctx_oclass;
 void nve4_grctx_generate_main(struct nvc0_graph_priv *, struct nvc0_grctx *);
-void nve4_grctx_generate_mods(struct nvc0_graph_priv *, struct nvc0_grctx *);
+void nve4_grctx_generate_bundle(struct nvc0_grctx *);
+void nve4_grctx_generate_pagepool(struct nvc0_grctx *);
 void nve4_grctx_generate_unkn(struct nvc0_graph_priv *);
 void nve4_grctx_generate_r418bb8(struct nvc0_graph_priv *);
 
 extern struct nouveau_oclass *nvf0_grctx_oclass;
+extern struct nouveau_oclass *gk110b_grctx_oclass;
 extern struct nouveau_oclass *nv108_grctx_oclass;
 extern struct nouveau_oclass *gm107_grctx_oclass;
 
@@ -160,16 +176,23 @@ extern const struct nvc0_graph_pack nve4_grctx_pack_ppc[];
 extern const struct nvc0_graph_pack nve4_grctx_pack_icmd[];
 extern const struct nvc0_graph_init nve4_grctx_init_a097_0[];
 
+extern const struct nvc0_graph_pack nvf0_grctx_pack_icmd[];
+
 extern const struct nvc0_graph_pack nvf0_grctx_pack_mthd[];
 
+extern const struct nvc0_graph_pack nvf0_grctx_pack_hub[];
 extern const struct nvc0_graph_init nvf0_grctx_init_pri_0[];
 extern const struct nvc0_graph_init nvf0_grctx_init_cwd_0[];
 
+extern const struct nvc0_graph_pack nvf0_grctx_pack_gpc[];
 extern const struct nvc0_graph_init nvf0_grctx_init_gpc_unk_2[];
 
+extern const struct nvc0_graph_init nvf0_grctx_init_tex_0[];
 extern const struct nvc0_graph_init nvf0_grctx_init_mpc_0[];
 extern const struct nvc0_graph_init nvf0_grctx_init_l1c_0[];
 
+extern const struct nvc0_graph_pack nvf0_grctx_pack_ppc[];
+
 extern const struct nvc0_graph_init nv108_grctx_init_rstr2d_0[];
 
 extern const struct nvc0_graph_init nv108_grctx_init_prop_0[];
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c
index 24a92c569c0a..c6ba8fed18f1 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c
@@ -727,38 +727,38 @@ nvc1_grctx_pack_tpc[] = {
  ******************************************************************************/
 
 void
-nvc1_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
+nvc1_grctx_generate_attrib(struct nvc0_grctx *info)
 {
+        struct nvc0_graph_priv *priv = info->priv;
+        const struct nvc0_grctx_oclass *impl = nvc0_grctx_impl(priv);
+        const u32  alpha = impl->alpha_nr;
+        const u32   beta = impl->attrib_nr;
+        const u32   size = 0x20 * (impl->attrib_nr_max + impl->alpha_nr_max);
+        const u32 access = NV_MEM_ACCESS_RW;
+        const int s = 12;
+        const int b = mmio_vram(info, size * priv->tpc_total, (1 << s), access);
+        const int timeslice_mode = 1;
+        const int max_batches = 0xffff;
+        u32 bo = 0;
+        u32 ao = bo + impl->attrib_nr_max * priv->tpc_total;
         int gpc, tpc;
-        u32 offset;
 
-        mmio_data(0x002000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x008000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x060000, 0x1000, NV_MEM_ACCESS_RW);
-        mmio_list(0x408004, 0x00000000,  8, 0);
-        mmio_list(0x408008, 0x80000018,  0, 0);
-        mmio_list(0x40800c, 0x00000000,  8, 1);
-        mmio_list(0x408010, 0x80000000,  0, 0);
-        mmio_list(0x418810, 0x80000000, 12, 2);
-        mmio_list(0x419848, 0x10000000, 12, 2);
-        mmio_list(0x419004, 0x00000000,  8, 1);
-        mmio_list(0x419008, 0x00000000,  0, 0);
-        mmio_list(0x418808, 0x00000000,  8, 0);
-        mmio_list(0x41880c, 0x80000018,  0, 0);
+        mmio_refn(info, 0x418810, 0x80000000, s, b);
+        mmio_refn(info, 0x419848, 0x10000000, s, b);
+        mmio_wr32(info, 0x405830, (beta << 16) | alpha);
+        mmio_wr32(info, 0x4064c4, ((alpha / 4) << 16) | max_batches);
 
-        mmio_list(0x405830, 0x02180218, 0, 0);
-        mmio_list(0x4064c4, 0x0086ffff, 0, 0);
-
-        for (gpc = 0, offset = 0; gpc < priv->gpc_nr; gpc++) {
-                for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
-                        u32 addr = TPC_UNIT(gpc, tpc, 0x0520);
-                        mmio_list(addr, 0x12180000 | offset, 0, 0);
-                        offset += 0x0324;
-                }
+        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
                 for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
-                        u32 addr = TPC_UNIT(gpc, tpc, 0x0544);
-                        mmio_list(addr, 0x02180000 | offset, 0, 0);
-                        offset += 0x0324;
+                        const u32 a = alpha;
+                        const u32 b =  beta;
+                        const u32 t = timeslice_mode;
+                        const u32 o = TPC_UNIT(gpc, tpc, 0x500);
+                        mmio_skip(info, o + 0x20, (t << 28) | (b << 16) | ++bo);
+                        mmio_wr32(info, o + 0x20, (t << 28) | (b << 16) | --bo);
+                        bo += impl->attrib_nr_max;
+                        mmio_wr32(info, o + 0x44, (a << 16) | ao);
+                        ao += impl->alpha_nr_max;
                 }
         }
 }
@@ -786,7 +786,6 @@ nvc1_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .wr32 = _nouveau_graph_context_wr32,
         },
         .main  = nvc0_grctx_generate_main,
-        .mods  = nvc1_grctx_generate_mods,
         .unkn  = nvc1_grctx_generate_unkn,
         .hub   = nvc1_grctx_pack_hub,
         .gpc   = nvc1_grctx_pack_gpc,
@@ -794,4 +793,13 @@ nvc1_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .tpc   = nvc1_grctx_pack_tpc,
         .icmd  = nvc1_grctx_pack_icmd,
         .mthd  = nvc1_grctx_pack_mthd,
+        .bundle = nvc0_grctx_generate_bundle,
+        .bundle_size = 0x1800,
+        .pagepool = nvc0_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = nvc1_grctx_generate_attrib,
+        .attrib_nr_max = 0x324,
+        .attrib_nr = 0x218,
+        .alpha_nr_max = 0x324,
+        .alpha_nr = 0x218,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc4.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc4.c
index e11ed5538193..41705c60cc47 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc4.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc4.c
@@ -92,7 +92,6 @@ nvc4_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .wr32 = _nouveau_graph_context_wr32,
         },
         .main  = nvc0_grctx_generate_main,
-        .mods  = nvc0_grctx_generate_mods,
         .unkn  = nvc0_grctx_generate_unkn,
         .hub   = nvc0_grctx_pack_hub,
         .gpc   = nvc0_grctx_pack_gpc,
@@ -100,4 +99,11 @@ nvc4_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .tpc   = nvc4_grctx_pack_tpc,
         .icmd  = nvc0_grctx_pack_icmd,
         .mthd  = nvc0_grctx_pack_mthd,
+        .bundle = nvc0_grctx_generate_bundle,
+        .bundle_size = 0x1800,
+        .pagepool = nvc0_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = nvc0_grctx_generate_attrib,
+        .attrib_nr_max = 0x324,
+        .attrib_nr = 0x218,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc8.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc8.c
index feebd58dfe8d..8f804cd8f9c7 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc8.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc8.c
@@ -343,7 +343,6 @@ nvc8_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .wr32 = _nouveau_graph_context_wr32,
         },
         .main  = nvc0_grctx_generate_main,
-        .mods  = nvc0_grctx_generate_mods,
         .unkn  = nvc0_grctx_generate_unkn,
         .hub   = nvc0_grctx_pack_hub,
         .gpc   = nvc8_grctx_pack_gpc,
@@ -351,4 +350,11 @@ nvc8_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .tpc   = nvc0_grctx_pack_tpc,
         .icmd  = nvc8_grctx_pack_icmd,
         .mthd  = nvc8_grctx_pack_mthd,
+        .bundle = nvc0_grctx_generate_bundle,
+        .bundle_size = 0x1800,
+        .pagepool = nvc0_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = nvc0_grctx_generate_attrib,
+        .attrib_nr_max = 0x324,
+        .attrib_nr = 0x218,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c
index 1dbc8d7f2e86..fcf534fd9e65 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c
@@ -177,44 +177,41 @@ nvd7_grctx_pack_ppc[] = {
  * PGRAPH context implementation
  ******************************************************************************/
 
-static void
-nvd7_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
+void
+nvd7_grctx_generate_attrib(struct nvc0_grctx *info)
 {
-        u32 magic[GPC_MAX][2];
-        u32 offset;
-        int gpc;
-
-        mmio_data(0x003000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x008000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x060000, 0x1000, NV_MEM_ACCESS_RW);
-        mmio_list(0x40800c, 0x00000000,  8, 1);
-        mmio_list(0x408010, 0x80000000,  0, 0);
-        mmio_list(0x419004, 0x00000000,  8, 1);
-        mmio_list(0x419008, 0x00000000,  0, 0);
-        mmio_list(0x408004, 0x00000000,  8, 0);
-        mmio_list(0x408008, 0x80000018,  0, 0);
-        mmio_list(0x418808, 0x00000000,  8, 0);
-        mmio_list(0x41880c, 0x80000018,  0, 0);
-        mmio_list(0x418810, 0x80000000, 12, 2);
-        mmio_list(0x419848, 0x10000000, 12, 2);
+        struct nvc0_graph_priv *priv = info->priv;
+        const struct nvc0_grctx_oclass *impl = nvc0_grctx_impl(priv);
+        const u32  alpha = impl->alpha_nr;
+        const u32   beta = impl->attrib_nr;
+        const u32   size = 0x20 * (impl->attrib_nr_max + impl->alpha_nr_max);
+        const u32 access = NV_MEM_ACCESS_RW;
+        const int s = 12;
+        const int b = mmio_vram(info, size * priv->tpc_total, (1 << s), access);
+        const int timeslice_mode = 1;
+        const int max_batches = 0xffff;
+        u32 bo = 0;
+        u32 ao = bo + impl->attrib_nr_max * priv->tpc_total;
+        int gpc, ppc;
 
-        mmio_list(0x405830, 0x02180324,  0, 0);
-        mmio_list(0x4064c4, 0x00c9ffff,  0, 0);
-
-        for (gpc = 0, offset = 0; gpc < priv->gpc_nr; gpc++) {
-                u16 magic0 = 0x0218 * priv->tpc_nr[gpc];
-                u16 magic1 = 0x0324 * priv->tpc_nr[gpc];
-                magic[gpc][0]  = 0x10000000 | (magic0 << 16) | offset;
-                magic[gpc][1]  = 0x00000000 | (magic1 << 16);
-                offset += 0x0324 * priv->tpc_nr[gpc];
-        }
+        mmio_refn(info, 0x418810, 0x80000000, s, b);
+        mmio_refn(info, 0x419848, 0x10000000, s, b);
+        mmio_wr32(info, 0x405830, (beta << 16) | alpha);
+        mmio_wr32(info, 0x4064c4, ((alpha / 4) << 16) | max_batches);
 
         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-                mmio_list(GPC_UNIT(gpc, 0x30c0), magic[gpc][0], 0, 0);
-                mmio_list(GPC_UNIT(gpc, 0x30e4), magic[gpc][1] | offset, 0, 0);
-                offset += 0x07ff * priv->tpc_nr[gpc];
+                for (ppc = 0; ppc < priv->ppc_nr[gpc]; ppc++) {
+                        const u32 a = alpha * priv->ppc_tpc_nr[gpc][ppc];
+                        const u32 b =  beta * priv->ppc_tpc_nr[gpc][ppc];
+                        const u32 t = timeslice_mode;
+                        const u32 o = PPC_UNIT(gpc, ppc, 0);
+                        mmio_skip(info, o + 0xc0, (t << 28) | (b << 16) | ++bo);
+                        mmio_wr32(info, o + 0xc0, (t << 28) | (b << 16) | --bo);
+                        bo += impl->attrib_nr_max * priv->ppc_tpc_nr[gpc][ppc];
+                        mmio_wr32(info, o + 0xe4, (a << 16) | ao);
+                        ao += impl->alpha_nr_max * priv->ppc_tpc_nr[gpc][ppc];
+                }
         }
-        mmio_list(0x17e91c, 0x03060609, 0, 0); /* different from kepler */
 }
 
 void
@@ -223,7 +220,7 @@ nvd7_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
         struct nvc0_grctx_oclass *oclass = (void *)nv_engine(priv)->cclass;
         int i;
 
-        nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
+        nouveau_mc(priv)->unk260(nouveau_mc(priv), 0);
 
         nvc0_graph_mmio(priv, oclass->hub);
         nvc0_graph_mmio(priv, oclass->gpc);
@@ -233,7 +230,9 @@ nvd7_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 
         nv_wr32(priv, 0x404154, 0x00000000);
 
-        oclass->mods(priv, info);
+        oclass->bundle(info);
+        oclass->pagepool(info);
+        oclass->attrib(info);
         oclass->unkn(priv);
 
         nvc0_grctx_generate_tpcid(priv);
@@ -248,7 +247,7 @@ nvd7_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
         nvc0_graph_icmd(priv, oclass->icmd);
         nv_wr32(priv, 0x404154, 0x00000400);
         nvc0_graph_mthd(priv, oclass->mthd);
-        nv_mask(priv, 0x000260, 0x00000001, 0x00000001);
+        nouveau_mc(priv)->unk260(nouveau_mc(priv), 1);
 }
 
 struct nouveau_oclass *
@@ -263,7 +262,6 @@ nvd7_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .wr32 = _nouveau_graph_context_wr32,
         },
         .main  = nvd7_grctx_generate_main,
-        .mods  = nvd7_grctx_generate_mods,
         .unkn  = nve4_grctx_generate_unkn,
         .hub   = nvd7_grctx_pack_hub,
         .gpc   = nvd7_grctx_pack_gpc,
@@ -272,4 +270,13 @@ nvd7_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .ppc   = nvd7_grctx_pack_ppc,
         .icmd  = nvd9_grctx_pack_icmd,
         .mthd  = nvd9_grctx_pack_mthd,
+        .bundle = nvc0_grctx_generate_bundle,
+        .bundle_size = 0x1800,
+        .pagepool = nvc0_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = nvd7_grctx_generate_attrib,
+        .attrib_nr_max = 0x324,
+        .attrib_nr = 0x218,
+        .alpha_nr_max = 0x7ff,
+        .alpha_nr = 0x324,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c
index c665fb7e4660..b9a301b6fd9f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c
@@ -511,7 +511,6 @@ nvd9_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .wr32 = _nouveau_graph_context_wr32,
         },
         .main  = nvc0_grctx_generate_main,
-        .mods  = nvc1_grctx_generate_mods,
         .unkn  = nvc1_grctx_generate_unkn,
         .hub   = nvd9_grctx_pack_hub,
         .gpc   = nvd9_grctx_pack_gpc,
@@ -519,4 +518,13 @@ nvd9_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .tpc   = nvd9_grctx_pack_tpc,
         .icmd  = nvd9_grctx_pack_icmd,
         .mthd  = nvd9_grctx_pack_mthd,
+        .bundle = nvc0_grctx_generate_bundle,
+        .bundle_size = 0x1800,
+        .pagepool = nvc0_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = nvc1_grctx_generate_attrib,
+        .attrib_nr_max = 0x324,
+        .attrib_nr = 0x218,
+        .alpha_nr_max = 0x324,
+        .alpha_nr = 0x218,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnve4.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnve4.c
index c5b249238587..ccac2ee1a1cb 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnve4.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnve4.c
@@ -839,47 +839,34 @@ nve4_grctx_pack_ppc[] = {
  ******************************************************************************/
 
 void
-nve4_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
+nve4_grctx_generate_bundle(struct nvc0_grctx *info)
 {
-        u32 magic[GPC_MAX][2];
-        u32 offset;
-        int gpc;
-
-        mmio_data(0x003000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x008000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x060000, 0x1000, NV_MEM_ACCESS_RW);
-        mmio_list(0x40800c, 0x00000000,  8, 1);
-        mmio_list(0x408010, 0x80000000,  0, 0);
-        mmio_list(0x419004, 0x00000000,  8, 1);
-        mmio_list(0x419008, 0x00000000,  0, 0);
-        mmio_list(0x4064cc, 0x80000000,  0, 0);
-        mmio_list(0x408004, 0x00000000,  8, 0);
-        mmio_list(0x408008, 0x80000030,  0, 0);
-        mmio_list(0x418808, 0x00000000,  8, 0);
-        mmio_list(0x41880c, 0x80000030,  0, 0);
-        mmio_list(0x4064c8, 0x01800600,  0, 0);
-        mmio_list(0x418810, 0x80000000, 12, 2);
-        mmio_list(0x419848, 0x10000000, 12, 2);
-
-        mmio_list(0x405830, 0x02180648,  0, 0);
-        mmio_list(0x4064c4, 0x0192ffff,  0, 0);
-
-        for (gpc = 0, offset = 0; gpc < priv->gpc_nr; gpc++) {
-                u16 magic0 = 0x0218 * priv->tpc_nr[gpc];
-                u16 magic1 = 0x0648 * priv->tpc_nr[gpc];
-                magic[gpc][0]  = 0x10000000 | (magic0 << 16) | offset;
-                magic[gpc][1]  = 0x00000000 | (magic1 << 16);
-                offset += 0x0324 * priv->tpc_nr[gpc];
-        }
-
-        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-                mmio_list(GPC_UNIT(gpc, 0x30c0), magic[gpc][0], 0, 0);
-                mmio_list(GPC_UNIT(gpc, 0x30e4), magic[gpc][1] | offset, 0, 0);
-                offset += 0x07ff * priv->tpc_nr[gpc];
-        }
+        const struct nvc0_grctx_oclass *impl = nvc0_grctx_impl(info->priv);
+        const u32 state_limit = min(impl->bundle_min_gpm_fifo_depth,
+                                    impl->bundle_size / 0x20);
+        const u32 token_limit = impl->bundle_token_limit;
+        const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
+        const int s = 8;
+        const int b = mmio_vram(info, impl->bundle_size, (1 << s), access);
+        mmio_refn(info, 0x408004, 0x00000000, s, b);
+        mmio_refn(info, 0x408008, 0x80000000 | (impl->bundle_size >> s), 0, b);
+        mmio_refn(info, 0x418808, 0x00000000, s, b);
+        mmio_refn(info, 0x41880c, 0x80000000 | (impl->bundle_size >> s), 0, b);
+        mmio_wr32(info, 0x4064c8, (state_limit << 16) | token_limit);
+}
 
-        mmio_list(0x17e91c, 0x06060609, 0, 0);
-        mmio_list(0x17e920, 0x00090a05, 0, 0);
+void
+nve4_grctx_generate_pagepool(struct nvc0_grctx *info)
+{
+        const struct nvc0_grctx_oclass *impl = nvc0_grctx_impl(info->priv);
+        const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
+        const int s = 8;
+        const int b = mmio_vram(info, impl->pagepool_size, (1 << s), access);
+        mmio_refn(info, 0x40800c, 0x00000000, s, b);
+        mmio_wr32(info, 0x408010, 0x80000000);
+        mmio_refn(info, 0x419004, 0x00000000, s, b);
+        mmio_wr32(info, 0x419008, 0x00000000);
+        mmio_wr32(info, 0x4064cc, 0x80000000);
 }
 
 void
@@ -957,7 +944,7 @@ nve4_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
         struct nvc0_grctx_oclass *oclass = (void *)nv_engine(priv)->cclass;
         int i;
 
-        nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
+        nouveau_mc(priv)->unk260(nouveau_mc(priv), 0);
 
         nvc0_graph_mmio(priv, oclass->hub);
         nvc0_graph_mmio(priv, oclass->gpc);
@@ -967,7 +954,9 @@ nve4_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 
         nv_wr32(priv, 0x404154, 0x00000000);
 
-        oclass->mods(priv, info);
+        oclass->bundle(info);
+        oclass->pagepool(info);
+        oclass->attrib(info);
         oclass->unkn(priv);
 
         nvc0_grctx_generate_tpcid(priv);
@@ -991,7 +980,7 @@ nve4_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
         nvc0_graph_icmd(priv, oclass->icmd);
         nv_wr32(priv, 0x404154, 0x00000400);
         nvc0_graph_mthd(priv, oclass->mthd);
-        nv_mask(priv, 0x000260, 0x00000001, 0x00000001);
+        nouveau_mc(priv)->unk260(nouveau_mc(priv), 1);
 
         nv_mask(priv, 0x418800, 0x00200000, 0x00200000);
         nv_mask(priv, 0x41be10, 0x00800000, 0x00800000);
@@ -1009,7 +998,6 @@ nve4_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .wr32 = _nouveau_graph_context_wr32,
         },
         .main  = nve4_grctx_generate_main,
-        .mods  = nve4_grctx_generate_mods,
         .unkn  = nve4_grctx_generate_unkn,
         .hub   = nve4_grctx_pack_hub,
         .gpc   = nve4_grctx_pack_gpc,
@@ -1018,4 +1006,15 @@ nve4_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .ppc   = nve4_grctx_pack_ppc,
         .icmd  = nve4_grctx_pack_icmd,
         .mthd  = nve4_grctx_pack_mthd,
+        .bundle = nve4_grctx_generate_bundle,
+        .bundle_size = 0x3000,
+        .bundle_min_gpm_fifo_depth = 0x180,
+        .bundle_token_limit = 0x600,
+        .pagepool = nve4_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = nvd7_grctx_generate_attrib,
+        .attrib_nr_max = 0x324,
+        .attrib_nr = 0x218,
+        .alpha_nr_max = 0x7ff,
+        .alpha_nr = 0x648,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvf0.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvf0.c
index dec03f04114d..e9b0dcf95a49 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvf0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvf0.c
@@ -279,7 +279,7 @@ nvf0_grctx_init_icmd_0[] = {
         {}
 };
 
-static const struct nvc0_graph_pack
+const struct nvc0_graph_pack
 nvf0_grctx_pack_icmd[] = {
         { nvf0_grctx_init_icmd_0 },
         {}
@@ -668,7 +668,7 @@ nvf0_grctx_init_be_0[] = {
         {}
 };
 
-static const struct nvc0_graph_pack
+const struct nvc0_graph_pack
 nvf0_grctx_pack_hub[] = {
         { nvc0_grctx_init_main_0 },
         { nvf0_grctx_init_fe_0 },
@@ -704,7 +704,7 @@ nvf0_grctx_init_gpc_unk_2[] = {
         {}
 };
 
-static const struct nvc0_graph_pack
+const struct nvc0_graph_pack
 nvf0_grctx_pack_gpc[] = {
         { nvc0_grctx_init_gpc_unk_0 },
         { nvd9_grctx_init_prop_0 },
@@ -718,7 +718,7 @@ nvf0_grctx_pack_gpc[] = {
         {}
 };
 
-static const struct nvc0_graph_init
+const struct nvc0_graph_init
 nvf0_grctx_init_tex_0[] = {
         { 0x419a00,   1, 0x04, 0x000000f0 },
         { 0x419a04,   1, 0x04, 0x00000001 },
@@ -797,7 +797,7 @@ nvf0_grctx_init_cbm_0[] = {
         {}
 };
 
-static const struct nvc0_graph_pack
+const struct nvc0_graph_pack
 nvf0_grctx_pack_ppc[] = {
         { nve4_grctx_init_pes_0 },
         { nvf0_grctx_init_cbm_0 },
@@ -809,58 +809,6 @@ nvf0_grctx_pack_ppc[] = {
  * PGRAPH context implementation
  ******************************************************************************/
 
-static void
-nvf0_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
-{
-        u32 magic[GPC_MAX][4];
-        u32 offset;
-        int gpc;
-
-        mmio_data(0x003000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x008000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
-        mmio_data(0x060000, 0x1000, NV_MEM_ACCESS_RW);
-        mmio_list(0x40800c, 0x00000000,  8, 1);
-        mmio_list(0x408010, 0x80000000,  0, 0);
-        mmio_list(0x419004, 0x00000000,  8, 1);
-        mmio_list(0x419008, 0x00000000,  0, 0);
-        mmio_list(0x4064cc, 0x80000000,  0, 0);
-        mmio_list(0x408004, 0x00000000,  8, 0);
-        mmio_list(0x408008, 0x80000030,  0, 0);
-        mmio_list(0x418808, 0x00000000,  8, 0);
-        mmio_list(0x41880c, 0x80000030,  0, 0);
-        mmio_list(0x4064c8, 0x01800600,  0, 0);
-        mmio_list(0x418810, 0x80000000, 12, 2);
-        mmio_list(0x419848, 0x10000000, 12, 2);
-
-        mmio_list(0x405830, 0x02180648,  0, 0);
-        mmio_list(0x4064c4, 0x0192ffff,  0, 0);
-
-        for (gpc = 0, offset = 0; gpc < priv->gpc_nr; gpc++) {
-                u16 magic0 = 0x0218 * (priv->tpc_nr[gpc] - 1);
-                u16 magic1 = 0x0648 * (priv->tpc_nr[gpc] - 1);
-                u16 magic2 = 0x0218;
-                u16 magic3 = 0x0648;
-                magic[gpc][0]  = 0x10000000 | (magic0 << 16) | offset;
-                magic[gpc][1]  = 0x00000000 | (magic1 << 16);
-                offset += 0x0324 * (priv->tpc_nr[gpc] - 1);
-                magic[gpc][2]  = 0x10000000 | (magic2 << 16) | offset;
-                magic[gpc][3]  = 0x00000000 | (magic3 << 16);
-                offset += 0x0324;
-        }
-
-        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-                mmio_list(GPC_UNIT(gpc, 0x30c0), magic[gpc][0], 0, 0);
-                mmio_list(GPC_UNIT(gpc, 0x30e4), magic[gpc][1] | offset, 0, 0);
-                offset += 0x07ff * (priv->tpc_nr[gpc] - 1);
-                mmio_list(GPC_UNIT(gpc, 0x32c0), magic[gpc][2], 0, 0);
-                mmio_list(GPC_UNIT(gpc, 0x32e4), magic[gpc][3] | offset, 0, 0);
-                offset += 0x07ff;
-        }
-
-        mmio_list(0x17e91c, 0x06060609, 0, 0);
-        mmio_list(0x17e920, 0x00090a05, 0, 0);
-}
-
 struct nouveau_oclass *
 nvf0_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .base.handle = NV_ENGCTX(GR, 0xf0),
@@ -873,7 +821,6 @@ nvf0_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .wr32 = _nouveau_graph_context_wr32,
         },
         .main  = nve4_grctx_generate_main,
-        .mods  = nvf0_grctx_generate_mods,
         .unkn  = nve4_grctx_generate_unkn,
         .hub   = nvf0_grctx_pack_hub,
         .gpc   = nvf0_grctx_pack_gpc,
@@ -882,4 +829,15 @@ nvf0_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .ppc   = nvf0_grctx_pack_ppc,
         .icmd  = nvf0_grctx_pack_icmd,
         .mthd  = nvf0_grctx_pack_mthd,
+        .bundle = nve4_grctx_generate_bundle,
+        .bundle_size = 0x3000,
+        .bundle_min_gpm_fifo_depth = 0x180,
+        .bundle_token_limit = 0x7c0,
+        .pagepool = nve4_grctx_generate_pagepool,
+        .pagepool_size = 0x8000,
+        .attrib = nvd7_grctx_generate_attrib,
+        .attrib_nr_max = 0x324,
+        .attrib_nr = 0x218,
+        .alpha_nr_max = 0x7ff,
+        .alpha_nr = 0x648,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/gk110b.c b/drivers/gpu/drm/nouveau/core/engine/graph/gk110b.c
new file mode 100644
index 000000000000..d07b19dc168d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/gk110b.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include "nvc0.h"
+#include "ctxnvc0.h"
+
+/*******************************************************************************
+ * PGRAPH register lists
+ ******************************************************************************/
+
+static const struct nvc0_graph_init
+gk110b_graph_init_l1c_0[] = {
+        { 0x419c98,   1, 0x04, 0x00000000 },
+        { 0x419ca8,   1, 0x04, 0x00000000 },
+        { 0x419cb0,   1, 0x04, 0x09000000 },
+        { 0x419cb4,   1, 0x04, 0x00000000 },
+        { 0x419cb8,   1, 0x04, 0x00b08bea },
+        { 0x419c84,   1, 0x04, 0x00010384 },
+        { 0x419cbc,   1, 0x04, 0x281b3646 },
+        { 0x419cc0,   2, 0x04, 0x00000000 },
+        { 0x419c80,   1, 0x04, 0x00020230 },
+        { 0x419ccc,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gk110b_graph_init_sm_0[] = {
+        { 0x419e00,   1, 0x04, 0x00000080 },
+        { 0x419ea0,   1, 0x04, 0x00000000 },
+        { 0x419ee4,   1, 0x04, 0x00000000 },
+        { 0x419ea4,   1, 0x04, 0x00000100 },
+        { 0x419ea8,   1, 0x04, 0x00000000 },
+        { 0x419eb4,   1, 0x04, 0x00000000 },
+        { 0x419ebc,   2, 0x04, 0x00000000 },
+        { 0x419edc,   1, 0x04, 0x00000000 },
+        { 0x419f00,   1, 0x04, 0x00000000 },
+        { 0x419ed0,   1, 0x04, 0x00002616 },
+        { 0x419f74,   1, 0x04, 0x00015555 },
+        { 0x419f80,   4, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_pack
+gk110b_graph_pack_mmio[] = {
+        { nve4_graph_init_main_0 },
+        { nvf0_graph_init_fe_0 },
+        { nvc0_graph_init_pri_0 },
+        { nvc0_graph_init_rstr2d_0 },
+        { nvd9_graph_init_pd_0 },
+        { nvf0_graph_init_ds_0 },
+        { nvc0_graph_init_scc_0 },
+        { nvf0_graph_init_sked_0 },
+        { nvf0_graph_init_cwd_0 },
+        { nvd9_graph_init_prop_0 },
+        { nvc1_graph_init_gpc_unk_0 },
+        { nvc0_graph_init_setup_0 },
+        { nvc0_graph_init_crstr_0 },
+        { nvc1_graph_init_setup_1 },
+        { nvc0_graph_init_zcull_0 },
+        { nvd9_graph_init_gpm_0 },
+        { nvf0_graph_init_gpc_unk_1 },
+        { nvc0_graph_init_gcc_0 },
+        { nve4_graph_init_tpccs_0 },
+        { nvf0_graph_init_tex_0 },
+        { nve4_graph_init_pe_0 },
+        { gk110b_graph_init_l1c_0 },
+        { nvc0_graph_init_mpc_0 },
+        { gk110b_graph_init_sm_0 },
+        { nvd7_graph_init_pes_0 },
+        { nvd7_graph_init_wwdx_0 },
+        { nvd7_graph_init_cbm_0 },
+        { nve4_graph_init_be_0 },
+        { nvc0_graph_init_fe_1 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+struct nouveau_oclass *
+gk110b_graph_oclass = &(struct nvc0_graph_oclass) {
+        .base.handle = NV_ENGINE(GR, 0xf1),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nvc0_graph_ctor,
+                .dtor = nvc0_graph_dtor,
+                .init = nve4_graph_init,
+                .fini = nvf0_graph_fini,
+        },
+        .cclass = &gk110b_grctx_oclass,
+        .sclass =  nvf0_graph_sclass,
+        .mmio = gk110b_graph_pack_mmio,
+        .fecs.ucode = &nvf0_graph_fecs_ucode,
+        .gpccs.ucode = &nvf0_graph_gpccs_ucode,
+        .ppc_nr = 2,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/gk20a.c b/drivers/gpu/drm/nouveau/core/engine/graph/gk20a.c
index 83048a56430d..7d0abe9f3fe7 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/gk20a.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/gk20a.c
@@ -27,8 +27,8 @@ static struct nouveau_oclass
 gk20a_graph_sclass[] = {
         { 0x902d, &nouveau_object_ofuncs },
         { 0xa040, &nouveau_object_ofuncs },
-        { 0xa297, &nouveau_object_ofuncs },
-        { 0xa0c0, &nouveau_object_ofuncs },
+        { KEPLER_C, &nvc0_fermi_ofuncs, nvc0_graph_9097_omthds },
+        { KEPLER_COMPUTE_A, &nouveau_object_ofuncs, nvc0_graph_90c0_omthds },
         {}
 };
 
@@ -39,9 +39,10 @@ gk20a_graph_oclass = &(struct nvc0_graph_oclass) {
                 .ctor = nvc0_graph_ctor,
                 .dtor = nvc0_graph_dtor,
                 .init = nve4_graph_init,
-                .fini = nve4_graph_fini,
+                .fini = _nouveau_graph_fini,
         },
         .cclass = &gk20a_grctx_oclass,
         .sclass = gk20a_graph_sclass,
         .mmio = nve4_graph_pack_mmio,
+        .ppc_nr = 1,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/gm107.c b/drivers/gpu/drm/nouveau/core/engine/graph/gm107.c
index 21c5f31d607f..4bdbdab2fd9a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/gm107.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/gm107.c
@@ -36,8 +36,8 @@ static struct nouveau_oclass
 gm107_graph_sclass[] = {
         { 0x902d, &nouveau_object_ofuncs },
         { 0xa140, &nouveau_object_ofuncs },
-        { 0xb097, &nouveau_object_ofuncs },
-        { 0xb0c0, &nouveau_object_ofuncs },
+        { MAXWELL_A, &nvc0_fermi_ofuncs, nvc0_graph_9097_omthds },
+        { MAXWELL_COMPUTE_A, &nouveau_object_ofuncs, nvc0_graph_90c0_omthds },
         {}
 };
 
@@ -425,6 +425,9 @@ gm107_graph_init(struct nouveau_object *object)
         nv_wr32(priv, 0x400134, 0xffffffff);
 
         nv_wr32(priv, 0x400054, 0x2c350f63);
+
+        nvc0_graph_zbc_init(priv);
+
         return nvc0_graph_init_ctxctl(priv);
 }
 
@@ -462,4 +465,5 @@ gm107_graph_oclass = &(struct nvc0_graph_oclass) {
         .mmio = gm107_graph_pack_mmio,
         .fecs.ucode = 0 ? &gm107_graph_fecs_ucode : NULL,
         .gpccs.ucode = &gm107_graph_gpccs_ucode,
+        .ppc_nr = 2,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv04.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv04.c
index ad13dcdd15f9..f70e2f67a4dd 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv04.c
@@ -24,7 +24,6 @@
 
 #include <core/client.h>
 #include <core/os.h>
-#include <core/class.h>
 #include <core/handle.h>
 #include <core/namedb.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv10.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv10.c
index 4532f7e5618c..2b12b09683c8 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv10.c
@@ -24,7 +24,6 @@
 
 #include <core/client.h>
 #include <core/os.h>
-#include <core/class.h>
 #include <core/handle.h>
 
 #include <subdev/fb.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv108.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv108.c
index 00ea1a089822..2b0e8f48c029 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv108.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv108.c
@@ -33,7 +33,7 @@ static struct nouveau_oclass
 nv108_graph_sclass[] = {
         { 0x902d, &nouveau_object_ofuncs },
         { 0xa140, &nouveau_object_ofuncs },
-        { 0xa197, &nouveau_object_ofuncs },
+        { KEPLER_B, &nvc0_fermi_ofuncs },
         { 0xa1c0, &nouveau_object_ofuncs },
         {}
 };
@@ -220,4 +220,5 @@ nv108_graph_oclass = &(struct nvc0_graph_oclass) {
         .mmio = nv108_graph_pack_mmio,
         .fecs.ucode = &nv108_graph_fecs_ucode,
         .gpccs.ucode = &nv108_graph_gpccs_ucode,
+        .ppc_nr = 1,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv20.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv20.c
index d145e080899a..ceb9c746d94e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv20.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv20.c
@@ -1,6 +1,5 @@
 #include <core/client.h>
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 #include <core/handle.h>
 #include <core/enum.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv25.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv25.c
index 7a80d005a974..f8a6fdd7d5e8 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv25.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv25.c
@@ -1,5 +1,4 @@
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 #include <core/enum.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv2a.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv2a.c
index 3e1f32ee43d4..5de9caa2ef67 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv2a.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv2a.c
@@ -1,5 +1,4 @@
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 #include <core/enum.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv30.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv30.c
index e451db32e92a..2f9dbc709389 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv30.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv30.c
@@ -1,5 +1,4 @@
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 #include <core/enum.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv34.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv34.c
index 9385ac7b44a4..34dd26c70b64 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv34.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv34.c
@@ -1,5 +1,4 @@
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 #include <core/enum.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv35.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv35.c
index 9ce84b73f86a..2fb5756d9f66 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv35.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv35.c
@@ -1,5 +1,4 @@
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 #include <core/enum.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv40.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv40.c
index 6477fbf6a550..4f401174868d 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv40.c
@@ -24,7 +24,6 @@
 
 #include <core/client.h>
 #include <core/os.h>
-#include <core/class.h>
 #include <core/handle.h>
 #include <core/engctx.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv50.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv50.c
index 20665c21d80e..38e0aa26f1cd 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv50.c
@@ -23,7 +23,6 @@
  */
 
 #include <core/os.h>
-#include <core/class.h>
 #include <core/client.h>
 #include <core/handle.h>
 #include <core/engctx.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
index aa0838916354..db19191176fa 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
@@ -26,15 +26,226 @@
 #include "ctxnvc0.h"
 
 /*******************************************************************************
+ * Zero Bandwidth Clear
+ ******************************************************************************/
+
+static void
+nvc0_graph_zbc_clear_color(struct nvc0_graph_priv *priv, int zbc)
+{
+        if (priv->zbc_color[zbc].format) {
+                nv_wr32(priv, 0x405804, priv->zbc_color[zbc].ds[0]);
+                nv_wr32(priv, 0x405808, priv->zbc_color[zbc].ds[1]);
+                nv_wr32(priv, 0x40580c, priv->zbc_color[zbc].ds[2]);
+                nv_wr32(priv, 0x405810, priv->zbc_color[zbc].ds[3]);
+        }
+        nv_wr32(priv, 0x405814, priv->zbc_color[zbc].format);
+        nv_wr32(priv, 0x405820, zbc);
+        nv_wr32(priv, 0x405824, 0x00000004); /* TRIGGER | WRITE | COLOR */
+}
+
+static int
+nvc0_graph_zbc_color_get(struct nvc0_graph_priv *priv, int format,
+                         const u32 ds[4], const u32 l2[4])
+{
+        struct nouveau_ltc *ltc = nouveau_ltc(priv);
+        int zbc = -ENOSPC, i;
+
+        for (i = ltc->zbc_min; i <= ltc->zbc_max; i++) {
+                if (priv->zbc_color[i].format) {
+                        if (priv->zbc_color[i].format != format)
+                                continue;
+                        if (memcmp(priv->zbc_color[i].ds, ds, sizeof(
+                                   priv->zbc_color[i].ds)))
+                                continue;
+                        if (memcmp(priv->zbc_color[i].l2, l2, sizeof(
+                                   priv->zbc_color[i].l2))) {
+                                WARN_ON(1);
+                                return -EINVAL;
+                        }
+                        return i;
+                } else {
+                        zbc = (zbc < 0) ? i : zbc;
+                }
+        }
+
+        memcpy(priv->zbc_color[zbc].ds, ds, sizeof(priv->zbc_color[zbc].ds));
+        memcpy(priv->zbc_color[zbc].l2, l2, sizeof(priv->zbc_color[zbc].l2));
+        priv->zbc_color[zbc].format = format;
+        ltc->zbc_color_get(ltc, zbc, l2);
+        nvc0_graph_zbc_clear_color(priv, zbc);
+        return zbc;
+}
+
+static void
+nvc0_graph_zbc_clear_depth(struct nvc0_graph_priv *priv, int zbc)
+{
+        if (priv->zbc_depth[zbc].format)
+                nv_wr32(priv, 0x405818, priv->zbc_depth[zbc].ds);
+        nv_wr32(priv, 0x40581c, priv->zbc_depth[zbc].format);
+        nv_wr32(priv, 0x405820, zbc);
+        nv_wr32(priv, 0x405824, 0x00000005); /* TRIGGER | WRITE | DEPTH */
+}
+
+static int
+nvc0_graph_zbc_depth_get(struct nvc0_graph_priv *priv, int format,
+                         const u32 ds, const u32 l2)
+{
+        struct nouveau_ltc *ltc = nouveau_ltc(priv);
+        int zbc = -ENOSPC, i;
+
+        for (i = ltc->zbc_min; i <= ltc->zbc_max; i++) {
+                if (priv->zbc_depth[i].format) {
+                        if (priv->zbc_depth[i].format != format)
+                                continue;
+                        if (priv->zbc_depth[i].ds != ds)
+                                continue;
+                        if (priv->zbc_depth[i].l2 != l2) {
+                                WARN_ON(1);
+                                return -EINVAL;
+                        }
+                        return i;
+                } else {
+                        zbc = (zbc < 0) ? i : zbc;
+                }
+        }
+
+        priv->zbc_depth[zbc].format = format;
+        priv->zbc_depth[zbc].ds = ds;
+        priv->zbc_depth[zbc].l2 = l2;
+        ltc->zbc_depth_get(ltc, zbc, l2);
+        nvc0_graph_zbc_clear_depth(priv, zbc);
+        return zbc;
+}
+
+/*******************************************************************************
  * Graphics object classes
  ******************************************************************************/
 
+static int
+nvc0_fermi_mthd_zbc_color(struct nouveau_object *object, void *data, u32 size)
+{
+        struct nvc0_graph_priv *priv = (void *)object->engine;
+        union {
+                struct fermi_a_zbc_color_v0 v0;
+        } *args = data;
+        int ret;
+
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                switch (args->v0.format) {
+                case FERMI_A_ZBC_COLOR_V0_FMT_ZERO:
+                case FERMI_A_ZBC_COLOR_V0_FMT_UNORM_ONE:
+                case FERMI_A_ZBC_COLOR_V0_FMT_RF32_GF32_BF32_AF32:
+                case FERMI_A_ZBC_COLOR_V0_FMT_R16_G16_B16_A16:
+                case FERMI_A_ZBC_COLOR_V0_FMT_RN16_GN16_BN16_AN16:
+                case FERMI_A_ZBC_COLOR_V0_FMT_RS16_GS16_BS16_AS16:
+                case FERMI_A_ZBC_COLOR_V0_FMT_RU16_GU16_BU16_AU16:
+                case FERMI_A_ZBC_COLOR_V0_FMT_RF16_GF16_BF16_AF16:
+                case FERMI_A_ZBC_COLOR_V0_FMT_A8R8G8B8:
+                case FERMI_A_ZBC_COLOR_V0_FMT_A8RL8GL8BL8:
+                case FERMI_A_ZBC_COLOR_V0_FMT_A2B10G10R10:
+                case FERMI_A_ZBC_COLOR_V0_FMT_AU2BU10GU10RU10:
+                case FERMI_A_ZBC_COLOR_V0_FMT_A8B8G8R8:
+                case FERMI_A_ZBC_COLOR_V0_FMT_A8BL8GL8RL8:
+                case FERMI_A_ZBC_COLOR_V0_FMT_AN8BN8GN8RN8:
+                case FERMI_A_ZBC_COLOR_V0_FMT_AS8BS8GS8RS8:
+                case FERMI_A_ZBC_COLOR_V0_FMT_AU8BU8GU8RU8:
+                case FERMI_A_ZBC_COLOR_V0_FMT_A2R10G10B10:
+                case FERMI_A_ZBC_COLOR_V0_FMT_BF10GF11RF11:
+                        ret = nvc0_graph_zbc_color_get(priv, args->v0.format,
+                                                             args->v0.ds,
+                                                             args->v0.l2);
+                        if (ret >= 0) {
+                                args->v0.index = ret;
+                                return 0;
+                        }
+                        break;
+                default:
+                        return -EINVAL;
+                }
+        }
+
+        return ret;
+}
+
+static int
+nvc0_fermi_mthd_zbc_depth(struct nouveau_object *object, void *data, u32 size)
+{
+        struct nvc0_graph_priv *priv = (void *)object->engine;
+        union {
+                struct fermi_a_zbc_depth_v0 v0;
+        } *args = data;
+        int ret;
+
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                switch (args->v0.format) {
+                case FERMI_A_ZBC_DEPTH_V0_FMT_FP32:
+                        ret = nvc0_graph_zbc_depth_get(priv, args->v0.format,
+                                                             args->v0.ds,
+                                                             args->v0.l2);
+                        return (ret >= 0) ? 0 : -ENOSPC;
+                default:
+                        return -EINVAL;
+                }
+        }
+
+        return ret;
+}
+
+static int
+nvc0_fermi_mthd(struct nouveau_object *object, u32 mthd, void *data, u32 size)
+{
+        switch (mthd) {
+        case FERMI_A_ZBC_COLOR:
+                return nvc0_fermi_mthd_zbc_color(object, data, size);
+        case FERMI_A_ZBC_DEPTH:
+                return nvc0_fermi_mthd_zbc_depth(object, data, size);
+        default:
+                break;
+        }
+        return -EINVAL;
+}
+
+struct nouveau_ofuncs
+nvc0_fermi_ofuncs = {
+        .ctor = _nouveau_object_ctor,
+        .dtor = nouveau_object_destroy,
+        .init = nouveau_object_init,
+        .fini = nouveau_object_fini,
+        .mthd = nvc0_fermi_mthd,
+};
+
+static int
+nvc0_graph_set_shader_exceptions(struct nouveau_object *object, u32 mthd,
+                                 void *pdata, u32 size)
+{
+        struct nvc0_graph_priv *priv = (void *)nv_engine(object);
+        if (size >= sizeof(u32)) {
+                u32 data = *(u32 *)pdata ? 0xffffffff : 0x00000000;
+                nv_wr32(priv, 0x419e44, data);
+                nv_wr32(priv, 0x419e4c, data);
+                return 0;
+        }
+        return -EINVAL;
+}
+
+struct nouveau_omthds
+nvc0_graph_9097_omthds[] = {
+        { 0x1528, 0x1528, nvc0_graph_set_shader_exceptions },
+        {}
+};
+
+struct nouveau_omthds
+nvc0_graph_90c0_omthds[] = {
+        { 0x1528, 0x1528, nvc0_graph_set_shader_exceptions },
+        {}
+};
+
 struct nouveau_oclass
 nvc0_graph_sclass[] = {
         { 0x902d, &nouveau_object_ofuncs },
         { 0x9039, &nouveau_object_ofuncs },
-        { 0x9097, &nouveau_object_ofuncs },
-        { 0x90c0, &nouveau_object_ofuncs },
+        { FERMI_A, &nvc0_fermi_ofuncs, nvc0_graph_9097_omthds },
+        { FERMI_COMPUTE_A, &nouveau_object_ofuncs, nvc0_graph_90c0_omthds },
         {}
 };
 
@@ -98,7 +309,7 @@ nvc0_graph_context_ctor(struct nouveau_object *parent,
                 u32 addr = mmio->addr;
                 u32 data = mmio->data;
 
-                if (mmio->shift) {
+                if (mmio->buffer >= 0) {
                         u64 info = chan->data[mmio->buffer].vma.offset;
                         data |= info >> mmio->shift;
                 }
@@ -407,6 +618,35 @@ nvc0_graph_pack_mmio[] = {
  ******************************************************************************/
 
 void
+nvc0_graph_zbc_init(struct nvc0_graph_priv *priv)
+{
+        const u32  zero[] = { 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+                              0x00000000, 0x00000000, 0x00000000, 0x00000000 };
+        const u32   one[] = { 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000,
+                              0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff };
+        const u32 f32_0[] = { 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+                              0x00000000, 0x00000000, 0x00000000, 0x00000000 };
+        const u32 f32_1[] = { 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000,
+                              0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 };
+        struct nouveau_ltc *ltc = nouveau_ltc(priv);
+        int index;
+
+        if (!priv->zbc_color[0].format) {
+                nvc0_graph_zbc_color_get(priv, 1,  & zero[0],   &zero[4]);
+                nvc0_graph_zbc_color_get(priv, 2,  &  one[0],    &one[4]);
+                nvc0_graph_zbc_color_get(priv, 4,  &f32_0[0],  &f32_0[4]);
+                nvc0_graph_zbc_color_get(priv, 4,  &f32_1[0],  &f32_1[4]);
+                nvc0_graph_zbc_depth_get(priv, 1, 0x00000000, 0x00000000);
+                nvc0_graph_zbc_depth_get(priv, 1, 0x3f800000, 0x3f800000);
+        }
+
+        for (index = ltc->zbc_min; index <= ltc->zbc_max; index++)
+                nvc0_graph_zbc_clear_color(priv, index);
+        for (index = ltc->zbc_min; index <= ltc->zbc_max; index++)
+                nvc0_graph_zbc_clear_depth(priv, index);
+}
+
+void
 nvc0_graph_mmio(struct nvc0_graph_priv *priv, const struct nvc0_graph_pack *p)
 {
         const struct nvc0_graph_pack *pack;
@@ -969,17 +1209,16 @@ nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
 {
         struct nvc0_graph_oclass *oclass = (void *)nv_object(priv)->oclass;
         struct nvc0_grctx_oclass *cclass = (void *)nv_engine(priv)->cclass;
-        u32 r000260;
         int i;
 
         if (priv->firmware) {
                 /* load fuc microcode */
-                r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
+                nouveau_mc(priv)->unk260(nouveau_mc(priv), 0);
                 nvc0_graph_init_fw(priv, 0x409000, &priv->fuc409c,
                                                    &priv->fuc409d);
                 nvc0_graph_init_fw(priv, 0x41a000, &priv->fuc41ac,
                                                    &priv->fuc41ad);
-                nv_wr32(priv, 0x000260, r000260);
+                nouveau_mc(priv)->unk260(nouveau_mc(priv), 1);
 
                 /* start both of them running */
                 nv_wr32(priv, 0x409840, 0xffffffff);
@@ -1066,7 +1305,7 @@ nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
         }
 
         /* load HUB microcode */
-        r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
+        nouveau_mc(priv)->unk260(nouveau_mc(priv), 0);
         nv_wr32(priv, 0x4091c0, 0x01000000);
         for (i = 0; i < oclass->fecs.ucode->data.size / 4; i++)
                 nv_wr32(priv, 0x4091c4, oclass->fecs.ucode->data.data[i]);
@@ -1089,7 +1328,7 @@ nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
                         nv_wr32(priv, 0x41a188, i >> 6);
                 nv_wr32(priv, 0x41a184, oclass->gpccs.ucode->code.data[i]);
         }
-        nv_wr32(priv, 0x000260, r000260);
+        nouveau_mc(priv)->unk260(nouveau_mc(priv), 1);
 
         /* load register lists */
         nvc0_graph_init_csdata(priv, cclass->hub, 0x409000, 0x000, 0x000000);
@@ -1224,6 +1463,9 @@ nvc0_graph_init(struct nouveau_object *object)
         nv_wr32(priv, 0x400134, 0xffffffff);
 
         nv_wr32(priv, 0x400054, 0x34ce3464);
+
+        nvc0_graph_zbc_init(priv);
+
         return nvc0_graph_init_ctxctl(priv);
 }
 
@@ -1287,7 +1529,7 @@ nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nouveau_device *device = nv_device(parent);
         struct nvc0_graph_priv *priv;
         bool use_ext_fw, enable;
-        int ret, i;
+        int ret, i, j;
 
         use_ext_fw = nouveau_boolopt(device->cfgopt, "NvGrUseFW",
                                      oclass->fecs.ucode == NULL);
@@ -1333,6 +1575,11 @@ nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         for (i = 0; i < priv->gpc_nr; i++) {
                 priv->tpc_nr[i]  = nv_rd32(priv, GPC_UNIT(i, 0x2608));
                 priv->tpc_total += priv->tpc_nr[i];
+                priv->ppc_nr[i]  = oclass->ppc_nr;
+                for (j = 0; j < priv->ppc_nr[i]; j++) {
+                        u8 mask = nv_rd32(priv, GPC_UNIT(i, 0x0c30 + (j * 4)));
+                        priv->ppc_tpc_nr[i][j] = hweight8(mask);
+                }
         }
 
         /*XXX: these need figuring out... though it might not even matter */
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.h b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.h
index ffc289198dd8..7ed9e89c3435 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.h
@@ -30,10 +30,15 @@
 #include <core/gpuobj.h>
 #include <core/option.h>
 
+#include <nvif/unpack.h>
+#include <nvif/class.h>
+
 #include <subdev/fb.h>
 #include <subdev/vm.h>
 #include <subdev/bar.h>
 #include <subdev/timer.h>
+#include <subdev/mc.h>
+#include <subdev/ltc.h>
 
 #include <engine/fifo.h>
 #include <engine/graph.h>
@@ -60,7 +65,7 @@ struct nvc0_graph_mmio {
         u32 addr;
         u32 data;
         u32 shift;
-        u32 buffer;
+        int buffer;
 };
 
 struct nvc0_graph_fuc {
@@ -68,6 +73,18 @@ struct nvc0_graph_fuc {
         u32  size;
 };
 
+struct nvc0_graph_zbc_color {
+        u32 format;
+        u32 ds[4];
+        u32 l2[4];
+};
+
+struct nvc0_graph_zbc_depth {
+        u32 format;
+        u32 ds;
+        u32 l2;
+};
+
 struct nvc0_graph_priv {
         struct nouveau_graph base;
 
@@ -77,10 +94,15 @@ struct nvc0_graph_priv {
         struct nvc0_graph_fuc fuc41ad;
         bool firmware;
 
+        struct nvc0_graph_zbc_color zbc_color[NOUVEAU_LTC_MAX_ZBC_CNT];
+        struct nvc0_graph_zbc_depth zbc_depth[NOUVEAU_LTC_MAX_ZBC_CNT];
+
         u8 rop_nr;
         u8 gpc_nr;
         u8 tpc_nr[GPC_MAX];
         u8 tpc_total;
+        u8 ppc_nr[GPC_MAX];
+        u8 ppc_tpc_nr[GPC_MAX][4];
 
         struct nouveau_gpuobj *unk4188b4;
         struct nouveau_gpuobj *unk4188b8;
@@ -118,12 +140,20 @@ int  nvc0_graph_ctor(struct nouveau_object *, struct nouveau_object *,
                      struct nouveau_object **);
 void nvc0_graph_dtor(struct nouveau_object *);
 int  nvc0_graph_init(struct nouveau_object *);
+void nvc0_graph_zbc_init(struct nvc0_graph_priv *);
+
 int  nve4_graph_fini(struct nouveau_object *, bool);
 int  nve4_graph_init(struct nouveau_object *);
 
-extern struct nouveau_oclass nvc0_graph_sclass[];
+int  nvf0_graph_fini(struct nouveau_object *, bool);
+
+extern struct nouveau_ofuncs nvc0_fermi_ofuncs;
 
+extern struct nouveau_oclass nvc0_graph_sclass[];
+extern struct nouveau_omthds nvc0_graph_9097_omthds[];
+extern struct nouveau_omthds nvc0_graph_90c0_omthds[];
 extern struct nouveau_oclass nvc8_graph_sclass[];
+extern struct nouveau_oclass nvf0_graph_sclass[];
 
 struct nvc0_graph_init {
         u32 addr;
@@ -149,6 +179,9 @@ struct nvc0_graph_ucode {
 extern struct nvc0_graph_ucode nvc0_graph_fecs_ucode;
 extern struct nvc0_graph_ucode nvc0_graph_gpccs_ucode;
 
+extern struct nvc0_graph_ucode nvf0_graph_fecs_ucode;
+extern struct nvc0_graph_ucode nvf0_graph_gpccs_ucode;
+
 struct nvc0_graph_oclass {
         struct nouveau_oclass base;
         struct nouveau_oclass **cclass;
@@ -160,6 +193,7 @@ struct nvc0_graph_oclass {
         struct {
                 struct nvc0_graph_ucode *ucode;
         } gpccs;
+        int ppc_nr;
 };
 
 void nvc0_graph_mmio(struct nvc0_graph_priv *, const struct nvc0_graph_pack *);
@@ -223,9 +257,11 @@ extern const struct nvc0_graph_init nve4_graph_init_be_0[];
 extern const struct nvc0_graph_pack nve4_graph_pack_mmio[];
 
 extern const struct nvc0_graph_init nvf0_graph_init_fe_0[];
+extern const struct nvc0_graph_init nvf0_graph_init_ds_0[];
 extern const struct nvc0_graph_init nvf0_graph_init_sked_0[];
 extern const struct nvc0_graph_init nvf0_graph_init_cwd_0[];
 extern const struct nvc0_graph_init nvf0_graph_init_gpc_unk_1[];
+extern const struct nvc0_graph_init nvf0_graph_init_tex_0[];
 extern const struct nvc0_graph_init nvf0_graph_init_sm_0[];
 
 extern const struct nvc0_graph_init nv108_graph_init_gpc_unk_0[];
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc1.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvc1.c
index 30cab0b2eba1..93d58e5b82c2 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc1.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc1.c
@@ -33,9 +33,9 @@ static struct nouveau_oclass
 nvc1_graph_sclass[] = {
         { 0x902d, &nouveau_object_ofuncs },
         { 0x9039, &nouveau_object_ofuncs },
-        { 0x9097, &nouveau_object_ofuncs },
-        { 0x90c0, &nouveau_object_ofuncs },
-        { 0x9197, &nouveau_object_ofuncs },
+        { FERMI_A, &nvc0_fermi_ofuncs, nvc0_graph_9097_omthds },
+        { FERMI_B, &nvc0_fermi_ofuncs, nvc0_graph_9097_omthds },
+        { FERMI_COMPUTE_A, &nouveau_object_ofuncs, nvc0_graph_90c0_omthds },
         {}
 };
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc8.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvc8.c
index a6bf783e1256..692e1eda0eb4 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc8.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc8.c
@@ -33,10 +33,10 @@ struct nouveau_oclass
 nvc8_graph_sclass[] = {
         { 0x902d, &nouveau_object_ofuncs },
         { 0x9039, &nouveau_object_ofuncs },
-        { 0x9097, &nouveau_object_ofuncs },
-        { 0x90c0, &nouveau_object_ofuncs },
-        { 0x9197, &nouveau_object_ofuncs },
-        { 0x9297, &nouveau_object_ofuncs },
+        { FERMI_A, &nvc0_fermi_ofuncs, nvc0_graph_9097_omthds },
+        { FERMI_B, &nvc0_fermi_ofuncs, nvc0_graph_9097_omthds },
+        { FERMI_C, &nvc0_fermi_ofuncs, nvc0_graph_9097_omthds },
+        { FERMI_COMPUTE_A, &nouveau_object_ofuncs, nvc0_graph_90c0_omthds },
         {}
 };
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvd7.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvd7.c
index 2a6a94e2a041..41e8445c7eea 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvd7.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvd7.c
@@ -133,4 +133,5 @@ nvd7_graph_oclass = &(struct nvc0_graph_oclass) {
         .mmio = nvd7_graph_pack_mmio,
         .fecs.ucode = &nvd7_graph_fecs_ucode,
         .gpccs.ucode = &nvd7_graph_gpccs_ucode,
+        .ppc_nr = 1,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nve4.c b/drivers/gpu/drm/nouveau/core/engine/graph/nve4.c
index 51e0c075ad34..0c71f5c67ae0 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nve4.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nve4.c
@@ -22,6 +22,8 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
+#include <subdev/pwr.h>
+
 #include "nvc0.h"
 #include "ctxnvc0.h"
 
@@ -33,8 +35,8 @@ static struct nouveau_oclass
 nve4_graph_sclass[] = {
         { 0x902d, &nouveau_object_ofuncs },
         { 0xa040, &nouveau_object_ofuncs },
-        { 0xa097, &nouveau_object_ofuncs },
-        { 0xa0c0, &nouveau_object_ofuncs },
+        { KEPLER_A, &nvc0_fermi_ofuncs, nvc0_graph_9097_omthds },
+        { KEPLER_COMPUTE_A, &nouveau_object_ofuncs, nvc0_graph_90c0_omthds },
         {}
 };
 
@@ -190,39 +192,20 @@ nve4_graph_pack_mmio[] = {
  ******************************************************************************/
 
 int
-nve4_graph_fini(struct nouveau_object *object, bool suspend)
-{
-        struct nvc0_graph_priv *priv = (void *)object;
-
-        /*XXX: this is a nasty hack to power on gr on certain boards
-         *     where it's disabled by therm, somehow.  ideally it'd
-         *     be nice to know when we should be doing this, and why,
-         *     but, it's yet to be determined.  for now we test for
-         *     the particular mmio error that occurs in the situation,
-         *     and then bash therm in the way nvidia do.
-         */
-        nv_mask(priv, 0x000200, 0x08001000, 0x08001000);
-        nv_rd32(priv, 0x000200);
-        if (nv_rd32(priv, 0x400700) == 0xbadf1000) {
-                nv_mask(priv, 0x000200, 0x08001000, 0x00000000);
-                nv_rd32(priv, 0x000200);
-                nv_mask(priv, 0x020004, 0xc0000000, 0x40000000);
-        }
-
-        return nouveau_graph_fini(&priv->base, suspend);
-}
-
-int
 nve4_graph_init(struct nouveau_object *object)
 {
         struct nvc0_graph_oclass *oclass = (void *)object->oclass;
         struct nvc0_graph_priv *priv = (void *)object;
+        struct nouveau_pwr *ppwr = nouveau_pwr(priv);
         const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
         u32 data[TPC_MAX / 8] = {};
         u8  tpcnr[GPC_MAX];
         int gpc, tpc, rop;
         int ret, i;
 
+        if (ppwr)
+                ppwr->pgob(ppwr, false);
+
         ret = nouveau_graph_init(&priv->base);
         if (ret)
                 return ret;
@@ -320,6 +303,9 @@ nve4_graph_init(struct nouveau_object *object)
         nv_wr32(priv, 0x400134, 0xffffffff);
 
         nv_wr32(priv, 0x400054, 0x34ce3464);
+
+        nvc0_graph_zbc_init(priv);
+
         return nvc0_graph_init_ctxctl(priv);
 }
 
@@ -350,11 +336,12 @@ nve4_graph_oclass = &(struct nvc0_graph_oclass) {
                 .ctor = nvc0_graph_ctor,
                 .dtor = nvc0_graph_dtor,
                 .init = nve4_graph_init,
-                .fini = nve4_graph_fini,
+                .fini = _nouveau_graph_fini,
         },
         .cclass = &nve4_grctx_oclass,
         .sclass = nve4_graph_sclass,
         .mmio = nve4_graph_pack_mmio,
         .fecs.ucode = &nve4_graph_fecs_ucode,
         .gpccs.ucode = &nve4_graph_gpccs_ucode,
+        .ppc_nr = 1,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvf0.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvf0.c
index c96762122b9b..c306c0f2fc84 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvf0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvf0.c
@@ -29,12 +29,12 @@
  * Graphics object classes
  ******************************************************************************/
 
-static struct nouveau_oclass
+struct nouveau_oclass
 nvf0_graph_sclass[] = {
         { 0x902d, &nouveau_object_ofuncs },
         { 0xa140, &nouveau_object_ofuncs },
-        { 0xa197, &nouveau_object_ofuncs },
-        { 0xa1c0, &nouveau_object_ofuncs },
+        { KEPLER_B, &nvc0_fermi_ofuncs, nvc0_graph_9097_omthds },
+        { KEPLER_COMPUTE_B, &nouveau_object_ofuncs, nvc0_graph_90c0_omthds },
         {}
 };
 
@@ -50,7 +50,7 @@ nvf0_graph_init_fe_0[] = {
         {}
 };
 
-static const struct nvc0_graph_init
+const struct nvc0_graph_init
 nvf0_graph_init_ds_0[] = {
         { 0x405844,   1, 0x04, 0x00ffffff },
         { 0x405850,   1, 0x04, 0x00000000 },
@@ -88,7 +88,7 @@ nvf0_graph_init_gpc_unk_1[] = {
         {}
 };
 
-static const struct nvc0_graph_init
+const struct nvc0_graph_init
 nvf0_graph_init_tex_0[] = {
         { 0x419ab0,   1, 0x04, 0x00000000 },
         { 0x419ac8,   1, 0x04, 0x00000000 },
@@ -170,7 +170,7 @@ nvf0_graph_pack_mmio[] = {
  * PGRAPH engine/subdev functions
  ******************************************************************************/
 
-static int
+int
 nvf0_graph_fini(struct nouveau_object *object, bool suspend)
 {
         struct nvc0_graph_priv *priv = (void *)object;
@@ -209,7 +209,7 @@ nvf0_graph_fini(struct nouveau_object *object, bool suspend)
 
 #include "fuc/hubnvf0.fuc.h"
 
-static struct nvc0_graph_ucode
+struct nvc0_graph_ucode
 nvf0_graph_fecs_ucode = {
         .code.data = nvf0_grhub_code,
         .code.size = sizeof(nvf0_grhub_code),
@@ -219,7 +219,7 @@ nvf0_graph_fecs_ucode = {
 
 #include "fuc/gpcnvf0.fuc.h"
 
-static struct nvc0_graph_ucode
+struct nvc0_graph_ucode
 nvf0_graph_gpccs_ucode = {
         .code.data = nvf0_grgpc_code,
         .code.size = sizeof(nvf0_grgpc_code),
@@ -241,4 +241,5 @@ nvf0_graph_oclass = &(struct nvc0_graph_oclass) {
         .mmio = nvf0_graph_pack_mmio,
         .fecs.ucode = &nvf0_graph_fecs_ucode,
         .gpccs.ucode = &nvf0_graph_gpccs_ucode,
+        .ppc_nr = 2,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.c
index 7eb6d94c84e2..d88c700b2f69 100644
--- a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.c
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.c
@@ -24,7 +24,6 @@
 
 #include <core/client.h>
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 #include <core/handle.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv40.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv40.c
index d4e7ec0ba68c..bdb2f20ff7b1 100644
--- a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv40.c
@@ -23,7 +23,6 @@
  */
 
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 
 #include <subdev/fb.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv44.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv44.c
index 3d8c2133e0e8..72c7f33fd29b 100644
--- a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv44.c
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv44.c
@@ -23,7 +23,6 @@
  */
 
 #include <core/os.h>
-#include <core/class.h>
 #include <core/client.h>
 #include <core/engctx.h>
 #include <core/handle.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv50.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv50.c
index 37a2bd9e8078..cae33f86b11a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv50.c
@@ -23,7 +23,6 @@
  */
 
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 
 #include <subdev/vm.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv84.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv84.c
index 96f5aa92677b..e9cc8b116a24 100644
--- a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv84.c
@@ -23,7 +23,6 @@
  */
 
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 
 #include <subdev/vm.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/base.c b/drivers/gpu/drm/nouveau/core/engine/perfmon/base.c
index e9c5e51943ef..63013812f7c9 100644
--- a/drivers/gpu/drm/nouveau/core/engine/perfmon/base.c
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/base.c
@@ -22,8 +22,11 @@
  * Authors: Ben Skeggs
  */
 
+#include <core/client.h>
 #include <core/option.h>
-#include <core/class.h>
+#include <nvif/unpack.h>
+#include <nvif/class.h>
+#include <nvif/ioctl.h>
 
 #include <subdev/clock.h>
 
@@ -101,24 +104,28 @@ nouveau_perfsig_wrap(struct nouveau_perfmon *ppm, const char *name,
  * Perfmon object classes
  ******************************************************************************/
 static int
-nouveau_perfctr_query(struct nouveau_object *object, u32 mthd,
-                      void *data, u32 size)
+nouveau_perfctr_query(struct nouveau_object *object, void *data, u32 size)
 {
+        union {
+                struct nvif_perfctr_query_v0 v0;
+        } *args = data;
         struct nouveau_device *device = nv_device(object);
         struct nouveau_perfmon *ppm = (void *)object->engine;
         struct nouveau_perfdom *dom = NULL, *chk;
-        struct nv_perfctr_query *args = data;
         const bool all = nouveau_boolopt(device->cfgopt, "NvPmShowAll", false);
         const bool raw = nouveau_boolopt(device->cfgopt, "NvPmUnnamed", all);
         const char *name;
         int tmp = 0, di, si;
-        char path[64];
-
-        if (size < sizeof(*args))
-                return -EINVAL;
+        int ret;
 
-        di = (args->iter & 0xff000000) >> 24;
-        si = (args->iter & 0x00ffffff) - 1;
+        nv_ioctl(object, "perfctr query size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "perfctr query vers %d iter %08x\n",
+                         args->v0.version, args->v0.iter);
+                di = (args->v0.iter & 0xff000000) >> 24;
+                si = (args->v0.iter & 0x00ffffff) - 1;
+        } else
+                return ret;
 
         list_for_each_entry(chk, &ppm->domains, head) {
                 if (tmp++ == di) {
@@ -132,19 +139,17 @@ nouveau_perfctr_query(struct nouveau_object *object, u32 mthd,
 
         if (si >= 0) {
                 if (raw || !(name = dom->signal[si].name)) {
-                        snprintf(path, sizeof(path), "/%s/%02x", dom->name, si);
-                        name = path;
+                        snprintf(args->v0.name, sizeof(args->v0.name),
+                                 "/%s/%02x", dom->name, si);
+                } else {
+                        strncpy(args->v0.name, name, sizeof(args->v0.name));
                 }
-
-                if (args->name)
-                        strncpy(args->name, name, args->size);
-                args->size = strlen(name) + 1;
         }
 
         do {
                 while (++si < dom->signal_nr) {
                         if (all || dom->signal[si].name) {
-                                args->iter = (di << 24) | ++si;
+                                args->v0.iter = (di << 24) | ++si;
                                 return 0;
                         }
                 }
@@ -153,21 +158,26 @@ nouveau_perfctr_query(struct nouveau_object *object, u32 mthd,
                 dom = list_entry(dom->head.next, typeof(*dom), head);
         } while (&dom->head != &ppm->domains);
 
-        args->iter = 0xffffffff;
+        args->v0.iter = 0xffffffff;
         return 0;
 }
 
 static int
-nouveau_perfctr_sample(struct nouveau_object *object, u32 mthd,
-                       void *data, u32 size)
+nouveau_perfctr_sample(struct nouveau_object *object, void *data, u32 size)
 {
+        union {
+                struct nvif_perfctr_sample none;
+        } *args = data;
         struct nouveau_perfmon *ppm = (void *)object->engine;
         struct nouveau_perfctr *ctr, *tmp;
         struct nouveau_perfdom *dom;
-        struct nv_perfctr_sample *args = data;
+        int ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(object, "perfctr sample size %d\n", size);
+        if (nvif_unvers(args->none)) {
+                nv_ioctl(object, "perfctr sample\n");
+        } else
+                return ret;
         ppm->sequence++;
 
         list_for_each_entry(dom, &ppm->domains, head) {
@@ -206,22 +216,45 @@ nouveau_perfctr_sample(struct nouveau_object *object, u32 mthd,
 }
 
 static int
-nouveau_perfctr_read(struct nouveau_object *object, u32 mthd,
-                     void *data, u32 size)
+nouveau_perfctr_read(struct nouveau_object *object, void *data, u32 size)
 {
+        union {
+                struct nvif_perfctr_read_v0 v0;
+        } *args = data;
         struct nouveau_perfctr *ctr = (void *)object;
-        struct nv_perfctr_read *args = data;
+        int ret;
+
+        nv_ioctl(object, "perfctr read size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(object, "perfctr read vers %d\n", args->v0.version);
+        } else
+                return ret;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
         if (!ctr->clk)
                 return -EAGAIN;
 
-        args->clk = ctr->clk;
-        args->ctr = ctr->ctr;
+        args->v0.clk = ctr->clk;
+        args->v0.ctr = ctr->ctr;
         return 0;
 }
 
+static int
+nouveau_perfctr_mthd(struct nouveau_object *object, u32 mthd,
+                     void *data, u32 size)
+{
+        switch (mthd) {
+        case NVIF_PERFCTR_V0_QUERY:
+                return nouveau_perfctr_query(object, data, size);
+        case NVIF_PERFCTR_V0_SAMPLE:
+                return nouveau_perfctr_sample(object, data, size);
+        case NVIF_PERFCTR_V0_READ:
+                return nouveau_perfctr_read(object, data, size);
+        default:
+                break;
+        }
+        return -EINVAL;
+}
+
 static void
 nouveau_perfctr_dtor(struct nouveau_object *object)
 {
@@ -237,19 +270,27 @@ nouveau_perfctr_ctor(struct nouveau_object *parent,
                      struct nouveau_oclass *oclass, void *data, u32 size,
                      struct nouveau_object **pobject)
 {
+        union {
+                struct nvif_perfctr_v0 v0;
+        } *args = data;
         struct nouveau_perfmon *ppm = (void *)engine;
         struct nouveau_perfdom *dom = NULL;
         struct nouveau_perfsig *sig[4] = {};
         struct nouveau_perfctr *ctr;
-        struct nv_perfctr_class *args = data;
         int ret, i;
 
-        if (size < sizeof(*args))
-                return -EINVAL;
+        nv_ioctl(parent, "create perfctr size %d\n", size);
+        if (nvif_unpack(args->v0, 0, 0, false)) {
+                nv_ioctl(parent, "create perfctr vers %d logic_op %04x\n",
+                         args->v0.version, args->v0.logic_op);
+        } else
+                return ret;
 
-        for (i = 0; i < ARRAY_SIZE(args->signal) && args->signal[i].name; i++) {
-                sig[i] = nouveau_perfsig_find(ppm, args->signal[i].name,
-                                              args->signal[i].size, &dom);
+        for (i = 0; i < ARRAY_SIZE(args->v0.name) && args->v0.name[i][0]; i++) {
+                sig[i] = nouveau_perfsig_find(ppm, args->v0.name[i],
+                                              strnlen(args->v0.name[i],
+                                              sizeof(args->v0.name[i])),
+                                              &dom);
                 if (!sig[i])
                         return -EINVAL;
         }
@@ -260,7 +301,7 @@ nouveau_perfctr_ctor(struct nouveau_object *parent,
                 return ret;
 
         ctr->slot = -1;
-        ctr->logic_op = args->logic_op;
+        ctr->logic_op = args->v0.logic_op;
         ctr->signal[0] = sig[0];
         ctr->signal[1] = sig[1];
         ctr->signal[2] = sig[2];
@@ -276,21 +317,13 @@ nouveau_perfctr_ofuncs = {
         .dtor = nouveau_perfctr_dtor,
         .init = nouveau_object_init,
         .fini = nouveau_object_fini,
-};
-
-static struct nouveau_omthds
-nouveau_perfctr_omthds[] = {
-        { NV_PERFCTR_QUERY, NV_PERFCTR_QUERY, nouveau_perfctr_query },
-        { NV_PERFCTR_SAMPLE, NV_PERFCTR_SAMPLE, nouveau_perfctr_sample },
-        { NV_PERFCTR_READ, NV_PERFCTR_READ, nouveau_perfctr_read },
-        {}
+        .mthd = nouveau_perfctr_mthd,
 };
 
 struct nouveau_oclass
 nouveau_perfmon_sclass[] = {
-        { .handle = NV_PERFCTR_CLASS,
+        { .handle = NVIF_IOCTL_NEW_V0_PERFCTR,
           .ofuncs = &nouveau_perfctr_ofuncs,
-          .omthds =  nouveau_perfctr_omthds,
         },
         {},
 };
@@ -303,6 +336,7 @@ nouveau_perfctx_dtor(struct nouveau_object *object)
 {
         struct nouveau_perfmon *ppm = (void *)object->engine;
         mutex_lock(&nv_subdev(ppm)->mutex);
+        nouveau_engctx_destroy(&ppm->context->base);
         ppm->context = NULL;
         mutex_unlock(&nv_subdev(ppm)->mutex);
 }
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nv04.c b/drivers/gpu/drm/nouveau/core/engine/software/nv04.c
index c571758e4a27..64df15c7f051 100644
--- a/drivers/gpu/drm/nouveau/core/engine/software/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nv04.c
@@ -23,7 +23,6 @@
  */
 
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 
 #include <engine/software.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nv10.c b/drivers/gpu/drm/nouveau/core/engine/software/nv10.c
index a62f11a78430..f54a2253deca 100644
--- a/drivers/gpu/drm/nouveau/core/engine/software/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nv10.c
@@ -23,7 +23,6 @@
  */
 
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 
 #include <engine/software.h>
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nv50.c b/drivers/gpu/drm/nouveau/core/engine/software/nv50.c
index f3b4d9dbf23c..4d2994d8cc32 100644
--- a/drivers/gpu/drm/nouveau/core/engine/software/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nv50.c
@@ -23,12 +23,12 @@
  */
 
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 #include <core/namedb.h>
 #include <core/handle.h>
 #include <core/gpuobj.h>
 #include <core/event.h>
+#include <nvif/event.h>
 
 #include <subdev/bar.h>
 
@@ -86,10 +86,10 @@ nv50_software_mthd_vblsem_release(struct nouveau_object *object, u32 mthd,
 {
         struct nv50_software_chan *chan = (void *)nv_engctx(object->parent);
         u32 head = *(u32 *)args;
-        if (head >= chan->vblank.nr_event)
+        if (head >= nouveau_disp(chan)->vblank.index_nr)
                 return -EINVAL;
 
-        nouveau_event_get(chan->vblank.event[head]);
+        nvkm_notify_get(&chan->vblank.notify[head]);
         return 0;
 }
 
@@ -124,9 +124,10 @@ nv50_software_sclass[] = {
  ******************************************************************************/
 
 static int
-nv50_software_vblsem_release(void *data, u32 type, int head)
+nv50_software_vblsem_release(struct nvkm_notify *notify)
 {
-        struct nv50_software_chan *chan = data;
+        struct nv50_software_chan *chan =
+                container_of(notify, typeof(*chan), vblank.notify[notify->index]);
         struct nv50_software_priv *priv = (void *)nv_object(chan)->engine;
         struct nouveau_bar *bar = nouveau_bar(priv);
 
@@ -142,7 +143,7 @@ nv50_software_vblsem_release(void *data, u32 type, int head)
                 nv_wr32(priv, 0x060014, chan->vblank.value);
         }
 
-        return NVKM_EVENT_DROP;
+        return NVKM_NOTIFY_DROP;
 }
 
 void
@@ -151,11 +152,8 @@ nv50_software_context_dtor(struct nouveau_object *object)
         struct nv50_software_chan *chan = (void *)object;
         int i;
 
-        if (chan->vblank.event) {
-                for (i = 0; i < chan->vblank.nr_event; i++)
-                        nouveau_event_ref(NULL, &chan->vblank.event[i]);
-                kfree(chan->vblank.event);
-        }
+        for (i = 0; i < ARRAY_SIZE(chan->vblank.notify); i++)
+                nvkm_notify_fini(&chan->vblank.notify[i]);
 
         nouveau_software_context_destroy(&chan->base);
 }
@@ -176,15 +174,14 @@ nv50_software_context_ctor(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
-        chan->vblank.nr_event = pdisp ? pdisp->vblank->index_nr : 0;
-        chan->vblank.event = kzalloc(chan->vblank.nr_event *
-                                     sizeof(*chan->vblank.event), GFP_KERNEL);
-        if (!chan->vblank.event)
-                return -ENOMEM;
-
-        for (i = 0; i < chan->vblank.nr_event; i++) {
-                ret = nouveau_event_new(pdisp->vblank, 1, i, pclass->vblank,
-                                        chan, &chan->vblank.event[i]);
+        for (i = 0; pdisp && i < pdisp->vblank.index_nr; i++) {
+                ret = nvkm_notify_init(&pdisp->vblank, pclass->vblank, false,
+                                       &(struct nvif_notify_head_req_v0) {
+                                        .head = i,
+                                       },
+                                       sizeof(struct nvif_notify_head_req_v0),
+                                       sizeof(struct nvif_notify_head_rep_v0),
+                                       &chan->vblank.notify[i]);
                 if (ret)
                         return ret;
         }
@@ -198,7 +195,7 @@ nv50_software_cclass = {
         .base.handle = NV_ENGCTX(SW, 0x50),
         .base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv50_software_context_ctor,
-                .dtor = _nouveau_software_context_dtor,
+                .dtor = nv50_software_context_dtor,
                 .init = _nouveau_software_context_init,
                 .fini = _nouveau_software_context_fini,
         },
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nv50.h b/drivers/gpu/drm/nouveau/core/engine/software/nv50.h
index bb49a7a20857..41542e725b4b 100644
--- a/drivers/gpu/drm/nouveau/core/engine/software/nv50.h
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nv50.h
@@ -19,14 +19,13 @@ int  nv50_software_ctor(struct nouveau_object *, struct nouveau_object *,
 
 struct nv50_software_cclass {
         struct nouveau_oclass base;
-        int (*vblank)(void *, u32, int);
+        int (*vblank)(struct nvkm_notify *);
 };
 
 struct nv50_software_chan {
         struct nouveau_software_chan base;
         struct {
-                struct nouveau_eventh **event;
-                int nr_event;
+                struct nvkm_notify notify[4];
                 u32 channel;
                 u32 ctxdma;
                 u64 offset;
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/software/nvc0.c
index 135c20f38356..6af370d3a06d 100644
--- a/drivers/gpu/drm/nouveau/core/engine/software/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nvc0.c
@@ -23,7 +23,6 @@
  */
 
 #include <core/os.h>
-#include <core/class.h>
 #include <core/engctx.h>
 #include <core/event.h>
 
@@ -104,9 +103,10 @@ nvc0_software_sclass[] = {
  ******************************************************************************/
 
 static int
-nvc0_software_vblsem_release(void *data, u32 type, int head)
+nvc0_software_vblsem_release(struct nvkm_notify *notify)
 {
-        struct nv50_software_chan *chan = data;
+        struct nv50_software_chan *chan =
+                container_of(notify, typeof(*chan), vblank.notify[notify->index]);
         struct nv50_software_priv *priv = (void *)nv_object(chan)->engine;
         struct nouveau_bar *bar = nouveau_bar(priv);
 
@@ -116,7 +116,7 @@ nvc0_software_vblsem_release(void *data, u32 type, int head)
         nv_wr32(priv, 0x060010, lower_32_bits(chan->vblank.offset));
         nv_wr32(priv, 0x060014, chan->vblank.value);
 
-        return NVKM_EVENT_DROP;
+        return NVKM_NOTIFY_DROP;
 }
 
 static struct nv50_software_cclass
@@ -124,7 +124,7 @@ nvc0_software_cclass = {
         .base.handle = NV_ENGCTX(SW, 0xc0),
         .base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv50_software_context_ctor,
-                .dtor = _nouveau_software_context_dtor,
+                .dtor = nv50_software_context_dtor,
                 .init = _nouveau_software_context_init,
                 .fini = _nouveau_software_context_fini,
         },
diff --git a/drivers/gpu/drm/nouveau/core/include/core/class.h b/drivers/gpu/drm/nouveau/core/include/core/class.h
deleted file mode 100644
index e0c812bc884f..000000000000
--- a/drivers/gpu/drm/nouveau/core/include/core/class.h
+++ /dev/null
@@ -1,470 +0,0 @@
-#ifndef __NOUVEAU_CLASS_H__
-#define __NOUVEAU_CLASS_H__
-
-/* Device class
- *
- * 0080: NV_DEVICE
- */
-#define NV_DEVICE_CLASS                                              0x00000080
-
-#define NV_DEVICE_DISABLE_IDENTIFY                        0x0000000000000001ULL
-#define NV_DEVICE_DISABLE_MMIO                            0x0000000000000002ULL
-#define NV_DEVICE_DISABLE_VBIOS                           0x0000000000000004ULL
-#define NV_DEVICE_DISABLE_CORE                            0x0000000000000008ULL
-#define NV_DEVICE_DISABLE_DISP                            0x0000000000010000ULL
-#define NV_DEVICE_DISABLE_FIFO                            0x0000000000020000ULL
-#define NV_DEVICE_DISABLE_GRAPH                           0x0000000100000000ULL
-#define NV_DEVICE_DISABLE_MPEG                            0x0000000200000000ULL
-#define NV_DEVICE_DISABLE_ME                              0x0000000400000000ULL
-#define NV_DEVICE_DISABLE_VP                              0x0000000800000000ULL
-#define NV_DEVICE_DISABLE_CRYPT                           0x0000001000000000ULL
-#define NV_DEVICE_DISABLE_BSP                             0x0000002000000000ULL
-#define NV_DEVICE_DISABLE_PPP                             0x0000004000000000ULL
-#define NV_DEVICE_DISABLE_COPY0                           0x0000008000000000ULL
-#define NV_DEVICE_DISABLE_COPY1                           0x0000010000000000ULL
-#define NV_DEVICE_DISABLE_VIC                             0x0000020000000000ULL
-#define NV_DEVICE_DISABLE_VENC                            0x0000040000000000ULL
-
-struct nv_device_class {
-        u64 device;     /* device identifier, ~0 for client default */
-        u64 disable;    /* disable particular subsystems */
-        u64 debug0;     /* as above, but *internal* ids, and *NOT* ABI */
-};
-
-/* DMA object classes
- *
- * 0002: NV_DMA_FROM_MEMORY
- * 0003: NV_DMA_TO_MEMORY
- * 003d: NV_DMA_IN_MEMORY
- */
-#define NV_DMA_FROM_MEMORY_CLASS                                     0x00000002
-#define NV_DMA_TO_MEMORY_CLASS                                       0x00000003
-#define NV_DMA_IN_MEMORY_CLASS                                       0x0000003d
-
-#define NV_DMA_TARGET_MASK                                           0x000000ff
-#define NV_DMA_TARGET_VM                                             0x00000000
-#define NV_DMA_TARGET_VRAM                                           0x00000001
-#define NV_DMA_TARGET_PCI                                            0x00000002
-#define NV_DMA_TARGET_PCI_US                                         0x00000003
-#define NV_DMA_TARGET_AGP                                            0x00000004
-#define NV_DMA_ACCESS_MASK                                           0x00000f00
-#define NV_DMA_ACCESS_VM                                             0x00000000
-#define NV_DMA_ACCESS_RD                                             0x00000100
-#define NV_DMA_ACCESS_WR                                             0x00000200
-#define NV_DMA_ACCESS_RDWR                                           0x00000300
-
-/* NV50:NVC0 */
-#define NV50_DMA_CONF0_ENABLE                                        0x80000000
-#define NV50_DMA_CONF0_PRIV                                          0x00300000
-#define NV50_DMA_CONF0_PRIV_VM                                       0x00000000
-#define NV50_DMA_CONF0_PRIV_US                                       0x00100000
-#define NV50_DMA_CONF0_PRIV__S                                       0x00200000
-#define NV50_DMA_CONF0_PART                                          0x00030000
-#define NV50_DMA_CONF0_PART_VM                                       0x00000000
-#define NV50_DMA_CONF0_PART_256                                      0x00010000
-#define NV50_DMA_CONF0_PART_1KB                                      0x00020000
-#define NV50_DMA_CONF0_COMP                                          0x00000180
-#define NV50_DMA_CONF0_COMP_NONE                                     0x00000000
-#define NV50_DMA_CONF0_COMP_VM                                       0x00000180
-#define NV50_DMA_CONF0_TYPE                                          0x0000007f
-#define NV50_DMA_CONF0_TYPE_LINEAR                                   0x00000000
-#define NV50_DMA_CONF0_TYPE_VM                                       0x0000007f
-
-/* NVC0:NVD9 */
-#define NVC0_DMA_CONF0_ENABLE                                        0x80000000
-#define NVC0_DMA_CONF0_PRIV                                          0x00300000
-#define NVC0_DMA_CONF0_PRIV_VM                                       0x00000000
-#define NVC0_DMA_CONF0_PRIV_US                                       0x00100000
-#define NVC0_DMA_CONF0_PRIV__S                                       0x00200000
-#define NVC0_DMA_CONF0_UNKN /* PART? */                              0x00030000
-#define NVC0_DMA_CONF0_TYPE                                          0x000000ff
-#define NVC0_DMA_CONF0_TYPE_LINEAR                                   0x00000000
-#define NVC0_DMA_CONF0_TYPE_VM                                       0x000000ff
-
-/* NVD9- */
-#define NVD0_DMA_CONF0_ENABLE                                        0x80000000
-#define NVD0_DMA_CONF0_PAGE                                          0x00000400
-#define NVD0_DMA_CONF0_PAGE_LP                                       0x00000000
-#define NVD0_DMA_CONF0_PAGE_SP                                       0x00000400
-#define NVD0_DMA_CONF0_TYPE                                          0x000000ff
-#define NVD0_DMA_CONF0_TYPE_LINEAR                                   0x00000000
-#define NVD0_DMA_CONF0_TYPE_VM                                       0x000000ff
-
-struct nv_dma_class {
-        u32 flags;
-        u32 pad0;
-        u64 start;
-        u64 limit;
-        u32 conf0;
-};
-
-/* Perfmon counter class
- *
- * XXXX: NV_PERFCTR
- */
-#define NV_PERFCTR_CLASS                                             0x0000ffff
-#define NV_PERFCTR_QUERY                                             0x00000000
-#define NV_PERFCTR_SAMPLE                                            0x00000001
-#define NV_PERFCTR_READ                                              0x00000002
-
-struct nv_perfctr_class {
-        u16 logic_op;
-        struct {
-                char __user *name; /*XXX: use cfu when exposed to userspace */
-                u32 size;
-        } signal[4];
-};
-
-struct nv_perfctr_query {
-        u32 iter;
-        u32 size;
-        char __user *name; /*XXX: use ctu when exposed to userspace */
-};
-
-struct nv_perfctr_sample {
-};
-
-struct nv_perfctr_read {
-        u32 ctr;
-        u32 clk;
-};
-
-/* Device control class
- *
- * XXXX: NV_CONTROL
- */
-#define NV_CONTROL_CLASS                                             0x0000fffe
-
-#define NV_CONTROL_PSTATE_INFO                                       0x00000000
-#define NV_CONTROL_PSTATE_INFO_USTATE_DISABLE                              (-1)
-#define NV_CONTROL_PSTATE_INFO_USTATE_PERFMON                              (-2)
-#define NV_CONTROL_PSTATE_INFO_PSTATE_UNKNOWN                              (-1)
-#define NV_CONTROL_PSTATE_INFO_PSTATE_PERFMON                              (-2)
-#define NV_CONTROL_PSTATE_ATTR                                       0x00000001
-#define NV_CONTROL_PSTATE_ATTR_STATE_CURRENT                               (-1)
-#define NV_CONTROL_PSTATE_USER                                       0x00000002
-#define NV_CONTROL_PSTATE_USER_STATE_UNKNOWN                               (-1)
-#define NV_CONTROL_PSTATE_USER_STATE_PERFMON                               (-2)
-
-struct nv_control_pstate_info {
-        u32 count; /* out: number of power states */
-        s32 ustate; /* out: current target pstate index */
-        u32 pstate; /* out: current pstate index */
-};
-
-struct nv_control_pstate_attr {
-        s32 state; /*  in: index of pstate to query
-                    * out: pstate identifier
-                    */
-        u32 index; /*  in: index of attribute to query
-                    * out: index of next attribute, or 0 if no more
-                    */
-        char name[32];
-        char unit[16];
-        u32 min;
-        u32 max;
-};
-
-struct nv_control_pstate_user {
-        s32 state; /*  in: pstate identifier */
-};
-
-/* DMA FIFO channel classes
- *
- * 006b: NV03_CHANNEL_DMA
- * 006e: NV10_CHANNEL_DMA
- * 176e: NV17_CHANNEL_DMA
- * 406e: NV40_CHANNEL_DMA
- * 506e: NV50_CHANNEL_DMA
- * 826e: NV84_CHANNEL_DMA
- */
-#define NV03_CHANNEL_DMA_CLASS                                       0x0000006b
-#define NV10_CHANNEL_DMA_CLASS                                       0x0000006e
-#define NV17_CHANNEL_DMA_CLASS                                       0x0000176e
-#define NV40_CHANNEL_DMA_CLASS                                       0x0000406e
-#define NV50_CHANNEL_DMA_CLASS                                       0x0000506e
-#define NV84_CHANNEL_DMA_CLASS                                       0x0000826e
-
-struct nv03_channel_dma_class {
-        u32 pushbuf;
-        u32 pad0;
-        u64 offset;
-};
-
-/* Indirect FIFO channel classes
- *
- * 506f: NV50_CHANNEL_IND
- * 826f: NV84_CHANNEL_IND
- * 906f: NVC0_CHANNEL_IND
- * a06f: NVE0_CHANNEL_IND
- */
-
-#define NV50_CHANNEL_IND_CLASS                                       0x0000506f
-#define NV84_CHANNEL_IND_CLASS                                       0x0000826f
-#define NVC0_CHANNEL_IND_CLASS                                       0x0000906f
-#define NVE0_CHANNEL_IND_CLASS                                       0x0000a06f
-
-struct nv50_channel_ind_class {
-        u32 pushbuf;
-        u32 ilength;
-        u64 ioffset;
-};
-
-#define NVE0_CHANNEL_IND_ENGINE_GR                                   0x00000001
-#define NVE0_CHANNEL_IND_ENGINE_VP                                   0x00000002
-#define NVE0_CHANNEL_IND_ENGINE_PPP                                  0x00000004
-#define NVE0_CHANNEL_IND_ENGINE_BSP                                  0x00000008
-#define NVE0_CHANNEL_IND_ENGINE_CE0                                  0x00000010
-#define NVE0_CHANNEL_IND_ENGINE_CE1                                  0x00000020
-#define NVE0_CHANNEL_IND_ENGINE_ENC                                  0x00000040
-
-struct nve0_channel_ind_class {
-        u32 pushbuf;
-        u32 ilength;
-        u64 ioffset;
-        u32 engine;
-};
-
-/* 0046: NV04_DISP
- */
-
-#define NV04_DISP_CLASS                                              0x00000046
-
-#define NV04_DISP_MTHD                                               0x00000000
-#define NV04_DISP_MTHD_HEAD                                          0x00000001
-
-#define NV04_DISP_SCANOUTPOS                                         0x00000000
-
-struct nv04_display_class {
-};
-
-struct nv04_display_scanoutpos {
-        s64 time[2];
-        u32 vblanks;
-        u32 vblanke;
-        u32 vtotal;
-        u32 vline;
-        u32 hblanks;
-        u32 hblanke;
-        u32 htotal;
-        u32 hline;
-};
-
-/* 5070: NV50_DISP
- * 8270: NV84_DISP
- * 8370: NVA0_DISP
- * 8870: NV94_DISP
- * 8570: NVA3_DISP
- * 9070: NVD0_DISP
- * 9170: NVE0_DISP
- * 9270: NVF0_DISP
- * 9470: GM107_DISP
- */
-
-#define NV50_DISP_CLASS                                              0x00005070
-#define NV84_DISP_CLASS                                              0x00008270
-#define NVA0_DISP_CLASS                                              0x00008370
-#define NV94_DISP_CLASS                                              0x00008870
-#define NVA3_DISP_CLASS                                              0x00008570
-#define NVD0_DISP_CLASS                                              0x00009070
-#define NVE0_DISP_CLASS                                              0x00009170
-#define NVF0_DISP_CLASS                                              0x00009270
-#define GM107_DISP_CLASS                                             0x00009470
-
-#define NV50_DISP_MTHD                                               0x00000000
-#define NV50_DISP_MTHD_HEAD                                          0x00000003
-
-#define NV50_DISP_SCANOUTPOS                                         0x00000000
-
-#define NV50_DISP_SOR_MTHD                                           0x00010000
-#define NV50_DISP_SOR_MTHD_TYPE                                      0x0000f000
-#define NV50_DISP_SOR_MTHD_HEAD                                      0x00000018
-#define NV50_DISP_SOR_MTHD_LINK                                      0x00000004
-#define NV50_DISP_SOR_MTHD_OR                                        0x00000003
-
-#define NV50_DISP_SOR_PWR                                            0x00010000
-#define NV50_DISP_SOR_PWR_STATE                                      0x00000001
-#define NV50_DISP_SOR_PWR_STATE_ON                                   0x00000001
-#define NV50_DISP_SOR_PWR_STATE_OFF                                  0x00000000
-#define NVA3_DISP_SOR_HDA_ELD                                        0x00010100
-#define NV84_DISP_SOR_HDMI_PWR                                       0x00012000
-#define NV84_DISP_SOR_HDMI_PWR_STATE                                 0x40000000
-#define NV84_DISP_SOR_HDMI_PWR_STATE_OFF                             0x00000000
-#define NV84_DISP_SOR_HDMI_PWR_STATE_ON                              0x40000000
-#define NV84_DISP_SOR_HDMI_PWR_MAX_AC_PACKET                         0x001f0000
-#define NV84_DISP_SOR_HDMI_PWR_REKEY                                 0x0000007f
-#define NV50_DISP_SOR_LVDS_SCRIPT                                    0x00013000
-#define NV50_DISP_SOR_LVDS_SCRIPT_ID                                 0x0000ffff
-#define NV94_DISP_SOR_DP_PWR                                         0x00016000
-#define NV94_DISP_SOR_DP_PWR_STATE                                   0x00000001
-#define NV94_DISP_SOR_DP_PWR_STATE_OFF                               0x00000000
-#define NV94_DISP_SOR_DP_PWR_STATE_ON                                0x00000001
-
-#define NV50_DISP_DAC_MTHD                                           0x00020000
-#define NV50_DISP_DAC_MTHD_TYPE                                      0x0000f000
-#define NV50_DISP_DAC_MTHD_OR                                        0x00000003
-
-#define NV50_DISP_DAC_PWR                                            0x00020000
-#define NV50_DISP_DAC_PWR_HSYNC                                      0x00000001
-#define NV50_DISP_DAC_PWR_HSYNC_ON                                   0x00000000
-#define NV50_DISP_DAC_PWR_HSYNC_LO                                   0x00000001
-#define NV50_DISP_DAC_PWR_VSYNC                                      0x00000004
-#define NV50_DISP_DAC_PWR_VSYNC_ON                                   0x00000000
-#define NV50_DISP_DAC_PWR_VSYNC_LO                                   0x00000004
-#define NV50_DISP_DAC_PWR_DATA                                       0x00000010
-#define NV50_DISP_DAC_PWR_DATA_ON                                    0x00000000
-#define NV50_DISP_DAC_PWR_DATA_LO                                    0x00000010
-#define NV50_DISP_DAC_PWR_STATE                                      0x00000040
-#define NV50_DISP_DAC_PWR_STATE_ON                                   0x00000000
-#define NV50_DISP_DAC_PWR_STATE_OFF                                  0x00000040
-#define NV50_DISP_DAC_LOAD                                           0x00020100
-#define NV50_DISP_DAC_LOAD_VALUE                                     0x00000007
-
-#define NV50_DISP_PIOR_MTHD                                          0x00030000
-#define NV50_DISP_PIOR_MTHD_TYPE                                     0x0000f000
-#define NV50_DISP_PIOR_MTHD_OR                                       0x00000003
-
-#define NV50_DISP_PIOR_PWR                                           0x00030000
-#define NV50_DISP_PIOR_PWR_STATE                                     0x00000001
-#define NV50_DISP_PIOR_PWR_STATE_ON                                  0x00000001
-#define NV50_DISP_PIOR_PWR_STATE_OFF                                 0x00000000
-#define NV50_DISP_PIOR_TMDS_PWR                                      0x00032000
-#define NV50_DISP_PIOR_TMDS_PWR_STATE                                0x00000001
-#define NV50_DISP_PIOR_TMDS_PWR_STATE_ON                             0x00000001
-#define NV50_DISP_PIOR_TMDS_PWR_STATE_OFF                            0x00000000
-#define NV50_DISP_PIOR_DP_PWR                                        0x00036000
-#define NV50_DISP_PIOR_DP_PWR_STATE                                  0x00000001
-#define NV50_DISP_PIOR_DP_PWR_STATE_ON                               0x00000001
-#define NV50_DISP_PIOR_DP_PWR_STATE_OFF                              0x00000000
-
-struct nv50_display_class {
-};
-
-/* 507a: NV50_DISP_CURS
- * 827a: NV84_DISP_CURS
- * 837a: NVA0_DISP_CURS
- * 887a: NV94_DISP_CURS
- * 857a: NVA3_DISP_CURS
- * 907a: NVD0_DISP_CURS
- * 917a: NVE0_DISP_CURS
- * 927a: NVF0_DISP_CURS
- * 947a: GM107_DISP_CURS
- */
-
-#define NV50_DISP_CURS_CLASS                                         0x0000507a
-#define NV84_DISP_CURS_CLASS                                         0x0000827a
-#define NVA0_DISP_CURS_CLASS                                         0x0000837a
-#define NV94_DISP_CURS_CLASS                                         0x0000887a
-#define NVA3_DISP_CURS_CLASS                                         0x0000857a
-#define NVD0_DISP_CURS_CLASS                                         0x0000907a
-#define NVE0_DISP_CURS_CLASS                                         0x0000917a
-#define NVF0_DISP_CURS_CLASS                                         0x0000927a
-#define GM107_DISP_CURS_CLASS                                        0x0000947a
-
-struct nv50_display_curs_class {
-        u32 head;
-};
-
-/* 507b: NV50_DISP_OIMM
- * 827b: NV84_DISP_OIMM
- * 837b: NVA0_DISP_OIMM
- * 887b: NV94_DISP_OIMM
- * 857b: NVA3_DISP_OIMM
- * 907b: NVD0_DISP_OIMM
- * 917b: NVE0_DISP_OIMM
- * 927b: NVE0_DISP_OIMM
- * 947b: GM107_DISP_OIMM
- */
-
-#define NV50_DISP_OIMM_CLASS                                         0x0000507b
-#define NV84_DISP_OIMM_CLASS                                         0x0000827b
-#define NVA0_DISP_OIMM_CLASS                                         0x0000837b
-#define NV94_DISP_OIMM_CLASS                                         0x0000887b
-#define NVA3_DISP_OIMM_CLASS                                         0x0000857b
-#define NVD0_DISP_OIMM_CLASS                                         0x0000907b
-#define NVE0_DISP_OIMM_CLASS                                         0x0000917b
-#define NVF0_DISP_OIMM_CLASS                                         0x0000927b
-#define GM107_DISP_OIMM_CLASS                                        0x0000947b
-
-struct nv50_display_oimm_class {
-        u32 head;
-};
-
-/* 507c: NV50_DISP_SYNC
- * 827c: NV84_DISP_SYNC
- * 837c: NVA0_DISP_SYNC
- * 887c: NV94_DISP_SYNC
- * 857c: NVA3_DISP_SYNC
- * 907c: NVD0_DISP_SYNC
- * 917c: NVE0_DISP_SYNC
- * 927c: NVF0_DISP_SYNC
- * 947c: GM107_DISP_SYNC
- */
-
-#define NV50_DISP_SYNC_CLASS                                         0x0000507c
-#define NV84_DISP_SYNC_CLASS                                         0x0000827c
-#define NVA0_DISP_SYNC_CLASS                                         0x0000837c
-#define NV94_DISP_SYNC_CLASS                                         0x0000887c
-#define NVA3_DISP_SYNC_CLASS                                         0x0000857c
-#define NVD0_DISP_SYNC_CLASS                                         0x0000907c
-#define NVE0_DISP_SYNC_CLASS                                         0x0000917c
-#define NVF0_DISP_SYNC_CLASS                                         0x0000927c
-#define GM107_DISP_SYNC_CLASS                                        0x0000947c
-
-struct nv50_display_sync_class {
-        u32 pushbuf;
-        u32 head;
-};
-
-/* 507d: NV50_DISP_MAST
- * 827d: NV84_DISP_MAST
- * 837d: NVA0_DISP_MAST
- * 887d: NV94_DISP_MAST
- * 857d: NVA3_DISP_MAST
- * 907d: NVD0_DISP_MAST
- * 917d: NVE0_DISP_MAST
- * 927d: NVF0_DISP_MAST
- * 947d: GM107_DISP_MAST
- */
-
-#define NV50_DISP_MAST_CLASS                                         0x0000507d
-#define NV84_DISP_MAST_CLASS                                         0x0000827d
-#define NVA0_DISP_MAST_CLASS                                         0x0000837d
-#define NV94_DISP_MAST_CLASS                                         0x0000887d
-#define NVA3_DISP_MAST_CLASS                                         0x0000857d
-#define NVD0_DISP_MAST_CLASS                                         0x0000907d
-#define NVE0_DISP_MAST_CLASS                                         0x0000917d
-#define NVF0_DISP_MAST_CLASS                                         0x0000927d
-#define GM107_DISP_MAST_CLASS                                        0x0000947d
-
-struct nv50_display_mast_class {
-        u32 pushbuf;
-};
-
-/* 507e: NV50_DISP_OVLY
- * 827e: NV84_DISP_OVLY
- * 837e: NVA0_DISP_OVLY
- * 887e: NV94_DISP_OVLY
- * 857e: NVA3_DISP_OVLY
- * 907e: NVD0_DISP_OVLY
- * 917e: NVE0_DISP_OVLY
- * 927e: NVF0_DISP_OVLY
- * 947e: GM107_DISP_OVLY
- */
-
-#define NV50_DISP_OVLY_CLASS                                         0x0000507e
-#define NV84_DISP_OVLY_CLASS                                         0x0000827e
-#define NVA0_DISP_OVLY_CLASS                                         0x0000837e
-#define NV94_DISP_OVLY_CLASS                                         0x0000887e
-#define NVA3_DISP_OVLY_CLASS                                         0x0000857e
-#define NVD0_DISP_OVLY_CLASS                                         0x0000907e
-#define NVE0_DISP_OVLY_CLASS                                         0x0000917e
-#define NVF0_DISP_OVLY_CLASS                                         0x0000927e
-#define GM107_DISP_OVLY_CLASS                                        0x0000947e
-
-struct nv50_display_ovly_class {
-        u32 pushbuf;
-        u32 head;
-};
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/client.h b/drivers/gpu/drm/nouveau/core/include/core/client.h
index c66eac513803..4fc6ab12382d 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/client.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/client.h
@@ -10,6 +10,11 @@ struct nouveau_client {
         char name[32];
         u32 debug;
         struct nouveau_vm *vm;
+        bool super;
+        void *data;
+
+        int (*ntfy)(const void *, u32, const void *, u32);
+        struct nvkm_client_notify *notify[8];
 };
 
 static inline struct nouveau_client *
@@ -43,4 +48,10 @@ int  nouveau_client_init(struct nouveau_client *);
 int  nouveau_client_fini(struct nouveau_client *, bool suspend);
 const char *nouveau_client_name(void *obj);
 
+int nvkm_client_notify_new(struct nouveau_client *, struct nvkm_event *,
+                           void *data, u32 size);
+int nvkm_client_notify_del(struct nouveau_client *, int index);
+int nvkm_client_notify_get(struct nouveau_client *, int index);
+int nvkm_client_notify_put(struct nouveau_client *, int index);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/device.h b/drivers/gpu/drm/nouveau/core/include/core/device.h
index a8a9a9cf16cb..8743766454a5 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/device.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/device.h
@@ -4,6 +4,7 @@
 #include <core/object.h>
 #include <core/subdev.h>
 #include <core/engine.h>
+#include <core/event.h>
 
 enum nv_subdev_type {
         NVDEV_ENGINE_DEVICE,
@@ -28,7 +29,7 @@ enum nv_subdev_type {
         NVDEV_SUBDEV_BUS,
         NVDEV_SUBDEV_TIMER,
         NVDEV_SUBDEV_FB,
-        NVDEV_SUBDEV_LTCG,
+        NVDEV_SUBDEV_LTC,
         NVDEV_SUBDEV_IBUS,
         NVDEV_SUBDEV_INSTMEM,
         NVDEV_SUBDEV_VM,
@@ -69,6 +70,8 @@ struct nouveau_device {
         struct platform_device *platformdev;
         u64 handle;
 
+        struct nvkm_event event;
+
         const char *cfgopt;
         const char *dbgopt;
         const char *name;
@@ -84,7 +87,6 @@ struct nouveau_device {
                 NV_40    = 0x40,
                 NV_50    = 0x50,
                 NV_C0    = 0xc0,
-                NV_D0    = 0xd0,
                 NV_E0    = 0xe0,
                 GM100    = 0x110,
         } card_type;
@@ -93,8 +95,14 @@ struct nouveau_device {
 
         struct nouveau_oclass *oclass[NVDEV_SUBDEV_NR];
         struct nouveau_object *subdev[NVDEV_SUBDEV_NR];
+
+        struct {
+                struct notifier_block nb;
+        } acpi;
 };
 
+int nouveau_device_list(u64 *name, int size);
+
 static inline struct nouveau_device *
 nv_device(void *obj)
 {
@@ -162,12 +170,6 @@ nv_device_resource_start(struct nouveau_device *device, unsigned int bar);
 resource_size_t
 nv_device_resource_len(struct nouveau_device *device, unsigned int bar);
 
-dma_addr_t
-nv_device_map_page(struct nouveau_device *device, struct page *page);
-
-void
-nv_device_unmap_page(struct nouveau_device *device, dma_addr_t addr);
-
 int
 nv_device_get_irq(struct nouveau_device *device, bool stall);
 
diff --git a/drivers/gpu/drm/nouveau/core/include/core/event.h b/drivers/gpu/drm/nouveau/core/include/core/event.h
index ba3f1a76a815..51e55d03330a 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/event.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/event.h
@@ -1,47 +1,34 @@
 #ifndef __NVKM_EVENT_H__
 #define __NVKM_EVENT_H__
 
-/* return codes from event handlers */
-#define NVKM_EVENT_DROP 0
-#define NVKM_EVENT_KEEP 1
+#include <core/notify.h>
 
-/* nouveau_eventh.flags bit #s */
-#define NVKM_EVENT_ENABLE 0
-
-struct nouveau_eventh {
-        struct nouveau_event *event;
-        struct list_head head;
-        unsigned long flags;
-        u32 types;
-        int index;
-        int (*func)(void *, u32, int);
-        void *priv;
+struct nvkm_event_func {
+        int  (*ctor)(void *data, u32 size, struct nvkm_notify *);
+        void (*send)(void *data, u32 size, struct nvkm_notify *);
+        void (*init)(struct nvkm_event *, int type, int index);
+        void (*fini)(struct nvkm_event *, int type, int index);
 };
 
-struct nouveau_event {
-        void *priv;
-        int (*check)(struct nouveau_event *, u32 type, int index);
-        void (*enable)(struct nouveau_event *, int type, int index);
-        void (*disable)(struct nouveau_event *, int type, int index);
+struct nvkm_event {
+        const struct nvkm_event_func *func;
 
         int types_nr;
         int index_nr;
 
-        spinlock_t list_lock;
-        struct list_head *list;
         spinlock_t refs_lock;
-        int refs[];
+        spinlock_t list_lock;
+        struct list_head list;
+        int *refs;
 };
 
-int  nouveau_event_create(int types_nr, int index_nr, struct nouveau_event **);
-void nouveau_event_destroy(struct nouveau_event **);
-void nouveau_event_trigger(struct nouveau_event *, u32 types, int index);
-
-int  nouveau_event_new(struct nouveau_event *, u32 types, int index,
-                       int (*func)(void *, u32, int), void *,
-                       struct nouveau_eventh **);
-void nouveau_event_ref(struct nouveau_eventh *, struct nouveau_eventh **);
-void nouveau_event_get(struct nouveau_eventh *);
-void nouveau_event_put(struct nouveau_eventh *);
+int  nvkm_event_init(const struct nvkm_event_func *func,
+                     int types_nr, int index_nr,
+                     struct nvkm_event *);
+void nvkm_event_fini(struct nvkm_event *);
+void nvkm_event_get(struct nvkm_event *, u32 types, int index);
+void nvkm_event_put(struct nvkm_event *, u32 types, int index);
+void nvkm_event_send(struct nvkm_event *, u32 types, int index,
+                     void *data, u32 size);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/handle.h b/drivers/gpu/drm/nouveau/core/include/core/handle.h
index 363674cdf8ab..ceb67d770875 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/handle.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/handle.h
@@ -10,6 +10,9 @@ struct nouveau_handle {
         u32 name;
         u32 priv;
 
+        u8  route;
+        u64 token;
+
         struct nouveau_handle *parent;
         struct nouveau_object *object;
 };
@@ -20,6 +23,11 @@ void nouveau_handle_destroy(struct nouveau_handle *);
 int  nouveau_handle_init(struct nouveau_handle *);
 int  nouveau_handle_fini(struct nouveau_handle *, bool suspend);
 
+int  nouveau_handle_new(struct nouveau_object *, u32 parent, u32 handle,
+                        u16 oclass, void *data, u32 size,
+                        struct nouveau_object **);
+int  nouveau_handle_del(struct nouveau_object *, u32 parent, u32 handle);
+
 struct nouveau_object *
 nouveau_handle_ref(struct nouveau_object *, u32 name);
 
diff --git a/drivers/gpu/drm/nouveau/core/include/core/ioctl.h b/drivers/gpu/drm/nouveau/core/include/core/ioctl.h
new file mode 100644
index 000000000000..ac7935c2474e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/ioctl.h
@@ -0,0 +1,6 @@
+#ifndef __NVKM_IOCTL_H__
+#define __NVKM_IOCTL_H__
+
+int nvkm_ioctl(struct nouveau_client *, bool, void *, u32, void **);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/notify.h b/drivers/gpu/drm/nouveau/core/include/core/notify.h
new file mode 100644
index 000000000000..1262d8f020f3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/notify.h
@@ -0,0 +1,36 @@
+#ifndef __NVKM_NOTIFY_H__
+#define __NVKM_NOTIFY_H__
+
+struct nvkm_notify {
+        struct nvkm_event *event;
+        struct list_head head;
+#define NVKM_NOTIFY_USER 0
+#define NVKM_NOTIFY_WORK 1
+        unsigned long flags;
+        int block;
+#define NVKM_NOTIFY_DROP 0
+#define NVKM_NOTIFY_KEEP 1
+        int (*func)(struct nvkm_notify *);
+
+        /* set by nvkm_event ctor */
+        u32 types;
+        int index;
+        u32 size;
+
+        struct work_struct work;
+        /* this is const for a *very* good reason - the data might be on the
+         * stack from an irq handler.  if you're not core/notify.c then you
+         * should probably think twice before casting it away...
+         */
+        const void *data;
+};
+
+int  nvkm_notify_init(struct nvkm_event *, int (*func)(struct nvkm_notify *),
+                      bool work, void *data, u32 size, u32 reply,
+                      struct nvkm_notify *);
+void nvkm_notify_fini(struct nvkm_notify *);
+void nvkm_notify_get(struct nvkm_notify *);
+void nvkm_notify_put(struct nvkm_notify *);
+void nvkm_notify_send(struct nvkm_notify *, void *data, u32 size);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/object.h b/drivers/gpu/drm/nouveau/core/include/core/object.h
index 62e68baef087..d7039482d6fd 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/object.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/object.h
@@ -48,6 +48,10 @@ void nouveau_object_destroy(struct nouveau_object *);
 int  nouveau_object_init(struct nouveau_object *);
 int  nouveau_object_fini(struct nouveau_object *, bool suspend);
 
+int _nouveau_object_ctor(struct nouveau_object *, struct nouveau_object *,
+                         struct nouveau_oclass *, void *, u32,
+                         struct nouveau_object **);
+
 extern struct nouveau_ofuncs nouveau_object_ofuncs;
 
 /* Don't allocate dynamically, because lockdep needs lock_class_keys to be in
@@ -78,6 +82,7 @@ struct nouveau_omthds {
         int (*call)(struct nouveau_object *, u32, void *, u32);
 };
 
+struct nvkm_event;
 struct nouveau_ofuncs {
         int  (*ctor)(struct nouveau_object *, struct nouveau_object *,
                      struct nouveau_oclass *, void *data, u32 size,
@@ -85,6 +90,9 @@ struct nouveau_ofuncs {
         void (*dtor)(struct nouveau_object *);
         int  (*init)(struct nouveau_object *);
         int  (*fini)(struct nouveau_object *, bool suspend);
+        int  (*mthd)(struct nouveau_object *, u32, void *, u32);
+        int  (*ntfy)(struct nouveau_object *, u32, struct nvkm_event **);
+        int  (* map)(struct nouveau_object *, u64 *, u32 *);
         u8   (*rd08)(struct nouveau_object *, u64 offset);
         u16  (*rd16)(struct nouveau_object *, u64 offset);
         u32  (*rd32)(struct nouveau_object *, u64 offset);
@@ -106,10 +114,6 @@ void nouveau_object_ref(struct nouveau_object *, struct nouveau_object **);
 int nouveau_object_inc(struct nouveau_object *);
 int nouveau_object_dec(struct nouveau_object *, bool suspend);
 
-int nouveau_object_new(struct nouveau_object *, u32 parent, u32 handle,
-                       u16 oclass, void *data, u32 size,
-                       struct nouveau_object **);
-int nouveau_object_del(struct nouveau_object *, u32 parent, u32 handle);
 void nouveau_object_debug(void);
 
 static inline int
@@ -199,4 +203,21 @@ nv_memcmp(void *obj, u32 addr, const char *str, u32 len)
         return 0;
 }
 
+#include <core/handle.h>
+
+static inline int
+nouveau_object_new(struct nouveau_object *client, u32 parent, u32 handle,
+                   u16 oclass, void *data, u32 size,
+                   struct nouveau_object **pobject)
+{
+        return nouveau_handle_new(client, parent, handle, oclass,
+                                  data, size, pobject);
+}
+
+static inline int
+nouveau_object_del(struct nouveau_object *client, u32 parent, u32 handle)
+{
+        return nouveau_handle_del(client, parent, handle);
+}
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/parent.h b/drivers/gpu/drm/nouveau/core/include/core/parent.h
index 9f5ea900ff00..12da418ec70a 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/parent.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/parent.h
@@ -57,5 +57,6 @@ void _nouveau_parent_dtor(struct nouveau_object *);
 int nouveau_parent_sclass(struct nouveau_object *, u16 handle,
                           struct nouveau_object **pengine,
                           struct nouveau_oclass **poclass);
+int nouveau_parent_lclass(struct nouveau_object *, u32 *, int);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/printk.h b/drivers/gpu/drm/nouveau/core/include/core/printk.h
index 0f9a37bd32b0..451b6ed20b7e 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/printk.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/printk.h
@@ -21,6 +21,7 @@ nv_printk_(struct nouveau_object *, int, const char *, ...);
 #define nv_debug(o,f,a...) nv_printk((o), DEBUG, f, ##a)
 #define nv_trace(o,f,a...) nv_printk((o), TRACE, f, ##a)
 #define nv_spam(o,f,a...) nv_printk((o), SPAM, f, ##a)
+#define nv_ioctl(o,f,a...) nv_trace(nouveau_client(o), "ioctl: "f, ##a)
 
 #define nv_assert(f,a...) do {                                                 \
         if (NV_DBG_FATAL <= CONFIG_NOUVEAU_DEBUG)                              \
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/disp.h b/drivers/gpu/drm/nouveau/core/include/engine/disp.h
index fde842896806..7a64f347b385 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/disp.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/disp.h
@@ -6,20 +6,13 @@
 #include <core/device.h>
 #include <core/event.h>
 
-enum nvkm_hpd_event {
-        NVKM_HPD_PLUG = 1,
-        NVKM_HPD_UNPLUG = 2,
-        NVKM_HPD_IRQ = 4,
-        NVKM_HPD = (NVKM_HPD_PLUG | NVKM_HPD_UNPLUG | NVKM_HPD_IRQ)
-};
-
 struct nouveau_disp {
         struct nouveau_engine base;
 
         struct list_head outp;
-        struct nouveau_event *hpd;
 
-        struct nouveau_event *vblank;
+        struct nvkm_event hpd;
+        struct nvkm_event vblank;
 };
 
 static inline struct nouveau_disp *
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/dmaobj.h b/drivers/gpu/drm/nouveau/core/include/engine/dmaobj.h
index b28914ed1752..1b283a7b78e6 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/dmaobj.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/dmaobj.h
@@ -12,37 +12,20 @@ struct nouveau_dmaobj {
         u32 access;
         u64 start;
         u64 limit;
-        u32 conf0;
 };
 
 struct nouveau_dmaeng {
         struct nouveau_engine base;
 
         /* creates a "physical" dma object from a struct nouveau_dmaobj */
-        int (*bind)(struct nouveau_dmaeng *dmaeng,
+        int (*bind)(struct nouveau_dmaobj *dmaobj,
                     struct nouveau_object *parent,
-                    struct nouveau_dmaobj *dmaobj,
                     struct nouveau_gpuobj **);
 };
 
-#define nouveau_dmaeng_create(p,e,c,d)                                         \
-        nouveau_engine_create((p), (e), (c), true, "DMAOBJ", "dmaobj", (d))
-#define nouveau_dmaeng_destroy(p)                                              \
-        nouveau_engine_destroy(&(p)->base)
-#define nouveau_dmaeng_init(p)                                                 \
-        nouveau_engine_init(&(p)->base)
-#define nouveau_dmaeng_fini(p,s)                                               \
-        nouveau_engine_fini(&(p)->base, (s))
-
-#define _nouveau_dmaeng_dtor _nouveau_engine_dtor
-#define _nouveau_dmaeng_init _nouveau_engine_init
-#define _nouveau_dmaeng_fini _nouveau_engine_fini
-
-extern struct nouveau_oclass nv04_dmaeng_oclass;
-extern struct nouveau_oclass nv50_dmaeng_oclass;
-extern struct nouveau_oclass nvc0_dmaeng_oclass;
-extern struct nouveau_oclass nvd0_dmaeng_oclass;
-
-extern struct nouveau_oclass nouveau_dmaobj_sclass[];
+extern struct nouveau_oclass *nv04_dmaeng_oclass;
+extern struct nouveau_oclass *nv50_dmaeng_oclass;
+extern struct nouveau_oclass *nvc0_dmaeng_oclass;
+extern struct nouveau_oclass *nvd0_dmaeng_oclass;
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/fifo.h b/drivers/gpu/drm/nouveau/core/include/engine/fifo.h
index b639eb2c74ff..e5e4d930b2c2 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/fifo.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/fifo.h
@@ -4,12 +4,14 @@
 #include <core/namedb.h>
 #include <core/gpuobj.h>
 #include <core/engine.h>
+#include <core/event.h>
 
 struct nouveau_fifo_chan {
         struct nouveau_namedb base;
         struct nouveau_dmaobj *pushdma;
         struct nouveau_gpuobj *pushgpu;
         void __iomem *user;
+        u64 addr;
         u32 size;
         u16 chid;
         atomic_t refcnt; /* NV04_NVSW_SET_REF */
@@ -40,8 +42,10 @@ void nouveau_fifo_channel_destroy(struct nouveau_fifo_chan *);
 #define _nouveau_fifo_channel_fini _nouveau_namedb_fini
 
 void _nouveau_fifo_channel_dtor(struct nouveau_object *);
+int  _nouveau_fifo_channel_map(struct nouveau_object *, u64 *, u32 *);
 u32  _nouveau_fifo_channel_rd32(struct nouveau_object *, u64);
 void _nouveau_fifo_channel_wr32(struct nouveau_object *, u64, u32);
+int  _nouveau_fifo_channel_ntfy(struct nouveau_object *, u32, struct nvkm_event **);
 
 struct nouveau_fifo_base {
         struct nouveau_gpuobj base;
@@ -65,8 +69,8 @@ struct nouveau_fifo_base {
 struct nouveau_fifo {
         struct nouveau_engine base;
 
-        struct nouveau_event *cevent; /* channel creation event */
-        struct nouveau_event *uevent; /* async user trigger */
+        struct nvkm_event cevent; /* channel creation event */
+        struct nvkm_event uevent; /* async user trigger */
 
         struct nouveau_object **channel;
         spinlock_t lock;
@@ -112,6 +116,9 @@ extern struct nouveau_oclass *nve0_fifo_oclass;
 extern struct nouveau_oclass *gk20a_fifo_oclass;
 extern struct nouveau_oclass *nv108_fifo_oclass;
 
+int  nouveau_fifo_uevent_ctor(void *, u32, struct nvkm_notify *);
+void nouveau_fifo_uevent(struct nouveau_fifo *);
+
 void nv04_fifo_intr(struct nouveau_subdev *);
 int  nv04_fifo_context_attach(struct nouveau_object *, struct nouveau_object *);
 
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/graph.h b/drivers/gpu/drm/nouveau/core/include/engine/graph.h
index 8c1d4772da0c..d5055570d01b 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/graph.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/graph.h
@@ -70,6 +70,7 @@ extern struct nouveau_oclass *nvd9_graph_oclass;
 extern struct nouveau_oclass *nve4_graph_oclass;
 extern struct nouveau_oclass *gk20a_graph_oclass;
 extern struct nouveau_oclass *nvf0_graph_oclass;
+extern struct nouveau_oclass *gk110b_graph_oclass;
 extern struct nouveau_oclass *nv108_graph_oclass;
 extern struct nouveau_oclass *gm107_graph_oclass;
 
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/perfmon.h b/drivers/gpu/drm/nouveau/core/include/engine/perfmon.h
index 49b0024910fe..88cc812baaa3 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/perfmon.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/perfmon.h
@@ -4,7 +4,6 @@
 #include <core/device.h>
 #include <core/engine.h>
 #include <core/engctx.h>
-#include <core/class.h>
 
 struct nouveau_perfdom;
 struct nouveau_perfctr;
diff --git a/drivers/gpu/drm/nouveau/core/include/nvif/class.h b/drivers/gpu/drm/nouveau/core/include/nvif/class.h
new file mode 120000
index 000000000000..f1ac4859edd4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/nvif/class.h
@@ -0,0 +1 @@
+../../../nvif/class.h
\ No newline at end of file
diff --git a/drivers/gpu/drm/nouveau/core/include/nvif/event.h b/drivers/gpu/drm/nouveau/core/include/nvif/event.h
new file mode 120000
index 000000000000..1b798538a725
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/nvif/event.h
@@ -0,0 +1 @@
+../../../nvif/event.h
\ No newline at end of file
diff --git a/drivers/gpu/drm/nouveau/core/include/nvif/ioctl.h b/drivers/gpu/drm/nouveau/core/include/nvif/ioctl.h
new file mode 120000
index 000000000000..8569c86907c5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/nvif/ioctl.h
@@ -0,0 +1 @@
+../../../nvif/ioctl.h
\ No newline at end of file
diff --git a/drivers/gpu/drm/nouveau/core/include/nvif/unpack.h b/drivers/gpu/drm/nouveau/core/include/nvif/unpack.h
new file mode 120000
index 000000000000..69d99292bca4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/nvif/unpack.h
@@ -0,0 +1 @@
+../../../nvif/unpack.h
\ No newline at end of file
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bar.h b/drivers/gpu/drm/nouveau/core/include/subdev/bar.h
index 9faa98e67ad8..be037fac534c 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/bar.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bar.h
@@ -20,6 +20,9 @@ struct nouveau_bar {
                     u32 flags, struct nouveau_vma *);
         void (*unmap)(struct nouveau_bar *, struct nouveau_vma *);
         void (*flush)(struct nouveau_bar *);
+
+        /* whether the BAR supports to be ioremapped WC or should be uncached */
+        bool iomap_uncached;
 };
 
 static inline struct nouveau_bar *
@@ -30,5 +33,6 @@ nouveau_bar(void *obj)
 
 extern struct nouveau_oclass nv50_bar_oclass;
 extern struct nouveau_oclass nvc0_bar_oclass;
+extern struct nouveau_oclass gk20a_bar_oclass;
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/clock.h b/drivers/gpu/drm/nouveau/core/include/subdev/clock.h
index c01e29c9f89a..a5ca00dd2f61 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/clock.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/clock.h
@@ -71,8 +71,15 @@ struct nouveau_clock {
         struct list_head states;
         int state_nr;
 
+        struct work_struct work;
+        wait_queue_head_t wait;
+        atomic_t waiting;
+
+        struct nvkm_notify pwrsrc_ntfy;
+        int pwrsrc;
         int pstate; /* current */
-        int ustate; /* user-requested (-1 disabled, -2 perfmon) */
+        int ustate_ac; /* user-requested (-1 disabled, -2 perfmon) */
+        int ustate_dc; /* user-requested (-1 disabled, -2 perfmon) */
         int astate; /* perfmon adjustment (base) */
         int tstate; /* thermal adjustment (max-) */
         int dstate; /* display adjustment (min+) */
@@ -108,8 +115,9 @@ struct nouveau_clocks {
         int mdiv;
 };
 
-#define nouveau_clock_create(p,e,o,i,r,d)                                      \
-        nouveau_clock_create_((p), (e), (o), (i), (r), sizeof(**d), (void **)d)
+#define nouveau_clock_create(p,e,o,i,r,s,n,d)                                  \
+        nouveau_clock_create_((p), (e), (o), (i), (r), (s), (n), sizeof(**d),  \
+                              (void **)d)
 #define nouveau_clock_destroy(p) ({                                            \
         struct nouveau_clock *clk = (p);                                       \
         _nouveau_clock_dtor(nv_object(clk));                                   \
@@ -118,15 +126,18 @@ struct nouveau_clocks {
         struct nouveau_clock *clk = (p);                                       \
         _nouveau_clock_init(nv_object(clk));                                   \
 })
-#define nouveau_clock_fini(p,s)                                                \
-        nouveau_subdev_fini(&(p)->base, (s))
+#define nouveau_clock_fini(p,s) ({                                             \
+        struct nouveau_clock *clk = (p);                                       \
+        _nouveau_clock_fini(nv_object(clk), (s));                              \
+})
 
 int  nouveau_clock_create_(struct nouveau_object *, struct nouveau_object *,
                            struct nouveau_oclass *,
-                           struct nouveau_clocks *, bool, int, void **);
+                           struct nouveau_clocks *, struct nouveau_pstate *,
+                           int, bool, int, void **);
 void _nouveau_clock_dtor(struct nouveau_object *);
-int _nouveau_clock_init(struct nouveau_object *);
-#define _nouveau_clock_fini _nouveau_subdev_fini
+int  _nouveau_clock_init(struct nouveau_object *);
+int  _nouveau_clock_fini(struct nouveau_object *, bool);
 
 extern struct nouveau_oclass nv04_clock_oclass;
 extern struct nouveau_oclass nv40_clock_oclass;
@@ -136,6 +147,7 @@ extern struct nouveau_oclass *nvaa_clock_oclass;
 extern struct nouveau_oclass nva3_clock_oclass;
 extern struct nouveau_oclass nvc0_clock_oclass;
 extern struct nouveau_oclass nve0_clock_oclass;
+extern struct nouveau_oclass gk20a_clock_oclass;
 
 int nv04_clock_pll_set(struct nouveau_clock *, u32 type, u32 freq);
 int nv04_clock_pll_calc(struct nouveau_clock *, struct nvbios_pll *,
@@ -145,7 +157,7 @@ int nv04_clock_pll_prog(struct nouveau_clock *, u32 reg1,
 int nva3_clock_pll_calc(struct nouveau_clock *, struct nvbios_pll *,
                         int clk, struct nouveau_pll_vals *);
 
-int nouveau_clock_ustate(struct nouveau_clock *, int req);
+int nouveau_clock_ustate(struct nouveau_clock *, int req, int pwr);
 int nouveau_clock_astate(struct nouveau_clock *, int req, int rel);
 int nouveau_clock_dstate(struct nouveau_clock *, int req, int rel);
 int nouveau_clock_tstate(struct nouveau_clock *, int req, int rel);
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/gpio.h b/drivers/gpu/drm/nouveau/core/include/subdev/gpio.h
index 612d82ab683d..b73733d21cc7 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/gpio.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/gpio.h
@@ -8,16 +8,22 @@
 #include <subdev/bios.h>
 #include <subdev/bios/gpio.h>
 
-enum nvkm_gpio_event {
-        NVKM_GPIO_HI = 1,
-        NVKM_GPIO_LO = 2,
-        NVKM_GPIO_TOGGLED = (NVKM_GPIO_HI | NVKM_GPIO_LO),
+struct nvkm_gpio_ntfy_req {
+#define NVKM_GPIO_HI                                                       0x01
+#define NVKM_GPIO_LO                                                       0x02
+#define NVKM_GPIO_TOGGLED                                                  0x03
+        u8 mask;
+        u8 line;
+};
+
+struct nvkm_gpio_ntfy_rep {
+        u8 mask;
 };
 
 struct nouveau_gpio {
         struct nouveau_subdev base;
 
-        struct nouveau_event *events;
+        struct nvkm_event event;
 
         void (*reset)(struct nouveau_gpio *, u8 func);
         int  (*find)(struct nouveau_gpio *, int idx, u8 tag, u8 line,
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h b/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
index 825f7bb46b67..1b937c2c25ae 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
@@ -14,15 +14,18 @@
 #define NV_I2C_TYPE_EXTDDC(e) (0x0005 | (e) << 8)
 #define NV_I2C_TYPE_EXTAUX(e) (0x0006 | (e) << 8)
 
-enum nvkm_i2c_event {
-        NVKM_I2C_PLUG = 1,
-        NVKM_I2C_UNPLUG = 2,
-        NVKM_I2C_IRQ = 4,
-        NVKM_I2C_DONE = 8,
-        NVKM_I2C_ANY = (NVKM_I2C_PLUG |
-                        NVKM_I2C_UNPLUG |
-                        NVKM_I2C_IRQ |
-                        NVKM_I2C_DONE),
+struct nvkm_i2c_ntfy_req {
+#define NVKM_I2C_PLUG                                                      0x01
+#define NVKM_I2C_UNPLUG                                                    0x02
+#define NVKM_I2C_IRQ                                                       0x04
+#define NVKM_I2C_DONE                                                      0x08
+#define NVKM_I2C_ANY                                                       0x0f
+        u8 mask;
+        u8 port;
+};
+
+struct nvkm_i2c_ntfy_rep {
+        u8 mask;
 };
 
 struct nouveau_i2c_port {
@@ -56,7 +59,7 @@ struct nouveau_i2c_board_info {
 
 struct nouveau_i2c {
         struct nouveau_subdev base;
-        struct nouveau_event *ntfy;
+        struct nvkm_event event;
 
         struct nouveau_i2c_port *(*find)(struct nouveau_i2c *, u8 index);
         struct nouveau_i2c_port *(*find_type)(struct nouveau_i2c *, u16 type);
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/ltc.h b/drivers/gpu/drm/nouveau/core/include/subdev/ltc.h
new file mode 100644
index 000000000000..b909a7363f6b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/ltc.h
@@ -0,0 +1,35 @@
+#ifndef __NOUVEAU_LTC_H__
+#define __NOUVEAU_LTC_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+#define NOUVEAU_LTC_MAX_ZBC_CNT 16
+
+struct nouveau_mm_node;
+
+struct nouveau_ltc {
+        struct nouveau_subdev base;
+
+        int  (*tags_alloc)(struct nouveau_ltc *, u32 count,
+                           struct nouveau_mm_node **);
+        void (*tags_free)(struct nouveau_ltc *, struct nouveau_mm_node **);
+        void (*tags_clear)(struct nouveau_ltc *, u32 first, u32 count);
+
+        int zbc_min;
+        int zbc_max;
+        int (*zbc_color_get)(struct nouveau_ltc *, int index, const u32[4]);
+        int (*zbc_depth_get)(struct nouveau_ltc *, int index, const u32);
+};
+
+static inline struct nouveau_ltc *
+nouveau_ltc(void *obj)
+{
+        return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_LTC];
+}
+
+extern struct nouveau_oclass *gf100_ltc_oclass;
+extern struct nouveau_oclass *gk104_ltc_oclass;
+extern struct nouveau_oclass *gm107_ltc_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/ltcg.h b/drivers/gpu/drm/nouveau/core/include/subdev/ltcg.h
deleted file mode 100644
index c9c1950b7743..000000000000
--- a/drivers/gpu/drm/nouveau/core/include/subdev/ltcg.h
+++ /dev/null
@@ -1,41 +0,0 @@
-#ifndef __NOUVEAU_LTCG_H__
-#define __NOUVEAU_LTCG_H__
-
-#include <core/subdev.h>
-#include <core/device.h>
-
-struct nouveau_mm_node;
-
-struct nouveau_ltcg {
-        struct nouveau_subdev base;
-
-        int  (*tags_alloc)(struct nouveau_ltcg *, u32 count,
-                           struct nouveau_mm_node **);
-        void (*tags_free)(struct nouveau_ltcg *, struct nouveau_mm_node **);
-        void (*tags_clear)(struct nouveau_ltcg *, u32 first, u32 count);
-};
-
-static inline struct nouveau_ltcg *
-nouveau_ltcg(void *obj)
-{
-        return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_LTCG];
-}
-
-#define nouveau_ltcg_create(p,e,o,d)                                           \
-        nouveau_subdev_create_((p), (e), (o), 0, "PLTCG", "level2",            \
-                               sizeof(**d), (void **)d)
-#define nouveau_ltcg_destroy(p)                                                \
-        nouveau_subdev_destroy(&(p)->base)
-#define nouveau_ltcg_init(p)                                                   \
-        nouveau_subdev_init(&(p)->base)
-#define nouveau_ltcg_fini(p,s)                                                 \
-        nouveau_subdev_fini(&(p)->base, (s))
-
-#define _nouveau_ltcg_dtor _nouveau_subdev_dtor
-#define _nouveau_ltcg_init _nouveau_subdev_init
-#define _nouveau_ltcg_fini _nouveau_subdev_fini
-
-extern struct nouveau_oclass *gf100_ltcg_oclass;
-extern struct nouveau_oclass *gm107_ltcg_oclass;
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/mc.h b/drivers/gpu/drm/nouveau/core/include/subdev/mc.h
index 72b176831be6..568e4dfc5e9e 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/mc.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/mc.h
@@ -4,15 +4,11 @@
 #include <core/subdev.h>
 #include <core/device.h>
 
-struct nouveau_mc_intr {
-        u32 stat;
-        u32 unit;
-};
-
 struct nouveau_mc {
         struct nouveau_subdev base;
         bool use_msi;
         unsigned int irq;
+        void (*unk260)(struct nouveau_mc *, u32);
 };
 
 static inline struct nouveau_mc *
@@ -21,30 +17,6 @@ nouveau_mc(void *obj)
         return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_MC];
 }
 
-#define nouveau_mc_create(p,e,o,d)                                             \
-        nouveau_mc_create_((p), (e), (o), sizeof(**d), (void **)d)
-#define nouveau_mc_destroy(p) ({                                               \
-        struct nouveau_mc *pmc = (p); _nouveau_mc_dtor(nv_object(pmc));        \
-})
-#define nouveau_mc_init(p) ({                                                  \
-        struct nouveau_mc *pmc = (p); _nouveau_mc_init(nv_object(pmc));        \
-})
-#define nouveau_mc_fini(p,s) ({                                                \
-        struct nouveau_mc *pmc = (p); _nouveau_mc_fini(nv_object(pmc), (s));   \
-})
-
-int  nouveau_mc_create_(struct nouveau_object *, struct nouveau_object *,
-                        struct nouveau_oclass *, int, void **);
-void _nouveau_mc_dtor(struct nouveau_object *);
-int  _nouveau_mc_init(struct nouveau_object *);
-int  _nouveau_mc_fini(struct nouveau_object *, bool);
-
-struct nouveau_mc_oclass {
-        struct nouveau_oclass base;
-        const struct nouveau_mc_intr *intr;
-        void (*msi_rearm)(struct nouveau_mc *);
-};
-
 extern struct nouveau_oclass *nv04_mc_oclass;
 extern struct nouveau_oclass *nv40_mc_oclass;
 extern struct nouveau_oclass *nv44_mc_oclass;
@@ -54,5 +26,6 @@ extern struct nouveau_oclass *nv94_mc_oclass;
 extern struct nouveau_oclass *nv98_mc_oclass;
 extern struct nouveau_oclass *nvc0_mc_oclass;
 extern struct nouveau_oclass *nvc3_mc_oclass;
+extern struct nouveau_oclass *gk20a_mc_oclass;
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h b/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h
index c5c92cbed33f..f73feec151db 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h
@@ -8,18 +8,6 @@ struct nouveau_pwr {
         struct nouveau_subdev base;
 
         struct {
-                u32 limit;
-                u32 *data;
-                u32  size;
-        } code;
-
-        struct {
-                u32 limit;
-                u32 *data;
-                u32  size;
-        } data;
-
-        struct {
                 u32 base;
                 u32 size;
         } send;
@@ -35,7 +23,8 @@ struct nouveau_pwr {
                 u32 data[2];
         } recv;
 
-        int (*message)(struct nouveau_pwr *, u32[2], u32, u32, u32, u32);
+        int  (*message)(struct nouveau_pwr *, u32[2], u32, u32, u32, u32);
+        void (*pgob)(struct nouveau_pwr *, bool);
 };
 
 static inline struct nouveau_pwr *
@@ -44,29 +33,11 @@ nouveau_pwr(void *obj)
         return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_PWR];
 }
 
-#define nouveau_pwr_create(p, e, o, d)                                         \
-        nouveau_pwr_create_((p), (e), (o), sizeof(**d), (void **)d)
-#define nouveau_pwr_destroy(p)                                                 \
-        nouveau_subdev_destroy(&(p)->base)
-#define nouveau_pwr_init(p) ({                                                 \
-        struct nouveau_pwr *ppwr = (p);                                        \
-        _nouveau_pwr_init(nv_object(ppwr));                                    \
-})
-#define nouveau_pwr_fini(p,s) ({                                               \
-        struct nouveau_pwr *ppwr = (p);                                        \
-        _nouveau_pwr_fini(nv_object(ppwr), (s));                               \
-})
-
-int nouveau_pwr_create_(struct nouveau_object *, struct nouveau_object *,
-                           struct nouveau_oclass *, int, void **);
-#define _nouveau_pwr_dtor _nouveau_subdev_dtor
-int _nouveau_pwr_init(struct nouveau_object *);
-int _nouveau_pwr_fini(struct nouveau_object *, bool);
-
-extern struct nouveau_oclass nva3_pwr_oclass;
-extern struct nouveau_oclass nvc0_pwr_oclass;
-extern struct nouveau_oclass nvd0_pwr_oclass;
-extern struct nouveau_oclass nv108_pwr_oclass;
+extern struct nouveau_oclass *nva3_pwr_oclass;
+extern struct nouveau_oclass *nvc0_pwr_oclass;
+extern struct nouveau_oclass *nvd0_pwr_oclass;
+extern struct nouveau_oclass *gk104_pwr_oclass;
+extern struct nouveau_oclass *nv108_pwr_oclass;
 
 /* interface to MEMX process running on PPWR */
 struct nouveau_memx;
diff --git a/drivers/gpu/drm/nouveau/core/os.h b/drivers/gpu/drm/nouveau/core/os.h
index d0ced94ca54c..ccfa21d72ddc 100644
--- a/drivers/gpu/drm/nouveau/core/os.h
+++ b/drivers/gpu/drm/nouveau/core/os.h
@@ -21,6 +21,8 @@
 #include <linux/interrupt.h>
 #include <linux/log2.h>
 #include <linux/pm_runtime.h>
+#include <linux/power_supply.h>
+#include <linux/clk.h>
 
 #include <asm/unaligned.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bar/gk20a.c b/drivers/gpu/drm/nouveau/core/subdev/bar/gk20a.c
new file mode 100644
index 000000000000..bf877af9d3bd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bar/gk20a.c
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <subdev/bar.h>
+
+#include "priv.h"
+
+int
+gk20a_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+               struct nouveau_oclass *oclass, void *data, u32 size,
+               struct nouveau_object **pobject)
+{
+        struct nouveau_bar *bar;
+        int ret;
+
+        ret = nvc0_bar_ctor(parent, engine, oclass, data, size, pobject);
+        if (ret)
+                return ret;
+
+        bar = (struct nouveau_bar *)*pobject;
+        bar->iomap_uncached = true;
+
+        return 0;
+}
+
+struct nouveau_oclass
+gk20a_bar_oclass = {
+        .handle = NV_SUBDEV(BAR, 0xea),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gk20a_bar_ctor,
+                .dtor = nvc0_bar_dtor,
+                .init = nvc0_bar_init,
+                .fini = _nouveau_bar_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bar/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/bar/nvc0.c
index ca8139b9ab27..0a44459844e3 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bar/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bar/nvc0.c
@@ -133,7 +133,7 @@ nvc0_bar_init_vm(struct nvc0_bar_priv *priv, struct nvc0_bar_priv_vm *bar_vm,
         return 0;
 }
 
-static int
+int
 nvc0_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
               struct nouveau_oclass *oclass, void *data, u32 size,
               struct nouveau_object **pobject)
@@ -169,7 +169,7 @@ nvc0_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-static void
+void
 nvc0_bar_dtor(struct nouveau_object *object)
 {
         struct nvc0_bar_priv *priv = (void *)object;
@@ -188,7 +188,7 @@ nvc0_bar_dtor(struct nouveau_object *object)
         nouveau_bar_destroy(&priv->base);
 }
 
-static int
+int
 nvc0_bar_init(struct nouveau_object *object)
 {
         struct nvc0_bar_priv *priv = (void *)object;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bar/priv.h b/drivers/gpu/drm/nouveau/core/subdev/bar/priv.h
index ffad8f337ead..3ee8b1476d00 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bar/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/bar/priv.h
@@ -23,4 +23,10 @@ int  nouveau_bar_alloc(struct nouveau_bar *, struct nouveau_object *,
 
 void nv84_bar_flush(struct nouveau_bar *);
 
+int nvc0_bar_ctor(struct nouveau_object *, struct nouveau_object *,
+                  struct nouveau_oclass *, void *, u32,
+                  struct nouveau_object **);
+void nvc0_bar_dtor(struct nouveau_object *);
+int nvc0_bar_init(struct nouveau_object *);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/base.c b/drivers/gpu/drm/nouveau/core/subdev/clock/base.c
index 22351f594d2a..a276a711294a 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/base.c
@@ -90,16 +90,20 @@ nouveau_cstate_prog(struct nouveau_clock *clk,
                 cstate = &pstate->base;
         }
 
-        ret = nouveau_therm_cstate(ptherm, pstate->fanspeed, +1);
-        if (ret && ret != -ENODEV) {
-                nv_error(clk, "failed to raise fan speed: %d\n", ret);
-                return ret;
+        if (ptherm) {
+                ret = nouveau_therm_cstate(ptherm, pstate->fanspeed, +1);
+                if (ret && ret != -ENODEV) {
+                        nv_error(clk, "failed to raise fan speed: %d\n", ret);
+                        return ret;
+                }
         }
 
-        ret = volt->set_id(volt, cstate->voltage, +1);
-        if (ret && ret != -ENODEV) {
-                nv_error(clk, "failed to raise voltage: %d\n", ret);
-                return ret;
+        if (volt) {
+                ret = volt->set_id(volt, cstate->voltage, +1);
+                if (ret && ret != -ENODEV) {
+                        nv_error(clk, "failed to raise voltage: %d\n", ret);
+                        return ret;
+                }
         }
 
         ret = clk->calc(clk, cstate);
@@ -108,13 +112,17 @@ nouveau_cstate_prog(struct nouveau_clock *clk,
                 clk->tidy(clk);
         }
 
-        ret = volt->set_id(volt, cstate->voltage, -1);
-        if (ret && ret != -ENODEV)
-                nv_error(clk, "failed to lower voltage: %d\n", ret);
+        if (volt) {
+                ret = volt->set_id(volt, cstate->voltage, -1);
+                if (ret && ret != -ENODEV)
+                        nv_error(clk, "failed to lower voltage: %d\n", ret);
+        }
 
-        ret = nouveau_therm_cstate(ptherm, pstate->fanspeed, -1);
-        if (ret && ret != -ENODEV)
-                nv_error(clk, "failed to lower fan speed: %d\n", ret);
+        if (ptherm) {
+                ret = nouveau_therm_cstate(ptherm, pstate->fanspeed, -1);
+                if (ret && ret != -ENODEV)
+                        nv_error(clk, "failed to lower fan speed: %d\n", ret);
+        }
 
         return 0;
 }
@@ -194,16 +202,23 @@ nouveau_pstate_prog(struct nouveau_clock *clk, int pstatei)
         return nouveau_cstate_prog(clk, pstate, 0);
 }
 
-static int
-nouveau_pstate_calc(struct nouveau_clock *clk)
+static void
+nouveau_pstate_work(struct work_struct *work)
 {
-        int pstate, ret = 0;
+        struct nouveau_clock *clk = container_of(work, typeof(*clk), work);
+        int pstate;
 
-        nv_trace(clk, "P %d U %d A %d T %d D %d\n", clk->pstate,
-                 clk->ustate, clk->astate, clk->tstate, clk->dstate);
+        if (!atomic_xchg(&clk->waiting, 0))
+                return;
+        clk->pwrsrc = power_supply_is_system_supplied();
 
-        if (clk->state_nr && clk->ustate != -1) {
-                pstate = (clk->ustate < 0) ? clk->astate : clk->ustate;
+        nv_trace(clk, "P %d PWR %d U(AC) %d U(DC) %d A %d T %d D %d\n",
+                 clk->pstate, clk->pwrsrc, clk->ustate_ac, clk->ustate_dc,
+                 clk->astate, clk->tstate, clk->dstate);
+
+        pstate = clk->pwrsrc ? clk->ustate_ac : clk->ustate_dc;
+        if (clk->state_nr && pstate != -1) {
+                pstate = (pstate < 0) ? clk->astate : pstate;
                 pstate = min(pstate, clk->state_nr - 1 - clk->tstate);
                 pstate = max(pstate, clk->dstate);
         } else {
@@ -211,9 +226,26 @@ nouveau_pstate_calc(struct nouveau_clock *clk)
         }
 
         nv_trace(clk, "-> %d\n", pstate);
-        if (pstate != clk->pstate)
-                ret = nouveau_pstate_prog(clk, pstate);
-        return ret;
+        if (pstate != clk->pstate) {
+                int ret = nouveau_pstate_prog(clk, pstate);
+                if (ret) {
+                        nv_error(clk, "error setting pstate %d: %d\n",
+                                 pstate, ret);
+                }
+        }
+
+        wake_up_all(&clk->wait);
+        nvkm_notify_get(&clk->pwrsrc_ntfy);
+}
+
+static int
+nouveau_pstate_calc(struct nouveau_clock *clk, bool wait)
+{
+        atomic_set(&clk->waiting, 1);
+        schedule_work(&clk->work);
+        if (wait)
+                wait_event(clk->wait, !atomic_read(&clk->waiting));
+        return 0;
 }
 
 static void
@@ -361,17 +393,40 @@ nouveau_clock_ustate_update(struct nouveau_clock *clk, int req)
                 req = i;
         }
 
-        clk->ustate = req;
-        return 0;
+        return req + 2;
+}
+
+static int
+nouveau_clock_nstate(struct nouveau_clock *clk, const char *mode, int arglen)
+{
+        int ret = 1;
+
+        if (strncasecmpz(mode, "disabled", arglen)) {
+                char save = mode[arglen];
+                long v;
+
+                ((char *)mode)[arglen] = '\0';
+                if (!kstrtol(mode, 0, &v)) {
+                        ret = nouveau_clock_ustate_update(clk, v);
+                        if (ret < 0)
+                                ret = 1;
+                }
+                ((char *)mode)[arglen] = save;
+        }
+
+        return ret - 2;
 }
 
 int
-nouveau_clock_ustate(struct nouveau_clock *clk, int req)
+nouveau_clock_ustate(struct nouveau_clock *clk, int req, int pwr)
 {
         int ret = nouveau_clock_ustate_update(clk, req);
-        if (ret)
-                return ret;
-        return nouveau_pstate_calc(clk);
+        if (ret >= 0) {
+                if (ret -= 2, pwr) clk->ustate_ac = ret;
+                else               clk->ustate_dc = ret;
+                return nouveau_pstate_calc(clk, true);
+        }
+        return ret;
 }
 
 int
@@ -381,7 +436,7 @@ nouveau_clock_astate(struct nouveau_clock *clk, int req, int rel)
         if ( rel) clk->astate += rel;
         clk->astate = min(clk->astate, clk->state_nr - 1);
         clk->astate = max(clk->astate, 0);
-        return nouveau_pstate_calc(clk);
+        return nouveau_pstate_calc(clk, true);
 }
 
 int
@@ -391,7 +446,7 @@ nouveau_clock_tstate(struct nouveau_clock *clk, int req, int rel)
         if ( rel) clk->tstate += rel;
         clk->tstate = min(clk->tstate, 0);
         clk->tstate = max(clk->tstate, -(clk->state_nr - 1));
-        return nouveau_pstate_calc(clk);
+        return nouveau_pstate_calc(clk, true);
 }
 
 int
@@ -401,12 +456,30 @@ nouveau_clock_dstate(struct nouveau_clock *clk, int req, int rel)
         if ( rel) clk->dstate += rel;
         clk->dstate = min(clk->dstate, clk->state_nr - 1);
         clk->dstate = max(clk->dstate, 0);
-        return nouveau_pstate_calc(clk);
+        return nouveau_pstate_calc(clk, true);
+}
+
+static int
+nouveau_clock_pwrsrc(struct nvkm_notify *notify)
+{
+        struct nouveau_clock *clk =
+                container_of(notify, typeof(*clk), pwrsrc_ntfy);
+        nouveau_pstate_calc(clk, false);
+        return NVKM_NOTIFY_DROP;
 }
 
 /******************************************************************************
  * subdev base class implementation
  *****************************************************************************/
+
+int
+_nouveau_clock_fini(struct nouveau_object *object, bool suspend)
+{
+        struct nouveau_clock *clk = (void *)object;
+        nvkm_notify_put(&clk->pwrsrc_ntfy);
+        return nouveau_subdev_fini(&clk->base, suspend);
+}
+
 int
 _nouveau_clock_init(struct nouveau_object *object)
 {
@@ -414,6 +487,10 @@ _nouveau_clock_init(struct nouveau_object *object)
         struct nouveau_clocks *clock = clk->domains;
         int ret;
 
+        ret = nouveau_subdev_init(&clk->base);
+        if (ret)
+                return ret;
+
         memset(&clk->bstate, 0x00, sizeof(clk->bstate));
         INIT_LIST_HEAD(&clk->bstate.list);
         clk->bstate.pstate = 0xff;
@@ -434,7 +511,7 @@ _nouveau_clock_init(struct nouveau_object *object)
         clk->tstate = 0;
         clk->dstate = 0;
         clk->pstate = -1;
-        nouveau_pstate_calc(clk);
+        nouveau_pstate_calc(clk, true);
         return 0;
 }
 
@@ -444,6 +521,8 @@ _nouveau_clock_dtor(struct nouveau_object *object)
         struct nouveau_clock *clk = (void *)object;
         struct nouveau_pstate *pstate, *temp;
 
+        nvkm_notify_fini(&clk->pwrsrc_ntfy);
+
         list_for_each_entry_safe(pstate, temp, &clk->states, head) {
                 nouveau_pstate_del(pstate);
         }
@@ -456,6 +535,7 @@ nouveau_clock_create_(struct nouveau_object *parent,
                       struct nouveau_object *engine,
                       struct nouveau_oclass *oclass,
                       struct nouveau_clocks *clocks,
+                      struct nouveau_pstate *pstates, int nb_pstates,
                       bool allow_reclock,
                       int length, void **object)
 {
@@ -472,29 +552,46 @@ nouveau_clock_create_(struct nouveau_object *parent,
 
         INIT_LIST_HEAD(&clk->states);
         clk->domains = clocks;
-        clk->ustate = -1;
+        clk->ustate_ac = -1;
+        clk->ustate_dc = -1;
+
+        INIT_WORK(&clk->work, nouveau_pstate_work);
+        init_waitqueue_head(&clk->wait);
+        atomic_set(&clk->waiting, 0);
 
-        idx = 0;
-        do {
-                ret = nouveau_pstate_new(clk, idx++);
-        } while (ret == 0);
+        /* If no pstates are provided, try and fetch them from the BIOS */
+        if (!pstates) {
+                idx = 0;
+                do {
+                        ret = nouveau_pstate_new(clk, idx++);
+                } while (ret == 0);
+        } else {
+                for (idx = 0; idx < nb_pstates; idx++)
+                        list_add_tail(&pstates[idx].head, &clk->states);
+                clk->state_nr = nb_pstates;
+        }
 
         clk->allow_reclock = allow_reclock;
 
+        ret = nvkm_notify_init(&device->event, nouveau_clock_pwrsrc, true,
+                               NULL, 0, 0, &clk->pwrsrc_ntfy);
+        if (ret)
+                return ret;
+
         mode = nouveau_stropt(device->cfgopt, "NvClkMode", &arglen);
         if (mode) {
-                if (!strncasecmpz(mode, "disabled", arglen)) {
-                        clk->ustate = -1;
-                } else {
-                        char save = mode[arglen];
-                        long v;
-
-                        ((char *)mode)[arglen] = '\0';
-                        if (!kstrtol(mode, 0, &v))
-                                nouveau_clock_ustate_update(clk, v);
-                        ((char *)mode)[arglen] = save;
-                }
+                clk->ustate_ac = nouveau_clock_nstate(clk, mode, arglen);
+                clk->ustate_dc = nouveau_clock_nstate(clk, mode, arglen);
         }
 
+        mode = nouveau_stropt(device->cfgopt, "NvClkModeAC", &arglen);
+        if (mode)
+                clk->ustate_ac = nouveau_clock_nstate(clk, mode, arglen);
+
+        mode = nouveau_stropt(device->cfgopt, "NvClkModeDC", &arglen);
+        if (mode)
+                clk->ustate_dc = nouveau_clock_nstate(clk, mode, arglen);
+
+
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/gk20a.c b/drivers/gpu/drm/nouveau/core/subdev/clock/gk20a.c
new file mode 100644
index 000000000000..425a8d5e9129
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/gk20a.c
@@ -0,0 +1,665 @@
+/*
+ * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Shamelessly ripped off from ChromeOS's gk20a/clk_pllg.c
+ *
+ */
+
+#define MHZ (1000 * 1000)
+
+#define MASK(w) ((1 << w) - 1)
+
+#define SYS_GPCPLL_CFG_BASE                     0x00137000
+#define GPC_BCASE_GPCPLL_CFG_BASE               0x00132800
+
+#define GPCPLL_CFG              (SYS_GPCPLL_CFG_BASE + 0)
+#define GPCPLL_CFG_ENABLE       BIT(0)
+#define GPCPLL_CFG_IDDQ         BIT(1)
+#define GPCPLL_CFG_LOCK_DET_OFF BIT(4)
+#define GPCPLL_CFG_LOCK         BIT(17)
+
+#define GPCPLL_COEFF            (SYS_GPCPLL_CFG_BASE + 4)
+#define GPCPLL_COEFF_M_SHIFT    0
+#define GPCPLL_COEFF_M_WIDTH    8
+#define GPCPLL_COEFF_N_SHIFT    8
+#define GPCPLL_COEFF_N_WIDTH    8
+#define GPCPLL_COEFF_P_SHIFT    16
+#define GPCPLL_COEFF_P_WIDTH    6
+
+#define GPCPLL_CFG2                     (SYS_GPCPLL_CFG_BASE + 0xc)
+#define GPCPLL_CFG2_SETUP2_SHIFT        16
+#define GPCPLL_CFG2_PLL_STEPA_SHIFT     24
+
+#define GPCPLL_CFG3                     (SYS_GPCPLL_CFG_BASE + 0x18)
+#define GPCPLL_CFG3_PLL_STEPB_SHIFT     16
+
+#define GPCPLL_NDIV_SLOWDOWN                    (SYS_GPCPLL_CFG_BASE + 0x1c)
+#define GPCPLL_NDIV_SLOWDOWN_NDIV_LO_SHIFT      0
+#define GPCPLL_NDIV_SLOWDOWN_NDIV_MID_SHIFT     8
+#define GPCPLL_NDIV_SLOWDOWN_STEP_SIZE_LO2MID_SHIFT     16
+#define GPCPLL_NDIV_SLOWDOWN_SLOWDOWN_USING_PLL_SHIFT   22
+#define GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT   31
+
+#define SEL_VCO                         (SYS_GPCPLL_CFG_BASE + 0x100)
+#define SEL_VCO_GPC2CLK_OUT_SHIFT       0
+
+#define GPC2CLK_OUT                     (SYS_GPCPLL_CFG_BASE + 0x250)
+#define GPC2CLK_OUT_SDIV14_INDIV4_WIDTH 1
+#define GPC2CLK_OUT_SDIV14_INDIV4_SHIFT 31
+#define GPC2CLK_OUT_SDIV14_INDIV4_MODE  1
+#define GPC2CLK_OUT_VCODIV_WIDTH        6
+#define GPC2CLK_OUT_VCODIV_SHIFT        8
+#define GPC2CLK_OUT_VCODIV1             0
+#define GPC2CLK_OUT_VCODIV_MASK         (MASK(GPC2CLK_OUT_VCODIV_WIDTH) << \
+                                        GPC2CLK_OUT_VCODIV_SHIFT)
+#define GPC2CLK_OUT_BYPDIV_WIDTH        6
+#define GPC2CLK_OUT_BYPDIV_SHIFT        0
+#define GPC2CLK_OUT_BYPDIV31            0x3c
+#define GPC2CLK_OUT_INIT_MASK   ((MASK(GPC2CLK_OUT_SDIV14_INDIV4_WIDTH) << \
+                GPC2CLK_OUT_SDIV14_INDIV4_SHIFT)\
+                | (MASK(GPC2CLK_OUT_VCODIV_WIDTH) << GPC2CLK_OUT_VCODIV_SHIFT)\
+                | (MASK(GPC2CLK_OUT_BYPDIV_WIDTH) << GPC2CLK_OUT_BYPDIV_SHIFT))
+#define GPC2CLK_OUT_INIT_VAL    ((GPC2CLK_OUT_SDIV14_INDIV4_MODE << \
+                GPC2CLK_OUT_SDIV14_INDIV4_SHIFT) \
+                | (GPC2CLK_OUT_VCODIV1 << GPC2CLK_OUT_VCODIV_SHIFT) \
+                | (GPC2CLK_OUT_BYPDIV31 << GPC2CLK_OUT_BYPDIV_SHIFT))
+
+#define GPC_BCAST_NDIV_SLOWDOWN_DEBUG   (GPC_BCASE_GPCPLL_CFG_BASE + 0xa0)
+#define GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_SHIFT     24
+#define GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_MASK \
+            (0x1 << GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_SHIFT)
+
+#include <subdev/clock.h>
+#include <subdev/timer.h>
+
+#ifdef __KERNEL__
+#include <nouveau_platform.h>
+#endif
+
+static const u8 pl_to_div[] = {
+/* PL:   0, 1, 2, 3, 4, 5, 6,  7,  8,  9, 10, 11, 12, 13, 14 */
+/* p: */ 1, 2, 3, 4, 5, 6, 8, 10, 12, 16, 12, 16, 20, 24, 32,
+};
+
+/* All frequencies in Mhz */
+struct gk20a_clk_pllg_params {
+        u32 min_vco, max_vco;
+        u32 min_u, max_u;
+        u32 min_m, max_m;
+        u32 min_n, max_n;
+        u32 min_pl, max_pl;
+};
+
+static const struct gk20a_clk_pllg_params gk20a_pllg_params = {
+        .min_vco = 1000, .max_vco = 1700,
+        .min_u = 12, .max_u = 38,
+        .min_m = 1, .max_m = 255,
+        .min_n = 8, .max_n = 255,
+        .min_pl = 1, .max_pl = 32,
+};
+
+struct gk20a_clock_priv {
+        struct nouveau_clock base;
+        const struct gk20a_clk_pllg_params *params;
+        u32 m, n, pl;
+        u32 parent_rate;
+};
+#define to_gk20a_clock(base) container_of(base, struct gk20a_clock_priv, base)
+
+static void
+gk20a_pllg_read_mnp(struct gk20a_clock_priv *priv)
+{
+        u32 val;
+
+        val = nv_rd32(priv, GPCPLL_COEFF);
+        priv->m = (val >> GPCPLL_COEFF_M_SHIFT) & MASK(GPCPLL_COEFF_M_WIDTH);
+        priv->n = (val >> GPCPLL_COEFF_N_SHIFT) & MASK(GPCPLL_COEFF_N_WIDTH);
+        priv->pl = (val >> GPCPLL_COEFF_P_SHIFT) & MASK(GPCPLL_COEFF_P_WIDTH);
+}
+
+static u32
+gk20a_pllg_calc_rate(struct gk20a_clock_priv *priv)
+{
+        u32 rate;
+        u32 divider;
+
+        rate = priv->parent_rate * priv->n;
+        divider = priv->m * pl_to_div[priv->pl];
+        do_div(rate, divider);
+
+        return rate / 2;
+}
+
+static int
+gk20a_pllg_calc_mnp(struct gk20a_clock_priv *priv, unsigned long rate)
+{
+        u32 target_clk_f, ref_clk_f, target_freq;
+        u32 min_vco_f, max_vco_f;
+        u32 low_pl, high_pl, best_pl;
+        u32 target_vco_f, vco_f;
+        u32 best_m, best_n;
+        u32 u_f;
+        u32 m, n, n2;
+        u32 delta, lwv, best_delta = ~0;
+        u32 pl;
+
+        target_clk_f = rate * 2 / MHZ;
+        ref_clk_f = priv->parent_rate / MHZ;
+
+        max_vco_f = priv->params->max_vco;
+        min_vco_f = priv->params->min_vco;
+        best_m = priv->params->max_m;
+        best_n = priv->params->min_n;
+        best_pl = priv->params->min_pl;
+
+        target_vco_f = target_clk_f + target_clk_f / 50;
+        if (max_vco_f < target_vco_f)
+                max_vco_f = target_vco_f;
+
+        /* min_pl <= high_pl <= max_pl */
+        high_pl = (max_vco_f + target_vco_f - 1) / target_vco_f;
+        high_pl = min(high_pl, priv->params->max_pl);
+        high_pl = max(high_pl, priv->params->min_pl);
+
+        /* min_pl <= low_pl <= max_pl */
+        low_pl = min_vco_f / target_vco_f;
+        low_pl = min(low_pl, priv->params->max_pl);
+        low_pl = max(low_pl, priv->params->min_pl);
+
+        /* Find Indices of high_pl and low_pl */
+        for (pl = 0; pl < ARRAY_SIZE(pl_to_div) - 1; pl++) {
+                if (pl_to_div[pl] >= low_pl) {
+                        low_pl = pl;
+                        break;
+                }
+        }
+        for (pl = 0; pl < ARRAY_SIZE(pl_to_div) - 1; pl++) {
+                if (pl_to_div[pl] >= high_pl) {
+                        high_pl = pl;
+                        break;
+                }
+        }
+
+        nv_debug(priv, "low_PL %d(div%d), high_PL %d(div%d)", low_pl,
+                 pl_to_div[low_pl], high_pl, pl_to_div[high_pl]);
+
+        /* Select lowest possible VCO */
+        for (pl = low_pl; pl <= high_pl; pl++) {
+                target_vco_f = target_clk_f * pl_to_div[pl];
+                for (m = priv->params->min_m; m <= priv->params->max_m; m++) {
+                        u_f = ref_clk_f / m;
+
+                        if (u_f < priv->params->min_u)
+                                break;
+                        if (u_f > priv->params->max_u)
+                                continue;
+
+                        n = (target_vco_f * m) / ref_clk_f;
+                        n2 = ((target_vco_f * m) + (ref_clk_f - 1)) / ref_clk_f;
+
+                        if (n > priv->params->max_n)
+                                break;
+
+                        for (; n <= n2; n++) {
+                                if (n < priv->params->min_n)
+                                        continue;
+                                if (n > priv->params->max_n)
+                                        break;
+
+                                vco_f = ref_clk_f * n / m;
+
+                                if (vco_f >= min_vco_f && vco_f <= max_vco_f) {
+                                        lwv = (vco_f + (pl_to_div[pl] / 2))
+                                                / pl_to_div[pl];
+                                        delta = abs(lwv - target_clk_f);
+
+                                        if (delta < best_delta) {
+                                                best_delta = delta;
+                                                best_m = m;
+                                                best_n = n;
+                                                best_pl = pl;
+
+                                                if (best_delta == 0)
+                                                        goto found_match;
+                                        }
+                                }
+                        }
+                }
+        }
+
+found_match:
+        WARN_ON(best_delta == ~0);
+
+        if (best_delta != 0)
+                nv_debug(priv, "no best match for target @ %dMHz on gpc_pll",
+                         target_clk_f);
+
+        priv->m = best_m;
+        priv->n = best_n;
+        priv->pl = best_pl;
+
+        target_freq = gk20a_pllg_calc_rate(priv) / MHZ;
+
+        nv_debug(priv, "actual target freq %d MHz, M %d, N %d, PL %d(div%d)\n",
+                 target_freq, priv->m, priv->n, priv->pl, pl_to_div[priv->pl]);
+
+        return 0;
+}
+
+static int
+gk20a_pllg_slide(struct gk20a_clock_priv *priv, u32 n)
+{
+        u32 val;
+        int ramp_timeout;
+
+        /* get old coefficients */
+        val = nv_rd32(priv, GPCPLL_COEFF);
+        /* do nothing if NDIV is the same */
+        if (n == ((val >> GPCPLL_COEFF_N_SHIFT) & MASK(GPCPLL_COEFF_N_WIDTH)))
+                return 0;
+
+        /* setup */
+        nv_mask(priv, GPCPLL_CFG2, 0xff << GPCPLL_CFG2_PLL_STEPA_SHIFT,
+                0x2b << GPCPLL_CFG2_PLL_STEPA_SHIFT);
+        nv_mask(priv, GPCPLL_CFG3, 0xff << GPCPLL_CFG3_PLL_STEPB_SHIFT,
+                0xb << GPCPLL_CFG3_PLL_STEPB_SHIFT);
+
+        /* pll slowdown mode */
+        nv_mask(priv, GPCPLL_NDIV_SLOWDOWN,
+                BIT(GPCPLL_NDIV_SLOWDOWN_SLOWDOWN_USING_PLL_SHIFT),
+                BIT(GPCPLL_NDIV_SLOWDOWN_SLOWDOWN_USING_PLL_SHIFT));
+
+        /* new ndiv ready for ramp */
+        val = nv_rd32(priv, GPCPLL_COEFF);
+        val &= ~(MASK(GPCPLL_COEFF_N_WIDTH) << GPCPLL_COEFF_N_SHIFT);
+        val |= (n & MASK(GPCPLL_COEFF_N_WIDTH)) << GPCPLL_COEFF_N_SHIFT;
+        udelay(1);
+        nv_wr32(priv, GPCPLL_COEFF, val);
+
+        /* dynamic ramp to new ndiv */
+        val = nv_rd32(priv, GPCPLL_NDIV_SLOWDOWN);
+        val |= 0x1 << GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT;
+        udelay(1);
+        nv_wr32(priv, GPCPLL_NDIV_SLOWDOWN, val);
+
+        for (ramp_timeout = 500; ramp_timeout > 0; ramp_timeout--) {
+                udelay(1);
+                val = nv_rd32(priv, GPC_BCAST_NDIV_SLOWDOWN_DEBUG);
+                if (val & GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_MASK)
+                        break;
+        }
+
+        /* exit slowdown mode */
+        nv_mask(priv, GPCPLL_NDIV_SLOWDOWN,
+                BIT(GPCPLL_NDIV_SLOWDOWN_SLOWDOWN_USING_PLL_SHIFT) |
+                BIT(GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT), 0);
+        nv_rd32(priv, GPCPLL_NDIV_SLOWDOWN);
+
+        if (ramp_timeout <= 0) {
+                nv_error(priv, "gpcpll dynamic ramp timeout\n");
+                return -ETIMEDOUT;
+        }
+
+        return 0;
+}
+
+static void
+_gk20a_pllg_enable(struct gk20a_clock_priv *priv)
+{
+        nv_mask(priv, GPCPLL_CFG, GPCPLL_CFG_ENABLE, GPCPLL_CFG_ENABLE);
+        nv_rd32(priv, GPCPLL_CFG);
+}
+
+static void
+_gk20a_pllg_disable(struct gk20a_clock_priv *priv)
+{
+        nv_mask(priv, GPCPLL_CFG, GPCPLL_CFG_ENABLE, 0);
+        nv_rd32(priv, GPCPLL_CFG);
+}
+
+static int
+_gk20a_pllg_program_mnp(struct gk20a_clock_priv *priv, bool allow_slide)
+{
+        u32 val, cfg;
+        u32 m_old, pl_old, n_lo;
+
+        /* get old coefficients */
+        val = nv_rd32(priv, GPCPLL_COEFF);
+        m_old = (val >> GPCPLL_COEFF_M_SHIFT) & MASK(GPCPLL_COEFF_M_WIDTH);
+        pl_old = (val >> GPCPLL_COEFF_P_SHIFT) & MASK(GPCPLL_COEFF_P_WIDTH);
+
+        /* do NDIV slide if there is no change in M and PL */
+        cfg = nv_rd32(priv, GPCPLL_CFG);
+        if (allow_slide && priv->m == m_old && priv->pl == pl_old &&
+            (cfg & GPCPLL_CFG_ENABLE)) {
+                return gk20a_pllg_slide(priv, priv->n);
+        }
+
+        /* slide down to NDIV_LO */
+        n_lo = DIV_ROUND_UP(m_old * priv->params->min_vco,
+                            priv->parent_rate / MHZ);
+        if (allow_slide && (cfg & GPCPLL_CFG_ENABLE)) {
+                int ret = gk20a_pllg_slide(priv, n_lo);
+
+                if (ret)
+                        return ret;
+        }
+
+        /* split FO-to-bypass jump in halfs by setting out divider 1:2 */
+        nv_mask(priv, GPC2CLK_OUT, GPC2CLK_OUT_VCODIV_MASK,
+                0x2 << GPC2CLK_OUT_VCODIV_SHIFT);
+
+        /* put PLL in bypass before programming it */
+        val = nv_rd32(priv, SEL_VCO);
+        val &= ~(BIT(SEL_VCO_GPC2CLK_OUT_SHIFT));
+        udelay(2);
+        nv_wr32(priv, SEL_VCO, val);
+
+        /* get out from IDDQ */
+        val = nv_rd32(priv, GPCPLL_CFG);
+        if (val & GPCPLL_CFG_IDDQ) {
+                val &= ~GPCPLL_CFG_IDDQ;
+                nv_wr32(priv, GPCPLL_CFG, val);
+                nv_rd32(priv, GPCPLL_CFG);
+                udelay(2);
+        }
+
+        _gk20a_pllg_disable(priv);
+
+        nv_debug(priv, "%s: m=%d n=%d pl=%d\n", __func__, priv->m, priv->n,
+                 priv->pl);
+
+        n_lo = DIV_ROUND_UP(priv->m * priv->params->min_vco,
+                            priv->parent_rate / MHZ);
+        val = priv->m << GPCPLL_COEFF_M_SHIFT;
+        val |= (allow_slide ? n_lo : priv->n) << GPCPLL_COEFF_N_SHIFT;
+        val |= priv->pl << GPCPLL_COEFF_P_SHIFT;
+        nv_wr32(priv, GPCPLL_COEFF, val);
+
+        _gk20a_pllg_enable(priv);
+
+        val = nv_rd32(priv, GPCPLL_CFG);
+        if (val & GPCPLL_CFG_LOCK_DET_OFF) {
+                val &= ~GPCPLL_CFG_LOCK_DET_OFF;
+                nv_wr32(priv, GPCPLL_CFG, val);
+        }
+
+        if (!nouveau_timer_wait_eq(priv, 300000, GPCPLL_CFG, GPCPLL_CFG_LOCK,
+                                   GPCPLL_CFG_LOCK)) {
+                nv_error(priv, "%s: timeout waiting for pllg lock\n", __func__);
+                return -ETIMEDOUT;
+        }
+
+        /* switch to VCO mode */
+        nv_mask(priv, SEL_VCO, 0, BIT(SEL_VCO_GPC2CLK_OUT_SHIFT));
+
+        /* restore out divider 1:1 */
+        val = nv_rd32(priv, GPC2CLK_OUT);
+        val &= ~GPC2CLK_OUT_VCODIV_MASK;
+        udelay(2);
+        nv_wr32(priv, GPC2CLK_OUT, val);
+
+        /* slide up to new NDIV */
+        return allow_slide ? gk20a_pllg_slide(priv, priv->n) : 0;
+}
+
+static int
+gk20a_pllg_program_mnp(struct gk20a_clock_priv *priv)
+{
+        int err;
+
+        err = _gk20a_pllg_program_mnp(priv, true);
+        if (err)
+                err = _gk20a_pllg_program_mnp(priv, false);
+
+        return err;
+}
+
+static void
+gk20a_pllg_disable(struct gk20a_clock_priv *priv)
+{
+        u32 val;
+
+        /* slide to VCO min */
+        val = nv_rd32(priv, GPCPLL_CFG);
+        if (val & GPCPLL_CFG_ENABLE) {
+                u32 coeff, m, n_lo;
+
+                coeff = nv_rd32(priv, GPCPLL_COEFF);
+                m = (coeff >> GPCPLL_COEFF_M_SHIFT) & MASK(GPCPLL_COEFF_M_WIDTH);
+                n_lo = DIV_ROUND_UP(m * priv->params->min_vco,
+                                    priv->parent_rate / MHZ);
+                gk20a_pllg_slide(priv, n_lo);
+        }
+
+        /* put PLL in bypass before disabling it */
+        nv_mask(priv, SEL_VCO, BIT(SEL_VCO_GPC2CLK_OUT_SHIFT), 0);
+
+        _gk20a_pllg_disable(priv);
+}
+
+#define GK20A_CLK_GPC_MDIV 1000
+
+static struct nouveau_clocks
+gk20a_domains[] = {
+        { nv_clk_src_crystal, 0xff },
+        { nv_clk_src_gpc, 0xff, 0, "core", GK20A_CLK_GPC_MDIV },
+        { nv_clk_src_max }
+};
+
+static struct nouveau_pstate
+gk20a_pstates[] = {
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 72000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 108000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 180000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 252000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 324000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 396000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 468000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 540000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 612000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 648000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 684000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 708000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 756000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 804000,
+                },
+        },
+        {
+                .base = {
+                        .domain[nv_clk_src_gpc] = 852000,
+                },
+        },
+};
+
+static int
+gk20a_clock_read(struct nouveau_clock *clk, enum nv_clk_src src)
+{
+        struct gk20a_clock_priv *priv = (void *)clk;
+
+        switch (src) {
+        case nv_clk_src_crystal:
+                return nv_device(clk)->crystal;
+        case nv_clk_src_gpc:
+                gk20a_pllg_read_mnp(priv);
+                return gk20a_pllg_calc_rate(priv) / GK20A_CLK_GPC_MDIV;
+        default:
+                nv_error(clk, "invalid clock source %d\n", src);
+                return -EINVAL;
+        }
+}
+
+static int
+gk20a_clock_calc(struct nouveau_clock *clk, struct nouveau_cstate *cstate)
+{
+        struct gk20a_clock_priv *priv = (void *)clk;
+
+        return gk20a_pllg_calc_mnp(priv, cstate->domain[nv_clk_src_gpc] *
+                                         GK20A_CLK_GPC_MDIV);
+}
+
+static int
+gk20a_clock_prog(struct nouveau_clock *clk)
+{
+        struct gk20a_clock_priv *priv = (void *)clk;
+
+        return gk20a_pllg_program_mnp(priv);
+}
+
+static void
+gk20a_clock_tidy(struct nouveau_clock *clk)
+{
+}
+
+static int
+gk20a_clock_fini(struct nouveau_object *object, bool suspend)
+{
+        struct gk20a_clock_priv *priv = (void *)object;
+        int ret;
+
+        ret = nouveau_clock_fini(&priv->base, false);
+
+        gk20a_pllg_disable(priv);
+
+        return ret;
+}
+
+static int
+gk20a_clock_init(struct nouveau_object *object)
+{
+        struct gk20a_clock_priv *priv = (void *)object;
+        int ret;
+
+        nv_mask(priv, GPC2CLK_OUT, GPC2CLK_OUT_INIT_MASK, GPC2CLK_OUT_INIT_VAL);
+
+        ret = nouveau_clock_init(&priv->base);
+        if (ret)
+                return ret;
+
+        ret = gk20a_clock_prog(&priv->base);
+        if (ret) {
+                nv_error(priv, "cannot initialize clock\n");
+                return ret;
+        }
+
+        return 0;
+}
+
+static int
+gk20a_clock_ctor(struct nouveau_object *parent,  struct nouveau_object *engine,
+                 struct nouveau_oclass *oclass, void *data, u32 size,
+                 struct nouveau_object **pobject)
+{
+        struct gk20a_clock_priv *priv;
+        struct nouveau_platform_device *plat;
+        int ret;
+        int i;
+
+        /* Finish initializing the pstates */
+        for (i = 0; i < ARRAY_SIZE(gk20a_pstates); i++) {
+                INIT_LIST_HEAD(&gk20a_pstates[i].list);
+                gk20a_pstates[i].pstate = i + 1;
+        }
+
+        ret = nouveau_clock_create(parent, engine, oclass, gk20a_domains,
+                        gk20a_pstates, ARRAY_SIZE(gk20a_pstates), true, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        priv->params = &gk20a_pllg_params;
+
+        plat = nv_device_to_platform(nv_device(parent));
+        priv->parent_rate = clk_get_rate(plat->gpu->clk);
+        nv_info(priv, "parent clock rate: %d Mhz\n", priv->parent_rate / MHZ);
+
+        priv->base.read = gk20a_clock_read;
+        priv->base.calc = gk20a_clock_calc;
+        priv->base.prog = gk20a_clock_prog;
+        priv->base.tidy = gk20a_clock_tidy;
+
+        return 0;
+}
+
+struct nouveau_oclass
+gk20a_clock_oclass = {
+        .handle = NV_SUBDEV(CLOCK, 0xea),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gk20a_clock_ctor,
+                .dtor = _nouveau_subdev_dtor,
+                .init = gk20a_clock_init,
+                .fini = gk20a_clock_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nv04.c
index eb2d4425a49e..4c48232686be 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nv04.c
@@ -82,8 +82,8 @@ nv04_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nv04_clock_priv *priv;
         int ret;
 
-        ret = nouveau_clock_create(parent, engine, oclass, nv04_domain, false,
-                                   &priv);
+        ret = nouveau_clock_create(parent, engine, oclass, nv04_domain, NULL, 0,
+                                   false, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nv40.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nv40.c
index 8a9e16839791..08368fe97029 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nv40.c
@@ -213,8 +213,8 @@ nv40_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nv40_clock_priv *priv;
         int ret;
 
-        ret = nouveau_clock_create(parent, engine, oclass, nv40_domain, true,
-                                   &priv);
+        ret = nouveau_clock_create(parent, engine, oclass, nv40_domain, NULL, 0,
+                                   true, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.c
index 8c132772ba9e..5070ebc260f8 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.c
@@ -507,7 +507,7 @@ nv50_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         int ret;
 
         ret = nouveau_clock_create(parent, engine, oclass, pclass->domains,
-                                   false, &priv);
+                                   NULL, 0, false, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
index 9fb58354a80b..087012b18956 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
@@ -302,8 +302,8 @@ nva3_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nva3_clock_priv *priv;
         int ret;
 
-        ret = nouveau_clock_create(parent, engine, oclass, nva3_domain, false,
-                                   &priv);
+        ret = nouveau_clock_create(parent, engine, oclass, nva3_domain, NULL, 0,
+                                   false, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nvaa.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nvaa.c
index 6a65fc9e9663..74e19731b1b7 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nvaa.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nvaa.c
@@ -421,8 +421,8 @@ nvaa_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nvaa_clock_priv *priv;
         int ret;
 
-        ret = nouveau_clock_create(parent, engine, oclass, nvaa_domains, true,
-                                   &priv);
+        ret = nouveau_clock_create(parent, engine, oclass, nvaa_domains, NULL,
+                                   0, true, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nvc0.c
index dbf8517f54da..1234abaab2db 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nvc0.c
@@ -437,8 +437,8 @@ nvc0_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nvc0_clock_priv *priv;
         int ret;
 
-        ret = nouveau_clock_create(parent, engine, oclass, nvc0_domain, false,
-                                   &priv);
+        ret = nouveau_clock_create(parent, engine, oclass, nvc0_domain, NULL, 0,
+                                   false, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nve0.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nve0.c
index 0e62a3240144..7eccad57512e 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nve0.c
@@ -475,8 +475,8 @@ nve0_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nve0_clock_priv *priv;
         int ret;
 
-        ret = nouveau_clock_create(parent, engine, oclass, nve0_domain, true,
-                                   &priv);
+        ret = nouveau_clock_create(parent, engine, oclass, nve0_domain, NULL, 0,
+                                   true, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c
index 1fc55c1e91a1..4150b0d10af8 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c
@@ -250,9 +250,11 @@ nv50_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         priv->r100c08_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
         if (priv->r100c08_page) {
-                priv->r100c08 = nv_device_map_page(device, priv->r100c08_page);
-                if (!priv->r100c08)
-                        nv_warn(priv, "failed 0x100c08 page map\n");
+                priv->r100c08 = dma_map_page(nv_device_base(device),
+                                             priv->r100c08_page, 0, PAGE_SIZE,
+                                             DMA_BIDIRECTIONAL);
+                if (dma_mapping_error(nv_device_base(device), priv->r100c08))
+                        return -EFAULT;
         } else {
                 nv_warn(priv, "failed 0x100c08 page alloc\n");
         }
@@ -268,7 +270,8 @@ nv50_fb_dtor(struct nouveau_object *object)
         struct nv50_fb_priv *priv = (void *)object;
 
         if (priv->r100c08_page) {
-                nv_device_unmap_page(device, priv->r100c08);
+                dma_unmap_page(nv_device_base(device), priv->r100c08, PAGE_SIZE,
+                               DMA_BIDIRECTIONAL);
                 __free_page(priv->r100c08_page);
         }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c
index 0670ae33ee45..b19a2b3c1081 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c
@@ -70,7 +70,8 @@ nvc0_fb_dtor(struct nouveau_object *object)
         struct nvc0_fb_priv *priv = (void *)object;
 
         if (priv->r100c10_page) {
-                nv_device_unmap_page(device, priv->r100c10);
+                dma_unmap_page(nv_device_base(device), priv->r100c10, PAGE_SIZE,
+                               DMA_BIDIRECTIONAL);
                 __free_page(priv->r100c10_page);
         }
 
@@ -93,8 +94,10 @@ nvc0_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         priv->r100c10_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
         if (priv->r100c10_page) {
-                priv->r100c10 = nv_device_map_page(device, priv->r100c10_page);
-                if (!priv->r100c10)
+                priv->r100c10 = dma_map_page(nv_device_base(device),
+                                             priv->r100c10_page, 0, PAGE_SIZE,
+                                             DMA_BIDIRECTIONAL);
+                if (dma_mapping_error(nv_device_base(device), priv->r100c10))
                         return -EFAULT;
         }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c
index 5a6a5027f749..946518572346 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c
@@ -26,7 +26,7 @@
 #include <subdev/bios/pll.h>
 #include <subdev/bios/rammap.h>
 #include <subdev/bios/timing.h>
-#include <subdev/ltcg.h>
+#include <subdev/ltc.h>
 
 #include <subdev/clock.h>
 #include <subdev/clock/pll.h>
@@ -425,7 +425,7 @@ extern const u8 nvc0_pte_storage_type_map[256];
 void
 nvc0_ram_put(struct nouveau_fb *pfb, struct nouveau_mem **pmem)
 {
-        struct nouveau_ltcg *ltcg = nouveau_ltcg(pfb);
+        struct nouveau_ltc *ltc = nouveau_ltc(pfb);
         struct nouveau_mem *mem = *pmem;
 
         *pmem = NULL;
@@ -434,7 +434,7 @@ nvc0_ram_put(struct nouveau_fb *pfb, struct nouveau_mem **pmem)
 
         mutex_lock(&pfb->base.mutex);
         if (mem->tag)
-                ltcg->tags_free(ltcg, &mem->tag);
+                ltc->tags_free(ltc, &mem->tag);
         __nv50_ram_put(pfb, mem);
         mutex_unlock(&pfb->base.mutex);
 
@@ -468,12 +468,12 @@ nvc0_ram_get(struct nouveau_fb *pfb, u64 size, u32 align, u32 ncmin,
 
         mutex_lock(&pfb->base.mutex);
         if (comp) {
-                struct nouveau_ltcg *ltcg = nouveau_ltcg(pfb);
+                struct nouveau_ltc *ltc = nouveau_ltc(pfb);
 
                 /* compression only works with lpages */
                 if (align == (1 << (17 - 12))) {
                         int n = size >> 5;
-                        ltcg->tags_alloc(ltcg, n, &mem->tag);
+                        ltc->tags_alloc(ltc, n, &mem->tag);
                 }
 
                 if (unlikely(!mem->tag))
diff --git a/drivers/gpu/drm/nouveau/core/subdev/gpio/base.c b/drivers/gpu/drm/nouveau/core/subdev/gpio/base.c
index 45e0202f3151..b1e3ed7c8beb 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/gpio/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/gpio/base.c
@@ -106,39 +106,59 @@ nouveau_gpio_get(struct nouveau_gpio *gpio, int idx, u8 tag, u8 line)
 }
 
 static void
-nouveau_gpio_intr_disable(struct nouveau_event *event, int type, int index)
+nouveau_gpio_intr_fini(struct nvkm_event *event, int type, int index)
 {
-        struct nouveau_gpio *gpio = nouveau_gpio(event->priv);
+        struct nouveau_gpio *gpio = container_of(event, typeof(*gpio), event);
         const struct nouveau_gpio_impl *impl = (void *)nv_object(gpio)->oclass;
         impl->intr_mask(gpio, type, 1 << index, 0);
 }
 
 static void
-nouveau_gpio_intr_enable(struct nouveau_event *event, int type, int index)
+nouveau_gpio_intr_init(struct nvkm_event *event, int type, int index)
 {
-        struct nouveau_gpio *gpio = nouveau_gpio(event->priv);
+        struct nouveau_gpio *gpio = container_of(event, typeof(*gpio), event);
         const struct nouveau_gpio_impl *impl = (void *)nv_object(gpio)->oclass;
         impl->intr_mask(gpio, type, 1 << index, 1 << index);
 }
 
+static int
+nouveau_gpio_intr_ctor(void *data, u32 size, struct nvkm_notify *notify)
+{
+        struct nvkm_gpio_ntfy_req *req = data;
+        if (!WARN_ON(size != sizeof(*req))) {
+                notify->size  = sizeof(struct nvkm_gpio_ntfy_rep);
+                notify->types = req->mask;
+                notify->index = req->line;
+                return 0;
+        }
+        return -EINVAL;
+}
+
 static void
 nouveau_gpio_intr(struct nouveau_subdev *subdev)
 {
         struct nouveau_gpio *gpio = nouveau_gpio(subdev);
         const struct nouveau_gpio_impl *impl = (void *)nv_object(gpio)->oclass;
-        u32 hi, lo, e, i;
+        u32 hi, lo, i;
 
         impl->intr_stat(gpio, &hi, &lo);
 
-        for (i = 0; e = 0, (hi | lo) && i < impl->lines; i++) {
-                if (hi & (1 << i))
-                        e |= NVKM_GPIO_HI;
-                if (lo & (1 << i))
-                        e |= NVKM_GPIO_LO;
-                nouveau_event_trigger(gpio->events, e, i);
+        for (i = 0; (hi | lo) && i < impl->lines; i++) {
+                struct nvkm_gpio_ntfy_rep rep = {
+                        .mask = (NVKM_GPIO_HI * !!(hi & (1 << i))) |
+                                (NVKM_GPIO_LO * !!(lo & (1 << i))),
+                };
+                nvkm_event_send(&gpio->event, rep.mask, i, &rep, sizeof(rep));
         }
 }
 
+static const struct nvkm_event_func
+nouveau_gpio_intr_func = {
+        .ctor = nouveau_gpio_intr_ctor,
+        .init = nouveau_gpio_intr_init,
+        .fini = nouveau_gpio_intr_fini,
+};
+
 int
 _nouveau_gpio_fini(struct nouveau_object *object, bool suspend)
 {
@@ -183,7 +203,7 @@ void
 _nouveau_gpio_dtor(struct nouveau_object *object)
 {
         struct nouveau_gpio *gpio = (void *)object;
-        nouveau_event_destroy(&gpio->events);
+        nvkm_event_fini(&gpio->event);
         nouveau_subdev_destroy(&gpio->base);
 }
 
@@ -208,13 +228,11 @@ nouveau_gpio_create_(struct nouveau_object *parent,
         gpio->get  = nouveau_gpio_get;
         gpio->reset = impl->reset;
 
-        ret = nouveau_event_create(2, impl->lines, &gpio->events);
+        ret = nvkm_event_init(&nouveau_gpio_intr_func, 2, impl->lines,
+                              &gpio->event);
         if (ret)
                 return ret;
 
-        gpio->events->priv = gpio;
-        gpio->events->enable = nouveau_gpio_intr_enable;
-        gpio->events->disable = nouveau_gpio_intr_disable;
         nv_subdev(gpio)->intr = nouveau_gpio_intr;
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
index 09ba2cc851cf..a652cafde3d6 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
@@ -326,9 +326,9 @@ nouveau_i2c_identify(struct nouveau_i2c *i2c, int index, const char *what,
 }
 
 static void
-nouveau_i2c_intr_disable(struct nouveau_event *event, int type, int index)
+nouveau_i2c_intr_fini(struct nvkm_event *event, int type, int index)
 {
-        struct nouveau_i2c *i2c = nouveau_i2c(event->priv);
+        struct nouveau_i2c *i2c = container_of(event, typeof(*i2c), event);
         struct nouveau_i2c_port *port = i2c->find(i2c, index);
         const struct nouveau_i2c_impl *impl = (void *)nv_object(i2c)->oclass;
         if (port && port->aux >= 0)
@@ -336,15 +336,28 @@ nouveau_i2c_intr_disable(struct nouveau_event *event, int type, int index)
 }
 
 static void
-nouveau_i2c_intr_enable(struct nouveau_event *event, int type, int index)
+nouveau_i2c_intr_init(struct nvkm_event *event, int type, int index)
 {
-        struct nouveau_i2c *i2c = nouveau_i2c(event->priv);
+        struct nouveau_i2c *i2c = container_of(event, typeof(*i2c), event);
         struct nouveau_i2c_port *port = i2c->find(i2c, index);
         const struct nouveau_i2c_impl *impl = (void *)nv_object(i2c)->oclass;
         if (port && port->aux >= 0)
                 impl->aux_mask(i2c, type, 1 << port->aux, 1 << port->aux);
 }
 
+static int
+nouveau_i2c_intr_ctor(void *data, u32 size, struct nvkm_notify *notify)
+{
+        struct nvkm_i2c_ntfy_req *req = data;
+        if (!WARN_ON(size != sizeof(*req))) {
+                notify->size  = sizeof(struct nvkm_i2c_ntfy_rep);
+                notify->types = req->mask;
+                notify->index = req->port;
+                return 0;
+        }
+        return -EINVAL;
+}
+
 static void
 nouveau_i2c_intr(struct nouveau_subdev *subdev)
 {
@@ -364,13 +377,26 @@ nouveau_i2c_intr(struct nouveau_subdev *subdev)
                                 if (lo & (1 << port->aux)) e |= NVKM_I2C_UNPLUG;
                                 if (rq & (1 << port->aux)) e |= NVKM_I2C_IRQ;
                                 if (tx & (1 << port->aux)) e |= NVKM_I2C_DONE;
-
-                                nouveau_event_trigger(i2c->ntfy, e, port->index);
+                                if (e) {
+                                        struct nvkm_i2c_ntfy_rep rep = {
+                                                .mask = e,
+                                        };
+                                        nvkm_event_send(&i2c->event, rep.mask,
+                                                        port->index, &rep,
+                                                        sizeof(rep));
+                                }
                         }
                 }
         }
 }
 
+static const struct nvkm_event_func
+nouveau_i2c_intr_func = {
+        .ctor = nouveau_i2c_intr_ctor,
+        .init = nouveau_i2c_intr_init,
+        .fini = nouveau_i2c_intr_fini,
+};
+
 int
 _nouveau_i2c_fini(struct nouveau_object *object, bool suspend)
 {
@@ -431,7 +457,7 @@ _nouveau_i2c_dtor(struct nouveau_object *object)
         struct nouveau_i2c *i2c = (void *)object;
         struct nouveau_i2c_port *port, *temp;
 
-        nouveau_event_destroy(&i2c->ntfy);
+        nvkm_event_fini(&i2c->event);
 
         list_for_each_entry_safe(port, temp, &i2c->ports, head) {
                 nouveau_object_ref(NULL, (struct nouveau_object **)&port);
@@ -547,13 +573,10 @@ nouveau_i2c_create_(struct nouveau_object *parent,
                 }
         }
 
-        ret = nouveau_event_create(4, index, &i2c->ntfy);
+        ret = nvkm_event_init(&nouveau_i2c_intr_func, 4, index, &i2c->event);
         if (ret)
                 return ret;
 
-        i2c->ntfy->priv = i2c;
-        i2c->ntfy->enable = nouveau_i2c_intr_enable;
-        i2c->ntfy->disable = nouveau_i2c_intr_disable;
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ltc/base.c b/drivers/gpu/drm/nouveau/core/subdev/ltc/base.c
new file mode 100644
index 000000000000..32ed442c5913
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/ltc/base.c
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include "priv.h"
+
+static int
+nvkm_ltc_tags_alloc(struct nouveau_ltc *ltc, u32 n,
+                    struct nouveau_mm_node **pnode)
+{
+        struct nvkm_ltc_priv *priv = (void *)ltc;
+        int ret;
+
+        ret = nouveau_mm_head(&priv->tags, 1, n, n, 1, pnode);
+        if (ret)
+                *pnode = NULL;
+
+        return ret;
+}
+
+static void
+nvkm_ltc_tags_free(struct nouveau_ltc *ltc, struct nouveau_mm_node **pnode)
+{
+        struct nvkm_ltc_priv *priv = (void *)ltc;
+        nouveau_mm_free(&priv->tags, pnode);
+}
+
+static void
+nvkm_ltc_tags_clear(struct nouveau_ltc *ltc, u32 first, u32 count)
+{
+        const struct nvkm_ltc_impl *impl = (void *)nv_oclass(ltc);
+        struct nvkm_ltc_priv *priv = (void *)ltc;
+        const u32 limit = first + count - 1;
+
+        BUG_ON((first > limit) || (limit >= priv->num_tags));
+
+        impl->cbc_clear(priv, first, limit);
+        impl->cbc_wait(priv);
+}
+
+static int
+nvkm_ltc_zbc_color_get(struct nouveau_ltc *ltc, int index, const u32 color[4])
+{
+        const struct nvkm_ltc_impl *impl = (void *)nv_oclass(ltc);
+        struct nvkm_ltc_priv *priv = (void *)ltc;
+        memcpy(priv->zbc_color[index], color, sizeof(priv->zbc_color[index]));
+        impl->zbc_clear_color(priv, index, color);
+        return index;
+}
+
+static int
+nvkm_ltc_zbc_depth_get(struct nouveau_ltc *ltc, int index, const u32 depth)
+{
+        const struct nvkm_ltc_impl *impl = (void *)nv_oclass(ltc);
+        struct nvkm_ltc_priv *priv = (void *)ltc;
+        priv->zbc_depth[index] = depth;
+        impl->zbc_clear_depth(priv, index, depth);
+        return index;
+}
+
+int
+_nvkm_ltc_init(struct nouveau_object *object)
+{
+        const struct nvkm_ltc_impl *impl = (void *)nv_oclass(object);
+        struct nvkm_ltc_priv *priv = (void *)object;
+        int ret, i;
+
+        ret = nouveau_subdev_init(&priv->base.base);
+        if (ret)
+                return ret;
+
+        for (i = priv->base.zbc_min; i <= priv->base.zbc_max; i++) {
+                impl->zbc_clear_color(priv, i, priv->zbc_color[i]);
+                impl->zbc_clear_depth(priv, i, priv->zbc_depth[i]);
+        }
+
+        return 0;
+}
+
+int
+nvkm_ltc_create_(struct nouveau_object *parent, struct nouveau_object *engine,
+                 struct nouveau_oclass *oclass, int length, void **pobject)
+{
+        const struct nvkm_ltc_impl *impl = (void *)oclass;
+        struct nvkm_ltc_priv *priv;
+        int ret;
+
+        ret = nouveau_subdev_create_(parent, engine, oclass, 0, "PLTCG",
+                                     "l2c", length, pobject);
+        priv = *pobject;
+        if (ret)
+                return ret;
+
+        memset(priv->zbc_color, 0x00, sizeof(priv->zbc_color));
+        memset(priv->zbc_depth, 0x00, sizeof(priv->zbc_depth));
+
+        priv->base.base.intr = impl->intr;
+        priv->base.tags_alloc = nvkm_ltc_tags_alloc;
+        priv->base.tags_free = nvkm_ltc_tags_free;
+        priv->base.tags_clear = nvkm_ltc_tags_clear;
+        priv->base.zbc_min = 1; /* reserve 0 for disabled */
+        priv->base.zbc_max = min(impl->zbc, NOUVEAU_LTC_MAX_ZBC_CNT) - 1;
+        priv->base.zbc_color_get = nvkm_ltc_zbc_color_get;
+        priv->base.zbc_depth_get = nvkm_ltc_zbc_depth_get;
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ltcg/gf100.c b/drivers/gpu/drm/nouveau/core/subdev/ltc/gf100.c
index f2f3338a967a..9e00a1ede120 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/ltcg/gf100.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/ltc/gf100.c
@@ -25,10 +25,45 @@
 #include <subdev/fb.h>
 #include <subdev/timer.h>
 
-#include "gf100.h"
+#include "priv.h"
+
+void
+gf100_ltc_cbc_clear(struct nvkm_ltc_priv *priv, u32 start, u32 limit)
+{
+        nv_wr32(priv, 0x17e8cc, start);
+        nv_wr32(priv, 0x17e8d0, limit);
+        nv_wr32(priv, 0x17e8c8, 0x00000004);
+}
+
+void
+gf100_ltc_cbc_wait(struct nvkm_ltc_priv *priv)
+{
+        int c, s;
+        for (c = 0; c < priv->ltc_nr; c++) {
+                for (s = 0; s < priv->lts_nr; s++)
+                        nv_wait(priv, 0x1410c8 + c * 0x2000 + s * 0x400, ~0, 0);
+        }
+}
+
+void
+gf100_ltc_zbc_clear_color(struct nvkm_ltc_priv *priv, int i, const u32 color[4])
+{
+        nv_mask(priv, 0x17ea44, 0x0000000f, i);
+        nv_wr32(priv, 0x17ea48, color[0]);
+        nv_wr32(priv, 0x17ea4c, color[1]);
+        nv_wr32(priv, 0x17ea50, color[2]);
+        nv_wr32(priv, 0x17ea54, color[3]);
+}
+
+void
+gf100_ltc_zbc_clear_depth(struct nvkm_ltc_priv *priv, int i, const u32 depth)
+{
+        nv_mask(priv, 0x17ea44, 0x0000000f, i);
+        nv_wr32(priv, 0x17ea58, depth);
+}
 
 static void
-gf100_ltcg_lts_isr(struct gf100_ltcg_priv *priv, int ltc, int lts)
+gf100_ltc_lts_isr(struct nvkm_ltc_priv *priv, int ltc, int lts)
 {
         u32 base = 0x141000 + (ltc * 0x2000) + (lts * 0x400);
         u32 stat = nv_rd32(priv, base + 0x020);
@@ -39,17 +74,17 @@ gf100_ltcg_lts_isr(struct gf100_ltcg_priv *priv, int ltc, int lts)
         }
 }
 
-static void
-gf100_ltcg_intr(struct nouveau_subdev *subdev)
+void
+gf100_ltc_intr(struct nouveau_subdev *subdev)
 {
-        struct gf100_ltcg_priv *priv = (void *)subdev;
+        struct nvkm_ltc_priv *priv = (void *)subdev;
         u32 mask;
 
         mask = nv_rd32(priv, 0x00017c);
         while (mask) {
                 u32 lts, ltc = __ffs(mask);
                 for (lts = 0; lts < priv->lts_nr; lts++)
-                        gf100_ltcg_lts_isr(priv, ltc, lts);
+                        gf100_ltc_lts_isr(priv, ltc, lts);
                 mask &= ~(1 << ltc);
         }
 
@@ -59,52 +94,38 @@ gf100_ltcg_intr(struct nouveau_subdev *subdev)
         nv_mask(priv, 0x000640, 0x02000000, 0x00000000);
 }
 
-int
-gf100_ltcg_tags_alloc(struct nouveau_ltcg *ltcg, u32 n,
-                     struct nouveau_mm_node **pnode)
+static int
+gf100_ltc_init(struct nouveau_object *object)
 {
-        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
+        struct nvkm_ltc_priv *priv = (void *)object;
         int ret;
 
-        ret = nouveau_mm_head(&priv->tags, 1, n, n, 1, pnode);
+        ret = nvkm_ltc_init(priv);
         if (ret)
-                *pnode = NULL;
+                return ret;
 
-        return ret;
+        nv_mask(priv, 0x17e820, 0x00100000, 0x00000000); /* INTR_EN &= ~0x10 */
+        nv_wr32(priv, 0x17e8d8, priv->ltc_nr);
+        nv_wr32(priv, 0x17e8d4, priv->tag_base);
+        return 0;
 }
 
 void
-gf100_ltcg_tags_free(struct nouveau_ltcg *ltcg, struct nouveau_mm_node **pnode)
-{
-        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
-
-        nouveau_mm_free(&priv->tags, pnode);
-}
-
-static void
-gf100_ltcg_tags_clear(struct nouveau_ltcg *ltcg, u32 first, u32 count)
+gf100_ltc_dtor(struct nouveau_object *object)
 {
-        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
-        u32 last = first + count - 1;
-        int p, i;
-
-        BUG_ON((first > last) || (last >= priv->num_tags));
+        struct nouveau_fb *pfb = nouveau_fb(object);
+        struct nvkm_ltc_priv *priv = (void *)object;
 
-        nv_wr32(priv, 0x17e8cc, first);
-        nv_wr32(priv, 0x17e8d0, last);
-        nv_wr32(priv, 0x17e8c8, 0x4); /* trigger clear */
+        nouveau_mm_fini(&priv->tags);
+        nouveau_mm_free(&pfb->vram, &priv->tag_ram);
 
-        /* wait until it's finished with clearing */
-        for (p = 0; p < priv->ltc_nr; ++p) {
-                for (i = 0; i < priv->lts_nr; ++i)
-                        nv_wait(priv, 0x1410c8 + p * 0x2000 + i * 0x400, ~0, 0);
-        }
+        nvkm_ltc_destroy(priv);
 }
 
 /* TODO: Figure out tag memory details and drop the over-cautious allocation.
  */
 int
-gf100_ltcg_init_tag_ram(struct nouveau_fb *pfb, struct gf100_ltcg_priv *priv)
+gf100_ltc_init_tag_ram(struct nouveau_fb *pfb, struct nvkm_ltc_priv *priv)
 {
         u32 tag_size, tag_margin, tag_align;
         int ret;
@@ -142,22 +163,22 @@ gf100_ltcg_init_tag_ram(struct nouveau_fb *pfb, struct gf100_ltcg_priv *priv)
 
                 priv->tag_base = tag_base;
         }
-        ret = nouveau_mm_init(&priv->tags, 0, priv->num_tags, 1);
 
+        ret = nouveau_mm_init(&priv->tags, 0, priv->num_tags, 1);
         return ret;
 }
 
-static int
-gf100_ltcg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+int
+gf100_ltc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
                struct nouveau_object **pobject)
 {
-        struct gf100_ltcg_priv *priv;
         struct nouveau_fb *pfb = nouveau_fb(parent);
+        struct nvkm_ltc_priv *priv;
         u32 parts, mask;
         int ret, i;
 
-        ret = nouveau_ltcg_create(parent, engine, oclass, &priv);
+        ret = nvkm_ltc_create(parent, engine, oclass, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
@@ -170,57 +191,27 @@ gf100_ltcg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         }
         priv->lts_nr = nv_rd32(priv, 0x17e8dc) >> 28;
 
-        ret = gf100_ltcg_init_tag_ram(pfb, priv);
+        ret = gf100_ltc_init_tag_ram(pfb, priv);
         if (ret)
                 return ret;
 
-        priv->base.tags_alloc = gf100_ltcg_tags_alloc;
-        priv->base.tags_free  = gf100_ltcg_tags_free;
-        priv->base.tags_clear = gf100_ltcg_tags_clear;
-
-        nv_subdev(priv)->intr = gf100_ltcg_intr;
-        return 0;
-}
-
-void
-gf100_ltcg_dtor(struct nouveau_object *object)
-{
-        struct nouveau_ltcg *ltcg = (struct nouveau_ltcg *)object;
-        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
-        struct nouveau_fb *pfb = nouveau_fb(ltcg->base.base.parent);
-
-        nouveau_mm_fini(&priv->tags);
-        nouveau_mm_free(&pfb->vram, &priv->tag_ram);
-
-        nouveau_ltcg_destroy(ltcg);
-}
-
-static int
-gf100_ltcg_init(struct nouveau_object *object)
-{
-        struct nouveau_ltcg *ltcg = (struct nouveau_ltcg *)object;
-        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
-        int ret;
-
-        ret = nouveau_ltcg_init(ltcg);
-        if (ret)
-                return ret;
-
-        nv_mask(priv, 0x17e820, 0x00100000, 0x00000000); /* INTR_EN &= ~0x10 */
-        nv_wr32(priv, 0x17e8d8, priv->ltc_nr);
-        if (nv_device(ltcg)->card_type >= NV_E0)
-                nv_wr32(priv, 0x17e000, priv->ltc_nr);
-        nv_wr32(priv, 0x17e8d4, priv->tag_base);
+        nv_subdev(priv)->intr = gf100_ltc_intr;
         return 0;
 }
 
 struct nouveau_oclass *
-gf100_ltcg_oclass = &(struct nouveau_oclass) {
-        .handle = NV_SUBDEV(LTCG, 0xc0),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = gf100_ltcg_ctor,
-                .dtor = gf100_ltcg_dtor,
-                .init = gf100_ltcg_init,
-                .fini = _nouveau_ltcg_fini,
+gf100_ltc_oclass = &(struct nvkm_ltc_impl) {
+        .base.handle = NV_SUBDEV(LTC, 0xc0),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gf100_ltc_ctor,
+                .dtor = gf100_ltc_dtor,
+                .init = gf100_ltc_init,
+                .fini = _nvkm_ltc_fini,
         },
-};
+        .intr = gf100_ltc_intr,
+        .cbc_clear = gf100_ltc_cbc_clear,
+        .cbc_wait = gf100_ltc_cbc_wait,
+        .zbc = 16,
+        .zbc_clear_color = gf100_ltc_zbc_clear_color,
+        .zbc_clear_depth = gf100_ltc_zbc_clear_depth,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ltc/gk104.c b/drivers/gpu/drm/nouveau/core/subdev/ltc/gk104.c
new file mode 100644
index 000000000000..ea716569745d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/ltc/gk104.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "priv.h"
+
+static int
+gk104_ltc_init(struct nouveau_object *object)
+{
+        struct nvkm_ltc_priv *priv = (void *)object;
+        int ret;
+
+        ret = nvkm_ltc_init(priv);
+        if (ret)
+                return ret;
+
+        nv_wr32(priv, 0x17e8d8, priv->ltc_nr);
+        nv_wr32(priv, 0x17e000, priv->ltc_nr);
+        nv_wr32(priv, 0x17e8d4, priv->tag_base);
+        return 0;
+}
+
+struct nouveau_oclass *
+gk104_ltc_oclass = &(struct nvkm_ltc_impl) {
+        .base.handle = NV_SUBDEV(LTC, 0xe4),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gf100_ltc_ctor,
+                .dtor = gf100_ltc_dtor,
+                .init = gk104_ltc_init,
+                .fini = _nvkm_ltc_fini,
+        },
+        .intr = gf100_ltc_intr,
+        .cbc_clear = gf100_ltc_cbc_clear,
+        .cbc_wait = gf100_ltc_cbc_wait,
+        .zbc = 16,
+        .zbc_clear_color = gf100_ltc_zbc_clear_color,
+        .zbc_clear_depth = gf100_ltc_zbc_clear_depth,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ltcg/gm107.c b/drivers/gpu/drm/nouveau/core/subdev/ltc/gm107.c
index e79d0e81de40..4761b2e9af00 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/ltcg/gm107.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/ltc/gm107.c
@@ -25,10 +25,45 @@
 #include <subdev/fb.h>
 #include <subdev/timer.h>
 
-#include "gf100.h"
+#include "priv.h"
 
 static void
-gm107_ltcg_lts_isr(struct gf100_ltcg_priv *priv, int ltc, int lts)
+gm107_ltc_cbc_clear(struct nvkm_ltc_priv *priv, u32 start, u32 limit)
+{
+        nv_wr32(priv, 0x17e270, start);
+        nv_wr32(priv, 0x17e274, limit);
+        nv_wr32(priv, 0x17e26c, 0x00000004);
+}
+
+static void
+gm107_ltc_cbc_wait(struct nvkm_ltc_priv *priv)
+{
+        int c, s;
+        for (c = 0; c < priv->ltc_nr; c++) {
+                for (s = 0; s < priv->lts_nr; s++)
+                        nv_wait(priv, 0x14046c + c * 0x2000 + s * 0x200, ~0, 0);
+        }
+}
+
+static void
+gm107_ltc_zbc_clear_color(struct nvkm_ltc_priv *priv, int i, const u32 color[4])
+{
+        nv_mask(priv, 0x17e338, 0x0000000f, i);
+        nv_wr32(priv, 0x17e33c, color[0]);
+        nv_wr32(priv, 0x17e340, color[1]);
+        nv_wr32(priv, 0x17e344, color[2]);
+        nv_wr32(priv, 0x17e348, color[3]);
+}
+
+static void
+gm107_ltc_zbc_clear_depth(struct nvkm_ltc_priv *priv, int i, const u32 depth)
+{
+        nv_mask(priv, 0x17e338, 0x0000000f, i);
+        nv_wr32(priv, 0x17e34c, depth);
+}
+
+static void
+gm107_ltc_lts_isr(struct nvkm_ltc_priv *priv, int ltc, int lts)
 {
         u32 base = 0x140000 + (ltc * 0x2000) + (lts * 0x400);
         u32 stat = nv_rd32(priv, base + 0x00c);
@@ -40,16 +75,16 @@ gm107_ltcg_lts_isr(struct gf100_ltcg_priv *priv, int ltc, int lts)
 }
 
 static void
-gm107_ltcg_intr(struct nouveau_subdev *subdev)
+gm107_ltc_intr(struct nouveau_subdev *subdev)
 {
-        struct gf100_ltcg_priv *priv = (void *)subdev;
+        struct nvkm_ltc_priv *priv = (void *)subdev;
         u32 mask;
 
         mask = nv_rd32(priv, 0x00017c);
         while (mask) {
                 u32 lts, ltc = __ffs(mask);
                 for (lts = 0; lts < priv->lts_nr; lts++)
-                        gm107_ltcg_lts_isr(priv, ltc, lts);
+                        gm107_ltc_lts_isr(priv, ltc, lts);
                 mask &= ~(1 << ltc);
         }
 
@@ -59,37 +94,32 @@ gm107_ltcg_intr(struct nouveau_subdev *subdev)
         nv_mask(priv, 0x000640, 0x02000000, 0x00000000);
 }
 
-static void
-gm107_ltcg_tags_clear(struct nouveau_ltcg *ltcg, u32 first, u32 count)
+static int
+gm107_ltc_init(struct nouveau_object *object)
 {
-        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
-        u32 last = first + count - 1;
-        int p, i;
-
-        BUG_ON((first > last) || (last >= priv->num_tags));
+        struct nvkm_ltc_priv *priv = (void *)object;
+        int ret;
 
-        nv_wr32(priv, 0x17e270, first);
-        nv_wr32(priv, 0x17e274, last);
-        nv_wr32(priv, 0x17e26c, 0x4); /* trigger clear */
+        ret = nvkm_ltc_init(priv);
+        if (ret)
+                return ret;
 
-        /* wait until it's finished with clearing */
-        for (p = 0; p < priv->ltc_nr; ++p) {
-                for (i = 0; i < priv->lts_nr; ++i)
-                        nv_wait(priv, 0x14046c + p * 0x2000 + i * 0x200, ~0, 0);
-        }
+        nv_wr32(priv, 0x17e27c, priv->ltc_nr);
+        nv_wr32(priv, 0x17e278, priv->tag_base);
+        return 0;
 }
 
 static int
-gm107_ltcg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+gm107_ltc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
                struct nouveau_object **pobject)
 {
-        struct gf100_ltcg_priv *priv;
         struct nouveau_fb *pfb = nouveau_fb(parent);
+        struct nvkm_ltc_priv *priv;
         u32 parts, mask;
         int ret, i;
 
-        ret = nouveau_ltcg_create(parent, engine, oclass, &priv);
+        ret = nvkm_ltc_create(parent, engine, oclass, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
@@ -102,41 +132,26 @@ gm107_ltcg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         }
         priv->lts_nr = nv_rd32(priv, 0x17e280) >> 28;
 
-        ret = gf100_ltcg_init_tag_ram(pfb, priv);
-        if (ret)
-                return ret;
-
-        priv->base.tags_alloc = gf100_ltcg_tags_alloc;
-        priv->base.tags_free  = gf100_ltcg_tags_free;
-        priv->base.tags_clear = gm107_ltcg_tags_clear;
-
-        nv_subdev(priv)->intr = gm107_ltcg_intr;
-        return 0;
-}
-
-static int
-gm107_ltcg_init(struct nouveau_object *object)
-{
-        struct nouveau_ltcg *ltcg = (struct nouveau_ltcg *)object;
-        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
-        int ret;
-
-        ret = nouveau_ltcg_init(ltcg);
+        ret = gf100_ltc_init_tag_ram(pfb, priv);
         if (ret)
                 return ret;
 
-        nv_wr32(priv, 0x17e27c, priv->ltc_nr);
-        nv_wr32(priv, 0x17e278, priv->tag_base);
         return 0;
 }
 
 struct nouveau_oclass *
-gm107_ltcg_oclass = &(struct nouveau_oclass) {
-        .handle = NV_SUBDEV(LTCG, 0xff),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = gm107_ltcg_ctor,
-                .dtor = gf100_ltcg_dtor,
-                .init = gm107_ltcg_init,
-                .fini = _nouveau_ltcg_fini,
+gm107_ltc_oclass = &(struct nvkm_ltc_impl) {
+        .base.handle = NV_SUBDEV(LTC, 0xff),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gm107_ltc_ctor,
+                .dtor = gf100_ltc_dtor,
+                .init = gm107_ltc_init,
+                .fini = _nvkm_ltc_fini,
         },
-};
+        .intr = gm107_ltc_intr,
+        .cbc_clear = gm107_ltc_cbc_clear,
+        .cbc_wait = gm107_ltc_cbc_wait,
+        .zbc = 16,
+        .zbc_clear_color = gm107_ltc_zbc_clear_color,
+        .zbc_clear_depth = gm107_ltc_zbc_clear_depth,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ltc/priv.h b/drivers/gpu/drm/nouveau/core/subdev/ltc/priv.h
new file mode 100644
index 000000000000..594924f39126
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/ltc/priv.h
@@ -0,0 +1,69 @@
+#ifndef __NVKM_LTC_PRIV_H__
+#define __NVKM_LTC_PRIV_H__
+
+#include <subdev/ltc.h>
+#include <subdev/fb.h>
+
+struct nvkm_ltc_priv {
+        struct nouveau_ltc base;
+        u32 ltc_nr;
+        u32 lts_nr;
+
+        u32 num_tags;
+        u32 tag_base;
+        struct nouveau_mm tags;
+        struct nouveau_mm_node *tag_ram;
+
+        u32 zbc_color[NOUVEAU_LTC_MAX_ZBC_CNT][4];
+        u32 zbc_depth[NOUVEAU_LTC_MAX_ZBC_CNT];
+};
+
+#define nvkm_ltc_create(p,e,o,d)                                               \
+        nvkm_ltc_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nvkm_ltc_destroy(p) ({                                                 \
+        struct nvkm_ltc_priv *_priv = (p);                                     \
+        _nvkm_ltc_dtor(nv_object(_priv));                                      \
+})
+#define nvkm_ltc_init(p) ({                                                    \
+        struct nvkm_ltc_priv *_priv = (p);                                     \
+        _nvkm_ltc_init(nv_object(_priv));                                      \
+})
+#define nvkm_ltc_fini(p,s) ({                                                  \
+        struct nvkm_ltc_priv *_priv = (p);                                     \
+        _nvkm_ltc_fini(nv_object(_priv), (s));                                 \
+})
+
+int  nvkm_ltc_create_(struct nouveau_object *, struct nouveau_object *,
+                      struct nouveau_oclass *, int, void **);
+
+#define _nvkm_ltc_dtor _nouveau_subdev_dtor
+int _nvkm_ltc_init(struct nouveau_object *);
+#define _nvkm_ltc_fini _nouveau_subdev_fini
+
+int  gf100_ltc_ctor(struct nouveau_object *, struct nouveau_object *,
+                    struct nouveau_oclass *, void *, u32,
+                    struct nouveau_object **);
+void gf100_ltc_dtor(struct nouveau_object *);
+int  gf100_ltc_init_tag_ram(struct nouveau_fb *, struct nvkm_ltc_priv *);
+int  gf100_ltc_tags_alloc(struct nouveau_ltc *, u32, struct nouveau_mm_node **);
+void gf100_ltc_tags_free(struct nouveau_ltc *, struct nouveau_mm_node **);
+
+struct nvkm_ltc_impl {
+        struct nouveau_oclass base;
+        void (*intr)(struct nouveau_subdev *);
+
+        void (*cbc_clear)(struct nvkm_ltc_priv *, u32 start, u32 limit);
+        void (*cbc_wait)(struct nvkm_ltc_priv *);
+
+        int zbc;
+        void (*zbc_clear_color)(struct nvkm_ltc_priv *, int, const u32[4]);
+        void (*zbc_clear_depth)(struct nvkm_ltc_priv *, int, const u32);
+};
+
+void gf100_ltc_intr(struct nouveau_subdev *);
+void gf100_ltc_cbc_clear(struct nvkm_ltc_priv *, u32, u32);
+void gf100_ltc_cbc_wait(struct nvkm_ltc_priv *);
+void gf100_ltc_zbc_clear_color(struct nvkm_ltc_priv *, int, const u32[4]);
+void gf100_ltc_zbc_clear_depth(struct nvkm_ltc_priv *, int, const u32);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ltcg/gf100.h b/drivers/gpu/drm/nouveau/core/subdev/ltcg/gf100.h
deleted file mode 100644
index 87b10b8412ea..000000000000
--- a/drivers/gpu/drm/nouveau/core/subdev/ltcg/gf100.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#ifndef __NVKM_LTCG_PRIV_GF100_H__
-#define __NVKM_LTCG_PRIV_GF100_H__
-
-#include <subdev/ltcg.h>
-
-struct gf100_ltcg_priv {
-        struct nouveau_ltcg base;
-        u32 ltc_nr;
-        u32 lts_nr;
-        u32 num_tags;
-        u32 tag_base;
-        struct nouveau_mm tags;
-        struct nouveau_mm_node *tag_ram;
-};
-
-void gf100_ltcg_dtor(struct nouveau_object *);
-int  gf100_ltcg_init_tag_ram(struct nouveau_fb *, struct gf100_ltcg_priv *);
-int  gf100_ltcg_tags_alloc(struct nouveau_ltcg *, u32, struct nouveau_mm_node **);
-void gf100_ltcg_tags_free(struct nouveau_ltcg *, struct nouveau_mm_node **);
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/base.c b/drivers/gpu/drm/nouveau/core/subdev/mc/base.c
index 8a5555192fa5..ca7cee3a314a 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mc/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/base.c
@@ -22,9 +22,17 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/mc.h>
+#include "priv.h"
 #include <core/option.h>
 
+static inline void
+nouveau_mc_unk260(struct nouveau_mc *pmc, u32 data)
+{
+        const struct nouveau_mc_oclass *impl = (void *)nv_oclass(pmc);
+        if (impl->unk260)
+                impl->unk260(pmc, data);
+}
+
 static inline u32
 nouveau_mc_intr_mask(struct nouveau_mc *pmc)
 {
@@ -114,6 +122,8 @@ nouveau_mc_create_(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
+        pmc->unk260 = nouveau_mc_unk260;
+
         if (nv_device_is_pci(device))
                 switch (device->pdev->device & 0x0ff0) {
                 case 0x00f0:
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/gk20a.c b/drivers/gpu/drm/nouveau/core/subdev/mc/gk20a.c
new file mode 100644
index 000000000000..b8d6cb435d0a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/gk20a.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv04.h"
+
+struct nouveau_oclass *
+gk20a_mc_oclass = &(struct nouveau_mc_oclass) {
+        .base.handle = NV_SUBDEV(MC, 0xea),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_mc_ctor,
+                .dtor = _nouveau_mc_dtor,
+                .init = nv50_mc_init,
+                .fini = _nouveau_mc_fini,
+        },
+        .intr = nvc0_mc_intr,
+        .msi_rearm = nv40_mc_msi_rearm,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.h b/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.h
index 81a408e7d034..4d9ea46c47c2 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.h
@@ -1,7 +1,7 @@
 #ifndef __NVKM_MC_NV04_H__
 #define __NVKM_MC_NV04_H__
 
-#include <subdev/mc.h>
+#include "priv.h"
 
 struct nv04_mc_priv {
         struct nouveau_mc base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nvc0.c
index f9c6a678b47d..15d41dc176ff 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mc/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nvc0.c
@@ -41,7 +41,7 @@ nvc0_mc_intr[] = {
         { 0x00200000, NVDEV_SUBDEV_GPIO },      /* PMGR->GPIO */
         { 0x00200000, NVDEV_SUBDEV_I2C },       /* PMGR->I2C/AUX */
         { 0x01000000, NVDEV_SUBDEV_PWR },
-        { 0x02000000, NVDEV_SUBDEV_LTCG },
+        { 0x02000000, NVDEV_SUBDEV_LTC },
         { 0x08000000, NVDEV_SUBDEV_FB },
         { 0x10000000, NVDEV_SUBDEV_BUS },
         { 0x40000000, NVDEV_SUBDEV_IBUS },
@@ -56,6 +56,12 @@ nvc0_mc_msi_rearm(struct nouveau_mc *pmc)
         nv_wr32(priv, 0x088704, 0x00000000);
 }
 
+void
+nvc0_mc_unk260(struct nouveau_mc *pmc, u32 data)
+{
+        nv_wr32(pmc, 0x000260, data);
+}
+
 struct nouveau_oclass *
 nvc0_mc_oclass = &(struct nouveau_mc_oclass) {
         .base.handle = NV_SUBDEV(MC, 0xc0),
@@ -67,4 +73,5 @@ nvc0_mc_oclass = &(struct nouveau_mc_oclass) {
         },
         .intr = nvc0_mc_intr,
         .msi_rearm = nvc0_mc_msi_rearm,
+        .unk260 = nvc0_mc_unk260,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nvc3.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nvc3.c
index 837e545aeb9f..68b5f61aadb5 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mc/nvc3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nvc3.c
@@ -35,4 +35,5 @@ nvc3_mc_oclass = &(struct nouveau_mc_oclass) {
         },
         .intr = nvc0_mc_intr,
         .msi_rearm = nv40_mc_msi_rearm,
+        .unk260 = nvc0_mc_unk260,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/priv.h b/drivers/gpu/drm/nouveau/core/subdev/mc/priv.h
new file mode 100644
index 000000000000..911e66392587
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/priv.h
@@ -0,0 +1,38 @@
+#ifndef __NVKM_MC_PRIV_H__
+#define __NVKM_MC_PRIV_H__
+
+#include <subdev/mc.h>
+
+#define nouveau_mc_create(p,e,o,d)                                             \
+        nouveau_mc_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nouveau_mc_destroy(p) ({                                               \
+        struct nouveau_mc *pmc = (p); _nouveau_mc_dtor(nv_object(pmc));        \
+})
+#define nouveau_mc_init(p) ({                                                  \
+        struct nouveau_mc *pmc = (p); _nouveau_mc_init(nv_object(pmc));        \
+})
+#define nouveau_mc_fini(p,s) ({                                                \
+        struct nouveau_mc *pmc = (p); _nouveau_mc_fini(nv_object(pmc), (s));   \
+})
+
+int  nouveau_mc_create_(struct nouveau_object *, struct nouveau_object *,
+                        struct nouveau_oclass *, int, void **);
+void _nouveau_mc_dtor(struct nouveau_object *);
+int  _nouveau_mc_init(struct nouveau_object *);
+int  _nouveau_mc_fini(struct nouveau_object *, bool);
+
+struct nouveau_mc_intr {
+        u32 stat;
+        u32 unit;
+};
+
+struct nouveau_mc_oclass {
+        struct nouveau_oclass base;
+        const struct nouveau_mc_intr *intr;
+        void (*msi_rearm)(struct nouveau_mc *);
+        void (*unk260)(struct nouveau_mc *, u32);
+};
+
+void nvc0_mc_unk260(struct nouveau_mc *, u32);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/base.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/base.c
index d4fd3bc9c66f..69f1f34f6931 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/base.c
@@ -22,9 +22,18 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/pwr.h>
 #include <subdev/timer.h>
 
+#include "priv.h"
+
+static void
+nouveau_pwr_pgob(struct nouveau_pwr *ppwr, bool enable)
+{
+        const struct nvkm_pwr_impl *impl = (void *)nv_oclass(ppwr);
+        if (impl->pgob)
+                impl->pgob(ppwr, enable);
+}
+
 static int
 nouveau_pwr_send(struct nouveau_pwr *ppwr, u32 reply[2],
                  u32 process, u32 message, u32 data0, u32 data1)
@@ -177,6 +186,7 @@ _nouveau_pwr_fini(struct nouveau_object *object, bool suspend)
 int
 _nouveau_pwr_init(struct nouveau_object *object)
 {
+        const struct nvkm_pwr_impl *impl = (void *)object->oclass;
         struct nouveau_pwr *ppwr = (void *)object;
         int ret, i;
 
@@ -186,6 +196,7 @@ _nouveau_pwr_init(struct nouveau_object *object)
 
         nv_subdev(ppwr)->intr = nouveau_pwr_intr;
         ppwr->message = nouveau_pwr_send;
+        ppwr->pgob = nouveau_pwr_pgob;
 
         /* prevent previous ucode from running, wait for idle, reset */
         nv_wr32(ppwr, 0x10a014, 0x0000ffff); /* INTR_EN_CLR = ALL */
@@ -195,15 +206,15 @@ _nouveau_pwr_init(struct nouveau_object *object)
 
         /* upload data segment */
         nv_wr32(ppwr, 0x10a1c0, 0x01000000);
-        for (i = 0; i < ppwr->data.size / 4; i++)
-                nv_wr32(ppwr, 0x10a1c4, ppwr->data.data[i]);
+        for (i = 0; i < impl->data.size / 4; i++)
+                nv_wr32(ppwr, 0x10a1c4, impl->data.data[i]);
 
         /* upload code segment */
         nv_wr32(ppwr, 0x10a180, 0x01000000);
-        for (i = 0; i < ppwr->code.size / 4; i++) {
+        for (i = 0; i < impl->code.size / 4; i++) {
                 if ((i & 0x3f) == 0)
                         nv_wr32(ppwr, 0x10a188, i >> 6);
-                nv_wr32(ppwr, 0x10a184, ppwr->code.data[i]);
+                nv_wr32(ppwr, 0x10a184, impl->code.data[i]);
         }
 
         /* start it running */
@@ -245,3 +256,15 @@ nouveau_pwr_create_(struct nouveau_object *parent,
         init_waitqueue_head(&ppwr->recv.wait);
         return 0;
 }
+
+int
+_nouveau_pwr_ctor(struct nouveau_object *parent,
+                  struct nouveau_object *engine,
+                  struct nouveau_oclass *oclass, void *data, u32 size,
+                  struct nouveau_object **pobject)
+{
+        struct nouveau_pwr *ppwr;
+        int ret = nouveau_pwr_create(parent, engine, oclass, &ppwr);
+        *pobject = nv_object(ppwr);
+        return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/macros.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/macros.fuc
index e2a63ac5422b..5668e045bac1 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/macros.fuc
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/macros.fuc
@@ -242,7 +242,7 @@
 */      push reg /*
 */      pop $r13 /*
 */      pop $r14 /*
-*/      call(wr32) /*
+*/      call(wr32)
 #else
 #define nv_wr32(addr,reg) /*
 */      sethi $r0 0x14000000 /*
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h
index 39a5dc150a05..986495d533dd 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h
@@ -46,8 +46,8 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x584d454d,
-        0x0000046f,
-        0x00000461,
+        0x00000464,
+        0x00000456,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -68,8 +68,8 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x46524550,
-        0x00000473,
-        0x00000471,
+        0x00000468,
+        0x00000466,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -90,8 +90,8 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x5f433249,
-        0x00000877,
-        0x0000071e,
+        0x0000086c,
+        0x00000713,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -112,8 +112,8 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x54534554,
-        0x00000898,
-        0x00000879,
+        0x0000088d,
+        0x0000086e,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -134,8 +134,8 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x454c4449,
-        0x000008a3,
-        0x000008a1,
+        0x00000898,
+        0x00000896,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -239,10 +239,10 @@ uint32_t nv108_pwr_data[] = {
         0x000003df,
         0x00040003,
         0x00000000,
-        0x00000407,
+        0x000003fc,
         0x00010004,
         0x00000000,
-        0x00000421,
+        0x00000416,
 /* 0x03ac: memx_func_tail */
 /* 0x03ac: memx_data_head */
         0x00000000,
@@ -1080,375 +1080,375 @@ uint32_t nv108_pwr_code[] = {
         0x50f960f9,
         0xe0fcd0fc,
         0x00002e7e,
-        0x140003f1,
-        0xa00506fd,
-        0xb604bd05,
-        0x1bf40242,
-/* 0x0407: memx_func_wait */
-        0x0800f8dd,
-        0x0088cf2c,
-        0x98001e98,
-        0x1c98011d,
-        0x031b9802,
-        0x7e1010b6,
-        0xf8000071,
-/* 0x0421: memx_func_delay */
+        0xf40242b6,
+        0x00f8e81b,
+/* 0x03fc: memx_func_wait */
+        0x88cf2c08,
         0x001e9800,
-        0x7e0410b6,
-        0xf800005d,
-/* 0x042d: memx_exec */
-        0xf9e0f900,
-        0xb2c1b2d0,
-/* 0x0435: memx_exec_next */
-        0x001398b2,
-        0x950410b6,
-        0x30f01034,
-        0xde35980c,
-        0x12a655f9,
-        0xfced1ef4,
-        0x7ee0fcd0,
-        0xf800023f,
-/* 0x0455: memx_info */
-        0x03ac4c00,
-        0x7e08004b,
-        0xf800023f,
-/* 0x0461: memx_recv */
-        0x01d6b000,
-        0xb0c90bf4,
-        0x0bf400d6,
-/* 0x046f: memx_init */
-        0xf800f8eb,
-/* 0x0471: perf_recv */
-/* 0x0473: perf_init */
-        0xf800f800,
-/* 0x0475: i2c_drive_scl */
-        0x0036b000,
-        0x400d0bf4,
-        0x01f607e0,
-        0xf804bd00,
-/* 0x0485: i2c_drive_scl_lo */
-        0x07e44000,
-        0xbd0001f6,
-/* 0x048f: i2c_drive_sda */
-        0xb000f804,
-        0x0bf40036,
-        0x07e0400d,
-        0xbd0002f6,
-/* 0x049f: i2c_drive_sda_lo */
-        0x4000f804,
-        0x02f607e4,
-        0xf804bd00,
-/* 0x04a9: i2c_sense_scl */
-        0x0132f400,
-        0xcf07c443,
-        0x31fd0033,
-        0x060bf404,
-/* 0x04bb: i2c_sense_scl_done */
-        0xf80131f4,
-/* 0x04bd: i2c_sense_sda */
-        0x0132f400,
-        0xcf07c443,
-        0x32fd0033,
-        0x060bf404,
-/* 0x04cf: i2c_sense_sda_done */
-        0xf80131f4,
-/* 0x04d1: i2c_raise_scl */
-        0x4440f900,
-        0x01030898,
-        0x0004757e,
-/* 0x04dc: i2c_raise_scl_wait */
-        0x7e03e84e,
-        0x7e00005d,
-        0xf40004a9,
-        0x42b60901,
-        0xef1bf401,
-/* 0x04f0: i2c_raise_scl_done */
-        0x00f840fc,
-/* 0x04f4: i2c_start */
-        0x0004a97e,
-        0x7e0d11f4,
-        0xf40004bd,
-        0x0ef40611,
-/* 0x0505: i2c_start_rep */
-        0x7e00032e,
-        0x03000475,
-        0x048f7e01,
-        0x0076bb00,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0xd17e50fc,
-        0x64b60004,
-        0x1d11f404,
-/* 0x0530: i2c_start_send */
-        0x8f7e0003,
-        0x884e0004,
-        0x005d7e13,
-        0x7e000300,
-        0x4e000475,
-        0x5d7e1388,
-/* 0x054a: i2c_start_out */
-        0x00f80000,
-/* 0x054c: i2c_stop */
-        0x757e0003,
-        0x00030004,
-        0x00048f7e,
-        0x7e03e84e,
-        0x0300005d,
-        0x04757e01,
-        0x13884e00,
+        0x98011d98,
+        0x1b98021c,
+        0x1010b603,
+        0x0000717e,
+/* 0x0416: memx_func_delay */
+        0x1e9800f8,
+        0x0410b600,
         0x00005d7e,
-        0x8f7e0103,
-        0x884e0004,
-        0x005d7e13,
-/* 0x057b: i2c_bitw */
-        0x7e00f800,
-        0x4e00048f,
-        0x5d7e03e8,
-        0x76bb0000,
+/* 0x0422: memx_exec */
+        0xe0f900f8,
+        0xc1b2d0f9,
+/* 0x042a: memx_exec_next */
+        0x1398b2b2,
+        0x0410b600,
+        0xf0103495,
+        0x35980c30,
+        0xa655f9de,
+        0xed1ef412,
+        0xe0fcd0fc,
+        0x00023f7e,
+/* 0x044a: memx_info */
+        0xac4c00f8,
+        0x08004b03,
+        0x00023f7e,
+/* 0x0456: memx_recv */
+        0xd6b000f8,
+        0xc90bf401,
+        0xf400d6b0,
+        0x00f8eb0b,
+/* 0x0464: memx_init */
+/* 0x0466: perf_recv */
+        0x00f800f8,
+/* 0x0468: perf_init */
+/* 0x046a: i2c_drive_scl */
+        0x36b000f8,
+        0x0d0bf400,
+        0xf607e040,
+        0x04bd0001,
+/* 0x047a: i2c_drive_scl_lo */
+        0xe44000f8,
+        0x0001f607,
+        0x00f804bd,
+/* 0x0484: i2c_drive_sda */
+        0xf40036b0,
+        0xe0400d0b,
+        0x0002f607,
+        0x00f804bd,
+/* 0x0494: i2c_drive_sda_lo */
+        0xf607e440,
+        0x04bd0002,
+/* 0x049e: i2c_sense_scl */
+        0x32f400f8,
+        0x07c44301,
+        0xfd0033cf,
+        0x0bf40431,
+        0x0131f406,
+/* 0x04b0: i2c_sense_scl_done */
+/* 0x04b2: i2c_sense_sda */
+        0x32f400f8,
+        0x07c44301,
+        0xfd0033cf,
+        0x0bf40432,
+        0x0131f406,
+/* 0x04c4: i2c_sense_sda_done */
+/* 0x04c6: i2c_raise_scl */
+        0x40f900f8,
+        0x03089844,
+        0x046a7e01,
+/* 0x04d1: i2c_raise_scl_wait */
+        0x03e84e00,
+        0x00005d7e,
+        0x00049e7e,
+        0xb60901f4,
+        0x1bf40142,
+/* 0x04e5: i2c_raise_scl_done */
+        0xf840fcef,
+/* 0x04e9: i2c_start */
+        0x049e7e00,
+        0x0d11f400,
+        0x0004b27e,
+        0xf40611f4,
+/* 0x04fa: i2c_start_rep */
+        0x00032e0e,
+        0x00046a7e,
+        0x847e0103,
+        0x76bb0004,
         0x0465b600,
         0x659450f9,
         0x0256bb04,
         0x75fd50bd,
         0x7e50fc04,
-        0xb60004d1,
+        0xb60004c6,
         0x11f40464,
-        0x13884e17,
+/* 0x0525: i2c_start_send */
+        0x7e00031d,
+        0x4e000484,
+        0x5d7e1388,
+        0x00030000,
+        0x00046a7e,
+        0x7e13884e,
+/* 0x053f: i2c_start_out */
+        0xf800005d,
+/* 0x0541: i2c_stop */
+        0x7e000300,
+        0x0300046a,
+        0x04847e00,
+        0x03e84e00,
         0x00005d7e,
-        0x757e0003,
+        0x6a7e0103,
         0x884e0004,
         0x005d7e13,
-/* 0x05b9: i2c_bitw_out */
-/* 0x05bb: i2c_bitr */
-        0x0300f800,
-        0x048f7e01,
-        0x03e84e00,
-        0x00005d7e,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0x04d17e50,
-        0x0464b600,
-        0x7e1a11f4,
-        0x030004bd,
-        0x04757e00,
-        0x13884e00,
-        0x00005d7e,
-        0xf4013cf0,
-/* 0x05fe: i2c_bitr_done */
-        0x00f80131,
-/* 0x0600: i2c_get_byte */
-        0x08040005,
-/* 0x0604: i2c_get_byte_next */
-        0xbb0154b6,
+        0x7e010300,
+        0x4e000484,
+        0x5d7e1388,
+        0x00f80000,
+/* 0x0570: i2c_bitw */
+        0x0004847e,
+        0x7e03e84e,
+        0xbb00005d,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x0005bb7e,
+        0x0004c67e,
         0xf40464b6,
-        0x53fd2a11,
-        0x0142b605,
-        0x03d81bf4,
-        0x0076bb01,
+        0x884e1711,
+        0x005d7e13,
+        0x7e000300,
+        0x4e00046a,
+        0x5d7e1388,
+/* 0x05ae: i2c_bitw_out */
+        0x00f80000,
+/* 0x05b0: i2c_bitr */
+        0x847e0103,
+        0xe84e0004,
+        0x005d7e03,
+        0x0076bb00,
         0xf90465b6,
         0x04659450,
         0xbd0256bb,
         0x0475fd50,
-        0x7b7e50fc,
-        0x64b60005,
-/* 0x064d: i2c_get_byte_done */
-/* 0x064f: i2c_put_byte */
-        0x0400f804,
-/* 0x0651: i2c_put_byte_next */
-        0x0142b608,
-        0xbb3854ff,
-        0x65b60076,
-        0x9450f904,
-        0x56bb0465,
-        0xfd50bd02,
-        0x50fc0475,
-        0x00057b7e,
-        0xf40464b6,
-        0x46b03411,
-        0xd81bf400,
+        0xc67e50fc,
+        0x64b60004,
+        0x1a11f404,
+        0x0004b27e,
+        0x6a7e0003,
+        0x884e0004,
+        0x005d7e13,
+        0x013cf000,
+/* 0x05f3: i2c_bitr_done */
+        0xf80131f4,
+/* 0x05f5: i2c_get_byte */
+        0x04000500,
+/* 0x05f9: i2c_get_byte_next */
+        0x0154b608,
         0xb60076bb,
         0x50f90465,
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0x05bb7e50,
+        0x05b07e50,
         0x0464b600,
-        0xbb0f11f4,
-        0x36b00076,
-        0x061bf401,
-/* 0x06a7: i2c_put_byte_done */
-        0xf80132f4,
-/* 0x06a9: i2c_addr */
-        0x0076bb00,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0xf47e50fc,
-        0x64b60004,
-        0x2911f404,
-        0x012ec3e7,
-        0xfd0134b6,
-        0x76bb0553,
+        0xfd2a11f4,
+        0x42b60553,
+        0xd81bf401,
+        0x76bb0103,
         0x0465b600,
         0x659450f9,
         0x0256bb04,
         0x75fd50bd,
         0x7e50fc04,
-        0xb600064f,
-/* 0x06ee: i2c_addr_done */
+        0xb6000570,
+/* 0x0642: i2c_get_byte_done */
         0x00f80464,
-/* 0x06f0: i2c_acquire_addr */
-        0xb6f8cec7,
-        0xe0b705e4,
-        0x00f8d014,
-/* 0x06fc: i2c_acquire */
-        0x0006f07e,
-        0x0000047e,
-        0x7e03d9f0,
-        0xf800002e,
-/* 0x070d: i2c_release */
-        0x06f07e00,
-        0x00047e00,
-        0x03daf000,
-        0x00002e7e,
-/* 0x071e: i2c_recv */
-        0x32f400f8,
-        0xf8c1c701,
-        0xb00214b6,
-        0x1ff52816,
-        0x13b80137,
-        0x98000bd4,
-        0x13b80032,
-        0x98000bac,
-        0x31f40031,
-        0xf9d0f902,
-        0xf1d0f9e0,
-        0xf1000067,
-        0x92100063,
-        0x76bb0167,
-        0x0465b600,
-        0x659450f9,
-        0x0256bb04,
-        0x75fd50bd,
-        0x7e50fc04,
-        0xb60006fc,
-        0xd0fc0464,
-        0xf500d6b0,
-        0x0500b01b,
-        0x0076bb00,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0xa97e50fc,
-        0x64b60006,
-        0xcc11f504,
-        0xe0c5c700,
+/* 0x0644: i2c_put_byte */
+/* 0x0646: i2c_put_byte_next */
+        0x42b60804,
+        0x3854ff01,
         0xb60076bb,
         0x50f90465,
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0x064f7e50,
+        0x05707e50,
         0x0464b600,
-        0x00a911f5,
-        0x76bb0105,
+        0xb03411f4,
+        0x1bf40046,
+        0x0076bbd8,
+        0xf90465b6,
+        0x04659450,
+        0xbd0256bb,
+        0x0475fd50,
+        0xb07e50fc,
+        0x64b60005,
+        0x0f11f404,
+        0xb00076bb,
+        0x1bf40136,
+        0x0132f406,
+/* 0x069c: i2c_put_byte_done */
+/* 0x069e: i2c_addr */
+        0x76bb00f8,
         0x0465b600,
         0x659450f9,
         0x0256bb04,
         0x75fd50bd,
         0x7e50fc04,
-        0xb60006a9,
-        0x11f50464,
-        0x76bb0087,
+        0xb60004e9,
+        0x11f40464,
+        0x2ec3e729,
+        0x0134b601,
+        0xbb0553fd,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x0006447e,
+/* 0x06e3: i2c_addr_done */
+        0xf80464b6,
+/* 0x06e5: i2c_acquire_addr */
+        0xf8cec700,
+        0xb705e4b6,
+        0xf8d014e0,
+/* 0x06f1: i2c_acquire */
+        0x06e57e00,
+        0x00047e00,
+        0x03d9f000,
+        0x00002e7e,
+/* 0x0702: i2c_release */
+        0xe57e00f8,
+        0x047e0006,
+        0xdaf00000,
+        0x002e7e03,
+/* 0x0713: i2c_recv */
+        0xf400f800,
+        0xc1c70132,
+        0x0214b6f8,
+        0xf52816b0,
+        0xb801371f,
+        0x000bd413,
+        0xb8003298,
+        0x000bac13,
+        0xf4003198,
+        0xd0f90231,
+        0xd0f9e0f9,
+        0x000067f1,
+        0x100063f1,
+        0xbb016792,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x0006f17e,
+        0xfc0464b6,
+        0x00d6b0d0,
+        0x00b01bf5,
+        0x76bb0005,
         0x0465b600,
         0x659450f9,
         0x0256bb04,
         0x75fd50bd,
         0x7e50fc04,
-        0xb6000600,
-        0x11f40464,
-        0xe05bcb67,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0x054c7e50,
-        0x0464b600,
-        0x74bd5bb2,
-/* 0x0823: i2c_recv_not_rd08 */
-        0xb0410ef4,
-        0x1bf401d6,
-        0x7e00053b,
-        0xf40006a9,
-        0xc5c73211,
-        0x064f7ee0,
-        0x2811f400,
-        0xa97e0005,
+        0xb600069e,
+        0x11f50464,
+        0xc5c700cc,
+        0x0076bbe0,
+        0xf90465b6,
+        0x04659450,
+        0xbd0256bb,
+        0x0475fd50,
+        0x447e50fc,
+        0x64b60006,
+        0xa911f504,
+        0xbb010500,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x00069e7e,
+        0xf50464b6,
+        0xbb008711,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x0005f57e,
+        0xf40464b6,
+        0x5bcb6711,
+        0x0076bbe0,
+        0xf90465b6,
+        0x04659450,
+        0xbd0256bb,
+        0x0475fd50,
+        0x417e50fc,
+        0x64b60005,
+        0xbd5bb204,
+        0x410ef474,
+/* 0x0818: i2c_recv_not_rd08 */
+        0xf401d6b0,
+        0x00053b1b,
+        0x00069e7e,
+        0xc73211f4,
+        0x447ee0c5,
         0x11f40006,
-        0xe0b5c71f,
-        0x00064f7e,
-        0x7e1511f4,
-        0xbd00054c,
-        0x08c5c774,
-        0xf4091bf4,
-        0x0ef40232,
-/* 0x0861: i2c_recv_not_wr08 */
-/* 0x0861: i2c_recv_done */
-        0xf8cec703,
-        0x00070d7e,
-        0xd0fce0fc,
-        0xb20912f4,
-        0x023f7e7c,
-/* 0x0875: i2c_recv_exit */
-/* 0x0877: i2c_init */
-        0xf800f800,
-/* 0x0879: test_recv */
-        0x04584100,
-        0xb60011cf,
-        0x58400110,
-        0x0001f604,
-        0xe7f104bd,
-        0xe3f1d900,
-        0x967e134f,
-        0x00f80001,
-/* 0x0898: test_init */
-        0x7e08004e,
-        0xf8000196,
-/* 0x08a1: idle_recv */
-/* 0x08a3: idle */
-        0xf400f800,
-        0x54410031,
+        0x7e000528,
+        0xf400069e,
+        0xb5c71f11,
+        0x06447ee0,
+        0x1511f400,
+        0x0005417e,
+        0xc5c774bd,
+        0x091bf408,
+        0xf40232f4,
+/* 0x0856: i2c_recv_not_wr08 */
+/* 0x0856: i2c_recv_done */
+        0xcec7030e,
+        0x07027ef8,
+        0xfce0fc00,
+        0x0912f4d0,
+        0x3f7e7cb2,
+/* 0x086a: i2c_recv_exit */
+        0x00f80002,
+/* 0x086c: i2c_init */
+/* 0x086e: test_recv */
+        0x584100f8,
         0x0011cf04,
         0x400110b6,
-        0x01f60454,
-/* 0x08b7: idle_loop */
-        0x0104bd00,
-        0x0232f458,
-/* 0x08bc: idle_proc */
-/* 0x08bc: idle_proc_exec */
-        0x1eb210f9,
-        0x0002487e,
-        0x11f410fc,
-        0x0231f409,
-/* 0x08cf: idle_proc_next */
-        0xb6f00ef4,
-        0x1fa65810,
-        0xf4e81bf4,
-        0x28f4e002,
-        0xc60ef400,
+        0x01f60458,
+        0xf104bd00,
+        0xf1d900e7,
+        0x7e134fe3,
+        0xf8000196,
+/* 0x088d: test_init */
+        0x08004e00,
+        0x0001967e,
+/* 0x0896: idle_recv */
+        0x00f800f8,
+/* 0x0898: idle */
+        0x410031f4,
+        0x11cf0454,
+        0x0110b600,
+        0xf6045440,
+        0x04bd0001,
+/* 0x08ac: idle_loop */
+        0x32f45801,
+/* 0x08b1: idle_proc */
+/* 0x08b1: idle_proc_exec */
+        0xb210f902,
+        0x02487e1e,
+        0xf410fc00,
+        0x31f40911,
+        0xf00ef402,
+/* 0x08c4: idle_proc_next */
+        0xa65810b6,
+        0xe81bf41f,
+        0xf4e002f4,
+        0x0ef40028,
+        0x000000c6,
+        0x00000000,
+        0x00000000,
         0x00000000,
         0x00000000,
         0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h
index 254205cd5166..e087ce3041be 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h
@@ -46,8 +46,8 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x584d454d,
-        0x0000054e,
-        0x00000540,
+        0x00000542,
+        0x00000534,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -68,8 +68,8 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x46524550,
-        0x00000552,
-        0x00000550,
+        0x00000546,
+        0x00000544,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -90,8 +90,8 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x5f433249,
-        0x00000982,
-        0x00000825,
+        0x00000976,
+        0x00000819,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -112,8 +112,8 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x54534554,
-        0x000009ab,
-        0x00000984,
+        0x0000099f,
+        0x00000978,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -134,8 +134,8 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x454c4449,
-        0x000009b7,
-        0x000009b5,
+        0x000009ab,
+        0x000009a9,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -239,10 +239,10 @@ uint32_t nva3_pwr_data[] = {
         0x000004b7,
         0x00040003,
         0x00000000,
-        0x000004df,
+        0x000004d3,
         0x00010004,
         0x00000000,
-        0x000004fc,
+        0x000004f0,
 /* 0x03ac: memx_func_tail */
 /* 0x03ac: memx_data_head */
         0x00000000,
@@ -1198,13 +1198,10 @@ uint32_t nva3_pwr_code[] = {
         0x0810b601,
         0x50f960f9,
         0xe0fcd0fc,
-        0xf13f21f4,
-        0xfd140003,
-        0x05800506,
-        0xb604bd00,
+        0xb63f21f4,
         0x1bf40242,
-/* 0x04df: memx_func_wait */
-        0xf000f8dd,
+/* 0x04d3: memx_func_wait */
+        0xf000f8e9,
         0x84b62c87,
         0x0088cf06,
         0x98001e98,
@@ -1212,14 +1209,14 @@ uint32_t nva3_pwr_code[] = {
         0x031b9802,
         0xf41010b6,
         0x00f89c21,
-/* 0x04fc: memx_func_delay */
+/* 0x04f0: memx_func_delay */
         0xb6001e98,
         0x21f40410,
-/* 0x0507: memx_exec */
+/* 0x04fb: memx_exec */
         0xf900f87f,
         0xb9d0f9e0,
         0xb2b902c1,
-/* 0x0511: memx_exec_next */
+/* 0x0505: memx_exec_next */
         0x00139802,
         0x950410b6,
         0x30f01034,
@@ -1228,112 +1225,112 @@ uint32_t nva3_pwr_code[] = {
         0xec1ef406,
         0xe0fcd0fc,
         0x02b921f5,
-/* 0x0532: memx_info */
+/* 0x0526: memx_info */
         0xc7f100f8,
         0xb7f103ac,
         0x21f50800,
         0x00f802b9,
-/* 0x0540: memx_recv */
+/* 0x0534: memx_recv */
         0xf401d6b0,
         0xd6b0c40b,
         0xe90bf400,
-/* 0x054e: memx_init */
+/* 0x0542: memx_init */
         0x00f800f8,
-/* 0x0550: perf_recv */
-/* 0x0552: perf_init */
+/* 0x0544: perf_recv */
+/* 0x0546: perf_init */
         0x00f800f8,
-/* 0x0554: i2c_drive_scl */
+/* 0x0548: i2c_drive_scl */
         0xf40036b0,
         0x07f1110b,
         0x04b607e0,
         0x0001d006,
         0x00f804bd,
-/* 0x0568: i2c_drive_scl_lo */
+/* 0x055c: i2c_drive_scl_lo */
         0x07e407f1,
         0xd00604b6,
         0x04bd0001,
-/* 0x0576: i2c_drive_sda */
+/* 0x056a: i2c_drive_sda */
         0x36b000f8,
         0x110bf400,
         0x07e007f1,
         0xd00604b6,
         0x04bd0002,
-/* 0x058a: i2c_drive_sda_lo */
+/* 0x057e: i2c_drive_sda_lo */
         0x07f100f8,
         0x04b607e4,
         0x0002d006,
         0x00f804bd,
-/* 0x0598: i2c_sense_scl */
+/* 0x058c: i2c_sense_scl */
         0xf10132f4,
         0xb607c437,
         0x33cf0634,
         0x0431fd00,
         0xf4060bf4,
-/* 0x05ae: i2c_sense_scl_done */
+/* 0x05a2: i2c_sense_scl_done */
         0x00f80131,
-/* 0x05b0: i2c_sense_sda */
+/* 0x05a4: i2c_sense_sda */
         0xf10132f4,
         0xb607c437,
         0x33cf0634,
         0x0432fd00,
         0xf4060bf4,
-/* 0x05c6: i2c_sense_sda_done */
+/* 0x05ba: i2c_sense_sda_done */
         0x00f80131,
-/* 0x05c8: i2c_raise_scl */
+/* 0x05bc: i2c_raise_scl */
         0x47f140f9,
         0x37f00898,
-        0x5421f501,
-/* 0x05d5: i2c_raise_scl_wait */
+        0x4821f501,
+/* 0x05c9: i2c_raise_scl_wait */
         0xe8e7f105,
         0x7f21f403,
-        0x059821f5,
+        0x058c21f5,
         0xb60901f4,
         0x1bf40142,
-/* 0x05e9: i2c_raise_scl_done */
+/* 0x05dd: i2c_raise_scl_done */
         0xf840fcef,
-/* 0x05ed: i2c_start */
-        0x9821f500,
+/* 0x05e1: i2c_start */
+        0x8c21f500,
         0x0d11f405,
-        0x05b021f5,
+        0x05a421f5,
         0xf40611f4,
-/* 0x05fe: i2c_start_rep */
+/* 0x05f2: i2c_start_rep */
         0x37f0300e,
-        0x5421f500,
+        0x4821f500,
         0x0137f005,
-        0x057621f5,
+        0x056a21f5,
         0xb60076bb,
         0x50f90465,
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0xc821f550,
+        0xbc21f550,
         0x0464b605,
-/* 0x062b: i2c_start_send */
+/* 0x061f: i2c_start_send */
         0xf01f11f4,
         0x21f50037,
-        0xe7f10576,
+        0xe7f1056a,
         0x21f41388,
         0x0037f07f,
-        0x055421f5,
+        0x054821f5,
         0x1388e7f1,
-/* 0x0647: i2c_start_out */
+/* 0x063b: i2c_start_out */
         0xf87f21f4,
-/* 0x0649: i2c_stop */
+/* 0x063d: i2c_stop */
         0x0037f000,
-        0x055421f5,
+        0x054821f5,
         0xf50037f0,
-        0xf1057621,
+        0xf1056a21,
         0xf403e8e7,
         0x37f07f21,
-        0x5421f501,
+        0x4821f501,
         0x88e7f105,
         0x7f21f413,
         0xf50137f0,
-        0xf1057621,
+        0xf1056a21,
         0xf41388e7,
         0x00f87f21,
-/* 0x067c: i2c_bitw */
-        0x057621f5,
+/* 0x0670: i2c_bitw */
+        0x056a21f5,
         0x03e8e7f1,
         0xbb7f21f4,
         0x65b60076,
@@ -1341,18 +1338,18 @@ uint32_t nva3_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x05c821f5,
+        0x05bc21f5,
         0xf40464b6,
         0xe7f11811,
         0x21f41388,
         0x0037f07f,
-        0x055421f5,
+        0x054821f5,
         0x1388e7f1,
-/* 0x06bb: i2c_bitw_out */
+/* 0x06af: i2c_bitw_out */
         0xf87f21f4,
-/* 0x06bd: i2c_bitr */
+/* 0x06b1: i2c_bitr */
         0x0137f000,
-        0x057621f5,
+        0x056a21f5,
         0x03e8e7f1,
         0xbb7f21f4,
         0x65b60076,
@@ -1360,19 +1357,19 @@ uint32_t nva3_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x05c821f5,
+        0x05bc21f5,
         0xf40464b6,
         0x21f51b11,
-        0x37f005b0,
-        0x5421f500,
+        0x37f005a4,
+        0x4821f500,
         0x88e7f105,
         0x7f21f413,
         0xf4013cf0,
-/* 0x0702: i2c_bitr_done */
+/* 0x06f6: i2c_bitr_done */
         0x00f80131,
-/* 0x0704: i2c_get_byte */
+/* 0x06f8: i2c_get_byte */
         0xf00057f0,
-/* 0x070a: i2c_get_byte_next */
+/* 0x06fe: i2c_get_byte_next */
         0x54b60847,
         0x0076bb01,
         0xf90465b6,
@@ -1380,7 +1377,7 @@ uint32_t nva3_pwr_code[] = {
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b606bd,
+        0x64b606b1,
         0x2b11f404,
         0xb60553fd,
         0x1bf40142,
@@ -1390,12 +1387,12 @@ uint32_t nva3_pwr_code[] = {
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0x7c21f550,
+        0x7021f550,
         0x0464b606,
-/* 0x0754: i2c_get_byte_done */
-/* 0x0756: i2c_put_byte */
+/* 0x0748: i2c_get_byte_done */
+/* 0x074a: i2c_put_byte */
         0x47f000f8,
-/* 0x0759: i2c_put_byte_next */
+/* 0x074d: i2c_put_byte_next */
         0x0142b608,
         0xbb3854ff,
         0x65b60076,
@@ -1403,7 +1400,7 @@ uint32_t nva3_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x067c21f5,
+        0x067021f5,
         0xf40464b6,
         0x46b03411,
         0xd81bf400,
@@ -1412,21 +1409,21 @@ uint32_t nva3_pwr_code[] = {
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0xbd21f550,
+        0xb121f550,
         0x0464b606,
         0xbb0f11f4,
         0x36b00076,
         0x061bf401,
-/* 0x07af: i2c_put_byte_done */
+/* 0x07a3: i2c_put_byte_done */
         0xf80132f4,
-/* 0x07b1: i2c_addr */
+/* 0x07a5: i2c_addr */
         0x0076bb00,
         0xf90465b6,
         0x04659450,
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b605ed,
+        0x64b605e1,
         0x2911f404,
         0x012ec3e7,
         0xfd0134b6,
@@ -1436,24 +1433,24 @@ uint32_t nva3_pwr_code[] = {
         0x0256bb04,
         0x75fd50bd,
         0xf550fc04,
-        0xb6075621,
-/* 0x07f6: i2c_addr_done */
+        0xb6074a21,
+/* 0x07ea: i2c_addr_done */
         0x00f80464,
-/* 0x07f8: i2c_acquire_addr */
+/* 0x07ec: i2c_acquire_addr */
         0xb6f8cec7,
         0xe0b702e4,
         0xee980bfc,
-/* 0x0807: i2c_acquire */
+/* 0x07fb: i2c_acquire */
         0xf500f800,
-        0xf407f821,
+        0xf407ec21,
         0xd9f00421,
         0x3f21f403,
-/* 0x0816: i2c_release */
+/* 0x080a: i2c_release */
         0x21f500f8,
-        0x21f407f8,
+        0x21f407ec,
         0x03daf004,
         0xf83f21f4,
-/* 0x0825: i2c_recv */
+/* 0x0819: i2c_recv */
         0x0132f400,
         0xb6f8c1c7,
         0x16b00214,
@@ -1472,7 +1469,7 @@ uint32_t nva3_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x080721f5,
+        0x07fb21f5,
         0xfc0464b6,
         0x00d6b0d0,
         0x00b31bf5,
@@ -1482,7 +1479,7 @@ uint32_t nva3_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x07b121f5,
+        0x07a521f5,
         0xf50464b6,
         0xc700d011,
         0x76bbe0c5,
@@ -1491,7 +1488,7 @@ uint32_t nva3_pwr_code[] = {
         0x0256bb04,
         0x75fd50bd,
         0xf550fc04,
-        0xb6075621,
+        0xb6074a21,
         0x11f50464,
         0x57f000ad,
         0x0076bb01,
@@ -1500,7 +1497,7 @@ uint32_t nva3_pwr_code[] = {
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b607b1,
+        0x64b607a5,
         0x8a11f504,
         0x0076bb00,
         0xf90465b6,
@@ -1508,7 +1505,7 @@ uint32_t nva3_pwr_code[] = {
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b60704,
+        0x64b606f8,
         0x6a11f404,
         0xbbe05bcb,
         0x65b60076,
@@ -1516,38 +1513,38 @@ uint32_t nva3_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x064921f5,
+        0x063d21f5,
         0xb90464b6,
         0x74bd025b,
-/* 0x092b: i2c_recv_not_rd08 */
+/* 0x091f: i2c_recv_not_rd08 */
         0xb0430ef4,
         0x1bf401d6,
         0x0057f03d,
-        0x07b121f5,
+        0x07a521f5,
         0xc73311f4,
         0x21f5e0c5,
-        0x11f40756,
+        0x11f4074a,
         0x0057f029,
-        0x07b121f5,
+        0x07a521f5,
         0xc71f11f4,
         0x21f5e0b5,
-        0x11f40756,
-        0x4921f515,
+        0x11f4074a,
+        0x3d21f515,
         0xc774bd06,
         0x1bf408c5,
         0x0232f409,
-/* 0x096b: i2c_recv_not_wr08 */
-/* 0x096b: i2c_recv_done */
+/* 0x095f: i2c_recv_not_wr08 */
+/* 0x095f: i2c_recv_done */
         0xc7030ef4,
         0x21f5f8ce,
-        0xe0fc0816,
+        0xe0fc080a,
         0x12f4d0fc,
         0x027cb90a,
         0x02b921f5,
-/* 0x0980: i2c_recv_exit */
-/* 0x0982: i2c_init */
+/* 0x0974: i2c_recv_exit */
+/* 0x0976: i2c_init */
         0x00f800f8,
-/* 0x0984: test_recv */
+/* 0x0978: test_recv */
         0x05d817f1,
         0xcf0614b6,
         0x10b60011,
@@ -1557,12 +1554,12 @@ uint32_t nva3_pwr_code[] = {
         0x00e7f104,
         0x4fe3f1d9,
         0xf521f513,
-/* 0x09ab: test_init */
+/* 0x099f: test_init */
         0xf100f801,
         0xf50800e7,
         0xf801f521,
-/* 0x09b5: idle_recv */
-/* 0x09b7: idle */
+/* 0x09a9: idle_recv */
+/* 0x09ab: idle */
         0xf400f800,
         0x17f10031,
         0x14b605d4,
@@ -1570,20 +1567,23 @@ uint32_t nva3_pwr_code[] = {
         0xf10110b6,
         0xb605d407,
         0x01d00604,
-/* 0x09d3: idle_loop */
+/* 0x09c7: idle_loop */
         0xf004bd00,
         0x32f45817,
-/* 0x09d9: idle_proc */
-/* 0x09d9: idle_proc_exec */
+/* 0x09cd: idle_proc */
+/* 0x09cd: idle_proc_exec */
         0xb910f902,
         0x21f5021e,
         0x10fc02c2,
         0xf40911f4,
         0x0ef40231,
-/* 0x09ed: idle_proc_next */
+/* 0x09e1: idle_proc_next */
         0x5810b6ef,
         0xf4061fb8,
         0x02f4e61b,
         0x0028f4dd,
         0x00bb0ef4,
+        0x00000000,
+        0x00000000,
+        0x00000000,
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h
index 7ac87405d01b..0773ff0e3dc3 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h
@@ -46,8 +46,8 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x584d454d,
-        0x0000054e,
-        0x00000540,
+        0x00000542,
+        0x00000534,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -68,8 +68,8 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x46524550,
-        0x00000552,
-        0x00000550,
+        0x00000546,
+        0x00000544,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -90,8 +90,8 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x5f433249,
-        0x00000982,
-        0x00000825,
+        0x00000976,
+        0x00000819,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -112,8 +112,8 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x54534554,
-        0x000009ab,
-        0x00000984,
+        0x0000099f,
+        0x00000978,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -134,8 +134,8 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x454c4449,
-        0x000009b7,
-        0x000009b5,
+        0x000009ab,
+        0x000009a9,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -239,10 +239,10 @@ uint32_t nvc0_pwr_data[] = {
         0x000004b7,
         0x00040003,
         0x00000000,
-        0x000004df,
+        0x000004d3,
         0x00010004,
         0x00000000,
-        0x000004fc,
+        0x000004f0,
 /* 0x03ac: memx_func_tail */
 /* 0x03ac: memx_data_head */
         0x00000000,
@@ -1198,13 +1198,10 @@ uint32_t nvc0_pwr_code[] = {
         0x0810b601,
         0x50f960f9,
         0xe0fcd0fc,
-        0xf13f21f4,
-        0xfd140003,
-        0x05800506,
-        0xb604bd00,
+        0xb63f21f4,
         0x1bf40242,
-/* 0x04df: memx_func_wait */
-        0xf000f8dd,
+/* 0x04d3: memx_func_wait */
+        0xf000f8e9,
         0x84b62c87,
         0x0088cf06,
         0x98001e98,
@@ -1212,14 +1209,14 @@ uint32_t nvc0_pwr_code[] = {
         0x031b9802,
         0xf41010b6,
         0x00f89c21,
-/* 0x04fc: memx_func_delay */
+/* 0x04f0: memx_func_delay */
         0xb6001e98,
         0x21f40410,
-/* 0x0507: memx_exec */
+/* 0x04fb: memx_exec */
         0xf900f87f,
         0xb9d0f9e0,
         0xb2b902c1,
-/* 0x0511: memx_exec_next */
+/* 0x0505: memx_exec_next */
         0x00139802,
         0x950410b6,
         0x30f01034,
@@ -1228,112 +1225,112 @@ uint32_t nvc0_pwr_code[] = {
         0xec1ef406,
         0xe0fcd0fc,
         0x02b921f5,
-/* 0x0532: memx_info */
+/* 0x0526: memx_info */
         0xc7f100f8,
         0xb7f103ac,
         0x21f50800,
         0x00f802b9,
-/* 0x0540: memx_recv */
+/* 0x0534: memx_recv */
         0xf401d6b0,
         0xd6b0c40b,
         0xe90bf400,
-/* 0x054e: memx_init */
+/* 0x0542: memx_init */
         0x00f800f8,
-/* 0x0550: perf_recv */
-/* 0x0552: perf_init */
+/* 0x0544: perf_recv */
+/* 0x0546: perf_init */
         0x00f800f8,
-/* 0x0554: i2c_drive_scl */
+/* 0x0548: i2c_drive_scl */
         0xf40036b0,
         0x07f1110b,
         0x04b607e0,
         0x0001d006,
         0x00f804bd,
-/* 0x0568: i2c_drive_scl_lo */
+/* 0x055c: i2c_drive_scl_lo */
         0x07e407f1,
         0xd00604b6,
         0x04bd0001,
-/* 0x0576: i2c_drive_sda */
+/* 0x056a: i2c_drive_sda */
         0x36b000f8,
         0x110bf400,
         0x07e007f1,
         0xd00604b6,
         0x04bd0002,
-/* 0x058a: i2c_drive_sda_lo */
+/* 0x057e: i2c_drive_sda_lo */
         0x07f100f8,
         0x04b607e4,
         0x0002d006,
         0x00f804bd,
-/* 0x0598: i2c_sense_scl */
+/* 0x058c: i2c_sense_scl */
         0xf10132f4,
         0xb607c437,
         0x33cf0634,
         0x0431fd00,
         0xf4060bf4,
-/* 0x05ae: i2c_sense_scl_done */
+/* 0x05a2: i2c_sense_scl_done */
         0x00f80131,
-/* 0x05b0: i2c_sense_sda */
+/* 0x05a4: i2c_sense_sda */
         0xf10132f4,
         0xb607c437,
         0x33cf0634,
         0x0432fd00,
         0xf4060bf4,
-/* 0x05c6: i2c_sense_sda_done */
+/* 0x05ba: i2c_sense_sda_done */
         0x00f80131,
-/* 0x05c8: i2c_raise_scl */
+/* 0x05bc: i2c_raise_scl */
         0x47f140f9,
         0x37f00898,
-        0x5421f501,
-/* 0x05d5: i2c_raise_scl_wait */
+        0x4821f501,
+/* 0x05c9: i2c_raise_scl_wait */
         0xe8e7f105,
         0x7f21f403,
-        0x059821f5,
+        0x058c21f5,
         0xb60901f4,
         0x1bf40142,
-/* 0x05e9: i2c_raise_scl_done */
+/* 0x05dd: i2c_raise_scl_done */
         0xf840fcef,
-/* 0x05ed: i2c_start */
-        0x9821f500,
+/* 0x05e1: i2c_start */
+        0x8c21f500,
         0x0d11f405,
-        0x05b021f5,
+        0x05a421f5,
         0xf40611f4,
-/* 0x05fe: i2c_start_rep */
+/* 0x05f2: i2c_start_rep */
         0x37f0300e,
-        0x5421f500,
+        0x4821f500,
         0x0137f005,
-        0x057621f5,
+        0x056a21f5,
         0xb60076bb,
         0x50f90465,
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0xc821f550,
+        0xbc21f550,
         0x0464b605,
-/* 0x062b: i2c_start_send */
+/* 0x061f: i2c_start_send */
         0xf01f11f4,
         0x21f50037,
-        0xe7f10576,
+        0xe7f1056a,
         0x21f41388,
         0x0037f07f,
-        0x055421f5,
+        0x054821f5,
         0x1388e7f1,
-/* 0x0647: i2c_start_out */
+/* 0x063b: i2c_start_out */
         0xf87f21f4,
-/* 0x0649: i2c_stop */
+/* 0x063d: i2c_stop */
         0x0037f000,
-        0x055421f5,
+        0x054821f5,
         0xf50037f0,
-        0xf1057621,
+        0xf1056a21,
         0xf403e8e7,
         0x37f07f21,
-        0x5421f501,
+        0x4821f501,
         0x88e7f105,
         0x7f21f413,
         0xf50137f0,
-        0xf1057621,
+        0xf1056a21,
         0xf41388e7,
         0x00f87f21,
-/* 0x067c: i2c_bitw */
-        0x057621f5,
+/* 0x0670: i2c_bitw */
+        0x056a21f5,
         0x03e8e7f1,
         0xbb7f21f4,
         0x65b60076,
@@ -1341,18 +1338,18 @@ uint32_t nvc0_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x05c821f5,
+        0x05bc21f5,
         0xf40464b6,
         0xe7f11811,
         0x21f41388,
         0x0037f07f,
-        0x055421f5,
+        0x054821f5,
         0x1388e7f1,
-/* 0x06bb: i2c_bitw_out */
+/* 0x06af: i2c_bitw_out */
         0xf87f21f4,
-/* 0x06bd: i2c_bitr */
+/* 0x06b1: i2c_bitr */
         0x0137f000,
-        0x057621f5,
+        0x056a21f5,
         0x03e8e7f1,
         0xbb7f21f4,
         0x65b60076,
@@ -1360,19 +1357,19 @@ uint32_t nvc0_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x05c821f5,
+        0x05bc21f5,
         0xf40464b6,
         0x21f51b11,
-        0x37f005b0,
-        0x5421f500,
+        0x37f005a4,
+        0x4821f500,
         0x88e7f105,
         0x7f21f413,
         0xf4013cf0,
-/* 0x0702: i2c_bitr_done */
+/* 0x06f6: i2c_bitr_done */
         0x00f80131,
-/* 0x0704: i2c_get_byte */
+/* 0x06f8: i2c_get_byte */
         0xf00057f0,
-/* 0x070a: i2c_get_byte_next */
+/* 0x06fe: i2c_get_byte_next */
         0x54b60847,
         0x0076bb01,
         0xf90465b6,
@@ -1380,7 +1377,7 @@ uint32_t nvc0_pwr_code[] = {
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b606bd,
+        0x64b606b1,
         0x2b11f404,
         0xb60553fd,
         0x1bf40142,
@@ -1390,12 +1387,12 @@ uint32_t nvc0_pwr_code[] = {
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0x7c21f550,
+        0x7021f550,
         0x0464b606,
-/* 0x0754: i2c_get_byte_done */
-/* 0x0756: i2c_put_byte */
+/* 0x0748: i2c_get_byte_done */
+/* 0x074a: i2c_put_byte */
         0x47f000f8,
-/* 0x0759: i2c_put_byte_next */
+/* 0x074d: i2c_put_byte_next */
         0x0142b608,
         0xbb3854ff,
         0x65b60076,
@@ -1403,7 +1400,7 @@ uint32_t nvc0_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x067c21f5,
+        0x067021f5,
         0xf40464b6,
         0x46b03411,
         0xd81bf400,
@@ -1412,21 +1409,21 @@ uint32_t nvc0_pwr_code[] = {
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0xbd21f550,
+        0xb121f550,
         0x0464b606,
         0xbb0f11f4,
         0x36b00076,
         0x061bf401,
-/* 0x07af: i2c_put_byte_done */
+/* 0x07a3: i2c_put_byte_done */
         0xf80132f4,
-/* 0x07b1: i2c_addr */
+/* 0x07a5: i2c_addr */
         0x0076bb00,
         0xf90465b6,
         0x04659450,
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b605ed,
+        0x64b605e1,
         0x2911f404,
         0x012ec3e7,
         0xfd0134b6,
@@ -1436,24 +1433,24 @@ uint32_t nvc0_pwr_code[] = {
         0x0256bb04,
         0x75fd50bd,
         0xf550fc04,
-        0xb6075621,
-/* 0x07f6: i2c_addr_done */
+        0xb6074a21,
+/* 0x07ea: i2c_addr_done */
         0x00f80464,
-/* 0x07f8: i2c_acquire_addr */
+/* 0x07ec: i2c_acquire_addr */
         0xb6f8cec7,
         0xe0b702e4,
         0xee980bfc,
-/* 0x0807: i2c_acquire */
+/* 0x07fb: i2c_acquire */
         0xf500f800,
-        0xf407f821,
+        0xf407ec21,
         0xd9f00421,
         0x3f21f403,
-/* 0x0816: i2c_release */
+/* 0x080a: i2c_release */
         0x21f500f8,
-        0x21f407f8,
+        0x21f407ec,
         0x03daf004,
         0xf83f21f4,
-/* 0x0825: i2c_recv */
+/* 0x0819: i2c_recv */
         0x0132f400,
         0xb6f8c1c7,
         0x16b00214,
@@ -1472,7 +1469,7 @@ uint32_t nvc0_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x080721f5,
+        0x07fb21f5,
         0xfc0464b6,
         0x00d6b0d0,
         0x00b31bf5,
@@ -1482,7 +1479,7 @@ uint32_t nvc0_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x07b121f5,
+        0x07a521f5,
         0xf50464b6,
         0xc700d011,
         0x76bbe0c5,
@@ -1491,7 +1488,7 @@ uint32_t nvc0_pwr_code[] = {
         0x0256bb04,
         0x75fd50bd,
         0xf550fc04,
-        0xb6075621,
+        0xb6074a21,
         0x11f50464,
         0x57f000ad,
         0x0076bb01,
@@ -1500,7 +1497,7 @@ uint32_t nvc0_pwr_code[] = {
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b607b1,
+        0x64b607a5,
         0x8a11f504,
         0x0076bb00,
         0xf90465b6,
@@ -1508,7 +1505,7 @@ uint32_t nvc0_pwr_code[] = {
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b60704,
+        0x64b606f8,
         0x6a11f404,
         0xbbe05bcb,
         0x65b60076,
@@ -1516,38 +1513,38 @@ uint32_t nvc0_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x064921f5,
+        0x063d21f5,
         0xb90464b6,
         0x74bd025b,
-/* 0x092b: i2c_recv_not_rd08 */
+/* 0x091f: i2c_recv_not_rd08 */
         0xb0430ef4,
         0x1bf401d6,
         0x0057f03d,
-        0x07b121f5,
+        0x07a521f5,
         0xc73311f4,
         0x21f5e0c5,
-        0x11f40756,
+        0x11f4074a,
         0x0057f029,
-        0x07b121f5,
+        0x07a521f5,
         0xc71f11f4,
         0x21f5e0b5,
-        0x11f40756,
-        0x4921f515,
+        0x11f4074a,
+        0x3d21f515,
         0xc774bd06,
         0x1bf408c5,
         0x0232f409,
-/* 0x096b: i2c_recv_not_wr08 */
-/* 0x096b: i2c_recv_done */
+/* 0x095f: i2c_recv_not_wr08 */
+/* 0x095f: i2c_recv_done */
         0xc7030ef4,
         0x21f5f8ce,
-        0xe0fc0816,
+        0xe0fc080a,
         0x12f4d0fc,
         0x027cb90a,
         0x02b921f5,
-/* 0x0980: i2c_recv_exit */
-/* 0x0982: i2c_init */
+/* 0x0974: i2c_recv_exit */
+/* 0x0976: i2c_init */
         0x00f800f8,
-/* 0x0984: test_recv */
+/* 0x0978: test_recv */
         0x05d817f1,
         0xcf0614b6,
         0x10b60011,
@@ -1557,12 +1554,12 @@ uint32_t nvc0_pwr_code[] = {
         0x00e7f104,
         0x4fe3f1d9,
         0xf521f513,
-/* 0x09ab: test_init */
+/* 0x099f: test_init */
         0xf100f801,
         0xf50800e7,
         0xf801f521,
-/* 0x09b5: idle_recv */
-/* 0x09b7: idle */
+/* 0x09a9: idle_recv */
+/* 0x09ab: idle */
         0xf400f800,
         0x17f10031,
         0x14b605d4,
@@ -1570,20 +1567,23 @@ uint32_t nvc0_pwr_code[] = {
         0xf10110b6,
         0xb605d407,
         0x01d00604,
-/* 0x09d3: idle_loop */
+/* 0x09c7: idle_loop */
         0xf004bd00,
         0x32f45817,
-/* 0x09d9: idle_proc */
-/* 0x09d9: idle_proc_exec */
+/* 0x09cd: idle_proc */
+/* 0x09cd: idle_proc_exec */
         0xb910f902,
         0x21f5021e,
         0x10fc02c2,
         0xf40911f4,
         0x0ef40231,
-/* 0x09ed: idle_proc_next */
+/* 0x09e1: idle_proc_next */
         0x5810b6ef,
         0xf4061fb8,
         0x02f4e61b,
         0x0028f4dd,
         0x00bb0ef4,
+        0x00000000,
+        0x00000000,
+        0x00000000,
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h
index cd9ff1a73284..8d369b3faaba 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h
@@ -46,8 +46,8 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x584d454d,
-        0x000004c4,
-        0x000004b6,
+        0x000004b8,
+        0x000004aa,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -68,8 +68,8 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x46524550,
-        0x000004c8,
-        0x000004c6,
+        0x000004bc,
+        0x000004ba,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -90,8 +90,8 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x5f433249,
-        0x000008e3,
-        0x00000786,
+        0x000008d7,
+        0x0000077a,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -112,8 +112,8 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x54534554,
-        0x00000906,
-        0x000008e5,
+        0x000008fa,
+        0x000008d9,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -134,8 +134,8 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x454c4449,
-        0x00000912,
-        0x00000910,
+        0x00000906,
+        0x00000904,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -239,10 +239,10 @@ uint32_t nvd0_pwr_data[] = {
         0x00000430,
         0x00040003,
         0x00000000,
-        0x00000458,
+        0x0000044c,
         0x00010004,
         0x00000000,
-        0x00000472,
+        0x00000466,
 /* 0x03ac: memx_func_tail */
 /* 0x03ac: memx_data_head */
         0x00000000,
@@ -1100,26 +1100,23 @@ uint32_t nvd0_pwr_code[] = {
         0xf960f908,
         0xfcd0fc50,
         0x3321f4e0,
-        0x140003f1,
-        0x800506fd,
-        0x04bd0005,
         0xf40242b6,
-        0x00f8dd1b,
-/* 0x0458: memx_func_wait */
+        0x00f8e91b,
+/* 0x044c: memx_func_wait */
         0xcf2c87f0,
         0x1e980088,
         0x011d9800,
         0x98021c98,
         0x10b6031b,
         0x7e21f410,
-/* 0x0472: memx_func_delay */
+/* 0x0466: memx_func_delay */
         0x1e9800f8,
         0x0410b600,
         0xf86721f4,
-/* 0x047d: memx_exec */
+/* 0x0471: memx_exec */
         0xf9e0f900,
         0x02c1b9d0,
-/* 0x0487: memx_exec_next */
+/* 0x047b: memx_exec_next */
         0x9802b2b9,
         0x10b60013,
         0x10349504,
@@ -1129,107 +1126,107 @@ uint32_t nvd0_pwr_code[] = {
         0xd0fcec1e,
         0x21f5e0fc,
         0x00f8026b,
-/* 0x04a8: memx_info */
+/* 0x049c: memx_info */
         0x03acc7f1,
         0x0800b7f1,
         0x026b21f5,
-/* 0x04b6: memx_recv */
+/* 0x04aa: memx_recv */
         0xd6b000f8,
         0xc40bf401,
         0xf400d6b0,
         0x00f8e90b,
-/* 0x04c4: memx_init */
-/* 0x04c6: perf_recv */
+/* 0x04b8: memx_init */
+/* 0x04ba: perf_recv */
         0x00f800f8,
-/* 0x04c8: perf_init */
-/* 0x04ca: i2c_drive_scl */
+/* 0x04bc: perf_init */
+/* 0x04be: i2c_drive_scl */
         0x36b000f8,
         0x0e0bf400,
         0x07e007f1,
         0xbd0001d0,
-/* 0x04db: i2c_drive_scl_lo */
+/* 0x04cf: i2c_drive_scl_lo */
         0xf100f804,
         0xd007e407,
         0x04bd0001,
-/* 0x04e6: i2c_drive_sda */
+/* 0x04da: i2c_drive_sda */
         0x36b000f8,
         0x0e0bf400,
         0x07e007f1,
         0xbd0002d0,
-/* 0x04f7: i2c_drive_sda_lo */
+/* 0x04eb: i2c_drive_sda_lo */
         0xf100f804,
         0xd007e407,
         0x04bd0002,
-/* 0x0502: i2c_sense_scl */
+/* 0x04f6: i2c_sense_scl */
         0x32f400f8,
         0xc437f101,
         0x0033cf07,
         0xf40431fd,
         0x31f4060b,
-/* 0x0515: i2c_sense_scl_done */
-/* 0x0517: i2c_sense_sda */
+/* 0x0509: i2c_sense_scl_done */
+/* 0x050b: i2c_sense_sda */
         0xf400f801,
         0x37f10132,
         0x33cf07c4,
         0x0432fd00,
         0xf4060bf4,
-/* 0x052a: i2c_sense_sda_done */
+/* 0x051e: i2c_sense_sda_done */
         0x00f80131,
-/* 0x052c: i2c_raise_scl */
+/* 0x0520: i2c_raise_scl */
         0x47f140f9,
         0x37f00898,
-        0xca21f501,
-/* 0x0539: i2c_raise_scl_wait */
+        0xbe21f501,
+/* 0x052d: i2c_raise_scl_wait */
         0xe8e7f104,
         0x6721f403,
-        0x050221f5,
+        0x04f621f5,
         0xb60901f4,
         0x1bf40142,
-/* 0x054d: i2c_raise_scl_done */
+/* 0x0541: i2c_raise_scl_done */
         0xf840fcef,
-/* 0x0551: i2c_start */
-        0x0221f500,
-        0x0d11f405,
-        0x051721f5,
+/* 0x0545: i2c_start */
+        0xf621f500,
+        0x0d11f404,
+        0x050b21f5,
         0xf40611f4,
-/* 0x0562: i2c_start_rep */
+/* 0x0556: i2c_start_rep */
         0x37f0300e,
-        0xca21f500,
+        0xbe21f500,
         0x0137f004,
-        0x04e621f5,
+        0x04da21f5,
         0xb60076bb,
         0x50f90465,
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0x2c21f550,
+        0x2021f550,
         0x0464b605,
-/* 0x058f: i2c_start_send */
+/* 0x0583: i2c_start_send */
         0xf01f11f4,
         0x21f50037,
-        0xe7f104e6,
+        0xe7f104da,
         0x21f41388,
         0x0037f067,
-        0x04ca21f5,
+        0x04be21f5,
         0x1388e7f1,
-/* 0x05ab: i2c_start_out */
+/* 0x059f: i2c_start_out */
         0xf86721f4,
-/* 0x05ad: i2c_stop */
+/* 0x05a1: i2c_stop */
         0x0037f000,
-        0x04ca21f5,
+        0x04be21f5,
         0xf50037f0,
-        0xf104e621,
+        0xf104da21,
         0xf403e8e7,
         0x37f06721,
-        0xca21f501,
+        0xbe21f501,
         0x88e7f104,
         0x6721f413,
         0xf50137f0,
-        0xf104e621,
+        0xf104da21,
         0xf41388e7,
         0x00f86721,
-/* 0x05e0: i2c_bitw */
-        0x04e621f5,
+/* 0x05d4: i2c_bitw */
+        0x04da21f5,
         0x03e8e7f1,
         0xbb6721f4,
         0x65b60076,
@@ -1237,18 +1234,18 @@ uint32_t nvd0_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x052c21f5,
+        0x052021f5,
         0xf40464b6,
         0xe7f11811,
         0x21f41388,
         0x0037f067,
-        0x04ca21f5,
+        0x04be21f5,
         0x1388e7f1,
-/* 0x061f: i2c_bitw_out */
+/* 0x0613: i2c_bitw_out */
         0xf86721f4,
-/* 0x0621: i2c_bitr */
+/* 0x0615: i2c_bitr */
         0x0137f000,
-        0x04e621f5,
+        0x04da21f5,
         0x03e8e7f1,
         0xbb6721f4,
         0x65b60076,
@@ -1256,19 +1253,19 @@ uint32_t nvd0_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x052c21f5,
+        0x052021f5,
         0xf40464b6,
         0x21f51b11,
-        0x37f00517,
-        0xca21f500,
+        0x37f0050b,
+        0xbe21f500,
         0x88e7f104,
         0x6721f413,
         0xf4013cf0,
-/* 0x0666: i2c_bitr_done */
+/* 0x065a: i2c_bitr_done */
         0x00f80131,
-/* 0x0668: i2c_get_byte */
+/* 0x065c: i2c_get_byte */
         0xf00057f0,
-/* 0x066e: i2c_get_byte_next */
+/* 0x0662: i2c_get_byte_next */
         0x54b60847,
         0x0076bb01,
         0xf90465b6,
@@ -1276,7 +1273,7 @@ uint32_t nvd0_pwr_code[] = {
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b60621,
+        0x64b60615,
         0x2b11f404,
         0xb60553fd,
         0x1bf40142,
@@ -1286,12 +1283,12 @@ uint32_t nvd0_pwr_code[] = {
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0xe021f550,
+        0xd421f550,
         0x0464b605,
-/* 0x06b8: i2c_get_byte_done */
-/* 0x06ba: i2c_put_byte */
+/* 0x06ac: i2c_get_byte_done */
+/* 0x06ae: i2c_put_byte */
         0x47f000f8,
-/* 0x06bd: i2c_put_byte_next */
+/* 0x06b1: i2c_put_byte_next */
         0x0142b608,
         0xbb3854ff,
         0x65b60076,
@@ -1299,7 +1296,7 @@ uint32_t nvd0_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x05e021f5,
+        0x05d421f5,
         0xf40464b6,
         0x46b03411,
         0xd81bf400,
@@ -1308,21 +1305,21 @@ uint32_t nvd0_pwr_code[] = {
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0x2121f550,
+        0x1521f550,
         0x0464b606,
         0xbb0f11f4,
         0x36b00076,
         0x061bf401,
-/* 0x0713: i2c_put_byte_done */
+/* 0x0707: i2c_put_byte_done */
         0xf80132f4,
-/* 0x0715: i2c_addr */
+/* 0x0709: i2c_addr */
         0x0076bb00,
         0xf90465b6,
         0x04659450,
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b60551,
+        0x64b60545,
         0x2911f404,
         0x012ec3e7,
         0xfd0134b6,
@@ -1332,23 +1329,23 @@ uint32_t nvd0_pwr_code[] = {
         0x0256bb04,
         0x75fd50bd,
         0xf550fc04,
-        0xb606ba21,
-/* 0x075a: i2c_addr_done */
+        0xb606ae21,
+/* 0x074e: i2c_addr_done */
         0x00f80464,
-/* 0x075c: i2c_acquire_addr */
+/* 0x0750: i2c_acquire_addr */
         0xb6f8cec7,
         0xe0b705e4,
         0x00f8d014,
-/* 0x0768: i2c_acquire */
-        0x075c21f5,
+/* 0x075c: i2c_acquire */
+        0x075021f5,
         0xf00421f4,
         0x21f403d9,
-/* 0x0777: i2c_release */
+/* 0x076b: i2c_release */
         0xf500f833,
-        0xf4075c21,
+        0xf4075021,
         0xdaf00421,
         0x3321f403,
-/* 0x0786: i2c_recv */
+/* 0x077a: i2c_recv */
         0x32f400f8,
         0xf8c1c701,
         0xb00214b6,
@@ -1367,7 +1364,7 @@ uint32_t nvd0_pwr_code[] = {
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0x6821f550,
+        0x5c21f550,
         0x0464b607,
         0xd6b0d0fc,
         0xb31bf500,
@@ -1377,7 +1374,7 @@ uint32_t nvd0_pwr_code[] = {
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0x1521f550,
+        0x0921f550,
         0x0464b607,
         0x00d011f5,
         0xbbe0c5c7,
@@ -1386,7 +1383,7 @@ uint32_t nvd0_pwr_code[] = {
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x06ba21f5,
+        0x06ae21f5,
         0xf50464b6,
         0xf000ad11,
         0x76bb0157,
@@ -1395,7 +1392,7 @@ uint32_t nvd0_pwr_code[] = {
         0x0256bb04,
         0x75fd50bd,
         0xf550fc04,
-        0xb6071521,
+        0xb6070921,
         0x11f50464,
         0x76bb008a,
         0x0465b600,
@@ -1403,7 +1400,7 @@ uint32_t nvd0_pwr_code[] = {
         0x0256bb04,
         0x75fd50bd,
         0xf550fc04,
-        0xb6066821,
+        0xb6065c21,
         0x11f40464,
         0xe05bcb6a,
         0xb60076bb,
@@ -1411,38 +1408,38 @@ uint32_t nvd0_pwr_code[] = {
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0xad21f550,
+        0xa121f550,
         0x0464b605,
         0xbd025bb9,
         0x430ef474,
-/* 0x088c: i2c_recv_not_rd08 */
+/* 0x0880: i2c_recv_not_rd08 */
         0xf401d6b0,
         0x57f03d1b,
-        0x1521f500,
+        0x0921f500,
         0x3311f407,
         0xf5e0c5c7,
-        0xf406ba21,
+        0xf406ae21,
         0x57f02911,
-        0x1521f500,
+        0x0921f500,
         0x1f11f407,
         0xf5e0b5c7,
-        0xf406ba21,
+        0xf406ae21,
         0x21f51511,
-        0x74bd05ad,
+        0x74bd05a1,
         0xf408c5c7,
         0x32f4091b,
         0x030ef402,
-/* 0x08cc: i2c_recv_not_wr08 */
-/* 0x08cc: i2c_recv_done */
+/* 0x08c0: i2c_recv_not_wr08 */
+/* 0x08c0: i2c_recv_done */
         0xf5f8cec7,
-        0xfc077721,
+        0xfc076b21,
         0xf4d0fce0,
         0x7cb90a12,
         0x6b21f502,
-/* 0x08e1: i2c_recv_exit */
-/* 0x08e3: i2c_init */
+/* 0x08d5: i2c_recv_exit */
+/* 0x08d7: i2c_init */
         0xf800f802,
-/* 0x08e5: test_recv */
+/* 0x08d9: test_recv */
         0xd817f100,
         0x0011cf05,
         0xf10110b6,
@@ -1451,28 +1448,28 @@ uint32_t nvd0_pwr_code[] = {
         0xd900e7f1,
         0x134fe3f1,
         0x01b621f5,
-/* 0x0906: test_init */
+/* 0x08fa: test_init */
         0xe7f100f8,
         0x21f50800,
         0x00f801b6,
-/* 0x0910: idle_recv */
-/* 0x0912: idle */
+/* 0x0904: idle_recv */
+/* 0x0906: idle */
         0x31f400f8,
         0xd417f100,
         0x0011cf05,
         0xf10110b6,
         0xd005d407,
         0x04bd0001,
-/* 0x0928: idle_loop */
+/* 0x091c: idle_loop */
         0xf45817f0,
-/* 0x092e: idle_proc */
-/* 0x092e: idle_proc_exec */
+/* 0x0922: idle_proc */
+/* 0x0922: idle_proc_exec */
         0x10f90232,
         0xf5021eb9,
         0xfc027421,
         0x0911f410,
         0xf40231f4,
-/* 0x0942: idle_proc_next */
+/* 0x0936: idle_proc_next */
         0x10b6ef0e,
         0x061fb858,
         0xf4e61bf4,
@@ -1521,4 +1518,7 @@ uint32_t nvd0_pwr_code[] = {
         0x00000000,
         0x00000000,
         0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/gk104.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/gk104.c
new file mode 100644
index 000000000000..d76612999b9f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/gk104.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "priv.h"
+
+#define nvd0_pwr_code gk104_pwr_code
+#define nvd0_pwr_data gk104_pwr_data
+#include "fuc/nvd0.fuc.h"
+
+static void
+gk104_pwr_pgob(struct nouveau_pwr *ppwr, bool enable)
+{
+        nv_mask(ppwr, 0x000200, 0x00001000, 0x00000000);
+        nv_rd32(ppwr, 0x000200);
+        nv_mask(ppwr, 0x000200, 0x08000000, 0x08000000);
+        msleep(50);
+
+        nv_mask(ppwr, 0x10a78c, 0x00000002, 0x00000002);
+        nv_mask(ppwr, 0x10a78c, 0x00000001, 0x00000001);
+        nv_mask(ppwr, 0x10a78c, 0x00000001, 0x00000000);
+
+        nv_mask(ppwr, 0x020004, 0xc0000000, enable ? 0xc0000000 : 0x40000000);
+        msleep(50);
+
+        nv_mask(ppwr, 0x10a78c, 0x00000002, 0x00000000);
+        nv_mask(ppwr, 0x10a78c, 0x00000001, 0x00000001);
+        nv_mask(ppwr, 0x10a78c, 0x00000001, 0x00000000);
+
+        nv_mask(ppwr, 0x000200, 0x08000000, 0x00000000);
+        nv_mask(ppwr, 0x000200, 0x00001000, 0x00001000);
+        nv_rd32(ppwr, 0x000200);
+}
+
+struct nouveau_oclass *
+gk104_pwr_oclass = &(struct nvkm_pwr_impl) {
+        .base.handle = NV_SUBDEV(PWR, 0xe4),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nouveau_pwr_ctor,
+                .dtor = _nouveau_pwr_dtor,
+                .init = _nouveau_pwr_init,
+                .fini = _nouveau_pwr_fini,
+        },
+        .code.data = gk104_pwr_code,
+        .code.size = sizeof(gk104_pwr_code),
+        .data.data = gk104_pwr_data,
+        .data.size = sizeof(gk104_pwr_data),
+        .pgob = gk104_pwr_pgob,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c
index 03de3107d29f..def6a9ac68cf 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c
@@ -1,8 +1,7 @@
 #ifndef __NVKM_PWR_MEMX_H__
 #define __NVKM_PWR_MEMX_H__
 
-#include <subdev/pwr.h>
-#include <subdev/pwr/fuc/os.h>
+#include "priv.h"
 
 struct nouveau_memx {
         struct nouveau_pwr *ppwr;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/nv108.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/nv108.c
index 52c85414866a..04ff7c3c34e9 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/nv108.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/nv108.c
@@ -22,41 +22,20 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/pwr.h>
-
+#include "priv.h"
 #include "fuc/nv108.fuc.h"
 
-struct nv108_pwr_priv {
-        struct nouveau_pwr base;
-};
-
-static int
-nv108_pwr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-               struct nouveau_oclass *oclass, void *data, u32 size,
-               struct nouveau_object **pobject)
-{
-        struct nv108_pwr_priv *priv;
-        int ret;
-
-        ret = nouveau_pwr_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.code.data = nv108_pwr_code;
-        priv->base.code.size = sizeof(nv108_pwr_code);
-        priv->base.data.data = nv108_pwr_data;
-        priv->base.data.size = sizeof(nv108_pwr_data);
-        return 0;
-}
-
-struct nouveau_oclass
-nv108_pwr_oclass = {
-        .handle = NV_SUBDEV(PWR, 0x00),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv108_pwr_ctor,
+struct nouveau_oclass *
+nv108_pwr_oclass = &(struct nvkm_pwr_impl) {
+        .base.handle = NV_SUBDEV(PWR, 0x00),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nouveau_pwr_ctor,
                 .dtor = _nouveau_pwr_dtor,
                 .init = _nouveau_pwr_init,
                 .fini = _nouveau_pwr_fini,
         },
-};
+        .code.data = nv108_pwr_code,
+        .code.size = sizeof(nv108_pwr_code),
+        .data.data = nv108_pwr_data,
+        .data.size = sizeof(nv108_pwr_data),
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/nva3.c
index c132b7ca9747..998d53076b8b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/nva3.c
@@ -22,50 +22,29 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/pwr.h>
-
+#include "priv.h"
 #include "fuc/nva3.fuc.h"
 
-struct nva3_pwr_priv {
-        struct nouveau_pwr base;
-};
-
 static int
 nva3_pwr_init(struct nouveau_object *object)
 {
-        struct nva3_pwr_priv *priv = (void *)object;
-        nv_mask(priv, 0x022210, 0x00000001, 0x00000000);
-        nv_mask(priv, 0x022210, 0x00000001, 0x00000001);
-        return nouveau_pwr_init(&priv->base);
-}
-
-static int
-nva3_pwr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-              struct nouveau_oclass *oclass, void *data, u32 size,
-              struct nouveau_object **pobject)
-{
-        struct nva3_pwr_priv *priv;
-        int ret;
-
-        ret = nouveau_pwr_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.code.data = nva3_pwr_code;
-        priv->base.code.size = sizeof(nva3_pwr_code);
-        priv->base.data.data = nva3_pwr_data;
-        priv->base.data.size = sizeof(nva3_pwr_data);
-        return 0;
+        struct nouveau_pwr *ppwr = (void *)object;
+        nv_mask(ppwr, 0x022210, 0x00000001, 0x00000000);
+        nv_mask(ppwr, 0x022210, 0x00000001, 0x00000001);
+        return nouveau_pwr_init(ppwr);
 }
 
-struct nouveau_oclass
-nva3_pwr_oclass = {
-        .handle = NV_SUBDEV(PWR, 0xa3),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nva3_pwr_ctor,
+struct nouveau_oclass *
+nva3_pwr_oclass = &(struct nvkm_pwr_impl) {
+        .base.handle = NV_SUBDEV(PWR, 0xa3),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nouveau_pwr_ctor,
                 .dtor = _nouveau_pwr_dtor,
                 .init = nva3_pwr_init,
                 .fini = _nouveau_pwr_fini,
         },
-};
+        .code.data = nva3_pwr_code,
+        .code.size = sizeof(nva3_pwr_code),
+        .data.data = nva3_pwr_data,
+        .data.size = sizeof(nva3_pwr_data),
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/nvc0.c
index 495f6857428d..9a773e66efa4 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/nvc0.c
@@ -22,41 +22,20 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/pwr.h>
-
+#include "priv.h"
 #include "fuc/nvc0.fuc.h"
 
-struct nvc0_pwr_priv {
-        struct nouveau_pwr base;
-};
-
-static int
-nvc0_pwr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-              struct nouveau_oclass *oclass, void *data, u32 size,
-              struct nouveau_object **pobject)
-{
-        struct nvc0_pwr_priv *priv;
-        int ret;
-
-        ret = nouveau_pwr_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.code.data = nvc0_pwr_code;
-        priv->base.code.size = sizeof(nvc0_pwr_code);
-        priv->base.data.data = nvc0_pwr_data;
-        priv->base.data.size = sizeof(nvc0_pwr_data);
-        return 0;
-}
-
-struct nouveau_oclass
-nvc0_pwr_oclass = {
-        .handle = NV_SUBDEV(PWR, 0xc0),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nvc0_pwr_ctor,
+struct nouveau_oclass *
+nvc0_pwr_oclass = &(struct nvkm_pwr_impl) {
+        .base.handle = NV_SUBDEV(PWR, 0xc0),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nouveau_pwr_ctor,
                 .dtor = _nouveau_pwr_dtor,
                 .init = _nouveau_pwr_init,
                 .fini = _nouveau_pwr_fini,
         },
-};
+        .code.data = nvc0_pwr_code,
+        .code.size = sizeof(nvc0_pwr_code),
+        .data.data = nvc0_pwr_data,
+        .data.size = sizeof(nvc0_pwr_data),
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/nvd0.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/nvd0.c
index 043aa142fe82..2b29be5d08ac 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/nvd0.c
@@ -22,41 +22,20 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/pwr.h>
-
+#include "priv.h"
 #include "fuc/nvd0.fuc.h"
 
-struct nvd0_pwr_priv {
-        struct nouveau_pwr base;
-};
-
-static int
-nvd0_pwr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-              struct nouveau_oclass *oclass, void *data, u32 size,
-              struct nouveau_object **pobject)
-{
-        struct nvd0_pwr_priv *priv;
-        int ret;
-
-        ret = nouveau_pwr_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.code.data = nvd0_pwr_code;
-        priv->base.code.size = sizeof(nvd0_pwr_code);
-        priv->base.data.data = nvd0_pwr_data;
-        priv->base.data.size = sizeof(nvd0_pwr_data);
-        return 0;
-}
-
-struct nouveau_oclass
-nvd0_pwr_oclass = {
-        .handle = NV_SUBDEV(PWR, 0xd0),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nvd0_pwr_ctor,
+struct nouveau_oclass *
+nvd0_pwr_oclass = &(struct nvkm_pwr_impl) {
+        .base.handle = NV_SUBDEV(PWR, 0xd0),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nouveau_pwr_ctor,
                 .dtor = _nouveau_pwr_dtor,
                 .init = _nouveau_pwr_init,
                 .fini = _nouveau_pwr_fini,
         },
-};
+        .code.data = nvd0_pwr_code,
+        .code.size = sizeof(nvd0_pwr_code),
+        .data.data = nvd0_pwr_data,
+        .data.size = sizeof(nvd0_pwr_data),
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/priv.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/priv.h
new file mode 100644
index 000000000000..3814a341db32
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/priv.h
@@ -0,0 +1,44 @@
+#ifndef __NVKM_PWR_PRIV_H__
+#define __NVKM_PWR_PRIV_H__
+
+#include <subdev/pwr.h>
+#include <subdev/pwr/fuc/os.h>
+
+#define nouveau_pwr_create(p, e, o, d)                                         \
+        nouveau_pwr_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nouveau_pwr_destroy(p)                                                 \
+        nouveau_subdev_destroy(&(p)->base)
+#define nouveau_pwr_init(p) ({                                                 \
+        struct nouveau_pwr *_ppwr = (p);                                       \
+        _nouveau_pwr_init(nv_object(_ppwr));                                   \
+})
+#define nouveau_pwr_fini(p,s) ({                                               \
+        struct nouveau_pwr *_ppwr = (p);                                       \
+        _nouveau_pwr_fini(nv_object(_ppwr), (s));                              \
+})
+
+int nouveau_pwr_create_(struct nouveau_object *, struct nouveau_object *,
+                        struct nouveau_oclass *, int, void **);
+
+int _nouveau_pwr_ctor(struct nouveau_object *, struct nouveau_object *,
+                      struct nouveau_oclass *, void *, u32,
+                      struct nouveau_object **);
+#define _nouveau_pwr_dtor _nouveau_subdev_dtor
+int _nouveau_pwr_init(struct nouveau_object *);
+int _nouveau_pwr_fini(struct nouveau_object *, bool);
+
+struct nvkm_pwr_impl {
+        struct nouveau_oclass base;
+        struct {
+                u32 *data;
+                u32  size;
+        } code;
+        struct {
+                u32 *data;
+                u32  size;
+        } data;
+
+        void (*pgob)(struct nouveau_pwr *, bool);
+};
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/vm/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/vm/nvc0.c
index 668cf964e4a9..2d0988755530 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/vm/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/vm/nvc0.c
@@ -28,7 +28,7 @@
 #include <subdev/timer.h>
 #include <subdev/fb.h>
 #include <subdev/vm.h>
-#include <subdev/ltcg.h>
+#include <subdev/ltc.h>
 #include <subdev/bar.h>
 
 struct nvc0_vmmgr_priv {
@@ -116,12 +116,12 @@ nvc0_vm_map(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
         pte <<= 3;
 
         if (mem->tag) {
-                struct nouveau_ltcg *ltcg =
-                        nouveau_ltcg(vma->vm->vmm->base.base.parent);
+                struct nouveau_ltc *ltc =
+                        nouveau_ltc(vma->vm->vmm->base.base.parent);
                 u32 tag = mem->tag->offset + (delta >> 17);
                 phys |= (u64)tag << (32 + 12);
                 next |= (u64)1   << (32 + 12);
-                ltcg->tags_clear(ltcg, tag, cnt);
+                ltc->tags_clear(ltc, tag, cnt);
         }
 
         while (cnt--) {
diff --git a/drivers/gpu/drm/nouveau/dispnv04/arb.c b/drivers/gpu/drm/nouveau/dispnv04/arb.c
index 2a15b98b4d2b..c6361422a0b2 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/arb.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/arb.c
@@ -198,12 +198,12 @@ nv04_update_arb(struct drm_device *dev, int VClk, int bpp,
                 int *burst, int *lwm)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         struct nv_fifo_info fifo_data;
         struct nv_sim_state sim_data;
         int MClk = nouveau_hw_get_clock(dev, PLL_MEMORY);
         int NVClk = nouveau_hw_get_clock(dev, PLL_CORE);
-        uint32_t cfg1 = nv_rd32(device, NV04_PFB_CFG1);
+        uint32_t cfg1 = nvif_rd32(device, NV04_PFB_CFG1);
 
         sim_data.pclk_khz = VClk;
         sim_data.mclk_khz = MClk;
@@ -221,13 +221,13 @@ nv04_update_arb(struct drm_device *dev, int VClk, int bpp,
                 sim_data.mem_latency = 3;
                 sim_data.mem_page_miss = 10;
         } else {
-                sim_data.memory_type = nv_rd32(device, NV04_PFB_CFG0) & 0x1;
-                sim_data.memory_width = (nv_rd32(device, NV_PEXTDEV_BOOT_0) & 0x10) ? 128 : 64;
+                sim_data.memory_type = nvif_rd32(device, NV04_PFB_CFG0) & 0x1;
+                sim_data.memory_width = (nvif_rd32(device, NV_PEXTDEV_BOOT_0) & 0x10) ? 128 : 64;
                 sim_data.mem_latency = cfg1 & 0xf;
                 sim_data.mem_page_miss = ((cfg1 >> 4) & 0xf) + ((cfg1 >> 31) & 0x1);
         }
 
-        if (nv_device(drm->device)->card_type == NV_04)
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_TNT)
                 nv04_calc_arb(&fifo_data, &sim_data);
         else
                 nv10_calc_arb(&fifo_data, &sim_data);
@@ -254,7 +254,7 @@ nouveau_calc_arb(struct drm_device *dev, int vclk, int bpp, int *burst, int *lwm
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
 
-        if (nv_device(drm->device)->card_type < NV_20)
+        if (drm->device.info.family < NV_DEVICE_INFO_V0_KELVIN)
                 nv04_update_arb(dev, vclk, bpp, burst, lwm);
         else if ((dev->pdev->device & 0xfff0) == 0x0240 /*CHIPSET_C51*/ ||
                  (dev->pdev->device & 0xfff0) == 0x03d0 /*CHIPSET_C512*/) {
diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
index 41be3424c906..b90aa5c1f90a 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
@@ -111,8 +111,8 @@ static void nv_crtc_calc_state_ext(struct drm_crtc *crtc, struct drm_display_mod
 {
         struct drm_device *dev = crtc->dev;
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_bios *bios = nouveau_bios(drm->device);
-        struct nouveau_clock *clk = nouveau_clock(drm->device);
+        struct nouveau_bios *bios = nvkm_bios(&drm->device);
+        struct nouveau_clock *clk = nvkm_clock(&drm->device);
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
         struct nv04_mode_state *state = &nv04_display(dev)->mode_reg;
         struct nv04_crtc_reg *regp = &state->crtc_reg[nv_crtc->index];
@@ -136,7 +136,7 @@ static void nv_crtc_calc_state_ext(struct drm_crtc *crtc, struct drm_display_mod
          * has yet been observed in allowing the use a single stage pll on all
          * nv43 however.  the behaviour of single stage use is untested on nv40
          */
-        if (nv_device(drm->device)->chipset > 0x40 && dot_clock <= (pll_lim.vco1.max_freq / 2))
+        if (drm->device.info.chipset > 0x40 && dot_clock <= (pll_lim.vco1.max_freq / 2))
                 memset(&pll_lim.vco2, 0, sizeof(pll_lim.vco2));
 
 
@@ -146,10 +146,10 @@ static void nv_crtc_calc_state_ext(struct drm_crtc *crtc, struct drm_display_mod
         state->pllsel &= PLLSEL_VPLL1_MASK | PLLSEL_VPLL2_MASK | PLLSEL_TV_MASK;
 
         /* The blob uses this always, so let's do the same */
-        if (nv_device(drm->device)->card_type == NV_40)
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE)
                 state->pllsel |= NV_PRAMDAC_PLL_COEFF_SELECT_USE_VPLL2_TRUE;
         /* again nv40 and some nv43 act more like nv3x as described above */
-        if (nv_device(drm->device)->chipset < 0x41)
+        if (drm->device.info.chipset < 0x41)
                 state->pllsel |= NV_PRAMDAC_PLL_COEFF_SELECT_SOURCE_PROG_MPLL |
                                  NV_PRAMDAC_PLL_COEFF_SELECT_SOURCE_PROG_NVPLL;
         state->pllsel |= nv_crtc->index ? PLLSEL_VPLL2_MASK : PLLSEL_VPLL1_MASK;
@@ -275,7 +275,7 @@ nv_crtc_mode_set_vga(struct drm_crtc *crtc, struct drm_display_mode *mode)
                 horizEnd = horizTotal - 2;
                 horizBlankEnd = horizTotal + 4;
 #if 0
-                if (dev->overlayAdaptor && nv_device(drm->device)->card_type >= NV_10)
+                if (dev->overlayAdaptor && drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS)
                         /* This reportedly works around some video overlay bandwidth problems */
                         horizTotal += 2;
 #endif
@@ -509,7 +509,7 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
         regp->cursor_cfg = NV_PCRTC_CURSOR_CONFIG_CUR_LINES_64 |
                              NV_PCRTC_CURSOR_CONFIG_CUR_PIXELS_64 |
                              NV_PCRTC_CURSOR_CONFIG_ADDRESS_SPACE_PNVM;
-        if (nv_device(drm->device)->chipset >= 0x11)
+        if (drm->device.info.chipset >= 0x11)
                 regp->cursor_cfg |= NV_PCRTC_CURSOR_CONFIG_CUR_BPP_32;
         if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
                 regp->cursor_cfg |= NV_PCRTC_CURSOR_CONFIG_DOUBLE_SCAN_ENABLE;
@@ -550,26 +550,26 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
          * 1 << 30 on 0x60.830), for no apparent reason */
         regp->CRTC[NV_CIO_CRE_59] = off_chip_digital;
 
-        if (nv_device(drm->device)->card_type >= NV_30)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_RANKINE)
                 regp->CRTC[0x9f] = off_chip_digital ? 0x11 : 0x1;
 
         regp->crtc_830 = mode->crtc_vdisplay - 3;
         regp->crtc_834 = mode->crtc_vdisplay - 1;
 
-        if (nv_device(drm->device)->card_type == NV_40)
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE)
                 /* This is what the blob does */
                 regp->crtc_850 = NVReadCRTC(dev, 0, NV_PCRTC_850);
 
-        if (nv_device(drm->device)->card_type >= NV_30)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_RANKINE)
                 regp->gpio_ext = NVReadCRTC(dev, 0, NV_PCRTC_GPIO_EXT);
 
-        if (nv_device(drm->device)->card_type >= NV_10)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS)
                 regp->crtc_cfg = NV10_PCRTC_CONFIG_START_ADDRESS_HSYNC;
         else
                 regp->crtc_cfg = NV04_PCRTC_CONFIG_START_ADDRESS_HSYNC;
 
         /* Some misc regs */
-        if (nv_device(drm->device)->card_type == NV_40) {
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE) {
                 regp->CRTC[NV_CIO_CRE_85] = 0xFF;
                 regp->CRTC[NV_CIO_CRE_86] = 0x1;
         }
@@ -581,7 +581,7 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
 
         /* Generic PRAMDAC regs */
 
-        if (nv_device(drm->device)->card_type >= NV_10)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS)
                 /* Only bit that bios and blob set. */
                 regp->nv10_cursync = (1 << 25);
 
@@ -590,7 +590,7 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
                                 NV_PRAMDAC_GENERAL_CONTROL_PIXMIX_ON;
         if (crtc->primary->fb->depth == 16)
                 regp->ramdac_gen_ctrl |= NV_PRAMDAC_GENERAL_CONTROL_ALT_MODE_SEL;
-        if (nv_device(drm->device)->chipset >= 0x11)
+        if (drm->device.info.chipset >= 0x11)
                 regp->ramdac_gen_ctrl |= NV_PRAMDAC_GENERAL_CONTROL_PIPE_LONG;
 
         regp->ramdac_630 = 0; /* turn off green mode (tv test pattern?) */
@@ -653,7 +653,7 @@ nv_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode,
 
         nv_crtc_mode_set_vga(crtc, adjusted_mode);
         /* calculated in nv04_dfp_prepare, nv40 needs it written before calculating PLLs */
-        if (nv_device(drm->device)->card_type == NV_40)
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE)
                 NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, nv04_display(dev)->mode_reg.sel_clk);
         nv_crtc_mode_set_regs(crtc, adjusted_mode);
         nv_crtc_calc_state_ext(crtc, mode, adjusted_mode->clock);
@@ -714,7 +714,7 @@ static void nv_crtc_prepare(struct drm_crtc *crtc)
 
         /* Some more preparation. */
         NVWriteCRTC(dev, nv_crtc->index, NV_PCRTC_CONFIG, NV_PCRTC_CONFIG_START_ADDRESS_NON_VGA);
-        if (nv_device(drm->device)->card_type == NV_40) {
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE) {
                 uint32_t reg900 = NVReadRAMDAC(dev, nv_crtc->index, NV_PRAMDAC_900);
                 NVWriteRAMDAC(dev, nv_crtc->index, NV_PRAMDAC_900, reg900 & ~0x10000);
         }
@@ -888,7 +888,7 @@ nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
         crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_FF_INDEX);
         crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_FFLWM__INDEX);
 
-        if (nv_device(drm->device)->card_type >= NV_20) {
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_KELVIN) {
                 regp->CRTC[NV_CIO_CRE_47] = arb_lwm >> 8;
                 crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_47);
         }
@@ -915,9 +915,9 @@ nv04_crtc_mode_set_base_atomic(struct drm_crtc *crtc,
         struct drm_device *dev = drm->dev;
 
         if (state == ENTER_ATOMIC_MODE_SET)
-                nouveau_fbcon_save_disable_accel(dev);
+                nouveau_fbcon_accel_save_disable(dev);
         else
-                nouveau_fbcon_restore_accel(dev);
+                nouveau_fbcon_accel_restore(dev);
 
         return nv04_crtc_do_mode_set_base(crtc, fb, x, y, true);
 }
@@ -969,7 +969,7 @@ static void nv11_cursor_upload(struct drm_device *dev, struct nouveau_bo *src,
                 {
                         struct nouveau_drm *drm = nouveau_drm(dev);
 
-                        if (nv_device(drm->device)->chipset == 0x11) {
+                        if (drm->device.info.chipset == 0x11) {
                                 pixel = ((pixel & 0x000000ff) << 24) |
                                         ((pixel & 0x0000ff00) << 8) |
                                         ((pixel & 0x00ff0000) >> 8) |
@@ -1010,7 +1010,7 @@ nv04_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
         if (ret)
                 goto out;
 
-        if (nv_device(drm->device)->chipset >= 0x11)
+        if (drm->device.info.chipset >= 0x11)
                 nv11_cursor_upload(dev, cursor, nv_crtc->cursor.nvbo);
         else
                 nv04_cursor_upload(dev, cursor, nv_crtc->cursor.nvbo);
diff --git a/drivers/gpu/drm/nouveau/dispnv04/cursor.c b/drivers/gpu/drm/nouveau/dispnv04/cursor.c
index a810303169de..4e61173c3353 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/cursor.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/cursor.c
@@ -55,7 +55,7 @@ nv04_cursor_set_offset(struct nouveau_crtc *nv_crtc, uint32_t offset)
         crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_HCUR_ADDR0_INDEX);
         crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_HCUR_ADDR1_INDEX);
         crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_HCUR_ADDR2_INDEX);
-        if (nv_device(drm->device)->card_type == NV_40)
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE)
                 nv_fix_nv40_hw_cursor(dev, nv_crtc->index);
 }
 
diff --git a/drivers/gpu/drm/nouveau/dispnv04/dac.c b/drivers/gpu/drm/nouveau/dispnv04/dac.c
index a96dda48718e..2d8056cde996 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/dac.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/dac.c
@@ -65,8 +65,8 @@ int nv04_dac_output_offset(struct drm_encoder *encoder)
 
 static int sample_load_twice(struct drm_device *dev, bool sense[2])
 {
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_timer *ptimer = nouveau_timer(device);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
+        struct nouveau_timer *ptimer = nvkm_timer(device);
         int i;
 
         for (i = 0; i < 2; i++) {
@@ -95,15 +95,15 @@ static int sample_load_twice(struct drm_device *dev, bool sense[2])
 
                 udelay(100);
                 /* when level triggers, sense is _LO_ */
-                sense_a = nv_rd08(device, NV_PRMCIO_INP0) & 0x10;
+                sense_a = nvif_rd08(device, NV_PRMCIO_INP0) & 0x10;
 
                 /* take another reading until it agrees with sense_a... */
                 do {
                         udelay(100);
-                        sense_b = nv_rd08(device, NV_PRMCIO_INP0) & 0x10;
+                        sense_b = nvif_rd08(device, NV_PRMCIO_INP0) & 0x10;
                         if (sense_a != sense_b) {
                                 sense_b_prime =
-                                        nv_rd08(device, NV_PRMCIO_INP0) & 0x10;
+                                        nvif_rd08(device, NV_PRMCIO_INP0) & 0x10;
                                 if (sense_b == sense_b_prime) {
                                         /* ... unless two consecutive subsequent
                                          * samples agree; sense_a is replaced */
@@ -128,7 +128,7 @@ static enum drm_connector_status nv04_dac_detect(struct drm_encoder *encoder,
                                                  struct drm_connector *connector)
 {
         struct drm_device *dev = encoder->dev;
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         struct nouveau_drm *drm = nouveau_drm(dev);
         uint8_t saved_seq1, saved_pi, saved_rpc1, saved_cr_mode;
         uint8_t saved_palette0[3], saved_palette_mask;
@@ -164,11 +164,11 @@ static enum drm_connector_status nv04_dac_detect(struct drm_encoder *encoder,
         saved_rpc1 = NVReadVgaCrtc(dev, 0, NV_CIO_CRE_RPC1_INDEX);
         NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_RPC1_INDEX, saved_rpc1 & ~0xc0);
 
-        nv_wr08(device, NV_PRMDIO_READ_MODE_ADDRESS, 0x0);
+        nvif_wr08(device, NV_PRMDIO_READ_MODE_ADDRESS, 0x0);
         for (i = 0; i < 3; i++)
-                saved_palette0[i] = nv_rd08(device, NV_PRMDIO_PALETTE_DATA);
-        saved_palette_mask = nv_rd08(device, NV_PRMDIO_PIXEL_MASK);
-        nv_wr08(device, NV_PRMDIO_PIXEL_MASK, 0);
+                saved_palette0[i] = nvif_rd08(device, NV_PRMDIO_PALETTE_DATA);
+        saved_palette_mask = nvif_rd08(device, NV_PRMDIO_PIXEL_MASK);
+        nvif_wr08(device, NV_PRMDIO_PIXEL_MASK, 0);
 
         saved_rgen_ctrl = NVReadRAMDAC(dev, 0, NV_PRAMDAC_GENERAL_CONTROL);
         NVWriteRAMDAC(dev, 0, NV_PRAMDAC_GENERAL_CONTROL,
@@ -181,11 +181,11 @@ static enum drm_connector_status nv04_dac_detect(struct drm_encoder *encoder,
         do {
                 bool sense_pair[2];
 
-                nv_wr08(device, NV_PRMDIO_WRITE_MODE_ADDRESS, 0);
-                nv_wr08(device, NV_PRMDIO_PALETTE_DATA, 0);
-                nv_wr08(device, NV_PRMDIO_PALETTE_DATA, 0);
+                nvif_wr08(device, NV_PRMDIO_WRITE_MODE_ADDRESS, 0);
+                nvif_wr08(device, NV_PRMDIO_PALETTE_DATA, 0);
+                nvif_wr08(device, NV_PRMDIO_PALETTE_DATA, 0);
                 /* testing blue won't find monochrome monitors.  I don't care */
-                nv_wr08(device, NV_PRMDIO_PALETTE_DATA, blue);
+                nvif_wr08(device, NV_PRMDIO_PALETTE_DATA, blue);
 
                 i = 0;
                 /* take sample pairs until both samples in the pair agree */
@@ -208,11 +208,11 @@ static enum drm_connector_status nv04_dac_detect(struct drm_encoder *encoder,
         } while (++blue < 0x18 && sense);
 
 out:
-        nv_wr08(device, NV_PRMDIO_PIXEL_MASK, saved_palette_mask);
+        nvif_wr08(device, NV_PRMDIO_PIXEL_MASK, saved_palette_mask);
         NVWriteRAMDAC(dev, 0, NV_PRAMDAC_GENERAL_CONTROL, saved_rgen_ctrl);
-        nv_wr08(device, NV_PRMDIO_WRITE_MODE_ADDRESS, 0);
+        nvif_wr08(device, NV_PRMDIO_WRITE_MODE_ADDRESS, 0);
         for (i = 0; i < 3; i++)
-                nv_wr08(device, NV_PRMDIO_PALETTE_DATA, saved_palette0[i]);
+                nvif_wr08(device, NV_PRMDIO_PALETTE_DATA, saved_palette0[i]);
         NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL, saved_rtest_ctrl);
         NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_PIXEL_INDEX, saved_pi);
         NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_RPC1_INDEX, saved_rpc1);
@@ -231,8 +231,8 @@ uint32_t nv17_dac_sample_load(struct drm_encoder *encoder)
 {
         struct drm_device *dev = encoder->dev;
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_gpio *gpio = nouveau_gpio(device);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
+        struct nouveau_gpio *gpio = nvkm_gpio(device);
         struct dcb_output *dcb = nouveau_encoder(encoder)->dcb;
         uint32_t sample, testval, regoffset = nv04_dac_output_offset(encoder);
         uint32_t saved_powerctrl_2 = 0, saved_powerctrl_4 = 0, saved_routput,
@@ -256,12 +256,12 @@ uint32_t nv17_dac_sample_load(struct drm_encoder *encoder)
         NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + regoffset,
                       saved_rtest_ctrl & ~NV_PRAMDAC_TEST_CONTROL_PWRDWN_DAC_OFF);
 
-        saved_powerctrl_2 = nv_rd32(device, NV_PBUS_POWERCTRL_2);
+        saved_powerctrl_2 = nvif_rd32(device, NV_PBUS_POWERCTRL_2);
 
-        nv_wr32(device, NV_PBUS_POWERCTRL_2, saved_powerctrl_2 & 0xd7ffffff);
+        nvif_wr32(device, NV_PBUS_POWERCTRL_2, saved_powerctrl_2 & 0xd7ffffff);
         if (regoffset == 0x68) {
-                saved_powerctrl_4 = nv_rd32(device, NV_PBUS_POWERCTRL_4);
-                nv_wr32(device, NV_PBUS_POWERCTRL_4, saved_powerctrl_4 & 0xffffffcf);
+                saved_powerctrl_4 = nvif_rd32(device, NV_PBUS_POWERCTRL_4);
+                nvif_wr32(device, NV_PBUS_POWERCTRL_4, saved_powerctrl_4 & 0xffffffcf);
         }
 
         if (gpio) {
@@ -283,7 +283,7 @@ uint32_t nv17_dac_sample_load(struct drm_encoder *encoder)
         /* nv driver and nv31 use 0xfffffeee, nv34 and 6600 use 0xfffffece */
         routput = (saved_routput & 0xfffffece) | head << 8;
 
-        if (nv_device(drm->device)->card_type >= NV_40) {
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_CURIE) {
                 if (dcb->type == DCB_OUTPUT_TV)
                         routput |= 0x1a << 16;
                 else
@@ -316,8 +316,8 @@ uint32_t nv17_dac_sample_load(struct drm_encoder *encoder)
         NVWriteRAMDAC(dev, 0, NV_PRAMDAC_DACCLK + regoffset, saved_routput);
         NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + regoffset, saved_rtest_ctrl);
         if (regoffset == 0x68)
-                nv_wr32(device, NV_PBUS_POWERCTRL_4, saved_powerctrl_4);
-        nv_wr32(device, NV_PBUS_POWERCTRL_2, saved_powerctrl_2);
+                nvif_wr32(device, NV_PBUS_POWERCTRL_4, saved_powerctrl_4);
+        nvif_wr32(device, NV_PBUS_POWERCTRL_2, saved_powerctrl_2);
 
         if (gpio) {
                 gpio->set(gpio, 0, DCB_GPIO_TVDAC1, 0xff, saved_gpio1);
@@ -398,7 +398,7 @@ static void nv04_dac_mode_set(struct drm_encoder *encoder,
         }
 
         /* This could use refinement for flatpanels, but it should work this way */
-        if (nv_device(drm->device)->chipset < 0x44)
+        if (drm->device.info.chipset < 0x44)
                 NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + nv04_dac_output_offset(encoder), 0xf0000000);
         else
                 NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + nv04_dac_output_offset(encoder), 0x00100000);
diff --git a/drivers/gpu/drm/nouveau/dispnv04/dfp.c b/drivers/gpu/drm/nouveau/dispnv04/dfp.c
index e57babb206d3..42a5435259f7 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/dfp.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/dfp.c
@@ -281,7 +281,7 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
                               struct drm_display_mode *adjusted_mode)
 {
         struct drm_device *dev = encoder->dev;
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
         struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
@@ -335,7 +335,7 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
                 regp->fp_control |= NV_PRAMDAC_FP_TG_CONTROL_MODE_NATIVE;
         else /* gpu needs to scale */
                 regp->fp_control |= NV_PRAMDAC_FP_TG_CONTROL_MODE_SCALE;
-        if (nv_rd32(device, NV_PEXTDEV_BOOT_0) & NV_PEXTDEV_BOOT_0_STRAP_FP_IFACE_12BIT)
+        if (nvif_rd32(device, NV_PEXTDEV_BOOT_0) & NV_PEXTDEV_BOOT_0_STRAP_FP_IFACE_12BIT)
                 regp->fp_control |= NV_PRAMDAC_FP_TG_CONTROL_WIDTH_12;
         if (nv_encoder->dcb->location != DCB_LOC_ON_CHIP &&
             output_mode->clock > 165000)
@@ -416,7 +416,7 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
         if ((nv_connector->dithering_mode == DITHERING_MODE_ON) ||
             (nv_connector->dithering_mode == DITHERING_MODE_AUTO &&
              encoder->crtc->primary->fb->depth > connector->display_info.bpc * 3)) {
-                if (nv_device(drm->device)->chipset == 0x11)
+                if (drm->device.info.chipset == 0x11)
                         regp->dither = savep->dither | 0x00010000;
                 else {
                         int i;
@@ -427,7 +427,7 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
                         }
                 }
         } else {
-                if (nv_device(drm->device)->chipset != 0x11) {
+                if (drm->device.info.chipset != 0x11) {
                         /* reset them */
                         int i;
                         for (i = 0; i < 3; i++) {
@@ -463,7 +463,7 @@ static void nv04_dfp_commit(struct drm_encoder *encoder)
                 NVReadRAMDAC(dev, head, NV_PRAMDAC_FP_TG_CONTROL);
 
         /* This could use refinement for flatpanels, but it should work this way */
-        if (nv_device(drm->device)->chipset < 0x44)
+        if (drm->device.info.chipset < 0x44)
                 NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + nv04_dac_output_offset(encoder), 0xf0000000);
         else
                 NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + nv04_dac_output_offset(encoder), 0x00100000);
@@ -485,7 +485,7 @@ static void nv04_dfp_update_backlight(struct drm_encoder *encoder, int mode)
 {
 #ifdef __powerpc__
         struct drm_device *dev = encoder->dev;
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
 
         /* BIOS scripts usually take care of the backlight, thanks
          * Apple for your consistency.
@@ -623,7 +623,7 @@ static void nv04_tmds_slave_init(struct drm_encoder *encoder)
         struct drm_device *dev = encoder->dev;
         struct dcb_output *dcb = nouveau_encoder(encoder)->dcb;
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+        struct nouveau_i2c *i2c = nvkm_i2c(&drm->device);
         struct nouveau_i2c_port *port = i2c->find(i2c, 2);
         struct nouveau_i2c_board_info info[] = {
                 {
diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.c b/drivers/gpu/drm/nouveau/dispnv04/disp.c
index 4342fdaee707..3d0afa1c6cff 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/disp.c
@@ -22,9 +22,6 @@
  * Author: Ben Skeggs
  */
 
-#include <core/object.h>
-#include <core/class.h>
-
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 
@@ -34,8 +31,6 @@
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
 
-#include <subdev/i2c.h>
-
 int
 nv04_display_early_init(struct drm_device *dev)
 {
@@ -58,7 +53,7 @@ int
 nv04_display_create(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+        struct nouveau_i2c *i2c = nvkm_i2c(&drm->device);
         struct dcb_table *dcb = &drm->vbios.dcb;
         struct drm_connector *connector, *ct;
         struct drm_encoder *encoder;
@@ -70,6 +65,8 @@ nv04_display_create(struct drm_device *dev)
         if (!disp)
                 return -ENOMEM;
 
+        nvif_object_map(nvif_object(&drm->device));
+
         nouveau_display(dev)->priv = disp;
         nouveau_display(dev)->dtor = nv04_display_destroy;
         nouveau_display(dev)->init = nv04_display_init;
@@ -144,6 +141,7 @@ void
 nv04_display_destroy(struct drm_device *dev)
 {
         struct nv04_display *disp = nv04_display(dev);
+        struct nouveau_drm *drm = nouveau_drm(dev);
         struct drm_encoder *encoder;
         struct drm_crtc *crtc;
 
@@ -170,6 +168,8 @@ nv04_display_destroy(struct drm_device *dev)
 
         nouveau_display(dev)->priv = NULL;
         kfree(disp);
+
+        nvif_object_unmap(nvif_object(&drm->device));
 }
 
 int
diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.h b/drivers/gpu/drm/nouveau/dispnv04/disp.h
index 4245fc3dab70..17b899d9aba3 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/disp.h
+++ b/drivers/gpu/drm/nouveau/dispnv04/disp.h
@@ -131,7 +131,7 @@ nv_two_heads(struct drm_device *dev)
         struct nouveau_drm *drm = nouveau_drm(dev);
         const int impl = dev->pdev->device & 0x0ff0;
 
-        if (nv_device(drm->device)->card_type >= NV_10 && impl != 0x0100 &&
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS && impl != 0x0100 &&
             impl != 0x0150 && impl != 0x01a0 && impl != 0x0200)
                 return true;
 
@@ -150,7 +150,7 @@ nv_two_reg_pll(struct drm_device *dev)
         struct nouveau_drm *drm = nouveau_drm(dev);
         const int impl = dev->pdev->device & 0x0ff0;
 
-        if (impl == 0x0310 || impl == 0x0340 || nv_device(drm->device)->card_type >= NV_40)
+        if (impl == 0x0310 || impl == 0x0340 || drm->device.info.family >= NV_DEVICE_INFO_V0_CURIE)
                 return true;
         return false;
 }
@@ -171,8 +171,8 @@ static inline void
 nouveau_bios_run_init_table(struct drm_device *dev, u16 table,
                             struct dcb_output *outp, int crtc)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_bios *bios = nouveau_bios(device);
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nouveau_bios *bios = nvkm_bios(&drm->device);
         struct nvbios_init init = {
                 .subdev = nv_subdev(bios),
                 .bios = bios,
diff --git a/drivers/gpu/drm/nouveau/dispnv04/hw.c b/drivers/gpu/drm/nouveau/dispnv04/hw.c
index aca76af115b3..3d4c19300768 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/hw.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/hw.c
@@ -27,9 +27,6 @@
 #include "hw.h"
 
 #include <subdev/bios/pll.h>
-#include <subdev/fb.h>
-#include <subdev/clock.h>
-#include <subdev/timer.h>
 
 #define CHIPSET_NFORCE 0x01a0
 #define CHIPSET_NFORCE2 0x01f0
@@ -92,7 +89,7 @@ NVSetOwner(struct drm_device *dev, int owner)
         if (owner == 1)
                 owner *= 3;
 
-        if (nv_device(drm->device)->chipset == 0x11) {
+        if (drm->device.info.chipset == 0x11) {
                 /* This might seem stupid, but the blob does it and
                  * omitting it often locks the system up.
                  */
@@ -103,7 +100,7 @@ NVSetOwner(struct drm_device *dev, int owner)
         /* CR44 is always changed on CRTC0 */
         NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_44, owner);
 
-        if (nv_device(drm->device)->chipset == 0x11) {  /* set me harder */
+        if (drm->device.info.chipset == 0x11) { /* set me harder */
                 NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_2E, owner);
                 NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_2E, owner);
         }
@@ -152,7 +149,7 @@ nouveau_hw_decode_pll(struct drm_device *dev, uint32_t reg1, uint32_t pll1,
                 pllvals->NM1 = pll1 & 0xffff;
                 if (nv_two_reg_pll(dev) && pll2 & NV31_RAMDAC_ENABLE_VCO2)
                         pllvals->NM2 = pll2 & 0xffff;
-                else if (nv_device(drm->device)->chipset == 0x30 || nv_device(drm->device)->chipset == 0x35) {
+                else if (drm->device.info.chipset == 0x30 || drm->device.info.chipset == 0x35) {
                         pllvals->M1 &= 0xf; /* only 4 bits */
                         if (pll1 & NV30_RAMDAC_ENABLE_VCO2) {
                                 pllvals->M2 = (pll1 >> 4) & 0x7;
@@ -168,8 +165,8 @@ nouveau_hw_get_pllvals(struct drm_device *dev, enum nvbios_pll_type plltype,
                        struct nouveau_pll_vals *pllvals)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
-        struct nouveau_bios *bios = nouveau_bios(device);
+        struct nvif_device *device = &drm->device;
+        struct nouveau_bios *bios = nvkm_bios(device);
         uint32_t reg1, pll1, pll2 = 0;
         struct nvbios_pll pll_lim;
         int ret;
@@ -178,16 +175,16 @@ nouveau_hw_get_pllvals(struct drm_device *dev, enum nvbios_pll_type plltype,
         if (ret || !(reg1 = pll_lim.reg))
                 return -ENOENT;
 
-        pll1 = nv_rd32(device, reg1);
+        pll1 = nvif_rd32(device, reg1);
         if (reg1 <= 0x405c)
-                pll2 = nv_rd32(device, reg1 + 4);
+                pll2 = nvif_rd32(device, reg1 + 4);
         else if (nv_two_reg_pll(dev)) {
                 uint32_t reg2 = reg1 + (reg1 == NV_RAMDAC_VPLL2 ? 0x5c : 0x70);
 
-                pll2 = nv_rd32(device, reg2);
+                pll2 = nvif_rd32(device, reg2);
         }
 
-        if (nv_device(drm->device)->card_type == 0x40 && reg1 >= NV_PRAMDAC_VPLL_COEFF) {
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CELSIUS && reg1 >= NV_PRAMDAC_VPLL_COEFF) {
                 uint32_t ramdac580 = NVReadRAMDAC(dev, 0, NV_PRAMDAC_580);
 
                 /* check whether vpll has been forced into single stage mode */
@@ -255,9 +252,9 @@ nouveau_hw_fix_bad_vpll(struct drm_device *dev, int head)
          */
 
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
-        struct nouveau_clock *clk = nouveau_clock(device);
-        struct nouveau_bios *bios = nouveau_bios(device);
+        struct nvif_device *device = &drm->device;
+        struct nouveau_clock *clk = nvkm_clock(device);
+        struct nouveau_bios *bios = nvkm_bios(device);
         struct nvbios_pll pll_lim;
         struct nouveau_pll_vals pv;
         enum nvbios_pll_type pll = head ? PLL_VPLL1 : PLL_VPLL0;
@@ -394,21 +391,21 @@ nv_save_state_ramdac(struct drm_device *dev, int head,
         struct nv04_crtc_reg *regp = &state->crtc_reg[head];
         int i;
 
-        if (nv_device(drm->device)->card_type >= NV_10)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS)
                 regp->nv10_cursync = NVReadRAMDAC(dev, head, NV_RAMDAC_NV10_CURSYNC);
 
         nouveau_hw_get_pllvals(dev, head ? PLL_VPLL1 : PLL_VPLL0, &regp->pllvals);
         state->pllsel = NVReadRAMDAC(dev, 0, NV_PRAMDAC_PLL_COEFF_SELECT);
         if (nv_two_heads(dev))
                 state->sel_clk = NVReadRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK);
-        if (nv_device(drm->device)->chipset == 0x11)
+        if (drm->device.info.chipset == 0x11)
                 regp->dither = NVReadRAMDAC(dev, head, NV_RAMDAC_DITHER_NV11);
 
         regp->ramdac_gen_ctrl = NVReadRAMDAC(dev, head, NV_PRAMDAC_GENERAL_CONTROL);
 
         if (nv_gf4_disp_arch(dev))
                 regp->ramdac_630 = NVReadRAMDAC(dev, head, NV_PRAMDAC_630);
-        if (nv_device(drm->device)->chipset >= 0x30)
+        if (drm->device.info.chipset >= 0x30)
                 regp->ramdac_634 = NVReadRAMDAC(dev, head, NV_PRAMDAC_634);
 
         regp->tv_setup = NVReadRAMDAC(dev, head, NV_PRAMDAC_TV_SETUP);
@@ -450,7 +447,7 @@ nv_save_state_ramdac(struct drm_device *dev, int head,
         if (nv_gf4_disp_arch(dev))
                 regp->ramdac_8c0 = NVReadRAMDAC(dev, head, NV_PRAMDAC_8C0);
 
-        if (nv_device(drm->device)->card_type == NV_40) {
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE) {
                 regp->ramdac_a20 = NVReadRAMDAC(dev, head, NV_PRAMDAC_A20);
                 regp->ramdac_a24 = NVReadRAMDAC(dev, head, NV_PRAMDAC_A24);
                 regp->ramdac_a34 = NVReadRAMDAC(dev, head, NV_PRAMDAC_A34);
@@ -466,26 +463,26 @@ nv_load_state_ramdac(struct drm_device *dev, int head,
                      struct nv04_mode_state *state)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_clock *clk = nouveau_clock(drm->device);
+        struct nouveau_clock *clk = nvkm_clock(&drm->device);
         struct nv04_crtc_reg *regp = &state->crtc_reg[head];
         uint32_t pllreg = head ? NV_RAMDAC_VPLL2 : NV_PRAMDAC_VPLL_COEFF;
         int i;
 
-        if (nv_device(drm->device)->card_type >= NV_10)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS)
                 NVWriteRAMDAC(dev, head, NV_RAMDAC_NV10_CURSYNC, regp->nv10_cursync);
 
         clk->pll_prog(clk, pllreg, &regp->pllvals);
         NVWriteRAMDAC(dev, 0, NV_PRAMDAC_PLL_COEFF_SELECT, state->pllsel);
         if (nv_two_heads(dev))
                 NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, state->sel_clk);
-        if (nv_device(drm->device)->chipset == 0x11)
+        if (drm->device.info.chipset == 0x11)
                 NVWriteRAMDAC(dev, head, NV_RAMDAC_DITHER_NV11, regp->dither);
 
         NVWriteRAMDAC(dev, head, NV_PRAMDAC_GENERAL_CONTROL, regp->ramdac_gen_ctrl);
 
         if (nv_gf4_disp_arch(dev))
                 NVWriteRAMDAC(dev, head, NV_PRAMDAC_630, regp->ramdac_630);
-        if (nv_device(drm->device)->chipset >= 0x30)
+        if (drm->device.info.chipset >= 0x30)
                 NVWriteRAMDAC(dev, head, NV_PRAMDAC_634, regp->ramdac_634);
 
         NVWriteRAMDAC(dev, head, NV_PRAMDAC_TV_SETUP, regp->tv_setup);
@@ -522,7 +519,7 @@ nv_load_state_ramdac(struct drm_device *dev, int head,
         if (nv_gf4_disp_arch(dev))
                 NVWriteRAMDAC(dev, head, NV_PRAMDAC_8C0, regp->ramdac_8c0);
 
-        if (nv_device(drm->device)->card_type == NV_40) {
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE) {
                 NVWriteRAMDAC(dev, head, NV_PRAMDAC_A20, regp->ramdac_a20);
                 NVWriteRAMDAC(dev, head, NV_PRAMDAC_A24, regp->ramdac_a24);
                 NVWriteRAMDAC(dev, head, NV_PRAMDAC_A34, regp->ramdac_a34);
@@ -603,10 +600,10 @@ nv_save_state_ext(struct drm_device *dev, int head,
         rd_cio_state(dev, head, regp, NV_CIO_CRE_FFLWM__INDEX);
         rd_cio_state(dev, head, regp, NV_CIO_CRE_21);
 
-        if (nv_device(drm->device)->card_type >= NV_20)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_KELVIN)
                 rd_cio_state(dev, head, regp, NV_CIO_CRE_47);
 
-        if (nv_device(drm->device)->card_type >= NV_30)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_RANKINE)
                 rd_cio_state(dev, head, regp, 0x9f);
 
         rd_cio_state(dev, head, regp, NV_CIO_CRE_49);
@@ -615,14 +612,14 @@ nv_save_state_ext(struct drm_device *dev, int head,
         rd_cio_state(dev, head, regp, NV_CIO_CRE_HCUR_ADDR2_INDEX);
         rd_cio_state(dev, head, regp, NV_CIO_CRE_ILACE__INDEX);
 
-        if (nv_device(drm->device)->card_type >= NV_10) {
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
                 regp->crtc_830 = NVReadCRTC(dev, head, NV_PCRTC_830);
                 regp->crtc_834 = NVReadCRTC(dev, head, NV_PCRTC_834);
 
-                if (nv_device(drm->device)->card_type >= NV_30)
+                if (drm->device.info.family >= NV_DEVICE_INFO_V0_RANKINE)
                         regp->gpio_ext = NVReadCRTC(dev, head, NV_PCRTC_GPIO_EXT);
 
-                if (nv_device(drm->device)->card_type == NV_40)
+                if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE)
                         regp->crtc_850 = NVReadCRTC(dev, head, NV_PCRTC_850);
 
                 if (nv_two_heads(dev))
@@ -634,7 +631,7 @@ nv_save_state_ext(struct drm_device *dev, int head,
 
         rd_cio_state(dev, head, regp, NV_CIO_CRE_SCRATCH3__INDEX);
         rd_cio_state(dev, head, regp, NV_CIO_CRE_SCRATCH4__INDEX);
-        if (nv_device(drm->device)->card_type >= NV_10) {
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
                 rd_cio_state(dev, head, regp, NV_CIO_CRE_EBR_INDEX);
                 rd_cio_state(dev, head, regp, NV_CIO_CRE_CSB);
                 rd_cio_state(dev, head, regp, NV_CIO_CRE_4B);
@@ -663,14 +660,13 @@ nv_load_state_ext(struct drm_device *dev, int head,
                   struct nv04_mode_state *state)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
-        struct nouveau_timer *ptimer = nouveau_timer(device);
-        struct nouveau_fb *pfb = nouveau_fb(device);
+        struct nvif_device *device = &drm->device;
+        struct nouveau_timer *ptimer = nvkm_timer(device);
         struct nv04_crtc_reg *regp = &state->crtc_reg[head];
         uint32_t reg900;
         int i;
 
-        if (nv_device(drm->device)->card_type >= NV_10) {
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
                 if (nv_two_heads(dev))
                         /* setting ENGINE_CTRL (EC) *must* come before
                          * CIO_CRE_LCD, as writing CRE_LCD sets bits 16 & 17 in
@@ -678,24 +674,24 @@ nv_load_state_ext(struct drm_device *dev, int head,
                          */
                         NVWriteCRTC(dev, head, NV_PCRTC_ENGINE_CTRL, regp->crtc_eng_ctrl);
 
-                nv_wr32(device, NV_PVIDEO_STOP, 1);
-                nv_wr32(device, NV_PVIDEO_INTR_EN, 0);
-                nv_wr32(device, NV_PVIDEO_OFFSET_BUFF(0), 0);
-                nv_wr32(device, NV_PVIDEO_OFFSET_BUFF(1), 0);
-                nv_wr32(device, NV_PVIDEO_LIMIT(0), pfb->ram->size - 1);
-                nv_wr32(device, NV_PVIDEO_LIMIT(1), pfb->ram->size - 1);
-                nv_wr32(device, NV_PVIDEO_UVPLANE_LIMIT(0), pfb->ram->size - 1);
-                nv_wr32(device, NV_PVIDEO_UVPLANE_LIMIT(1), pfb->ram->size - 1);
-                nv_wr32(device, NV_PBUS_POWERCTRL_2, 0);
+                nvif_wr32(device, NV_PVIDEO_STOP, 1);
+                nvif_wr32(device, NV_PVIDEO_INTR_EN, 0);
+                nvif_wr32(device, NV_PVIDEO_OFFSET_BUFF(0), 0);
+                nvif_wr32(device, NV_PVIDEO_OFFSET_BUFF(1), 0);
+                nvif_wr32(device, NV_PVIDEO_LIMIT(0), device->info.ram_size - 1);
+                nvif_wr32(device, NV_PVIDEO_LIMIT(1), device->info.ram_size - 1);
+                nvif_wr32(device, NV_PVIDEO_UVPLANE_LIMIT(0), device->info.ram_size - 1);
+                nvif_wr32(device, NV_PVIDEO_UVPLANE_LIMIT(1), device->info.ram_size - 1);
+                nvif_wr32(device, NV_PBUS_POWERCTRL_2, 0);
 
                 NVWriteCRTC(dev, head, NV_PCRTC_CURSOR_CONFIG, regp->cursor_cfg);
                 NVWriteCRTC(dev, head, NV_PCRTC_830, regp->crtc_830);
                 NVWriteCRTC(dev, head, NV_PCRTC_834, regp->crtc_834);
 
-                if (nv_device(drm->device)->card_type >= NV_30)
+                if (drm->device.info.family >= NV_DEVICE_INFO_V0_RANKINE)
                         NVWriteCRTC(dev, head, NV_PCRTC_GPIO_EXT, regp->gpio_ext);
 
-                if (nv_device(drm->device)->card_type == NV_40) {
+                if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE) {
                         NVWriteCRTC(dev, head, NV_PCRTC_850, regp->crtc_850);
 
                         reg900 = NVReadRAMDAC(dev, head, NV_PRAMDAC_900);
@@ -718,23 +714,23 @@ nv_load_state_ext(struct drm_device *dev, int head,
         wr_cio_state(dev, head, regp, NV_CIO_CRE_FF_INDEX);
         wr_cio_state(dev, head, regp, NV_CIO_CRE_FFLWM__INDEX);
 
-        if (nv_device(drm->device)->card_type >= NV_20)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_KELVIN)
                 wr_cio_state(dev, head, regp, NV_CIO_CRE_47);
 
-        if (nv_device(drm->device)->card_type >= NV_30)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_RANKINE)
                 wr_cio_state(dev, head, regp, 0x9f);
 
         wr_cio_state(dev, head, regp, NV_CIO_CRE_49);
         wr_cio_state(dev, head, regp, NV_CIO_CRE_HCUR_ADDR0_INDEX);
         wr_cio_state(dev, head, regp, NV_CIO_CRE_HCUR_ADDR1_INDEX);
         wr_cio_state(dev, head, regp, NV_CIO_CRE_HCUR_ADDR2_INDEX);
-        if (nv_device(drm->device)->card_type == NV_40)
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE)
                 nv_fix_nv40_hw_cursor(dev, head);
         wr_cio_state(dev, head, regp, NV_CIO_CRE_ILACE__INDEX);
 
         wr_cio_state(dev, head, regp, NV_CIO_CRE_SCRATCH3__INDEX);
         wr_cio_state(dev, head, regp, NV_CIO_CRE_SCRATCH4__INDEX);
-        if (nv_device(drm->device)->card_type >= NV_10) {
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
                 wr_cio_state(dev, head, regp, NV_CIO_CRE_EBR_INDEX);
                 wr_cio_state(dev, head, regp, NV_CIO_CRE_CSB);
                 wr_cio_state(dev, head, regp, NV_CIO_CRE_4B);
@@ -742,7 +738,7 @@ nv_load_state_ext(struct drm_device *dev, int head,
         }
         /* NV11 and NV20 stop at 0x52. */
         if (nv_gf4_disp_arch(dev)) {
-                if (nv_device(drm->device)->card_type < NV_20) {
+                if (drm->device.info.family < NV_DEVICE_INFO_V0_KELVIN) {
                         /* Not waiting for vertical retrace before modifying
                            CRE_53/CRE_54 causes lockups. */
                         nouveau_timer_wait_eq(ptimer, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x8);
@@ -769,15 +765,15 @@ static void
 nv_save_state_palette(struct drm_device *dev, int head,
                       struct nv04_mode_state *state)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         int head_offset = head * NV_PRMDIO_SIZE, i;
 
-        nv_wr08(device, NV_PRMDIO_PIXEL_MASK + head_offset,
+        nvif_wr08(device, NV_PRMDIO_PIXEL_MASK + head_offset,
                                 NV_PRMDIO_PIXEL_MASK_MASK);
-        nv_wr08(device, NV_PRMDIO_READ_MODE_ADDRESS + head_offset, 0x0);
+        nvif_wr08(device, NV_PRMDIO_READ_MODE_ADDRESS + head_offset, 0x0);
 
         for (i = 0; i < 768; i++) {
-                state->crtc_reg[head].DAC[i] = nv_rd08(device,
+                state->crtc_reg[head].DAC[i] = nvif_rd08(device,
                                 NV_PRMDIO_PALETTE_DATA + head_offset);
         }
 
@@ -788,15 +784,15 @@ void
 nouveau_hw_load_state_palette(struct drm_device *dev, int head,
                               struct nv04_mode_state *state)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         int head_offset = head * NV_PRMDIO_SIZE, i;
 
-        nv_wr08(device, NV_PRMDIO_PIXEL_MASK + head_offset,
+        nvif_wr08(device, NV_PRMDIO_PIXEL_MASK + head_offset,
                                 NV_PRMDIO_PIXEL_MASK_MASK);
-        nv_wr08(device, NV_PRMDIO_WRITE_MODE_ADDRESS + head_offset, 0x0);
+        nvif_wr08(device, NV_PRMDIO_WRITE_MODE_ADDRESS + head_offset, 0x0);
 
         for (i = 0; i < 768; i++) {
-                nv_wr08(device, NV_PRMDIO_PALETTE_DATA + head_offset,
+                nvif_wr08(device, NV_PRMDIO_PALETTE_DATA + head_offset,
                                 state->crtc_reg[head].DAC[i]);
         }
 
@@ -808,7 +804,7 @@ void nouveau_hw_save_state(struct drm_device *dev, int head,
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
 
-        if (nv_device(drm->device)->chipset == 0x11)
+        if (drm->device.info.chipset == 0x11)
                 /* NB: no attempt is made to restore the bad pll later on */
                 nouveau_hw_fix_bad_vpll(dev, head);
         nv_save_state_ramdac(dev, head, state);
diff --git a/drivers/gpu/drm/nouveau/dispnv04/hw.h b/drivers/gpu/drm/nouveau/dispnv04/hw.h
index eeb70d912d99..7f53c571f31f 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/hw.h
+++ b/drivers/gpu/drm/nouveau/dispnv04/hw.h
@@ -60,41 +60,41 @@ extern void nouveau_calc_arb(struct drm_device *, int vclk, int bpp,
 static inline uint32_t NVReadCRTC(struct drm_device *dev,
                                         int head, uint32_t reg)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         uint32_t val;
         if (head)
                 reg += NV_PCRTC0_SIZE;
-        val = nv_rd32(device, reg);
+        val = nvif_rd32(device, reg);
         return val;
 }
 
 static inline void NVWriteCRTC(struct drm_device *dev,
                                         int head, uint32_t reg, uint32_t val)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         if (head)
                 reg += NV_PCRTC0_SIZE;
-        nv_wr32(device, reg, val);
+        nvif_wr32(device, reg, val);
 }
 
 static inline uint32_t NVReadRAMDAC(struct drm_device *dev,
                                         int head, uint32_t reg)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         uint32_t val;
         if (head)
                 reg += NV_PRAMDAC0_SIZE;
-        val = nv_rd32(device, reg);
+        val = nvif_rd32(device, reg);
         return val;
 }
 
 static inline void NVWriteRAMDAC(struct drm_device *dev,
                                         int head, uint32_t reg, uint32_t val)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         if (head)
                 reg += NV_PRAMDAC0_SIZE;
-        nv_wr32(device, reg, val);
+        nvif_wr32(device, reg, val);
 }
 
 static inline uint8_t nv_read_tmds(struct drm_device *dev,
@@ -120,18 +120,18 @@ static inline void nv_write_tmds(struct drm_device *dev,
 static inline void NVWriteVgaCrtc(struct drm_device *dev,
                                         int head, uint8_t index, uint8_t value)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
-        nv_wr08(device, NV_PRMCIO_CRX__COLOR + head * NV_PRMCIO_SIZE, index);
-        nv_wr08(device, NV_PRMCIO_CR__COLOR + head * NV_PRMCIO_SIZE, value);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
+        nvif_wr08(device, NV_PRMCIO_CRX__COLOR + head * NV_PRMCIO_SIZE, index);
+        nvif_wr08(device, NV_PRMCIO_CR__COLOR + head * NV_PRMCIO_SIZE, value);
 }
 
 static inline uint8_t NVReadVgaCrtc(struct drm_device *dev,
                                         int head, uint8_t index)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         uint8_t val;
-        nv_wr08(device, NV_PRMCIO_CRX__COLOR + head * NV_PRMCIO_SIZE, index);
-        val = nv_rd08(device, NV_PRMCIO_CR__COLOR + head * NV_PRMCIO_SIZE);
+        nvif_wr08(device, NV_PRMCIO_CRX__COLOR + head * NV_PRMCIO_SIZE, index);
+        val = nvif_rd08(device, NV_PRMCIO_CR__COLOR + head * NV_PRMCIO_SIZE);
         return val;
 }
 
@@ -165,74 +165,74 @@ static inline uint8_t NVReadVgaCrtc5758(struct drm_device *dev, int head, uint8_
 static inline uint8_t NVReadPRMVIO(struct drm_device *dev,
                                         int head, uint32_t reg)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         struct nouveau_drm *drm = nouveau_drm(dev);
         uint8_t val;
 
         /* Only NV4x have two pvio ranges; other twoHeads cards MUST call
          * NVSetOwner for the relevant head to be programmed */
-        if (head && nv_device(drm->device)->card_type == NV_40)
+        if (head && drm->device.info.family == NV_DEVICE_INFO_V0_CURIE)
                 reg += NV_PRMVIO_SIZE;
 
-        val = nv_rd08(device, reg);
+        val = nvif_rd08(device, reg);
         return val;
 }
 
 static inline void NVWritePRMVIO(struct drm_device *dev,
                                         int head, uint32_t reg, uint8_t value)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         struct nouveau_drm *drm = nouveau_drm(dev);
 
         /* Only NV4x have two pvio ranges; other twoHeads cards MUST call
          * NVSetOwner for the relevant head to be programmed */
-        if (head && nv_device(drm->device)->card_type == NV_40)
+        if (head && drm->device.info.family == NV_DEVICE_INFO_V0_CURIE)
                 reg += NV_PRMVIO_SIZE;
 
-        nv_wr08(device, reg, value);
+        nvif_wr08(device, reg, value);
 }
 
 static inline void NVSetEnablePalette(struct drm_device *dev, int head, bool enable)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
-        nv_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
-        nv_wr08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, enable ? 0 : 0x20);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
+        nvif_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
+        nvif_wr08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, enable ? 0 : 0x20);
 }
 
 static inline bool NVGetEnablePalette(struct drm_device *dev, int head)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
-        nv_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
-        return !(nv_rd08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE) & 0x20);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
+        nvif_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
+        return !(nvif_rd08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE) & 0x20);
 }
 
 static inline void NVWriteVgaAttr(struct drm_device *dev,
                                         int head, uint8_t index, uint8_t value)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         if (NVGetEnablePalette(dev, head))
                 index &= ~0x20;
         else
                 index |= 0x20;
 
-        nv_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
-        nv_wr08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, index);
-        nv_wr08(device, NV_PRMCIO_AR__WRITE + head * NV_PRMCIO_SIZE, value);
+        nvif_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
+        nvif_wr08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, index);
+        nvif_wr08(device, NV_PRMCIO_AR__WRITE + head * NV_PRMCIO_SIZE, value);
 }
 
 static inline uint8_t NVReadVgaAttr(struct drm_device *dev,
                                         int head, uint8_t index)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         uint8_t val;
         if (NVGetEnablePalette(dev, head))
                 index &= ~0x20;
         else
                 index |= 0x20;
 
-        nv_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
-        nv_wr08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, index);
-        val = nv_rd08(device, NV_PRMCIO_AR__READ + head * NV_PRMCIO_SIZE);
+        nvif_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
+        nvif_wr08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, index);
+        val = nvif_rd08(device, NV_PRMCIO_AR__READ + head * NV_PRMCIO_SIZE);
         return val;
 }
 
@@ -259,11 +259,11 @@ static inline void NVVgaProtect(struct drm_device *dev, int head, bool protect)
 static inline bool
 nv_heads_tied(struct drm_device *dev)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         struct nouveau_drm *drm = nouveau_drm(dev);
 
-        if (nv_device(drm->device)->chipset == 0x11)
-                return !!(nv_rd32(device, NV_PBUS_DEBUG_1) & (1 << 28));
+        if (drm->device.info.chipset == 0x11)
+                return !!(nvif_rd32(device, NV_PBUS_DEBUG_1) & (1 << 28));
 
         return NVReadVgaCrtc(dev, 0, NV_CIO_CRE_44) & 0x4;
 }
@@ -318,7 +318,7 @@ NVLockVgaCrtcs(struct drm_device *dev, bool lock)
         NVWriteVgaCrtc(dev, 0, NV_CIO_SR_LOCK_INDEX,
                        lock ? NV_CIO_SR_LOCK_VALUE : NV_CIO_SR_UNLOCK_RW_VALUE);
         /* NV11 has independently lockable extended crtcs, except when tied */
-        if (nv_device(drm->device)->chipset == 0x11 && !nv_heads_tied(dev))
+        if (drm->device.info.chipset == 0x11 && !nv_heads_tied(dev))
                 NVWriteVgaCrtc(dev, 1, NV_CIO_SR_LOCK_INDEX,
                                lock ? NV_CIO_SR_LOCK_VALUE :
                                       NV_CIO_SR_UNLOCK_RW_VALUE);
@@ -335,7 +335,7 @@ static inline int nv_cursor_width(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
 
-        return nv_device(drm->device)->card_type >= NV_10 ? NV10_CURSOR_SIZE : NV04_CURSOR_SIZE;
+        return drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS ? NV10_CURSOR_SIZE : NV04_CURSOR_SIZE;
 }
 
 static inline void
@@ -357,7 +357,7 @@ nv_set_crtc_base(struct drm_device *dev, int head, uint32_t offset)
 
         NVWriteCRTC(dev, head, NV_PCRTC_START, offset);
 
-        if (nv_device(drm->device)->card_type == NV_04) {
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_TNT) {
                 /*
                  * Hilarious, the 24th bit doesn't want to stick to
                  * PCRTC_START...
@@ -382,7 +382,7 @@ nv_show_cursor(struct drm_device *dev, int head, bool show)
                 *curctl1 &= ~MASK(NV_CIO_CRE_HCUR_ADDR1_ENABLE);
         NVWriteVgaCrtc(dev, head, NV_CIO_CRE_HCUR_ADDR1_INDEX, *curctl1);
 
-        if (nv_device(drm->device)->card_type == NV_40)
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE)
                 nv_fix_nv40_hw_cursor(dev, head);
 }
 
@@ -398,7 +398,7 @@ nv_pitch_align(struct drm_device *dev, uint32_t width, int bpp)
                 bpp = 8;
 
         /* Alignment requirements taken from the Haiku driver */
-        if (nv_device(drm->device)->card_type == NV_04)
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_TNT)
                 mask = 128 / bpp - 1;
         else
                 mask = 512 / bpp - 1;
diff --git a/drivers/gpu/drm/nouveau/dispnv04/overlay.c b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
index ab03f7719d2d..b36afcbbc83f 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/overlay.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
@@ -96,7 +96,7 @@ nv10_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
                   uint32_t src_x, uint32_t src_y,
                   uint32_t src_w, uint32_t src_h)
 {
-        struct nouveau_device *dev = nouveau_dev(plane->dev);
+        struct nvif_device *dev = &nouveau_drm(plane->dev)->device;
         struct nouveau_plane *nv_plane = (struct nouveau_plane *)plane;
         struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb);
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
@@ -117,7 +117,7 @@ nv10_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
         if (format > 0xffff)
                 return -ERANGE;
 
-        if (dev->chipset >= 0x30) {
+        if (dev->info.chipset >= 0x30) {
                 if (crtc_w < (src_w >> 1) || crtc_h < (src_h >> 1))
                         return -ERANGE;
         } else {
@@ -131,17 +131,17 @@ nv10_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 
         nv_plane->cur = nv_fb->nvbo;
 
-        nv_mask(dev, NV_PCRTC_ENGINE_CTRL + soff, NV_CRTC_FSEL_OVERLAY, NV_CRTC_FSEL_OVERLAY);
-        nv_mask(dev, NV_PCRTC_ENGINE_CTRL + soff2, NV_CRTC_FSEL_OVERLAY, 0);
+        nvif_mask(dev, NV_PCRTC_ENGINE_CTRL + soff, NV_CRTC_FSEL_OVERLAY, NV_CRTC_FSEL_OVERLAY);
+        nvif_mask(dev, NV_PCRTC_ENGINE_CTRL + soff2, NV_CRTC_FSEL_OVERLAY, 0);
 
-        nv_wr32(dev, NV_PVIDEO_BASE(flip), 0);
-        nv_wr32(dev, NV_PVIDEO_OFFSET_BUFF(flip), nv_fb->nvbo->bo.offset);
-        nv_wr32(dev, NV_PVIDEO_SIZE_IN(flip), src_h << 16 | src_w);
-        nv_wr32(dev, NV_PVIDEO_POINT_IN(flip), src_y << 16 | src_x);
-        nv_wr32(dev, NV_PVIDEO_DS_DX(flip), (src_w << 20) / crtc_w);
-        nv_wr32(dev, NV_PVIDEO_DT_DY(flip), (src_h << 20) / crtc_h);
-        nv_wr32(dev, NV_PVIDEO_POINT_OUT(flip), crtc_y << 16 | crtc_x);
-        nv_wr32(dev, NV_PVIDEO_SIZE_OUT(flip), crtc_h << 16 | crtc_w);
+        nvif_wr32(dev, NV_PVIDEO_BASE(flip), 0);
+        nvif_wr32(dev, NV_PVIDEO_OFFSET_BUFF(flip), nv_fb->nvbo->bo.offset);
+        nvif_wr32(dev, NV_PVIDEO_SIZE_IN(flip), src_h << 16 | src_w);
+        nvif_wr32(dev, NV_PVIDEO_POINT_IN(flip), src_y << 16 | src_x);
+        nvif_wr32(dev, NV_PVIDEO_DS_DX(flip), (src_w << 20) / crtc_w);
+        nvif_wr32(dev, NV_PVIDEO_DT_DY(flip), (src_h << 20) / crtc_h);
+        nvif_wr32(dev, NV_PVIDEO_POINT_OUT(flip), crtc_y << 16 | crtc_x);
+        nvif_wr32(dev, NV_PVIDEO_SIZE_OUT(flip), crtc_h << 16 | crtc_w);
 
         if (fb->pixel_format != DRM_FORMAT_UYVY)
                 format |= NV_PVIDEO_FORMAT_COLOR_LE_CR8YB8CB8YA8;
@@ -153,14 +153,14 @@ nv10_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
                 format |= NV_PVIDEO_FORMAT_DISPLAY_COLOR_KEY;
 
         if (fb->pixel_format == DRM_FORMAT_NV12) {
-                nv_wr32(dev, NV_PVIDEO_UVPLANE_BASE(flip), 0);
-                nv_wr32(dev, NV_PVIDEO_UVPLANE_OFFSET_BUFF(flip),
+                nvif_wr32(dev, NV_PVIDEO_UVPLANE_BASE(flip), 0);
+                nvif_wr32(dev, NV_PVIDEO_UVPLANE_OFFSET_BUFF(flip),
                         nv_fb->nvbo->bo.offset + fb->offsets[1]);
         }
-        nv_wr32(dev, NV_PVIDEO_FORMAT(flip), format);
-        nv_wr32(dev, NV_PVIDEO_STOP, 0);
+        nvif_wr32(dev, NV_PVIDEO_FORMAT(flip), format);
+        nvif_wr32(dev, NV_PVIDEO_STOP, 0);
         /* TODO: wait for vblank? */
-        nv_wr32(dev, NV_PVIDEO_BUFFER, flip ? 0x10 : 0x1);
+        nvif_wr32(dev, NV_PVIDEO_BUFFER, flip ? 0x10 : 0x1);
         nv_plane->flip = !flip;
 
         if (cur)
@@ -172,10 +172,10 @@ nv10_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 static int
 nv10_disable_plane(struct drm_plane *plane)
 {
-        struct nouveau_device *dev = nouveau_dev(plane->dev);
+        struct nvif_device *dev = &nouveau_drm(plane->dev)->device;
         struct nouveau_plane *nv_plane = (struct nouveau_plane *)plane;
 
-        nv_wr32(dev, NV_PVIDEO_STOP, 1);
+        nvif_wr32(dev, NV_PVIDEO_STOP, 1);
         if (nv_plane->cur) {
                 nouveau_bo_unpin(nv_plane->cur);
                 nv_plane->cur = NULL;
@@ -195,24 +195,24 @@ nv_destroy_plane(struct drm_plane *plane)
 static void
 nv10_set_params(struct nouveau_plane *plane)
 {
-        struct nouveau_device *dev = nouveau_dev(plane->base.dev);
+        struct nvif_device *dev = &nouveau_drm(plane->base.dev)->device;
         u32 luma = (plane->brightness - 512) << 16 | plane->contrast;
         u32 chroma = ((sin_mul(plane->hue, plane->saturation) & 0xffff) << 16) |
                 (cos_mul(plane->hue, plane->saturation) & 0xffff);
         u32 format = 0;
 
-        nv_wr32(dev, NV_PVIDEO_LUMINANCE(0), luma);
-        nv_wr32(dev, NV_PVIDEO_LUMINANCE(1), luma);
-        nv_wr32(dev, NV_PVIDEO_CHROMINANCE(0), chroma);
-        nv_wr32(dev, NV_PVIDEO_CHROMINANCE(1), chroma);
-        nv_wr32(dev, NV_PVIDEO_COLOR_KEY, plane->colorkey & 0xffffff);
+        nvif_wr32(dev, NV_PVIDEO_LUMINANCE(0), luma);
+        nvif_wr32(dev, NV_PVIDEO_LUMINANCE(1), luma);
+        nvif_wr32(dev, NV_PVIDEO_CHROMINANCE(0), chroma);
+        nvif_wr32(dev, NV_PVIDEO_CHROMINANCE(1), chroma);
+        nvif_wr32(dev, NV_PVIDEO_COLOR_KEY, plane->colorkey & 0xffffff);
 
         if (plane->cur) {
                 if (plane->iturbt_709)
                         format |= NV_PVIDEO_FORMAT_MATRIX_ITURBT709;
                 if (plane->colorkey & (1 << 24))
                         format |= NV_PVIDEO_FORMAT_DISPLAY_COLOR_KEY;
-                nv_mask(dev, NV_PVIDEO_FORMAT(plane->flip),
+                nvif_mask(dev, NV_PVIDEO_FORMAT(plane->flip),
                         NV_PVIDEO_FORMAT_MATRIX_ITURBT709 |
                         NV_PVIDEO_FORMAT_DISPLAY_COLOR_KEY,
                         format);
@@ -256,7 +256,7 @@ static const struct drm_plane_funcs nv10_plane_funcs = {
 static void
 nv10_overlay_init(struct drm_device *device)
 {
-        struct nouveau_device *dev = nouveau_dev(device);
+        struct nouveau_drm *drm = nouveau_drm(device);
         struct nouveau_plane *plane = kzalloc(sizeof(struct nouveau_plane), GFP_KERNEL);
         int num_formats = ARRAY_SIZE(formats);
         int ret;
@@ -264,7 +264,7 @@ nv10_overlay_init(struct drm_device *device)
         if (!plane)
                 return;
 
-        switch (dev->chipset) {
+        switch (drm->device.info.chipset) {
         case 0x10:
         case 0x11:
         case 0x15:
@@ -333,7 +333,7 @@ cleanup:
         drm_plane_cleanup(&plane->base);
 err:
         kfree(plane);
-        nv_error(dev, "Failed to create plane\n");
+        NV_ERROR(drm, "Failed to create plane\n");
 }
 
 static int
@@ -343,7 +343,7 @@ nv04_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
                   uint32_t src_x, uint32_t src_y,
                   uint32_t src_w, uint32_t src_h)
 {
-        struct nouveau_device *dev = nouveau_dev(plane->dev);
+        struct nvif_device *dev = &nouveau_drm(plane->dev)->device;
         struct nouveau_plane *nv_plane = (struct nouveau_plane *)plane;
         struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb);
         struct nouveau_bo *cur = nv_plane->cur;
@@ -375,43 +375,43 @@ nv04_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 
         nv_plane->cur = nv_fb->nvbo;
 
-        nv_wr32(dev, NV_PVIDEO_OE_STATE, 0);
-        nv_wr32(dev, NV_PVIDEO_SU_STATE, 0);
-        nv_wr32(dev, NV_PVIDEO_RM_STATE, 0);
+        nvif_wr32(dev, NV_PVIDEO_OE_STATE, 0);
+        nvif_wr32(dev, NV_PVIDEO_SU_STATE, 0);
+        nvif_wr32(dev, NV_PVIDEO_RM_STATE, 0);
 
         for (i = 0; i < 2; i++) {
-                nv_wr32(dev, NV_PVIDEO_BUFF0_START_ADDRESS + 4 * i,
+                nvif_wr32(dev, NV_PVIDEO_BUFF0_START_ADDRESS + 4 * i,
                         nv_fb->nvbo->bo.offset);
-                nv_wr32(dev, NV_PVIDEO_BUFF0_PITCH_LENGTH + 4 * i, pitch);
-                nv_wr32(dev, NV_PVIDEO_BUFF0_OFFSET + 4 * i, 0);
+                nvif_wr32(dev, NV_PVIDEO_BUFF0_PITCH_LENGTH + 4 * i, pitch);
+                nvif_wr32(dev, NV_PVIDEO_BUFF0_OFFSET + 4 * i, 0);
         }
-        nv_wr32(dev, NV_PVIDEO_WINDOW_START, crtc_y << 16 | crtc_x);
-        nv_wr32(dev, NV_PVIDEO_WINDOW_SIZE, crtc_h << 16 | crtc_w);
-        nv_wr32(dev, NV_PVIDEO_STEP_SIZE,
+        nvif_wr32(dev, NV_PVIDEO_WINDOW_START, crtc_y << 16 | crtc_x);
+        nvif_wr32(dev, NV_PVIDEO_WINDOW_SIZE, crtc_h << 16 | crtc_w);
+        nvif_wr32(dev, NV_PVIDEO_STEP_SIZE,
                 (uint32_t)(((src_h - 1) << 11) / (crtc_h - 1)) << 16 | (uint32_t)(((src_w - 1) << 11) / (crtc_w - 1)));
 
         /* It should be possible to convert hue/contrast to this */
-        nv_wr32(dev, NV_PVIDEO_RED_CSC_OFFSET, 0x69 - brightness);
-        nv_wr32(dev, NV_PVIDEO_GREEN_CSC_OFFSET, 0x3e + brightness);
-        nv_wr32(dev, NV_PVIDEO_BLUE_CSC_OFFSET, 0x89 - brightness);
-        nv_wr32(dev, NV_PVIDEO_CSC_ADJUST, 0);
+        nvif_wr32(dev, NV_PVIDEO_RED_CSC_OFFSET, 0x69 - brightness);
+        nvif_wr32(dev, NV_PVIDEO_GREEN_CSC_OFFSET, 0x3e + brightness);
+        nvif_wr32(dev, NV_PVIDEO_BLUE_CSC_OFFSET, 0x89 - brightness);
+        nvif_wr32(dev, NV_PVIDEO_CSC_ADJUST, 0);
 
-        nv_wr32(dev, NV_PVIDEO_CONTROL_Y, 0x001); /* (BLUR_ON, LINE_HALF) */
-        nv_wr32(dev, NV_PVIDEO_CONTROL_X, 0x111); /* (WEIGHT_HEAVY, SHARPENING_ON, SMOOTHING_ON) */
+        nvif_wr32(dev, NV_PVIDEO_CONTROL_Y, 0x001); /* (BLUR_ON, LINE_HALF) */
+        nvif_wr32(dev, NV_PVIDEO_CONTROL_X, 0x111); /* (WEIGHT_HEAVY, SHARPENING_ON, SMOOTHING_ON) */
 
-        nv_wr32(dev, NV_PVIDEO_FIFO_BURST_LENGTH, 0x03);
-        nv_wr32(dev, NV_PVIDEO_FIFO_THRES_SIZE, 0x38);
+        nvif_wr32(dev, NV_PVIDEO_FIFO_BURST_LENGTH, 0x03);
+        nvif_wr32(dev, NV_PVIDEO_FIFO_THRES_SIZE, 0x38);
 
-        nv_wr32(dev, NV_PVIDEO_KEY, nv_plane->colorkey);
+        nvif_wr32(dev, NV_PVIDEO_KEY, nv_plane->colorkey);
 
         if (nv_plane->colorkey & (1 << 24))
                 overlay |= 0x10;
         if (fb->pixel_format == DRM_FORMAT_YUYV)
                 overlay |= 0x100;
 
-        nv_wr32(dev, NV_PVIDEO_OVERLAY, overlay);
+        nvif_wr32(dev, NV_PVIDEO_OVERLAY, overlay);
 
-        nv_wr32(dev, NV_PVIDEO_SU_STATE, nv_rd32(dev, NV_PVIDEO_SU_STATE) ^ (1 << 16));
+        nvif_wr32(dev, NV_PVIDEO_SU_STATE, nvif_rd32(dev, NV_PVIDEO_SU_STATE) ^ (1 << 16));
 
         if (cur)
                 nouveau_bo_unpin(cur);
@@ -422,13 +422,13 @@ nv04_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 static int
 nv04_disable_plane(struct drm_plane *plane)
 {
-        struct nouveau_device *dev = nouveau_dev(plane->dev);
+        struct nvif_device *dev = &nouveau_drm(plane->dev)->device;
         struct nouveau_plane *nv_plane = (struct nouveau_plane *)plane;
 
-        nv_mask(dev, NV_PVIDEO_OVERLAY, 1, 0);
-        nv_wr32(dev, NV_PVIDEO_OE_STATE, 0);
-        nv_wr32(dev, NV_PVIDEO_SU_STATE, 0);
-        nv_wr32(dev, NV_PVIDEO_RM_STATE, 0);
+        nvif_mask(dev, NV_PVIDEO_OVERLAY, 1, 0);
+        nvif_wr32(dev, NV_PVIDEO_OE_STATE, 0);
+        nvif_wr32(dev, NV_PVIDEO_SU_STATE, 0);
+        nvif_wr32(dev, NV_PVIDEO_RM_STATE, 0);
         if (nv_plane->cur) {
                 nouveau_bo_unpin(nv_plane->cur);
                 nv_plane->cur = NULL;
@@ -447,7 +447,7 @@ static const struct drm_plane_funcs nv04_plane_funcs = {
 static void
 nv04_overlay_init(struct drm_device *device)
 {
-        struct nouveau_device *dev = nouveau_dev(device);
+        struct nouveau_drm *drm = nouveau_drm(device);
         struct nouveau_plane *plane = kzalloc(sizeof(struct nouveau_plane), GFP_KERNEL);
         int ret;
 
@@ -483,15 +483,15 @@ cleanup:
         drm_plane_cleanup(&plane->base);
 err:
         kfree(plane);
-        nv_error(dev, "Failed to create plane\n");
+        NV_ERROR(drm, "Failed to create plane\n");
 }
 
 void
 nouveau_overlay_init(struct drm_device *device)
 {
-        struct nouveau_device *dev = nouveau_dev(device);
-        if (dev->chipset < 0x10)
+        struct nvif_device *dev = &nouveau_drm(device)->device;
+        if (dev->info.chipset < 0x10)
                 nv04_overlay_init(device);
-        else if (dev->chipset <= 0x40)
+        else if (dev->info.chipset <= 0x40)
                 nv10_overlay_init(device);
 }
diff --git a/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c b/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c
index 8667620b703a..8061d8d0ce79 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c
@@ -35,8 +35,6 @@
 
 #include <drm/i2c/ch7006.h>
 
-#include <subdev/i2c.h>
-
 static struct nouveau_i2c_board_info nv04_tv_encoder_info[] = {
         {
                 {
@@ -56,7 +54,7 @@ static struct nouveau_i2c_board_info nv04_tv_encoder_info[] = {
 int nv04_tv_identify(struct drm_device *dev, int i2c_index)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+        struct nouveau_i2c *i2c = nvkm_i2c(&drm->device);
 
         return i2c->identify(i2c, i2c_index, "TV encoder",
                              nv04_tv_encoder_info, NULL, NULL);
@@ -206,7 +204,7 @@ nv04_tv_create(struct drm_connector *connector, struct dcb_output *entry)
         struct drm_encoder *encoder;
         struct drm_device *dev = connector->dev;
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+        struct nouveau_i2c *i2c = nvkm_i2c(&drm->device);
         struct nouveau_i2c_port *port = i2c->find(i2c, entry->i2c_index);
         int type, ret;
 
diff --git a/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c b/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c
index 195bd8e86c6a..72d2ab04db47 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c
@@ -34,11 +34,6 @@
 #include "hw.h"
 #include "tvnv17.h"
 
-#include <core/device.h>
-
-#include <subdev/bios/gpio.h>
-#include <subdev/gpio.h>
-
 MODULE_PARM_DESC(tv_norm, "Default TV norm.\n"
                  "\t\tSupported: PAL, PAL-M, PAL-N, PAL-Nc, NTSC-M, NTSC-J,\n"
                  "\t\t\thd480i, hd480p, hd576i, hd576p, hd720p, hd1080i.\n"
@@ -51,7 +46,7 @@ static uint32_t nv42_tv_sample_load(struct drm_encoder *encoder)
 {
         struct drm_device *dev = encoder->dev;
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
+        struct nouveau_gpio *gpio = nvkm_gpio(&drm->device);
         uint32_t testval, regoffset = nv04_dac_output_offset(encoder);
         uint32_t gpio0, gpio1, fp_htotal, fp_hsync_start, fp_hsync_end,
                 fp_control, test_ctrl, dacclk, ctv_14, ctv_1c, ctv_6c;
@@ -135,17 +130,17 @@ static bool
 get_tv_detect_quirks(struct drm_device *dev, uint32_t *pin_mask)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_object *device = drm->device;
+        struct nvif_device *device = &drm->device;
 
         /* Zotac FX5200 */
-        if (nv_device_match(device, 0x0322, 0x19da, 0x1035) ||
-            nv_device_match(device, 0x0322, 0x19da, 0x2035)) {
+        if (nv_device_match(nvkm_object(device), 0x0322, 0x19da, 0x1035) ||
+            nv_device_match(nvkm_object(device), 0x0322, 0x19da, 0x2035)) {
                 *pin_mask = 0xc;
                 return false;
         }
 
         /* MSI nForce2 IGP */
-        if (nv_device_match(device, 0x01f0, 0x1462, 0x5710)) {
+        if (nv_device_match(nvkm_object(device), 0x01f0, 0x1462, 0x5710)) {
                 *pin_mask = 0xc;
                 return false;
         }
@@ -167,8 +162,8 @@ nv17_tv_detect(struct drm_encoder *encoder, struct drm_connector *connector)
                 return connector_status_disconnected;
 
         if (reliable) {
-                if (nv_device(drm->device)->chipset == 0x42 ||
-                    nv_device(drm->device)->chipset == 0x43)
+                if (drm->device.info.chipset == 0x42 ||
+                    drm->device.info.chipset == 0x43)
                         tv_enc->pin_mask =
                                 nv42_tv_sample_load(encoder) >> 28 & 0xe;
                 else
@@ -375,7 +370,7 @@ static void  nv17_tv_dpms(struct drm_encoder *encoder, int mode)
 {
         struct drm_device *dev = encoder->dev;
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
+        struct nouveau_gpio *gpio = nvkm_gpio(&drm->device);
         struct nv17_tv_state *regs = &to_tv_enc(encoder)->state;
         struct nv17_tv_norm_params *tv_norm = get_tv_norm(encoder);
 
@@ -448,7 +443,7 @@ static void nv17_tv_prepare(struct drm_encoder *encoder)
         /* Set the DACCLK register */
         dacclk = (NVReadRAMDAC(dev, 0, dacclk_off) & ~0x30) | 0x1;
 
-        if (nv_device(drm->device)->card_type == NV_40)
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CURIE)
                 dacclk |= 0x1a << 16;
 
         if (tv_norm->kind == CTV_ENC_MODE) {
@@ -505,7 +500,7 @@ static void nv17_tv_mode_set(struct drm_encoder *encoder,
                         tv_regs->ptv_614 = 0x13;
                 }
 
-                if (nv_device(drm->device)->card_type >= NV_30) {
+                if (drm->device.info.family >= NV_DEVICE_INFO_V0_RANKINE) {
                         tv_regs->ptv_500 = 0xe8e0;
                         tv_regs->ptv_504 = 0x1710;
                         tv_regs->ptv_604 = 0x0;
@@ -600,7 +595,7 @@ static void nv17_tv_commit(struct drm_encoder *encoder)
         nv17_tv_state_load(dev, &to_tv_enc(encoder)->state);
 
         /* This could use refinement for flatpanels, but it should work */
-        if (nv_device(drm->device)->chipset < 0x44)
+        if (drm->device.info.chipset < 0x44)
                 NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL +
                                         nv04_dac_output_offset(encoder),
                                         0xf0000000);
diff --git a/drivers/gpu/drm/nouveau/dispnv04/tvnv17.h b/drivers/gpu/drm/nouveau/dispnv04/tvnv17.h
index 7b331543a41b..225894cdcac2 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/tvnv17.h
+++ b/drivers/gpu/drm/nouveau/dispnv04/tvnv17.h
@@ -130,14 +130,14 @@ void nv17_ctv_update_rescaler(struct drm_encoder *encoder);
 static inline void nv_write_ptv(struct drm_device *dev, uint32_t reg,
                                 uint32_t val)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
-        nv_wr32(device, reg, val);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
+        nvif_wr32(device, reg, val);
 }
 
 static inline uint32_t nv_read_ptv(struct drm_device *dev, uint32_t reg)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
-        return nv_rd32(device, reg);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
+        return nvif_rd32(device, reg);
 }
 
 static inline void nv_write_tv_enc(struct drm_device *dev, uint8_t reg,
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index b13f441c6431..615714c1727d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -21,16 +21,10 @@
  *
  */
 
-#include <core/object.h>
-#include <core/client.h>
-#include <core/device.h>
-#include <core/class.h>
-#include <core/mm.h>
-
-#include <subdev/fb.h>
-#include <subdev/timer.h>
-#include <subdev/instmem.h>
-#include <engine/graph.h>
+#include <nvif/client.h>
+#include <nvif/driver.h>
+#include <nvif/ioctl.h>
+#include <nvif/class.h>
 
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
@@ -47,20 +41,20 @@ nouveau_abi16_get(struct drm_file *file_priv, struct drm_device *dev)
                 struct nouveau_abi16 *abi16;
                 cli->abi16 = abi16 = kzalloc(sizeof(*abi16), GFP_KERNEL);
                 if (cli->abi16) {
+                        struct nv_device_v0 args = {
+                                .device = ~0ULL,
+                        };
+
                         INIT_LIST_HEAD(&abi16->channels);
-                        abi16->client = nv_object(cli);
 
                         /* allocate device object targeting client's default
                          * device (ie. the one that belongs to the fd it
                          * opened)
                          */
-                        if (nouveau_object_new(abi16->client, NVDRM_CLIENT,
-                                               NVDRM_DEVICE, 0x0080,
-                                               &(struct nv_device_class) {
-                                                .device = ~0ULL,
-                                               },
-                                               sizeof(struct nv_device_class),
-                                               &abi16->device) == 0)
+                        if (nvif_device_init(&cli->base.base, NULL,
+                                             NOUVEAU_ABI16_DEVICE, NV_DEVICE,
+                                             &args, sizeof(args),
+                                             &abi16->device) == 0)
                                 return cli->abi16;
 
                         kfree(cli->abi16);
@@ -75,7 +69,7 @@ nouveau_abi16_get(struct drm_file *file_priv, struct drm_device *dev)
 int
 nouveau_abi16_put(struct nouveau_abi16 *abi16, int ret)
 {
-        struct nouveau_cli *cli = (void *)abi16->client;
+        struct nouveau_cli *cli = (void *)nvif_client(&abi16->device.base);
         mutex_unlock(&cli->mutex);
         return ret;
 }
@@ -83,21 +77,19 @@ nouveau_abi16_put(struct nouveau_abi16 *abi16, int ret)
 u16
 nouveau_abi16_swclass(struct nouveau_drm *drm)
 {
-        switch (nv_device(drm->device)->card_type) {
-        case NV_04:
+        switch (drm->device.info.family) {
+        case NV_DEVICE_INFO_V0_TNT:
                 return 0x006e;
-        case NV_10:
-        case NV_11:
-        case NV_20:
-        case NV_30:
-        case NV_40:
+        case NV_DEVICE_INFO_V0_CELSIUS:
+        case NV_DEVICE_INFO_V0_KELVIN:
+        case NV_DEVICE_INFO_V0_RANKINE:
+        case NV_DEVICE_INFO_V0_CURIE:
                 return 0x016e;
-        case NV_50:
+        case NV_DEVICE_INFO_V0_TESLA:
                 return 0x506e;
-        case NV_C0:
-        case NV_D0:
-        case NV_E0:
-        case GM100:
+        case NV_DEVICE_INFO_V0_FERMI:
+        case NV_DEVICE_INFO_V0_KEPLER:
+        case NV_DEVICE_INFO_V0_MAXWELL:
                 return 0x906e;
         }
 
@@ -140,7 +132,7 @@ nouveau_abi16_chan_fini(struct nouveau_abi16 *abi16,
 
         /* destroy channel object, all children will be killed too */
         if (chan->chan) {
-                abi16->handles &= ~(1ULL << (chan->chan->handle & 0xffff));
+                abi16->handles &= ~(1ULL << (chan->chan->object->handle & 0xffff));
                 nouveau_channel_del(&chan->chan);
         }
 
@@ -151,7 +143,7 @@ nouveau_abi16_chan_fini(struct nouveau_abi16 *abi16,
 void
 nouveau_abi16_fini(struct nouveau_abi16 *abi16)
 {
-        struct nouveau_cli *cli = (void *)abi16->client;
+        struct nouveau_cli *cli = (void *)nvif_client(&abi16->device.base);
         struct nouveau_abi16_chan *chan, *temp;
 
         /* cleanup channels */
@@ -160,7 +152,7 @@ nouveau_abi16_fini(struct nouveau_abi16 *abi16)
         }
 
         /* destroy the device object */
-        nouveau_object_del(abi16->client, NVDRM_CLIENT, NVDRM_DEVICE);
+        nvif_device_fini(&abi16->device);
 
         kfree(cli->abi16);
         cli->abi16 = NULL;
@@ -169,30 +161,31 @@ nouveau_abi16_fini(struct nouveau_abi16 *abi16)
 int
 nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS)
 {
+        struct nouveau_cli *cli = nouveau_cli(file_priv);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
-        struct nouveau_timer *ptimer = nouveau_timer(device);
-        struct nouveau_graph *graph = (void *)nouveau_engine(device, NVDEV_ENGINE_GR);
+        struct nvif_device *device = &drm->device;
+        struct nouveau_timer *ptimer = nvkm_timer(device);
+        struct nouveau_graph *graph = nvkm_gr(device);
         struct drm_nouveau_getparam *getparam = data;
 
         switch (getparam->param) {
         case NOUVEAU_GETPARAM_CHIPSET_ID:
-                getparam->value = device->chipset;
+                getparam->value = device->info.chipset;
                 break;
         case NOUVEAU_GETPARAM_PCI_VENDOR:
-                if (nv_device_is_pci(device))
+                if (nv_device_is_pci(nvkm_device(device)))
                         getparam->value = dev->pdev->vendor;
                 else
                         getparam->value = 0;
                 break;
         case NOUVEAU_GETPARAM_PCI_DEVICE:
-                if (nv_device_is_pci(device))
+                if (nv_device_is_pci(nvkm_device(device)))
                         getparam->value = dev->pdev->device;
                 else
                         getparam->value = 0;
                 break;
         case NOUVEAU_GETPARAM_BUS_TYPE:
-                if (!nv_device_is_pci(device))
+                if (!nv_device_is_pci(nvkm_device(device)))
                         getparam->value = 3;
                 else
                 if (drm_pci_device_is_agp(dev))
@@ -225,7 +218,7 @@ nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS)
                 getparam->value = graph->units ? graph->units(graph) : 0;
                 break;
         default:
-                nv_debug(device, "unknown parameter %lld\n", getparam->param);
+                NV_PRINTK(debug, cli, "unknown parameter %lld\n", getparam->param);
                 return -EINVAL;
         }
 
@@ -246,10 +239,7 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
         struct nouveau_abi16_chan *chan;
-        struct nouveau_client *client;
-        struct nouveau_device *device;
-        struct nouveau_instmem *imem;
-        struct nouveau_fb *pfb;
+        struct nvif_device *device;
         int ret;
 
         if (unlikely(!abi16))
@@ -258,21 +248,18 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
         if (!drm->channel)
                 return nouveau_abi16_put(abi16, -ENODEV);
 
-        client = nv_client(abi16->client);
-        device = nv_device(abi16->device);
-        imem   = nouveau_instmem(device);
-        pfb    = nouveau_fb(device);
+        device = &abi16->device;
 
         /* hack to allow channel engine type specification on kepler */
-        if (device->card_type >= NV_E0) {
+        if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
                 if (init->fb_ctxdma_handle != ~0)
-                        init->fb_ctxdma_handle = NVE0_CHANNEL_IND_ENGINE_GR;
+                        init->fb_ctxdma_handle = KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_GR;
                 else
                         init->fb_ctxdma_handle = init->tt_ctxdma_handle;
 
                 /* allow flips to be executed if this is a graphics channel */
                 init->tt_ctxdma_handle = 0;
-                if (init->fb_ctxdma_handle == NVE0_CHANNEL_IND_ENGINE_GR)
+                if (init->fb_ctxdma_handle == KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_GR)
                         init->tt_ctxdma_handle = 1;
         }
 
@@ -293,13 +280,14 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
         abi16->handles |= (1ULL << init->channel);
 
         /* create channel object and initialise dma and fence management */
-        ret = nouveau_channel_new(drm, cli, NVDRM_DEVICE, NVDRM_CHAN |
-                                  init->channel, init->fb_ctxdma_handle,
+        ret = nouveau_channel_new(drm, device,
+                                  NOUVEAU_ABI16_CHAN(init->channel),
+                                  init->fb_ctxdma_handle,
                                   init->tt_ctxdma_handle, &chan->chan);
         if (ret)
                 goto done;
 
-        if (device->card_type >= NV_50)
+        if (device->info.family >= NV_DEVICE_INFO_V0_TESLA)
                 init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_VRAM |
                                         NOUVEAU_GEM_DOMAIN_GART;
         else
@@ -308,10 +296,10 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
         else
                 init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_GART;
 
-        if (device->card_type < NV_10) {
+        if (device->info.family < NV_DEVICE_INFO_V0_CELSIUS) {
                 init->subchan[0].handle = 0x00000000;
                 init->subchan[0].grclass = 0x0000;
-                init->subchan[1].handle = NvSw;
+                init->subchan[1].handle = chan->chan->nvsw.handle;
                 init->subchan[1].grclass = 0x506e;
                 init->nr_subchan = 2;
         }
@@ -324,8 +312,8 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
         if (ret)
                 goto done;
 
-        if (device->card_type >= NV_50) {
-                ret = nouveau_bo_vma_add(chan->ntfy, client->vm,
+        if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) {
+                ret = nouveau_bo_vma_add(chan->ntfy, cli->vm,
                                         &chan->ntfy_vma);
                 if (ret)
                         goto done;
@@ -343,6 +331,18 @@ done:
         return nouveau_abi16_put(abi16, ret);
 }
 
+static struct nouveau_abi16_chan *
+nouveau_abi16_chan(struct nouveau_abi16 *abi16, int channel)
+{
+        struct nouveau_abi16_chan *chan;
+
+        list_for_each_entry(chan, &abi16->channels, head) {
+                if (chan->chan->object->handle == NOUVEAU_ABI16_CHAN(channel))
+                        return chan;
+        }
+
+        return NULL;
+}
 
 int
 nouveau_abi16_ioctl_channel_free(ABI16_IOCTL_ARGS)
@@ -350,28 +350,38 @@ nouveau_abi16_ioctl_channel_free(ABI16_IOCTL_ARGS)
         struct drm_nouveau_channel_free *req = data;
         struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
         struct nouveau_abi16_chan *chan;
-        int ret = -ENOENT;
 
         if (unlikely(!abi16))
                 return -ENOMEM;
 
-        list_for_each_entry(chan, &abi16->channels, head) {
-                if (chan->chan->handle == (NVDRM_CHAN | req->channel)) {
-                        nouveau_abi16_chan_fini(abi16, chan);
-                        return nouveau_abi16_put(abi16, 0);
-                }
-        }
-
-        return nouveau_abi16_put(abi16, ret);
+        chan = nouveau_abi16_chan(abi16, req->channel);
+        if (!chan)
+                return nouveau_abi16_put(abi16, -ENOENT);
+        nouveau_abi16_chan_fini(abi16, chan);
+        return nouveau_abi16_put(abi16, 0);
 }
 
 int
 nouveau_abi16_ioctl_grobj_alloc(ABI16_IOCTL_ARGS)
 {
         struct drm_nouveau_grobj_alloc *init = data;
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_new_v0 new;
+        } args = {
+                .ioctl.owner = NVIF_IOCTL_V0_OWNER_ANY,
+                .ioctl.type = NVIF_IOCTL_V0_NEW,
+                .ioctl.path_nr = 3,
+                .ioctl.path[2] = NOUVEAU_ABI16_CLIENT,
+                .ioctl.path[1] = NOUVEAU_ABI16_DEVICE,
+                .ioctl.path[0] = NOUVEAU_ABI16_CHAN(init->channel),
+                .new.route = NVDRM_OBJECT_ABI16,
+                .new.handle = init->handle,
+                .new.oclass = init->class,
+        };
         struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_object *object;
+        struct nvif_client *client;
         int ret;
 
         if (unlikely(!abi16))
@@ -379,6 +389,7 @@ nouveau_abi16_ioctl_grobj_alloc(ABI16_IOCTL_ARGS)
 
         if (init->handle == ~0)
                 return nouveau_abi16_put(abi16, -EINVAL);
+        client = nvif_client(nvif_object(&abi16->device));
 
         /* compatibility with userspace that assumes 506e for all chipsets */
         if (init->class == 0x506e) {
@@ -387,8 +398,7 @@ nouveau_abi16_ioctl_grobj_alloc(ABI16_IOCTL_ARGS)
                         return nouveau_abi16_put(abi16, 0);
         }
 
-        ret = nouveau_object_new(abi16->client, NVDRM_CHAN | init->channel,
-                                  init->handle, init->class, NULL, 0, &object);
+        ret = nvif_client_ioctl(client, &args, sizeof(args));
         return nouveau_abi16_put(abi16, ret);
 }
 
@@ -396,29 +406,38 @@ int
 nouveau_abi16_ioctl_notifierobj_alloc(ABI16_IOCTL_ARGS)
 {
         struct drm_nouveau_notifierobj_alloc *info = data;
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_new_v0 new;
+                struct nv_dma_v0 ctxdma;
+        } args = {
+                .ioctl.owner = NVIF_IOCTL_V0_OWNER_ANY,
+                .ioctl.type = NVIF_IOCTL_V0_NEW,
+                .ioctl.path_nr = 3,
+                .ioctl.path[2] = NOUVEAU_ABI16_CLIENT,
+                .ioctl.path[1] = NOUVEAU_ABI16_DEVICE,
+                .ioctl.path[0] = NOUVEAU_ABI16_CHAN(info->channel),
+                .new.route = NVDRM_OBJECT_ABI16,
+                .new.handle = info->handle,
+                .new.oclass = NV_DMA_IN_MEMORY,
+        };
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
         struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
-        struct nouveau_abi16_chan *chan = NULL, *temp;
+        struct nouveau_abi16_chan *chan;
         struct nouveau_abi16_ntfy *ntfy;
-        struct nouveau_object *object;
-        struct nv_dma_class args = {};
+        struct nvif_device *device = &abi16->device;
+        struct nvif_client *client;
         int ret;
 
         if (unlikely(!abi16))
                 return -ENOMEM;
 
         /* completely unnecessary for these chipsets... */
-        if (unlikely(nv_device(abi16->device)->card_type >= NV_C0))
+        if (unlikely(device->info.family >= NV_DEVICE_INFO_V0_FERMI))
                 return nouveau_abi16_put(abi16, -EINVAL);
+        client = nvif_client(nvif_object(&abi16->device));
 
-        list_for_each_entry(temp, &abi16->channels, head) {
-                if (temp->chan->handle == (NVDRM_CHAN | info->channel)) {
-                        chan = temp;
-                        break;
-                }
-        }
-
+        chan = nouveau_abi16_chan(abi16, info->channel);
         if (!chan)
                 return nouveau_abi16_put(abi16, -ENOENT);
 
@@ -434,26 +453,29 @@ nouveau_abi16_ioctl_notifierobj_alloc(ABI16_IOCTL_ARGS)
         if (ret)
                 goto done;
 
-        args.start = ntfy->node->offset;
-        args.limit = ntfy->node->offset + ntfy->node->length - 1;
-        if (device->card_type >= NV_50) {
-                args.flags  = NV_DMA_TARGET_VM | NV_DMA_ACCESS_VM;
-                args.start += chan->ntfy_vma.offset;
-                args.limit += chan->ntfy_vma.offset;
+        args.ctxdma.start = ntfy->node->offset;
+        args.ctxdma.limit = ntfy->node->offset + ntfy->node->length - 1;
+        if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) {
+                args.ctxdma.target = NV_DMA_V0_TARGET_VM;
+                args.ctxdma.access = NV_DMA_V0_ACCESS_VM;
+                args.ctxdma.start += chan->ntfy_vma.offset;
+                args.ctxdma.limit += chan->ntfy_vma.offset;
         } else
         if (drm->agp.stat == ENABLED) {
-                args.flags  = NV_DMA_TARGET_AGP | NV_DMA_ACCESS_RDWR;
-                args.start += drm->agp.base + chan->ntfy->bo.offset;
-                args.limit += drm->agp.base + chan->ntfy->bo.offset;
+                args.ctxdma.target = NV_DMA_V0_TARGET_AGP;
+                args.ctxdma.access = NV_DMA_V0_ACCESS_RDWR;
+                args.ctxdma.start += drm->agp.base + chan->ntfy->bo.offset;
+                args.ctxdma.limit += drm->agp.base + chan->ntfy->bo.offset;
+                client->super = true;
         } else {
-                args.flags  = NV_DMA_TARGET_VM | NV_DMA_ACCESS_RDWR;
-                args.start += chan->ntfy->bo.offset;
-                args.limit += chan->ntfy->bo.offset;
+                args.ctxdma.target = NV_DMA_V0_TARGET_VM;
+                args.ctxdma.access = NV_DMA_V0_ACCESS_RDWR;
+                args.ctxdma.start += chan->ntfy->bo.offset;
+                args.ctxdma.limit += chan->ntfy->bo.offset;
         }
 
-        ret = nouveau_object_new(abi16->client, chan->chan->handle,
-                                 ntfy->handle, 0x003d, &args,
-                                 sizeof(args), &object);
+        ret = nvif_client_ioctl(client, &args, sizeof(args));
+        client->super = false;
         if (ret)
                 goto done;
 
@@ -469,28 +491,36 @@ int
 nouveau_abi16_ioctl_gpuobj_free(ABI16_IOCTL_ARGS)
 {
         struct drm_nouveau_gpuobj_free *fini = data;
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_del del;
+        } args = {
+                .ioctl.owner = NVDRM_OBJECT_ABI16,
+                .ioctl.type = NVIF_IOCTL_V0_DEL,
+                .ioctl.path_nr = 4,
+                .ioctl.path[3] = NOUVEAU_ABI16_CLIENT,
+                .ioctl.path[2] = NOUVEAU_ABI16_DEVICE,
+                .ioctl.path[1] = NOUVEAU_ABI16_CHAN(fini->channel),
+                .ioctl.path[0] = fini->handle,
+        };
         struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
-        struct nouveau_abi16_chan *chan = NULL, *temp;
+        struct nouveau_abi16_chan *chan;
         struct nouveau_abi16_ntfy *ntfy;
+        struct nvif_client *client;
         int ret;
 
         if (unlikely(!abi16))
                 return -ENOMEM;
 
-        list_for_each_entry(temp, &abi16->channels, head) {
-                if (temp->chan->handle == (NVDRM_CHAN | fini->channel)) {
-                        chan = temp;
-                        break;
-                }
-        }
-
+        chan = nouveau_abi16_chan(abi16, fini->channel);
         if (!chan)
                 return nouveau_abi16_put(abi16, -ENOENT);
+        client = nvif_client(nvif_object(&abi16->device));
 
         /* synchronize with the user channel and destroy the gpu object */
         nouveau_channel_idle(chan->chan);
 
-        ret = nouveau_object_del(abi16->client, chan->chan->handle, fini->handle);
+        ret = nvif_client_ioctl(client, &args, sizeof(args));
         if (ret)
                 return nouveau_abi16_put(abi16, ret);
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.h b/drivers/gpu/drm/nouveau/nouveau_abi16.h
index 90004081a501..39844e6bfbff 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.h
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.h
@@ -28,8 +28,7 @@ struct nouveau_abi16_chan {
 };
 
 struct nouveau_abi16 {
-        struct nouveau_object *client;
-        struct nouveau_object *device;
+        struct nvif_device device;
         struct list_head channels;
         u64 handles;
 };
diff --git a/drivers/gpu/drm/nouveau/nouveau_agp.c b/drivers/gpu/drm/nouveau/nouveau_agp.c
index 51666daddb94..1f6f6ba6847a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_agp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_agp.c
@@ -1,7 +1,5 @@
 #include <linux/module.h>
 
-#include <core/device.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_agp.h"
 #include "nouveau_reg.h"
@@ -29,7 +27,7 @@ static struct nouveau_agpmode_quirk nouveau_agpmode_quirk_list[] = {
 static unsigned long
 get_agp_mode(struct nouveau_drm *drm, const struct drm_agp_info *info)
 {
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         struct nouveau_agpmode_quirk *quirk = nouveau_agpmode_quirk_list;
         int agpmode = nouveau_agpmode;
         unsigned long mode = info->mode;
@@ -38,7 +36,7 @@ get_agp_mode(struct nouveau_drm *drm, const struct drm_agp_info *info)
          * FW seems to be broken on nv18, it makes the card lock up
          * randomly.
          */
-        if (device->chipset == 0x18)
+        if (device->info.chipset == 0x18)
                 mode &= ~PCI_AGP_COMMAND_FW;
 
         /*
@@ -47,10 +45,10 @@ get_agp_mode(struct nouveau_drm *drm, const struct drm_agp_info *info)
         while (agpmode == -1 && quirk->hostbridge_vendor) {
                 if (info->id_vendor == quirk->hostbridge_vendor &&
                     info->id_device == quirk->hostbridge_device &&
-                    device->pdev->vendor == quirk->chip_vendor &&
-                    device->pdev->device == quirk->chip_device) {
+                    nvkm_device(device)->pdev->vendor == quirk->chip_vendor &&
+                    nvkm_device(device)->pdev->device == quirk->chip_device) {
                         agpmode = quirk->mode;
-                        nv_info(device, "Forcing agp mode to %dX. Use agpmode to override.\n",
+                        NV_INFO(drm, "Forcing agp mode to %dX. Use agpmode to override.\n",
                                 agpmode);
                         break;
                 }
@@ -104,7 +102,7 @@ void
 nouveau_agp_reset(struct nouveau_drm *drm)
 {
 #if __OS_HAS_AGP
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         struct drm_device *dev = drm->dev;
         u32 save[2];
         int ret;
@@ -115,7 +113,7 @@ nouveau_agp_reset(struct nouveau_drm *drm)
         /* First of all, disable fast writes, otherwise if it's
          * already enabled in the AGP bridge and we disable the card's
          * AGP controller we might be locking ourselves out of it. */
-        if ((nv_rd32(device, NV04_PBUS_PCI_NV_19) |
+        if ((nvif_rd32(device, NV04_PBUS_PCI_NV_19) |
              dev->agp->mode) & PCI_AGP_COMMAND_FW) {
                 struct drm_agp_info info;
                 struct drm_agp_mode mode;
@@ -134,15 +132,15 @@ nouveau_agp_reset(struct nouveau_drm *drm)
 
 
         /* clear busmaster bit, and disable AGP */
-        save[0] = nv_mask(device, NV04_PBUS_PCI_NV_1, 0x00000004, 0x00000000);
-        nv_wr32(device, NV04_PBUS_PCI_NV_19, 0);
+        save[0] = nvif_mask(device, NV04_PBUS_PCI_NV_1, 0x00000004, 0x00000000);
+        nvif_wr32(device, NV04_PBUS_PCI_NV_19, 0);
 
         /* reset PGRAPH, PFIFO and PTIMER */
-        save[1] = nv_mask(device, 0x000200, 0x00011100, 0x00000000);
-        nv_mask(device, 0x000200, 0x00011100, save[1]);
+        save[1] = nvif_mask(device, 0x000200, 0x00011100, 0x00000000);
+        nvif_mask(device, 0x000200, 0x00011100, save[1]);
 
         /* and restore bustmaster bit (gives effect of resetting AGP) */
-        nv_wr32(device, NV04_PBUS_PCI_NV_1, save[0]);
+        nvif_wr32(device, NV04_PBUS_PCI_NV_1, save[0]);
 #endif
 }
 
@@ -150,7 +148,6 @@ void
 nouveau_agp_init(struct nouveau_drm *drm)
 {
 #if __OS_HAS_AGP
-        struct nouveau_device *device = nv_device(drm->device);
         struct drm_device *dev = drm->dev;
         struct drm_agp_info info;
         struct drm_agp_mode mode;
@@ -162,13 +159,13 @@ nouveau_agp_init(struct nouveau_drm *drm)
 
         ret = drm_agp_acquire(dev);
         if (ret) {
-                nv_error(device, "unable to acquire AGP: %d\n", ret);
+                NV_ERROR(drm, "unable to acquire AGP: %d\n", ret);
                 return;
         }
 
         ret = drm_agp_info(dev, &info);
         if (ret) {
-                nv_error(device, "unable to get AGP info: %d\n", ret);
+                NV_ERROR(drm, "unable to get AGP info: %d\n", ret);
                 return;
         }
 
@@ -177,7 +174,7 @@ nouveau_agp_init(struct nouveau_drm *drm)
 
         ret = drm_agp_enable(dev, mode);
         if (ret) {
-                nv_error(device, "unable to enable AGP: %d\n", ret);
+                NV_ERROR(drm, "unable to enable AGP: %d\n", ret);
                 return;
         }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c
index 2c1e4aad7da3..e566c5b53651 100644
--- a/drivers/gpu/drm/nouveau/nouveau_backlight.c
+++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c
@@ -40,8 +40,8 @@ static int
 nv40_get_intensity(struct backlight_device *bd)
 {
         struct nouveau_drm *drm = bl_get_data(bd);
-        struct nouveau_device *device = nv_device(drm->device);
-        int val = (nv_rd32(device, NV40_PMC_BACKLIGHT) &
+        struct nvif_device *device = &drm->device;
+        int val = (nvif_rd32(device, NV40_PMC_BACKLIGHT) &
                                    NV40_PMC_BACKLIGHT_MASK) >> 16;
 
         return val;
@@ -51,11 +51,11 @@ static int
 nv40_set_intensity(struct backlight_device *bd)
 {
         struct nouveau_drm *drm = bl_get_data(bd);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         int val = bd->props.brightness;
-        int reg = nv_rd32(device, NV40_PMC_BACKLIGHT);
+        int reg = nvif_rd32(device, NV40_PMC_BACKLIGHT);
 
-        nv_wr32(device, NV40_PMC_BACKLIGHT,
+        nvif_wr32(device, NV40_PMC_BACKLIGHT,
                  (val << 16) | (reg & ~NV40_PMC_BACKLIGHT_MASK));
 
         return 0;
@@ -71,11 +71,11 @@ static int
 nv40_backlight_init(struct drm_connector *connector)
 {
         struct nouveau_drm *drm = nouveau_drm(connector->dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         struct backlight_properties props;
         struct backlight_device *bd;
 
-        if (!(nv_rd32(device, NV40_PMC_BACKLIGHT) & NV40_PMC_BACKLIGHT_MASK))
+        if (!(nvif_rd32(device, NV40_PMC_BACKLIGHT) & NV40_PMC_BACKLIGHT_MASK))
                 return 0;
 
         memset(&props, 0, sizeof(struct backlight_properties));
@@ -97,12 +97,12 @@ nv50_get_intensity(struct backlight_device *bd)
 {
         struct nouveau_encoder *nv_encoder = bl_get_data(bd);
         struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         int or = nv_encoder->or;
         u32 div = 1025;
         u32 val;
 
-        val  = nv_rd32(device, NV50_PDISP_SOR_PWM_CTL(or));
+        val  = nvif_rd32(device, NV50_PDISP_SOR_PWM_CTL(or));
         val &= NV50_PDISP_SOR_PWM_CTL_VAL;
         return ((val * 100) + (div / 2)) / div;
 }
@@ -112,12 +112,12 @@ nv50_set_intensity(struct backlight_device *bd)
 {
         struct nouveau_encoder *nv_encoder = bl_get_data(bd);
         struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         int or = nv_encoder->or;
         u32 div = 1025;
         u32 val = (bd->props.brightness * div) / 100;
 
-        nv_wr32(device, NV50_PDISP_SOR_PWM_CTL(or),
+        nvif_wr32(device, NV50_PDISP_SOR_PWM_CTL(or),
                         NV50_PDISP_SOR_PWM_CTL_NEW | val);
         return 0;
 }
@@ -133,12 +133,12 @@ nva3_get_intensity(struct backlight_device *bd)
 {
         struct nouveau_encoder *nv_encoder = bl_get_data(bd);
         struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         int or = nv_encoder->or;
         u32 div, val;
 
-        div  = nv_rd32(device, NV50_PDISP_SOR_PWM_DIV(or));
-        val  = nv_rd32(device, NV50_PDISP_SOR_PWM_CTL(or));
+        div  = nvif_rd32(device, NV50_PDISP_SOR_PWM_DIV(or));
+        val  = nvif_rd32(device, NV50_PDISP_SOR_PWM_CTL(or));
         val &= NVA3_PDISP_SOR_PWM_CTL_VAL;
         if (div && div >= val)
                 return ((val * 100) + (div / 2)) / div;
@@ -151,14 +151,14 @@ nva3_set_intensity(struct backlight_device *bd)
 {
         struct nouveau_encoder *nv_encoder = bl_get_data(bd);
         struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         int or = nv_encoder->or;
         u32 div, val;
 
-        div = nv_rd32(device, NV50_PDISP_SOR_PWM_DIV(or));
+        div = nvif_rd32(device, NV50_PDISP_SOR_PWM_DIV(or));
         val = (bd->props.brightness * div) / 100;
         if (div) {
-                nv_wr32(device, NV50_PDISP_SOR_PWM_CTL(or), val |
+                nvif_wr32(device, NV50_PDISP_SOR_PWM_CTL(or), val |
                                 NV50_PDISP_SOR_PWM_CTL_NEW |
                                 NVA3_PDISP_SOR_PWM_CTL_UNK);
                 return 0;
@@ -177,7 +177,7 @@ static int
 nv50_backlight_init(struct drm_connector *connector)
 {
         struct nouveau_drm *drm = nouveau_drm(connector->dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         struct nouveau_encoder *nv_encoder;
         struct backlight_properties props;
         struct backlight_device *bd;
@@ -190,12 +190,12 @@ nv50_backlight_init(struct drm_connector *connector)
                         return -ENODEV;
         }
 
-        if (!nv_rd32(device, NV50_PDISP_SOR_PWM_CTL(nv_encoder->or)))
+        if (!nvif_rd32(device, NV50_PDISP_SOR_PWM_CTL(nv_encoder->or)))
                 return 0;
 
-        if (device->chipset <= 0xa0 ||
-            device->chipset == 0xaa ||
-            device->chipset == 0xac)
+        if (device->info.chipset <= 0xa0 ||
+            device->info.chipset == 0xaa ||
+            device->info.chipset == 0xac)
                 ops = &nv50_bl_ops;
         else
                 ops = &nva3_bl_ops;
@@ -218,7 +218,7 @@ int
 nouveau_backlight_init(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         struct drm_connector *connector;
 
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
@@ -226,13 +226,12 @@ nouveau_backlight_init(struct drm_device *dev)
                     connector->connector_type != DRM_MODE_CONNECTOR_eDP)
                         continue;
 
-                switch (device->card_type) {
-                case NV_40:
+                switch (device->info.family) {
+                case NV_DEVICE_INFO_V0_CURIE:
                         return nv40_backlight_init(connector);
-                case NV_50:
-                case NV_C0:
-                case NV_D0:
-                case NV_E0:
+                case NV_DEVICE_INFO_V0_TESLA:
+                case NV_DEVICE_INFO_V0_FERMI:
+                case NV_DEVICE_INFO_V0_KEPLER:
                         return nv50_backlight_init(connector);
                 default:
                         break;
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index 8268a4ccac15..dae2c96deef8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -22,8 +22,6 @@
  * SOFTWARE.
  */
 
-#include <subdev/bios.h>
-
 #include <drm/drmP.h>
 
 #include "nouveau_drm.h"
@@ -217,7 +215,7 @@ int call_lvds_script(struct drm_device *dev, struct dcb_output *dcbent, int head
          */
 
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         struct nvbios *bios = &drm->vbios;
         uint8_t lvds_ver = bios->data[bios->fp.lvdsmanufacturerpointer];
         uint32_t sel_clk_binding, sel_clk;
@@ -240,7 +238,7 @@ int call_lvds_script(struct drm_device *dev, struct dcb_output *dcbent, int head
         NV_INFO(drm, "Calling LVDS script %d:\n", script);
 
         /* don't let script change pll->head binding */
-        sel_clk_binding = nv_rd32(device, NV_PRAMDAC_SEL_CLK) & 0x50000;
+        sel_clk_binding = nvif_rd32(device, NV_PRAMDAC_SEL_CLK) & 0x50000;
 
         if (lvds_ver < 0x30)
                 ret = call_lvds_manufacturer_script(dev, dcbent, head, script);
@@ -252,7 +250,7 @@ int call_lvds_script(struct drm_device *dev, struct dcb_output *dcbent, int head
         sel_clk = NVReadRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK) & ~0x50000;
         NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, sel_clk | sel_clk_binding);
         /* some scripts set a value in NV_PBUS_POWERCTRL_2 and break video overlay */
-        nv_wr32(device, NV_PBUS_POWERCTRL_2, 0);
+        nvif_wr32(device, NV_PBUS_POWERCTRL_2, 0);
 
         return ret;
 }
@@ -320,7 +318,7 @@ static int parse_lvds_manufacturer_table_header(struct drm_device *dev, struct n
 static int
 get_fp_strap(struct drm_device *dev, struct nvbios *bios)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
 
         /*
          * The fp strap is normally dictated by the "User Strap" in
@@ -334,10 +332,10 @@ get_fp_strap(struct drm_device *dev, struct nvbios *bios)
         if (bios->major_version < 5 && bios->data[0x48] & 0x4)
                 return NVReadVgaCrtc5758(dev, 0, 0xf) & 0xf;
 
-        if (device->card_type >= NV_50)
-                return (nv_rd32(device, NV_PEXTDEV_BOOT_0) >> 24) & 0xf;
+        if (device->info.family >= NV_DEVICE_INFO_V0_TESLA)
+                return (nvif_rd32(device, NV_PEXTDEV_BOOT_0) >> 24) & 0xf;
         else
-                return (nv_rd32(device, NV_PEXTDEV_BOOT_0) >> 16) & 0xf;
+                return (nvif_rd32(device, NV_PEXTDEV_BOOT_0) >> 16) & 0xf;
 }
 
 static int parse_fp_mode_table(struct drm_device *dev, struct nvbios *bios)
@@ -636,7 +634,7 @@ int run_tmds_table(struct drm_device *dev, struct dcb_output *dcbent, int head,
          */
 
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         struct nvbios *bios = &drm->vbios;
         int cv = bios->chip_version;
         uint16_t clktable = 0, scriptptr;
@@ -670,7 +668,7 @@ int run_tmds_table(struct drm_device *dev, struct dcb_output *dcbent, int head,
         }
 
         /* don't let script change pll->head binding */
-        sel_clk_binding = nv_rd32(device, NV_PRAMDAC_SEL_CLK) & 0x50000;
+        sel_clk_binding = nvif_rd32(device, NV_PRAMDAC_SEL_CLK) & 0x50000;
         run_digital_op_script(dev, scriptptr, dcbent, head, pxclk >= 165000);
         sel_clk = NVReadRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK) & ~0x50000;
         NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, sel_clk | sel_clk_binding);
@@ -1253,7 +1251,7 @@ olddcb_table(struct drm_device *dev)
         struct nouveau_drm *drm = nouveau_drm(dev);
         u8 *dcb = NULL;
 
-        if (nv_device(drm->device)->card_type > NV_04)
+        if (drm->device.info.family > NV_DEVICE_INFO_V0_TNT)
                 dcb = ROMPTR(dev, drm->vbios.data[0x36]);
         if (!dcb) {
                 NV_WARN(drm, "No DCB data found in VBIOS\n");
@@ -1399,6 +1397,7 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
                   uint32_t conn, uint32_t conf, struct dcb_output *entry)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
+        int link = 0;
 
         entry->type = conn & 0xf;
         entry->i2c_index = (conn >> 4) & 0xf;
@@ -1444,6 +1443,7 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
                         if (conf & 0x4)
                                 entry->lvdsconf.use_power_scripts = true;
                         entry->lvdsconf.sor.link = (conf & 0x00000030) >> 4;
+                        link = entry->lvdsconf.sor.link;
                 }
                 if (conf & mask) {
                         /*
@@ -1492,17 +1492,18 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
                         entry->dpconf.link_nr = 1;
                         break;
                 }
+                link = entry->dpconf.sor.link;
                 break;
         case DCB_OUTPUT_TMDS:
                 if (dcb->version >= 0x40) {
                         entry->tmdsconf.sor.link = (conf & 0x00000030) >> 4;
                         entry->extdev = (conf & 0x0000ff00) >> 8;
+                        link = entry->tmdsconf.sor.link;
                 }
                 else if (dcb->version >= 0x30)
                         entry->tmdsconf.slave_addr = (conf & 0x00000700) >> 8;
                 else if (dcb->version >= 0x22)
                         entry->tmdsconf.slave_addr = (conf & 0x00000070) >> 4;
-
                 break;
         case DCB_OUTPUT_EOL:
                 /* weird g80 mobile type that "nv" treats as a terminator */
@@ -1526,6 +1527,8 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
         if (conf & 0x100000)
                 entry->i2c_upper_default = true;
 
+        entry->hasht = (entry->location << 4) | entry->type;
+        entry->hashm = (entry->heads << 8) | (link << 6) | entry->or;
         return true;
 }
 
@@ -1908,7 +1911,7 @@ static int load_nv17_hwsq_ucode_entry(struct drm_device *dev, struct nvbios *bio
          */
 
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         uint8_t bytes_to_write;
         uint16_t hwsq_entry_offset;
         int i;
@@ -1931,15 +1934,15 @@ static int load_nv17_hwsq_ucode_entry(struct drm_device *dev, struct nvbios *bio
         hwsq_entry_offset = hwsq_offset + 2 + entry * bytes_to_write;
 
         /* set sequencer control */
-        nv_wr32(device, 0x00001304, ROM32(bios->data[hwsq_entry_offset]));
+        nvif_wr32(device, 0x00001304, ROM32(bios->data[hwsq_entry_offset]));
         bytes_to_write -= 4;
 
         /* write ucode */
         for (i = 0; i < bytes_to_write; i += 4)
-                nv_wr32(device, 0x00001400 + i, ROM32(bios->data[hwsq_entry_offset + i + 4]));
+                nvif_wr32(device, 0x00001400 + i, ROM32(bios->data[hwsq_entry_offset + i + 4]));
 
         /* twiddle NV_PBUS_DEBUG_4 */
-        nv_wr32(device, NV_PBUS_DEBUG_4, nv_rd32(device, NV_PBUS_DEBUG_4) | 0x18);
+        nvif_wr32(device, NV_PBUS_DEBUG_4, nvif_rd32(device, NV_PBUS_DEBUG_4) | 0x18);
 
         return 0;
 }
@@ -2002,7 +2005,7 @@ uint8_t *nouveau_bios_embedded_edid(struct drm_device *dev)
 static bool NVInitVBIOS(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_bios *bios = nouveau_bios(drm->device);
+        struct nouveau_bios *bios = nvkm_bios(&drm->device);
         struct nvbios *legacy = &drm->vbios;
 
         memset(legacy, 0, sizeof(struct nvbios));
@@ -2054,7 +2057,7 @@ nouveau_bios_posted(struct drm_device *dev)
         struct nouveau_drm *drm = nouveau_drm(dev);
         unsigned htotal;
 
-        if (nv_device(drm->device)->card_type >= NV_50)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
                 return true;
 
         htotal  = NVReadVgaCrtc(dev, 0, 0x06);
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index ba29a701ca1d..da5d631aa5b9 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -27,13 +27,9 @@
  *          Jeremy Kolb  <jkolb@brandeis.edu>
  */
 
-#include <core/engine.h>
+#include <linux/dma-mapping.h>
 #include <linux/swiotlb.h>
 
-#include <subdev/fb.h>
-#include <subdev/vm.h>
-#include <subdev/bar.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_fence.h"
@@ -52,7 +48,7 @@ nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
         int i = reg - drm->tile.reg;
-        struct nouveau_fb *pfb = nouveau_fb(drm->device);
+        struct nouveau_fb *pfb = nvkm_fb(&drm->device);
         struct nouveau_fb_tile *tile = &pfb->tile.region[i];
         struct nouveau_engine *engine;
 
@@ -109,7 +105,7 @@ nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
                    u32 size, u32 pitch, u32 flags)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_fb *pfb = nouveau_fb(drm->device);
+        struct nouveau_fb *pfb = nvkm_fb(&drm->device);
         struct nouveau_drm_tile *tile, *found = NULL;
         int i;
 
@@ -153,23 +149,23 @@ nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
                        int *align, int *size)
 {
         struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
 
-        if (device->card_type < NV_50) {
+        if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
                 if (nvbo->tile_mode) {
-                        if (device->chipset >= 0x40) {
+                        if (device->info.chipset >= 0x40) {
                                 *align = 65536;
                                 *size = roundup(*size, 64 * nvbo->tile_mode);
 
-                        } else if (device->chipset >= 0x30) {
+                        } else if (device->info.chipset >= 0x30) {
                                 *align = 32768;
                                 *size = roundup(*size, 64 * nvbo->tile_mode);
 
-                        } else if (device->chipset >= 0x20) {
+                        } else if (device->info.chipset >= 0x20) {
                                 *align = 16384;
                                 *size = roundup(*size, 64 * nvbo->tile_mode);
 
-                        } else if (device->chipset >= 0x10) {
+                        } else if (device->info.chipset >= 0x10) {
                                 *align = 16384;
                                 *size = roundup(*size, 32 * nvbo->tile_mode);
                         }
@@ -196,12 +192,12 @@ nouveau_bo_new(struct drm_device *dev, int size, int align,
         int lpg_shift = 12;
         int max_size;
 
-        if (drm->client.base.vm)
-                lpg_shift = drm->client.base.vm->vmm->lpg_shift;
+        if (drm->client.vm)
+                lpg_shift = drm->client.vm->vmm->lpg_shift;
         max_size = INT_MAX & ~((1 << lpg_shift) - 1);
 
         if (size <= 0 || size > max_size) {
-                nv_warn(drm, "skipped size %x\n", (u32)size);
+                NV_WARN(drm, "skipped size %x\n", (u32)size);
                 return -EINVAL;
         }
 
@@ -219,9 +215,9 @@ nouveau_bo_new(struct drm_device *dev, int size, int align,
         nvbo->bo.bdev = &drm->ttm.bdev;
 
         nvbo->page_shift = 12;
-        if (drm->client.base.vm) {
+        if (drm->client.vm) {
                 if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
-                        nvbo->page_shift = drm->client.base.vm->vmm->lpg_shift;
+                        nvbo->page_shift = drm->client.vm->vmm->lpg_shift;
         }
 
         nouveau_bo_fixup_align(nvbo, flags, &align, &size);
@@ -261,11 +257,9 @@ static void
 set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
 {
         struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
-        struct nouveau_fb *pfb = nouveau_fb(drm->device);
-        u32 vram_pages = pfb->ram->size >> PAGE_SHIFT;
+        u32 vram_pages = drm->device.info.ram_size >> PAGE_SHIFT;
 
-        if ((nv_device(drm->device)->card_type == NV_10 ||
-             nv_device(drm->device)->card_type == NV_11) &&
+        if (drm->device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
             nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
             nvbo->bo.mem.num_pages < vram_pages / 4) {
                 /*
@@ -500,21 +494,28 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
                 man->default_caching = TTM_PL_FLAG_CACHED;
                 break;
         case TTM_PL_VRAM:
-                if (nv_device(drm->device)->card_type >= NV_50) {
+                man->flags = TTM_MEMTYPE_FLAG_FIXED |
+                             TTM_MEMTYPE_FLAG_MAPPABLE;
+                man->available_caching = TTM_PL_FLAG_UNCACHED |
+                                         TTM_PL_FLAG_WC;
+                man->default_caching = TTM_PL_FLAG_WC;
+
+                if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
+                        /* Some BARs do not support being ioremapped WC */
+                        if (nvkm_bar(&drm->device)->iomap_uncached) {
+                                man->available_caching = TTM_PL_FLAG_UNCACHED;
+                                man->default_caching = TTM_PL_FLAG_UNCACHED;
+                        }
+
                         man->func = &nouveau_vram_manager;
                         man->io_reserve_fastpath = false;
                         man->use_io_reserve_lru = true;
                 } else {
                         man->func = &ttm_bo_manager_func;
                 }
-                man->flags = TTM_MEMTYPE_FLAG_FIXED |
-                             TTM_MEMTYPE_FLAG_MAPPABLE;
-                man->available_caching = TTM_PL_FLAG_UNCACHED |
-                                         TTM_PL_FLAG_WC;
-                man->default_caching = TTM_PL_FLAG_WC;
                 break;
         case TTM_PL_TT:
-                if (nv_device(drm->device)->card_type >= NV_50)
+                if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
                         man->func = &nouveau_gart_manager;
                 else
                 if (drm->agp.stat != ENABLED)
@@ -763,9 +764,9 @@ nv50_bo_move_init(struct nouveau_channel *chan, u32 handle)
                 BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
                 OUT_RING  (chan, handle);
                 BEGIN_NV04(chan, NvSubCopy, 0x0180, 3);
-                OUT_RING  (chan, NvNotify0);
-                OUT_RING  (chan, NvDmaFB);
-                OUT_RING  (chan, NvDmaFB);
+                OUT_RING  (chan, chan->drm->ntfy.handle);
+                OUT_RING  (chan, chan->vram.handle);
+                OUT_RING  (chan, chan->vram.handle);
         }
 
         return ret;
@@ -852,7 +853,7 @@ nv04_bo_move_init(struct nouveau_channel *chan, u32 handle)
                 BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
                 OUT_RING  (chan, handle);
                 BEGIN_NV04(chan, NvSubCopy, 0x0180, 1);
-                OUT_RING  (chan, NvNotify0);
+                OUT_RING  (chan, chan->drm->ntfy.handle);
         }
 
         return ret;
@@ -864,7 +865,7 @@ nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
 {
         if (mem->mem_type == TTM_PL_TT)
                 return NvDmaTT;
-        return NvDmaFB;
+        return chan->vram.handle;
 }
 
 static int
@@ -922,12 +923,12 @@ nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,
         u64 size = (u64)mem->num_pages << PAGE_SHIFT;
         int ret;
 
-        ret = nouveau_vm_get(nv_client(drm)->vm, size, old_node->page_shift,
+        ret = nouveau_vm_get(drm->client.vm, size, old_node->page_shift,
                              NV_MEM_ACCESS_RW, &old_node->vma[0]);
         if (ret)
                 return ret;
 
-        ret = nouveau_vm_get(nv_client(drm)->vm, size, new_node->page_shift,
+        ret = nouveau_vm_get(drm->client.vm, size, new_node->page_shift,
                              NV_MEM_ACCESS_RW, &old_node->vma[1]);
         if (ret) {
                 nouveau_vm_put(&old_node->vma[0]);
@@ -945,6 +946,7 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
 {
         struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
         struct nouveau_channel *chan = drm->ttm.chan;
+        struct nouveau_cli *cli = (void *)nvif_client(&chan->device->base);
         struct nouveau_fence *fence;
         int ret;
 
@@ -952,13 +954,13 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
          * old nouveau_mem node, these will get cleaned up after ttm has
          * destroyed the ttm_mem_reg
          */
-        if (nv_device(drm->device)->card_type >= NV_50) {
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
                 ret = nouveau_bo_move_prep(drm, bo, new_mem);
                 if (ret)
                         return ret;
         }
 
-        mutex_lock_nested(&chan->cli->mutex, SINGLE_DEPTH_NESTING);
+        mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING);
         ret = nouveau_fence_sync(bo->sync_obj, chan);
         if (ret == 0) {
                 ret = drm->ttm.move(chan, bo, &bo->mem, new_mem);
@@ -973,7 +975,7 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
                         }
                 }
         }
-        mutex_unlock(&chan->cli->mutex);
+        mutex_unlock(&cli->mutex);
         return ret;
 }
 
@@ -1005,9 +1007,7 @@ nouveau_bo_move_init(struct nouveau_drm *drm)
         int ret;
 
         do {
-                struct nouveau_object *object;
                 struct nouveau_channel *chan;
-                u32 handle = (mthd->engine << 16) | mthd->oclass;
 
                 if (mthd->engine)
                         chan = drm->cechan;
@@ -1016,13 +1016,14 @@ nouveau_bo_move_init(struct nouveau_drm *drm)
                 if (chan == NULL)
                         continue;
 
-                ret = nouveau_object_new(nv_object(drm), chan->handle, handle,
-                                         mthd->oclass, NULL, 0, &object);
+                ret = nvif_object_init(chan->object, NULL,
+                                       mthd->oclass | (mthd->engine << 16),
+                                       mthd->oclass, NULL, 0,
+                                       &drm->ttm.copy);
                 if (ret == 0) {
-                        ret = mthd->init(chan, handle);
+                        ret = mthd->init(chan, drm->ttm.copy.handle);
                         if (ret) {
-                                nouveau_object_del(nv_object(drm),
-                                                   chan->handle, handle);
+                                nvif_object_fini(&drm->ttm.copy);
                                 continue;
                         }
 
@@ -1135,7 +1136,7 @@ nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
         if (new_mem->mem_type != TTM_PL_VRAM)
                 return 0;
 
-        if (nv_device(drm->device)->card_type >= NV_10) {
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
                 *new_tile = nv10_bo_set_tiling(dev, offset, new_mem->size,
                                                 nvbo->tile_mode,
                                                 nvbo->tile_flags);
@@ -1166,7 +1167,7 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
         struct nouveau_drm_tile *new_tile = NULL;
         int ret = 0;
 
-        if (nv_device(drm->device)->card_type < NV_50) {
+        if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
                 ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
                 if (ret)
                         return ret;
@@ -1203,7 +1204,7 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
                 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
 
 out:
-        if (nv_device(drm->device)->card_type < NV_50) {
+        if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
                 if (ret)
                         nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
                 else
@@ -1249,16 +1250,16 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
                         mem->bus.is_iomem = !dev->agp->cant_use_aperture;
                 }
 #endif
-                if (nv_device(drm->device)->card_type < NV_50 || !node->memtype)
+                if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA || !node->memtype)
                         /* untiled */
                         break;
                 /* fallthrough, tiled memory */
         case TTM_PL_VRAM:
                 mem->bus.offset = mem->start << PAGE_SHIFT;
-                mem->bus.base = nv_device_resource_start(nouveau_dev(dev), 1);
+                mem->bus.base = nv_device_resource_start(nvkm_device(&drm->device), 1);
                 mem->bus.is_iomem = true;
-                if (nv_device(drm->device)->card_type >= NV_50) {
-                        struct nouveau_bar *bar = nouveau_bar(drm->device);
+                if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
+                        struct nouveau_bar *bar = nvkm_bar(&drm->device);
 
                         ret = bar->umap(bar, node, NV_MEM_ACCESS_RW,
                                         &node->bar_vma);
@@ -1278,7 +1279,7 @@ static void
 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 {
         struct nouveau_drm *drm = nouveau_bdev(bdev);
-        struct nouveau_bar *bar = nouveau_bar(drm->device);
+        struct nouveau_bar *bar = nvkm_bar(&drm->device);
         struct nouveau_mem *node = mem->mm_node;
 
         if (!node->bar_vma.node)
@@ -1292,15 +1293,15 @@ nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
 {
         struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
         struct nouveau_bo *nvbo = nouveau_bo(bo);
-        struct nouveau_device *device = nv_device(drm->device);
-        u32 mappable = nv_device_resource_len(device, 1) >> PAGE_SHIFT;
+        struct nvif_device *device = &drm->device;
+        u32 mappable = nv_device_resource_len(nvkm_device(device), 1) >> PAGE_SHIFT;
         int ret;
 
         /* as long as the bo isn't in vram, and isn't tiled, we've got
          * nothing to do here.
          */
         if (bo->mem.mem_type != TTM_PL_VRAM) {
-                if (nv_device(drm->device)->card_type < NV_50 ||
+                if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA ||
                     !nouveau_bo_tile_layout(nvbo))
                         return 0;
 
@@ -1315,7 +1316,7 @@ nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
         }
 
         /* make sure bo is in mappable vram */
-        if (nv_device(drm->device)->card_type >= NV_50 ||
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA ||
             bo->mem.start + bo->mem.num_pages < mappable)
                 return 0;
 
@@ -1333,6 +1334,7 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm)
         struct nouveau_drm *drm;
         struct nouveau_device *device;
         struct drm_device *dev;
+        struct device *pdev;
         unsigned i;
         int r;
         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
@@ -1349,8 +1351,9 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm)
         }
 
         drm = nouveau_bdev(ttm->bdev);
-        device = nv_device(drm->device);
+        device = nvkm_device(&drm->device);
         dev = drm->dev;
+        pdev = nv_device_base(device);
 
 #if __OS_HAS_AGP
         if (drm->agp.stat == ENABLED) {
@@ -1370,17 +1373,22 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm)
         }
 
         for (i = 0; i < ttm->num_pages; i++) {
-                ttm_dma->dma_address[i] = nv_device_map_page(device,
-                                                             ttm->pages[i]);
-                if (!ttm_dma->dma_address[i]) {
+                dma_addr_t addr;
+
+                addr = dma_map_page(pdev, ttm->pages[i], 0, PAGE_SIZE,
+                                    DMA_BIDIRECTIONAL);
+
+                if (dma_mapping_error(pdev, addr)) {
                         while (--i) {
-                                nv_device_unmap_page(device,
-                                                     ttm_dma->dma_address[i]);
+                                dma_unmap_page(pdev, ttm_dma->dma_address[i],
+                                               PAGE_SIZE, DMA_BIDIRECTIONAL);
                                 ttm_dma->dma_address[i] = 0;
                         }
                         ttm_pool_unpopulate(ttm);
                         return -EFAULT;
                 }
+
+                ttm_dma->dma_address[i] = addr;
         }
         return 0;
 }
@@ -1392,6 +1400,7 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
         struct nouveau_drm *drm;
         struct nouveau_device *device;
         struct drm_device *dev;
+        struct device *pdev;
         unsigned i;
         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
 
@@ -1399,8 +1408,9 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
                 return;
 
         drm = nouveau_bdev(ttm->bdev);
-        device = nv_device(drm->device);
+        device = nvkm_device(&drm->device);
         dev = drm->dev;
+        pdev = nv_device_base(device);
 
 #if __OS_HAS_AGP
         if (drm->agp.stat == ENABLED) {
@@ -1418,7 +1428,8 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
 
         for (i = 0; i < ttm->num_pages; i++) {
                 if (ttm_dma->dma_address[i]) {
-                        nv_device_unmap_page(device, ttm_dma->dma_address[i]);
+                        dma_unmap_page(pdev, ttm_dma->dma_address[i], PAGE_SIZE,
+                                       DMA_BIDIRECTIONAL);
                 }
         }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c
index ccb6b452d6d0..99cd9e4a2aa6 100644
--- a/drivers/gpu/drm/nouveau/nouveau_chan.c
+++ b/drivers/gpu/drm/nouveau/nouveau_chan.c
@@ -22,16 +22,11 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/object.h>
-#include <core/client.h>
-#include <core/device.h>
-#include <core/class.h>
-
-#include <subdev/fb.h>
-#include <subdev/vm.h>
-#include <subdev/instmem.h>
+#include <nvif/os.h>
+#include <nvif/class.h>
 
-#include <engine/software.h>
+/*XXX*/
+#include <core/client.h>
 
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
@@ -47,7 +42,7 @@ module_param_named(vram_pushbuf, nouveau_vram_pushbuf, int, 0400);
 int
 nouveau_channel_idle(struct nouveau_channel *chan)
 {
-        struct nouveau_cli *cli = chan->cli;
+        struct nouveau_cli *cli = (void *)nvif_client(chan->object);
         struct nouveau_fence *fence = NULL;
         int ret;
 
@@ -58,8 +53,8 @@ nouveau_channel_idle(struct nouveau_channel *chan)
         }
 
         if (ret)
-                NV_ERROR(cli, "failed to idle channel 0x%08x [%s]\n",
-                         chan->handle, cli->base.name);
+                NV_PRINTK(error, cli, "failed to idle channel 0x%08x [%s]\n",
+                          chan->object->handle, nvkm_client(&cli->base)->name);
         return ret;
 }
 
@@ -68,36 +63,34 @@ nouveau_channel_del(struct nouveau_channel **pchan)
 {
         struct nouveau_channel *chan = *pchan;
         if (chan) {
-                struct nouveau_object *client = nv_object(chan->cli);
                 if (chan->fence) {
                         nouveau_channel_idle(chan);
                         nouveau_fence(chan->drm)->context_del(chan);
                 }
-                nouveau_object_del(client, NVDRM_DEVICE, chan->handle);
-                nouveau_object_del(client, NVDRM_DEVICE, chan->push.handle);
+                nvif_object_fini(&chan->nvsw);
+                nvif_object_fini(&chan->gart);
+                nvif_object_fini(&chan->vram);
+                nvif_object_ref(NULL, &chan->object);
+                nvif_object_fini(&chan->push.ctxdma);
                 nouveau_bo_vma_del(chan->push.buffer, &chan->push.vma);
                 nouveau_bo_unmap(chan->push.buffer);
                 if (chan->push.buffer && chan->push.buffer->pin_refcnt)
                         nouveau_bo_unpin(chan->push.buffer);
                 nouveau_bo_ref(NULL, &chan->push.buffer);
+                nvif_device_ref(NULL, &chan->device);
                 kfree(chan);
         }
         *pchan = NULL;
 }
 
 static int
-nouveau_channel_prep(struct nouveau_drm *drm, struct nouveau_cli *cli,
-                     u32 parent, u32 handle, u32 size,
-                     struct nouveau_channel **pchan)
+nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device,
+                     u32 handle, u32 size, struct nouveau_channel **pchan)
 {
-        struct nouveau_device *device = nv_device(drm->device);
-        struct nouveau_instmem *imem = nouveau_instmem(device);
-        struct nouveau_vmmgr *vmm = nouveau_vmmgr(device);
-        struct nouveau_fb *pfb = nouveau_fb(device);
-        struct nouveau_client *client = &cli->base;
-        struct nv_dma_class args = {};
+        struct nouveau_cli *cli = (void *)nvif_client(&device->base);
+        struct nouveau_vmmgr *vmm = nvkm_vmmgr(device);
+        struct nv_dma_v0 args = {};
         struct nouveau_channel *chan;
-        struct nouveau_object *push;
         u32 target;
         int ret;
 
@@ -105,9 +98,8 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nouveau_cli *cli,
         if (!chan)
                 return -ENOMEM;
 
-        chan->cli = cli;
+        nvif_device_ref(device, &chan->device);
         chan->drm = drm;
-        chan->handle = handle;
 
         /* allocate memory for dma push buffer */
         target = TTM_PL_FLAG_TT;
@@ -132,51 +124,54 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nouveau_cli *cli,
          * we be able to call out to other (indirect) push buffers
          */
         chan->push.vma.offset = chan->push.buffer->bo.offset;
-        chan->push.handle = NVDRM_PUSH | (handle & 0xffff);
 
-        if (device->card_type >= NV_50) {
-                ret = nouveau_bo_vma_add(chan->push.buffer, client->vm,
+        if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) {
+                ret = nouveau_bo_vma_add(chan->push.buffer, cli->vm,
                                         &chan->push.vma);
                 if (ret) {
                         nouveau_channel_del(pchan);
                         return ret;
                 }
 
-                args.flags = NV_DMA_TARGET_VM | NV_DMA_ACCESS_VM;
+                args.target = NV_DMA_V0_TARGET_VM;
+                args.access = NV_DMA_V0_ACCESS_VM;
                 args.start = 0;
-                args.limit = client->vm->vmm->limit - 1;
+                args.limit = cli->vm->vmm->limit - 1;
         } else
         if (chan->push.buffer->bo.mem.mem_type == TTM_PL_VRAM) {
-                u64 limit = pfb->ram->size - imem->reserved - 1;
-                if (device->card_type == NV_04) {
+                if (device->info.family == NV_DEVICE_INFO_V0_TNT) {
                         /* nv04 vram pushbuf hack, retarget to its location in
                          * the framebuffer bar rather than direct vram access..
                          * nfi why this exists, it came from the -nv ddx.
                          */
-                        args.flags = NV_DMA_TARGET_PCI | NV_DMA_ACCESS_RDWR;
-                        args.start = nv_device_resource_start(device, 1);
-                        args.limit = args.start + limit;
+                        args.target = NV_DMA_V0_TARGET_PCI;
+                        args.access = NV_DMA_V0_ACCESS_RDWR;
+                        args.start = nv_device_resource_start(nvkm_device(device), 1);
+                        args.limit = args.start + device->info.ram_user - 1;
                 } else {
-                        args.flags = NV_DMA_TARGET_VRAM | NV_DMA_ACCESS_RDWR;
+                        args.target = NV_DMA_V0_TARGET_VRAM;
+                        args.access = NV_DMA_V0_ACCESS_RDWR;
                         args.start = 0;
-                        args.limit = limit;
+                        args.limit = device->info.ram_user - 1;
                 }
         } else {
                 if (chan->drm->agp.stat == ENABLED) {
-                        args.flags = NV_DMA_TARGET_AGP | NV_DMA_ACCESS_RDWR;
+                        args.target = NV_DMA_V0_TARGET_AGP;
+                        args.access = NV_DMA_V0_ACCESS_RDWR;
                         args.start = chan->drm->agp.base;
                         args.limit = chan->drm->agp.base +
                                      chan->drm->agp.size - 1;
                 } else {
-                        args.flags = NV_DMA_TARGET_VM | NV_DMA_ACCESS_RDWR;
+                        args.target = NV_DMA_V0_TARGET_VM;
+                        args.access = NV_DMA_V0_ACCESS_RDWR;
                         args.start = 0;
                         args.limit = vmm->limit - 1;
                 }
         }
 
-        ret = nouveau_object_new(nv_object(chan->cli), parent,
-                                 chan->push.handle, 0x0002,
-                                 &args, sizeof(args), &push);
+        ret = nvif_object_init(nvif_object(device), NULL, NVDRM_PUSH |
+                               (handle & 0xffff), NV_DMA_FROM_MEMORY,
+                               &args, sizeof(args), &chan->push.ctxdma);
         if (ret) {
                 nouveau_channel_del(pchan);
                 return ret;
@@ -186,38 +181,56 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nouveau_cli *cli,
 }
 
 static int
-nouveau_channel_ind(struct nouveau_drm *drm, struct nouveau_cli *cli,
-                    u32 parent, u32 handle, u32 engine,
-                    struct nouveau_channel **pchan)
+nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device,
+                    u32 handle, u32 engine, struct nouveau_channel **pchan)
 {
-        static const u16 oclasses[] = { NVE0_CHANNEL_IND_CLASS,
-                                        NVC0_CHANNEL_IND_CLASS,
-                                        NV84_CHANNEL_IND_CLASS,
-                                        NV50_CHANNEL_IND_CLASS,
+        static const u16 oclasses[] = { KEPLER_CHANNEL_GPFIFO_A,
+                                        FERMI_CHANNEL_GPFIFO,
+                                        G82_CHANNEL_GPFIFO,
+                                        NV50_CHANNEL_GPFIFO,
                                         0 };
         const u16 *oclass = oclasses;
-        struct nve0_channel_ind_class args;
+        union {
+                struct nv50_channel_gpfifo_v0 nv50;
+                struct kepler_channel_gpfifo_a_v0 kepler;
+        } args, *retn;
         struct nouveau_channel *chan;
+        u32 size;
         int ret;
 
         /* allocate dma push buffer */
-        ret = nouveau_channel_prep(drm, cli, parent, handle, 0x12000, &chan);
+        ret = nouveau_channel_prep(drm, device, handle, 0x12000, &chan);
         *pchan = chan;
         if (ret)
                 return ret;
 
         /* create channel object */
-        args.pushbuf = chan->push.handle;
-        args.ioffset = 0x10000 + chan->push.vma.offset;
-        args.ilength = 0x02000;
-        args.engine  = engine;
-
         do {
-                ret = nouveau_object_new(nv_object(cli), parent, handle,
-                                         *oclass++, &args, sizeof(args),
-                                         &chan->object);
-                if (ret == 0)
+                if (oclass[0] >= KEPLER_CHANNEL_GPFIFO_A) {
+                        args.kepler.version = 0;
+                        args.kepler.engine  = engine;
+                        args.kepler.pushbuf = chan->push.ctxdma.handle;
+                        args.kepler.ilength = 0x02000;
+                        args.kepler.ioffset = 0x10000 + chan->push.vma.offset;
+                        size = sizeof(args.kepler);
+                } else {
+                        args.nv50.version = 0;
+                        args.nv50.pushbuf = chan->push.ctxdma.handle;
+                        args.nv50.ilength = 0x02000;
+                        args.nv50.ioffset = 0x10000 + chan->push.vma.offset;
+                        size = sizeof(args.nv50);
+                }
+
+                ret = nvif_object_new(nvif_object(device), handle, *oclass++,
+                                      &args, size, &chan->object);
+                if (ret == 0) {
+                        retn = chan->object->data;
+                        if (chan->object->oclass >= KEPLER_CHANNEL_GPFIFO_A)
+                                chan->chid = retn->kepler.chid;
+                        else
+                                chan->chid = retn->nv50.chid;
                         return ret;
+                }
         } while (*oclass);
 
         nouveau_channel_del(pchan);
@@ -225,35 +238,38 @@ nouveau_channel_ind(struct nouveau_drm *drm, struct nouveau_cli *cli,
 }
 
 static int
-nouveau_channel_dma(struct nouveau_drm *drm, struct nouveau_cli *cli,
-                    u32 parent, u32 handle, struct nouveau_channel **pchan)
+nouveau_channel_dma(struct nouveau_drm *drm, struct nvif_device *device,
+                    u32 handle, struct nouveau_channel **pchan)
 {
-        static const u16 oclasses[] = { NV40_CHANNEL_DMA_CLASS,
-                                        NV17_CHANNEL_DMA_CLASS,
-                                        NV10_CHANNEL_DMA_CLASS,
-                                        NV03_CHANNEL_DMA_CLASS,
+        static const u16 oclasses[] = { NV40_CHANNEL_DMA,
+                                        NV17_CHANNEL_DMA,
+                                        NV10_CHANNEL_DMA,
+                                        NV03_CHANNEL_DMA,
                                         0 };
         const u16 *oclass = oclasses;
-        struct nv03_channel_dma_class args;
+        struct nv03_channel_dma_v0 args, *retn;
         struct nouveau_channel *chan;
         int ret;
 
         /* allocate dma push buffer */
-        ret = nouveau_channel_prep(drm, cli, parent, handle, 0x10000, &chan);
+        ret = nouveau_channel_prep(drm, device, handle, 0x10000, &chan);
         *pchan = chan;
         if (ret)
                 return ret;
 
         /* create channel object */
-        args.pushbuf = chan->push.handle;
+        args.version = 0;
+        args.pushbuf = chan->push.ctxdma.handle;
         args.offset = chan->push.vma.offset;
 
         do {
-                ret = nouveau_object_new(nv_object(cli), parent, handle,
-                                         *oclass++, &args, sizeof(args),
-                                         &chan->object);
-                if (ret == 0)
+                ret = nvif_object_new(nvif_object(device), handle, *oclass++,
+                                      &args, sizeof(args), &chan->object);
+                if (ret == 0) {
+                        retn = chan->object->data;
+                        chan->chid = retn->chid;
                         return ret;
+                }
         } while (ret && *oclass);
 
         nouveau_channel_del(pchan);
@@ -263,60 +279,63 @@ nouveau_channel_dma(struct nouveau_drm *drm, struct nouveau_cli *cli,
 static int
 nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart)
 {
-        struct nouveau_client *client = nv_client(chan->cli);
-        struct nouveau_device *device = nv_device(chan->drm->device);
-        struct nouveau_instmem *imem = nouveau_instmem(device);
-        struct nouveau_vmmgr *vmm = nouveau_vmmgr(device);
-        struct nouveau_fb *pfb = nouveau_fb(device);
+        struct nvif_device *device = chan->device;
+        struct nouveau_cli *cli = (void *)nvif_client(&device->base);
+        struct nouveau_vmmgr *vmm = nvkm_vmmgr(device);
         struct nouveau_software_chan *swch;
-        struct nouveau_object *object;
-        struct nv_dma_class args = {};
+        struct nv_dma_v0 args = {};
         int ret, i;
 
+        nvif_object_map(chan->object);
+
         /* allocate dma objects to cover all allowed vram, and gart */
-        if (device->card_type < NV_C0) {
-                if (device->card_type >= NV_50) {
-                        args.flags = NV_DMA_TARGET_VM | NV_DMA_ACCESS_VM;
+        if (device->info.family < NV_DEVICE_INFO_V0_FERMI) {
+                if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) {
+                        args.target = NV_DMA_V0_TARGET_VM;
+                        args.access = NV_DMA_V0_ACCESS_VM;
                         args.start = 0;
-                        args.limit = client->vm->vmm->limit - 1;
+                        args.limit = cli->vm->vmm->limit - 1;
                 } else {
-                        args.flags = NV_DMA_TARGET_VRAM | NV_DMA_ACCESS_RDWR;
+                        args.target = NV_DMA_V0_TARGET_VRAM;
+                        args.access = NV_DMA_V0_ACCESS_RDWR;
                         args.start = 0;
-                        args.limit = pfb->ram->size - imem->reserved - 1;
+                        args.limit = device->info.ram_user - 1;
                 }
 
-                ret = nouveau_object_new(nv_object(client), chan->handle, vram,
-                                         0x003d, &args, sizeof(args), &object);
+                ret = nvif_object_init(chan->object, NULL, vram,
+                                       NV_DMA_IN_MEMORY, &args,
+                                       sizeof(args), &chan->vram);
                 if (ret)
                         return ret;
 
-                if (device->card_type >= NV_50) {
-                        args.flags = NV_DMA_TARGET_VM | NV_DMA_ACCESS_VM;
+                if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) {
+                        args.target = NV_DMA_V0_TARGET_VM;
+                        args.access = NV_DMA_V0_ACCESS_VM;
                         args.start = 0;
-                        args.limit = client->vm->vmm->limit - 1;
+                        args.limit = cli->vm->vmm->limit - 1;
                 } else
                 if (chan->drm->agp.stat == ENABLED) {
-                        args.flags = NV_DMA_TARGET_AGP | NV_DMA_ACCESS_RDWR;
+                        args.target = NV_DMA_V0_TARGET_AGP;
+                        args.access = NV_DMA_V0_ACCESS_RDWR;
                         args.start = chan->drm->agp.base;
                         args.limit = chan->drm->agp.base +
                                      chan->drm->agp.size - 1;
                 } else {
-                        args.flags = NV_DMA_TARGET_VM | NV_DMA_ACCESS_RDWR;
+                        args.target = NV_DMA_V0_TARGET_VM;
+                        args.access = NV_DMA_V0_ACCESS_RDWR;
                         args.start = 0;
                         args.limit = vmm->limit - 1;
                 }
 
-                ret = nouveau_object_new(nv_object(client), chan->handle, gart,
-                                         0x003d, &args, sizeof(args), &object);
+                ret = nvif_object_init(chan->object, NULL, gart,
+                                       NV_DMA_IN_MEMORY, &args,
+                                       sizeof(args), &chan->gart);
                 if (ret)
                         return ret;
-
-                chan->vram = vram;
-                chan->gart = gart;
         }
 
         /* initialise dma tracking parameters */
-        switch (nv_hclass(chan->object) & 0x00ff) {
+        switch (chan->object->oclass & 0x00ff) {
         case 0x006b:
         case 0x006e:
                 chan->user_put = 0x40;
@@ -347,13 +366,13 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart)
                 OUT_RING(chan, 0x00000000);
 
         /* allocate software object class (used for fences on <= nv05) */
-        if (device->card_type < NV_10) {
-                ret = nouveau_object_new(nv_object(client), chan->handle,
-                                         NvSw, 0x006e, NULL, 0, &object);
+        if (device->info.family < NV_DEVICE_INFO_V0_CELSIUS) {
+                ret = nvif_object_init(chan->object, NULL, 0x006e, 0x006e,
+                                       NULL, 0, &chan->nvsw);
                 if (ret)
                         return ret;
 
-                swch = (void *)object->parent;
+                swch = (void *)nvkm_object(&chan->nvsw)->parent;
                 swch->flip = nouveau_flip_complete;
                 swch->flip_data = chan;
 
@@ -362,7 +381,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart)
                         return ret;
 
                 BEGIN_NV04(chan, NvSubSw, 0x0000, 1);
-                OUT_RING  (chan, NvSw);
+                OUT_RING  (chan, chan->nvsw.handle);
                 FIRE_RING (chan);
         }
 
@@ -371,25 +390,26 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart)
 }
 
 int
-nouveau_channel_new(struct nouveau_drm *drm, struct nouveau_cli *cli,
-                    u32 parent, u32 handle, u32 arg0, u32 arg1,
+nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device,
+                    u32 handle, u32 arg0, u32 arg1,
                     struct nouveau_channel **pchan)
 {
+        struct nouveau_cli *cli = (void *)nvif_client(&device->base);
         int ret;
 
-        ret = nouveau_channel_ind(drm, cli, parent, handle, arg0, pchan);
+        ret = nouveau_channel_ind(drm, device, handle, arg0, pchan);
         if (ret) {
-                NV_DEBUG(cli, "ib channel create, %d\n", ret);
-                ret = nouveau_channel_dma(drm, cli, parent, handle, pchan);
+                NV_PRINTK(debug, cli, "ib channel create, %d\n", ret);
+                ret = nouveau_channel_dma(drm, device, handle, pchan);
                 if (ret) {
-                        NV_DEBUG(cli, "dma channel create, %d\n", ret);
+                        NV_PRINTK(debug, cli, "dma channel create, %d\n", ret);
                         return ret;
                 }
         }
 
         ret = nouveau_channel_init(*pchan, arg0, arg1);
         if (ret) {
-                NV_ERROR(cli, "channel failed to initialise, %d\n", ret);
+                NV_PRINTK(error, cli, "channel failed to initialise, %d\n", ret);
                 nouveau_channel_del(pchan);
                 return ret;
         }
diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.h b/drivers/gpu/drm/nouveau/nouveau_chan.h
index 40f97e2c47b6..20163709d608 100644
--- a/drivers/gpu/drm/nouveau/nouveau_chan.h
+++ b/drivers/gpu/drm/nouveau/nouveau_chan.h
@@ -1,20 +1,23 @@
 #ifndef __NOUVEAU_CHAN_H__
 #define __NOUVEAU_CHAN_H__
 
-struct nouveau_cli;
+#include <nvif/object.h>
+struct nvif_device;
 
 struct nouveau_channel {
-        struct nouveau_cli *cli;
+        struct nvif_device *device;
         struct nouveau_drm *drm;
 
-        u32 handle;
-        u32 vram;
-        u32 gart;
+        int chid;
+
+        struct nvif_object vram;
+        struct nvif_object gart;
+        struct nvif_object nvsw;
 
         struct {
                 struct nouveau_bo *buffer;
                 struct nouveau_vma vma;
-                u32 handle;
+                struct nvif_object ctxdma;
         } push;
 
         /* TODO: this will be reworked in the near future */
@@ -34,12 +37,12 @@ struct nouveau_channel {
         u32 user_get;
         u32 user_put;
 
-        struct nouveau_object *object;
+        struct nvif_object *object;
 };
 
 
-int  nouveau_channel_new(struct nouveau_drm *, struct nouveau_cli *,
-                         u32 parent, u32 handle, u32 arg0, u32 arg1,
+int  nouveau_channel_new(struct nouveau_drm *, struct nvif_device *,
+                         u32 handle, u32 arg0, u32 arg1,
                          struct nouveau_channel **);
 void nouveau_channel_del(struct nouveau_channel **);
 int  nouveau_channel_idle(struct nouveau_channel *);
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index dbdc9ad59546..1ec44c83e919 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -42,9 +42,7 @@
 #include "nouveau_encoder.h"
 #include "nouveau_crtc.h"
 
-#include <subdev/i2c.h>
-#include <subdev/gpio.h>
-#include <engine/disp.h>
+#include <nvif/event.h>
 
 MODULE_PARM_DESC(tv_disable, "Disable TV-out detection");
 static int nouveau_tv_disable = 0;
@@ -102,7 +100,7 @@ static void
 nouveau_connector_destroy(struct drm_connector *connector)
 {
         struct nouveau_connector *nv_connector = nouveau_connector(connector);
-        nouveau_event_ref(NULL, &nv_connector->hpd);
+        nvif_notify_fini(&nv_connector->hpd);
         kfree(nv_connector->edid);
         drm_connector_unregister(connector);
         drm_connector_cleanup(connector);
@@ -117,7 +115,7 @@ nouveau_connector_ddc_detect(struct drm_connector *connector)
         struct drm_device *dev = connector->dev;
         struct nouveau_connector *nv_connector = nouveau_connector(connector);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
+        struct nouveau_gpio *gpio = nvkm_gpio(&drm->device);
         struct nouveau_encoder *nv_encoder;
         struct drm_encoder *encoder;
         int i, panel = -ENODEV;
@@ -206,7 +204,7 @@ nouveau_connector_set_encoder(struct drm_connector *connector,
                 return;
         nv_connector->detected_encoder = nv_encoder;
 
-        if (nv_device(drm->device)->card_type >= NV_50) {
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
                 connector->interlace_allowed = true;
                 connector->doublescan_allowed = true;
         } else
@@ -216,9 +214,8 @@ nouveau_connector_set_encoder(struct drm_connector *connector,
                 connector->interlace_allowed = false;
         } else {
                 connector->doublescan_allowed = true;
-                if (nv_device(drm->device)->card_type == NV_20 ||
-                    ((nv_device(drm->device)->card_type == NV_10 ||
-                      nv_device(drm->device)->card_type == NV_11) &&
+                if (drm->device.info.family == NV_DEVICE_INFO_V0_KELVIN ||
+                    (drm->device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
                      (dev->pdev->device & 0x0ff0) != 0x0100 &&
                      (dev->pdev->device & 0x0ff0) != 0x0150))
                         /* HW is broken */
@@ -802,11 +799,11 @@ get_tmds_link_bandwidth(struct drm_connector *connector)
         struct dcb_output *dcb = nv_connector->detected_encoder->dcb;
 
         if (dcb->location != DCB_LOC_ON_CHIP ||
-            nv_device(drm->device)->chipset >= 0x46)
+            drm->device.info.chipset >= 0x46)
                 return 165000;
-        else if (nv_device(drm->device)->chipset >= 0x40)
+        else if (drm->device.info.chipset >= 0x40)
                 return 155000;
-        else if (nv_device(drm->device)->chipset >= 0x18)
+        else if (drm->device.info.chipset >= 0x18)
                 return 135000;
         else
                 return 112000;
@@ -939,18 +936,19 @@ nouveau_connector_funcs_dp = {
         .force = nouveau_connector_force
 };
 
-static void
-nouveau_connector_hotplug_work(struct work_struct *work)
+static int
+nouveau_connector_hotplug(struct nvif_notify *notify)
 {
         struct nouveau_connector *nv_connector =
-                container_of(work, typeof(*nv_connector), work);
+                container_of(notify, typeof(*nv_connector), hpd);
         struct drm_connector *connector = &nv_connector->base;
         struct nouveau_drm *drm = nouveau_drm(connector->dev);
+        const struct nvif_notify_conn_rep_v0 *rep = notify->data;
         const char *name = connector->name;
 
-        if (nv_connector->status & NVKM_HPD_IRQ) {
+        if (rep->mask & NVIF_NOTIFY_CONN_V0_IRQ) {
         } else {
-                bool plugged = (nv_connector->status != NVKM_HPD_UNPLUG);
+                bool plugged = (rep->mask != NVIF_NOTIFY_CONN_V0_UNPLUG);
 
                 NV_DEBUG(drm, "%splugged %s\n", plugged ? "" : "un", name);
 
@@ -961,16 +959,7 @@ nouveau_connector_hotplug_work(struct work_struct *work)
                 drm_helper_hpd_irq_event(connector->dev);
         }
 
-        nouveau_event_get(nv_connector->hpd);
-}
-
-static int
-nouveau_connector_hotplug(void *data, u32 type, int index)
-{
-        struct nouveau_connector *nv_connector = data;
-        nv_connector->status = type;
-        schedule_work(&nv_connector->work);
-        return NVKM_EVENT_DROP;
+        return NVIF_NOTIFY_KEEP;
 }
 
 static ssize_t
@@ -1040,7 +1029,6 @@ nouveau_connector_create(struct drm_device *dev, int index)
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_display *disp = nouveau_display(dev);
         struct nouveau_connector *nv_connector = NULL;
-        struct nouveau_disp *pdisp = nouveau_disp(drm->device);
         struct drm_connector *connector;
         int type, ret = 0;
         bool dummy;
@@ -1194,7 +1182,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
 
         switch (nv_connector->type) {
         case DCB_CONNECTOR_VGA:
-                if (nv_device(drm->device)->card_type >= NV_50) {
+                if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
                         drm_object_attach_property(&connector->base,
                                         dev->mode_config.scaling_mode_property,
                                         nv_connector->scaling_mode);
@@ -1226,16 +1214,20 @@ nouveau_connector_create(struct drm_device *dev, int index)
                 break;
         }
 
-        ret = nouveau_event_new(pdisp->hpd, NVKM_HPD, index,
-                                nouveau_connector_hotplug,
-                                nv_connector, &nv_connector->hpd);
+        ret = nvif_notify_init(&disp->disp, NULL, nouveau_connector_hotplug,
+                                true, NV04_DISP_NTFY_CONN,
+                               &(struct nvif_notify_conn_req_v0) {
+                                .mask = NVIF_NOTIFY_CONN_V0_ANY,
+                                .conn = index,
+                               },
+                               sizeof(struct nvif_notify_conn_req_v0),
+                               sizeof(struct nvif_notify_conn_rep_v0),
+                               &nv_connector->hpd);
         if (ret)
                 connector->polled = DRM_CONNECTOR_POLL_CONNECT;
         else
                 connector->polled = DRM_CONNECTOR_POLL_HPD;
 
-        INIT_WORK(&nv_connector->work, nouveau_connector_hotplug_work);
-
         drm_connector_register(connector);
         return connector;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h
index 8861b6c579ad..68029d041dd2 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.h
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.h
@@ -27,14 +27,12 @@
 #ifndef __NOUVEAU_CONNECTOR_H__
 #define __NOUVEAU_CONNECTOR_H__
 
+#include <nvif/notify.h>
+
 #include <drm/drm_edid.h>
 #include <drm/drm_dp_helper.h>
 #include "nouveau_crtc.h"
 
-#include <core/event.h>
-
-#include <subdev/bios.h>
-
 struct nouveau_i2c_port;
 
 enum nouveau_underscan_type {
@@ -67,9 +65,7 @@ struct nouveau_connector {
         u8 index;
         u8 *dcb;
 
-        struct nouveau_eventh *hpd;
-        u32 status;
-        struct work_struct work;
+        struct nvif_notify hpd;
 
         struct drm_dp_aux aux;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_crtc.h b/drivers/gpu/drm/nouveau/nouveau_crtc.h
index a0534489d23f..f19cb1c5fc5a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_crtc.h
+++ b/drivers/gpu/drm/nouveau/nouveau_crtc.h
@@ -27,10 +27,13 @@
 #ifndef __NOUVEAU_CRTC_H__
 #define __NOUVEAU_CRTC_H__
 
+#include <nvif/notify.h>
+
 struct nouveau_crtc {
         struct drm_crtc base;
 
         int index;
+        struct nvif_notify vblank;
 
         uint32_t dpms_saved_fp_control;
         uint32_t fp_users;
@@ -46,7 +49,7 @@ struct nouveau_crtc {
                 int cpp;
                 bool blanked;
                 uint32_t offset;
-                uint32_t tile_flags;
+                uint32_t handle;
         } fb;
 
         struct {
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 47ad74255bf1..1cc7b603c753 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -27,6 +27,8 @@
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 
+#include <nvif/class.h>
+
 #include "nouveau_fbcon.h"
 #include "dispnv04/hw.h"
 #include "nouveau_crtc.h"
@@ -37,35 +39,42 @@
 
 #include "nouveau_fence.h"
 
-#include <engine/disp.h>
-
-#include <core/class.h>
+#include <nvif/event.h>
 
 static int
-nouveau_display_vblank_handler(void *data, u32 type, int head)
+nouveau_display_vblank_handler(struct nvif_notify *notify)
 {
-        struct nouveau_drm *drm = data;
-        drm_handle_vblank(drm->dev, head);
-        return NVKM_EVENT_KEEP;
+        struct nouveau_crtc *nv_crtc =
+                container_of(notify, typeof(*nv_crtc), vblank);
+        drm_handle_vblank(nv_crtc->base.dev, nv_crtc->index);
+        return NVIF_NOTIFY_KEEP;
 }
 
 int
 nouveau_display_vblank_enable(struct drm_device *dev, int head)
 {
-        struct nouveau_display *disp = nouveau_display(dev);
-        if (disp) {
-                nouveau_event_get(disp->vblank[head]);
-                return 0;
+        struct drm_crtc *crtc;
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+                if (nv_crtc->index == head) {
+                        nvif_notify_get(&nv_crtc->vblank);
+                        return 0;
+                }
         }
-        return -EIO;
+        return -EINVAL;
 }
 
 void
 nouveau_display_vblank_disable(struct drm_device *dev, int head)
 {
-        struct nouveau_display *disp = nouveau_display(dev);
-        if (disp)
-                nouveau_event_put(disp->vblank[head]);
+        struct drm_crtc *crtc;
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+                if (nv_crtc->index == head) {
+                        nvif_notify_put(&nv_crtc->vblank);
+                        return;
+                }
+        }
 }
 
 static inline int
@@ -86,17 +95,22 @@ int
 nouveau_display_scanoutpos_head(struct drm_crtc *crtc, int *vpos, int *hpos,
                                 ktime_t *stime, ktime_t *etime)
 {
-        const u32 mthd = NV04_DISP_SCANOUTPOS + nouveau_crtc(crtc)->index;
+        struct {
+                struct nv04_disp_mthd_v0 base;
+                struct nv04_disp_scanoutpos_v0 scan;
+        } args = {
+                .base.method = NV04_DISP_SCANOUTPOS,
+                .base.head = nouveau_crtc(crtc)->index,
+        };
         struct nouveau_display *disp = nouveau_display(crtc->dev);
-        struct nv04_display_scanoutpos args;
         int ret, retry = 1;
 
         do {
-                ret = nv_exec(disp->core, mthd, &args, sizeof(args));
+                ret = nvif_mthd(&disp->disp, 0, &args, sizeof(args));
                 if (ret != 0)
                         return 0;
 
-                if (args.vline) {
+                if (args.scan.vline) {
                         ret |= DRM_SCANOUTPOS_ACCURATE;
                         ret |= DRM_SCANOUTPOS_VALID;
                         break;
@@ -105,10 +119,11 @@ nouveau_display_scanoutpos_head(struct drm_crtc *crtc, int *vpos, int *hpos,
                 if (retry) ndelay(crtc->linedur_ns);
         } while (retry--);
 
-        *hpos = args.hline;
-        *vpos = calc(args.vblanks, args.vblanke, args.vtotal, args.vline);
-        if (stime) *stime = ns_to_ktime(args.time[0]);
-        if (etime) *etime = ns_to_ktime(args.time[1]);
+        *hpos = args.scan.hline;
+        *vpos = calc(args.scan.vblanks, args.scan.vblanke,
+                     args.scan.vtotal, args.scan.vline);
+        if (stime) *stime = ns_to_ktime(args.scan.time[0]);
+        if (etime) *etime = ns_to_ktime(args.scan.time[1]);
 
         if (*vpos < 0)
                 ret |= DRM_SCANOUTPOS_INVBL;
@@ -151,16 +166,13 @@ nouveau_display_vblstamp(struct drm_device *dev, int head, int *max_error,
 static void
 nouveau_display_vblank_fini(struct drm_device *dev)
 {
-        struct nouveau_display *disp = nouveau_display(dev);
-        int i;
+        struct drm_crtc *crtc;
 
         drm_vblank_cleanup(dev);
 
-        if (disp->vblank) {
-                for (i = 0; i < dev->mode_config.num_crtc; i++)
-                        nouveau_event_ref(NULL, &disp->vblank[i]);
-                kfree(disp->vblank);
-                disp->vblank = NULL;
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+                nvif_notify_fini(&nv_crtc->vblank);
         }
 }
 
@@ -168,19 +180,20 @@ static int
 nouveau_display_vblank_init(struct drm_device *dev)
 {
         struct nouveau_display *disp = nouveau_display(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_disp *pdisp = nouveau_disp(drm->device);
-        int ret, i;
-
-        disp->vblank = kzalloc(dev->mode_config.num_crtc *
-                               sizeof(*disp->vblank), GFP_KERNEL);
-        if (!disp->vblank)
-                return -ENOMEM;
+        struct drm_crtc *crtc;
+        int ret;
 
-        for (i = 0; i < dev->mode_config.num_crtc; i++) {
-                ret = nouveau_event_new(pdisp->vblank, 1, i,
-                                        nouveau_display_vblank_handler,
-                                        drm, &disp->vblank[i]);
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+                ret = nvif_notify_init(&disp->disp, NULL,
+                                       nouveau_display_vblank_handler, false,
+                                       NV04_DISP_NTFY_VBLANK,
+                                       &(struct nvif_notify_head_req_v0) {
+                                        .head = nv_crtc->index,
+                                       },
+                                       sizeof(struct nvif_notify_head_req_v0),
+                                       sizeof(struct nvif_notify_head_rep_v0),
+                                       &nv_crtc->vblank);
                 if (ret) {
                         nouveau_display_vblank_fini(dev);
                         return ret;
@@ -200,6 +213,10 @@ static void
 nouveau_user_framebuffer_destroy(struct drm_framebuffer *drm_fb)
 {
         struct nouveau_framebuffer *fb = nouveau_framebuffer(drm_fb);
+        struct nouveau_display *disp = nouveau_display(drm_fb->dev);
+
+        if (disp->fb_dtor)
+                disp->fb_dtor(drm_fb);
 
         if (fb->nvbo)
                 drm_gem_object_unreference_unlocked(&fb->nvbo->gem);
@@ -229,63 +246,24 @@ nouveau_framebuffer_init(struct drm_device *dev,
                          struct drm_mode_fb_cmd2 *mode_cmd,
                          struct nouveau_bo *nvbo)
 {
-        struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nouveau_display *disp = nouveau_display(dev);
         struct drm_framebuffer *fb = &nv_fb->base;
         int ret;
 
         drm_helper_mode_fill_fb_struct(fb, mode_cmd);
         nv_fb->nvbo = nvbo;
 
-        if (nv_device(drm->device)->card_type >= NV_50) {
-                u32 tile_flags = nouveau_bo_tile_layout(nvbo);
-                if (tile_flags == 0x7a00 ||
-                    tile_flags == 0xfe00)
-                        nv_fb->r_dma = NvEvoFB32;
-                else
-                if (tile_flags == 0x7000)
-                        nv_fb->r_dma = NvEvoFB16;
-                else
-                        nv_fb->r_dma = NvEvoVRAM_LP;
-
-                switch (fb->depth) {
-                case  8: nv_fb->r_format = 0x1e00; break;
-                case 15: nv_fb->r_format = 0xe900; break;
-                case 16: nv_fb->r_format = 0xe800; break;
-                case 24:
-                case 32: nv_fb->r_format = 0xcf00; break;
-                case 30: nv_fb->r_format = 0xd100; break;
-                default:
-                         NV_ERROR(drm, "unknown depth %d\n", fb->depth);
-                         return -EINVAL;
-                }
-
-                if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG) {
-                        NV_ERROR(drm, "framebuffer requires contiguous bo\n");
-                        return -EINVAL;
-                }
-
-                if (nv_device(drm->device)->chipset == 0x50)
-                        nv_fb->r_format |= (tile_flags << 8);
-
-                if (!tile_flags) {
-                        if (nv_device(drm->device)->card_type < NV_D0)
-                                nv_fb->r_pitch = 0x00100000 | fb->pitches[0];
-                        else
-                                nv_fb->r_pitch = 0x01000000 | fb->pitches[0];
-                } else {
-                        u32 mode = nvbo->tile_mode;
-                        if (nv_device(drm->device)->card_type >= NV_C0)
-                                mode >>= 4;
-                        nv_fb->r_pitch = ((fb->pitches[0] / 4) << 4) | mode;
-                }
-        }
-
         ret = drm_framebuffer_init(dev, fb, &nouveau_framebuffer_funcs);
-        if (ret) {
+        if (ret)
                 return ret;
+
+        if (disp->fb_ctor) {
+                ret = disp->fb_ctor(fb);
+                if (ret)
+                        disp->fb_dtor(fb);
         }
 
-        return 0;
+        return ret;
 }
 
 static struct drm_framebuffer *
@@ -393,7 +371,7 @@ nouveau_display_init(struct drm_device *dev)
         /* enable hotplug interrupts */
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
                 struct nouveau_connector *conn = nouveau_connector(connector);
-                if (conn->hpd) nouveau_event_get(conn->hpd);
+                nvif_notify_get(&conn->hpd);
         }
 
         return ret;
@@ -404,37 +382,32 @@ nouveau_display_fini(struct drm_device *dev)
 {
         struct nouveau_display *disp = nouveau_display(dev);
         struct drm_connector *connector;
+        int head;
+
+        /* Make sure that drm and hw vblank irqs get properly disabled. */
+        for (head = 0; head < dev->mode_config.num_crtc; head++)
+                drm_vblank_off(dev, head);
 
         /* disable hotplug interrupts */
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
                 struct nouveau_connector *conn = nouveau_connector(connector);
-                if (conn->hpd) nouveau_event_put(conn->hpd);
+                nvif_notify_put(&conn->hpd);
         }
 
         drm_kms_helper_poll_disable(dev);
         disp->fini(dev);
 }
 
-int
-nouveau_display_create(struct drm_device *dev)
+static void
+nouveau_display_create_properties(struct drm_device *dev)
 {
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_display *disp;
-        int ret, gen;
-
-        disp = drm->display = kzalloc(sizeof(*disp), GFP_KERNEL);
-        if (!disp)
-                return -ENOMEM;
-
-        drm_mode_config_init(dev);
-        drm_mode_create_scaling_mode_property(dev);
-        drm_mode_create_dvi_i_properties(dev);
+        struct nouveau_display *disp = nouveau_display(dev);
+        int gen;
 
-        if (nv_device(drm->device)->card_type < NV_50)
+        if (disp->disp.oclass < NV50_DISP)
                 gen = 0;
         else
-        if (nv_device(drm->device)->card_type < NV_D0)
+        if (disp->disp.oclass < GF110_DISP)
                 gen = 1;
         else
                 gen = 2;
@@ -449,26 +422,43 @@ nouveau_display_create(struct drm_device *dev)
         disp->underscan_vborder_property =
                 drm_property_create_range(dev, 0, "underscan vborder", 0, 128);
 
-        if (gen >= 1) {
-                /* -90..+90 */
-                disp->vibrant_hue_property =
-                        drm_property_create_range(dev, 0, "vibrant hue", 0, 180);
+        if (gen < 1)
+                return;
 
-                /* -100..+100 */
-                disp->color_vibrance_property =
-                        drm_property_create_range(dev, 0, "color vibrance", 0, 200);
-        }
+        /* -90..+90 */
+        disp->vibrant_hue_property =
+                drm_property_create_range(dev, 0, "vibrant hue", 0, 180);
+
+        /* -100..+100 */
+        disp->color_vibrance_property =
+                drm_property_create_range(dev, 0, "color vibrance", 0, 200);
+}
+
+int
+nouveau_display_create(struct drm_device *dev)
+{
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nouveau_display *disp;
+        int ret;
+
+        disp = drm->display = kzalloc(sizeof(*disp), GFP_KERNEL);
+        if (!disp)
+                return -ENOMEM;
+
+        drm_mode_config_init(dev);
+        drm_mode_create_scaling_mode_property(dev);
+        drm_mode_create_dvi_i_properties(dev);
 
         dev->mode_config.funcs = &nouveau_mode_config_funcs;
-        dev->mode_config.fb_base = nv_device_resource_start(device, 1);
+        dev->mode_config.fb_base = nv_device_resource_start(nvkm_device(&drm->device), 1);
 
         dev->mode_config.min_width = 0;
         dev->mode_config.min_height = 0;
-        if (nv_device(drm->device)->card_type < NV_10) {
+        if (drm->device.info.family < NV_DEVICE_INFO_V0_CELSIUS) {
                 dev->mode_config.max_width = 2048;
                 dev->mode_config.max_height = 2048;
         } else
-        if (nv_device(drm->device)->card_type < NV_50) {
+        if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
                 dev->mode_config.max_width = 4096;
                 dev->mode_config.max_height = 4096;
         } else {
@@ -479,7 +469,7 @@ nouveau_display_create(struct drm_device *dev)
         dev->mode_config.preferred_depth = 24;
         dev->mode_config.prefer_shadow = 1;
 
-        if (nv_device(drm->device)->chipset < 0x11)
+        if (drm->device.info.chipset < 0x11)
                 dev->mode_config.async_page_flip = false;
         else
                 dev->mode_config.async_page_flip = true;
@@ -487,29 +477,30 @@ nouveau_display_create(struct drm_device *dev)
         drm_kms_helper_poll_init(dev);
         drm_kms_helper_poll_disable(dev);
 
-        if (drm->vbios.dcb.entries) {
+        if (nouveau_modeset != 2 && drm->vbios.dcb.entries) {
                 static const u16 oclass[] = {
-                        GM107_DISP_CLASS,
-                        NVF0_DISP_CLASS,
-                        NVE0_DISP_CLASS,
-                        NVD0_DISP_CLASS,
-                        NVA3_DISP_CLASS,
-                        NV94_DISP_CLASS,
-                        NVA0_DISP_CLASS,
-                        NV84_DISP_CLASS,
-                        NV50_DISP_CLASS,
-                        NV04_DISP_CLASS,
+                        GM107_DISP,
+                        GK110_DISP,
+                        GK104_DISP,
+                        GF110_DISP,
+                        GT214_DISP,
+                        GT206_DISP,
+                        GT200_DISP,
+                        G82_DISP,
+                        NV50_DISP,
+                        NV04_DISP,
                 };
                 int i;
 
                 for (i = 0, ret = -ENODEV; ret && i < ARRAY_SIZE(oclass); i++) {
-                        ret = nouveau_object_new(nv_object(drm), NVDRM_DEVICE,
-                                                 NVDRM_DISPLAY, oclass[i],
-                                                 NULL, 0, &disp->core);
+                        ret = nvif_object_init(nvif_object(&drm->device), NULL,
+                                               NVDRM_DISPLAY, oclass[i],
+                                               NULL, 0, &disp->disp);
                 }
 
                 if (ret == 0) {
-                        if (nv_mclass(disp->core) < NV50_DISP_CLASS)
+                        nouveau_display_create_properties(dev);
+                        if (disp->disp.oclass < NV50_DISP)
                                 ret = nv04_display_create(dev);
                         else
                                 ret = nv50_display_create(dev);
@@ -542,7 +533,6 @@ void
 nouveau_display_destroy(struct drm_device *dev)
 {
         struct nouveau_display *disp = nouveau_display(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
 
         nouveau_backlight_exit(dev);
         nouveau_display_vblank_fini(dev);
@@ -553,7 +543,7 @@ nouveau_display_destroy(struct drm_device *dev)
         if (disp->dtor)
                 disp->dtor(dev);
 
-        nouveau_object_del(nv_object(drm), NVDRM_DEVICE, NVDRM_DISPLAY);
+        nvif_object_fini(&disp->disp);
 
         nouveau_drm(dev)->display = NULL;
         kfree(disp);
@@ -620,6 +610,8 @@ void
 nouveau_display_resume(struct drm_device *dev)
 {
         struct drm_crtc *crtc;
+        int head;
+
         nouveau_display_init(dev);
 
         /* Force CLUT to get re-loaded during modeset */
@@ -629,6 +621,10 @@ nouveau_display_resume(struct drm_device *dev)
                 nv_crtc->lut.depth = 0;
         }
 
+        /* Make sure that drm and hw vblank irqs get resumed if needed. */
+        for (head = 0; head < dev->mode_config.num_crtc; head++)
+                drm_vblank_on(dev, head);
+
         drm_helper_resume_force_mode(dev);
 
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
@@ -669,7 +665,7 @@ nouveau_page_flip_emit(struct nouveau_channel *chan,
         if (ret)
                 goto fail;
 
-        if (nv_device(drm->device)->card_type < NV_C0)
+        if (drm->device.info.family < NV_DEVICE_INFO_V0_FERMI)
                 BEGIN_NV04(chan, NvSubSw, NV_SW_PAGE_FLIP, 1);
         else
                 BEGIN_NVC0(chan, FermiSw, NV_SW_PAGE_FLIP, 1);
@@ -698,12 +694,15 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         struct nouveau_bo *old_bo = nouveau_framebuffer(crtc->primary->fb)->nvbo;
         struct nouveau_bo *new_bo = nouveau_framebuffer(fb)->nvbo;
         struct nouveau_page_flip_state *s;
-        struct nouveau_channel *chan = drm->channel;
+        struct nouveau_channel *chan;
+        struct nouveau_cli *cli;
         struct nouveau_fence *fence;
         int ret;
 
-        if (!drm->channel)
+        chan = drm->channel;
+        if (!chan)
                 return -ENODEV;
+        cli = (void *)nvif_client(&chan->device->base);
 
         s = kzalloc(sizeof(*s), GFP_KERNEL);
         if (!s)
@@ -715,7 +714,7 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                         goto fail_free;
         }
 
-        mutex_lock(&chan->cli->mutex);
+        mutex_lock(&cli->mutex);
 
         /* synchronise rendering channel with the kernel's channel */
         spin_lock(&new_bo->bo.bdev->fence_lock);
@@ -740,7 +739,7 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         drm_vblank_get(dev, nouveau_crtc(crtc)->index);
 
         /* Emit a page flip */
-        if (nv_device(drm->device)->card_type >= NV_50) {
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
                 ret = nv50_display_flip_next(crtc, fb, chan, swap_interval);
                 if (ret)
                         goto fail_unreserve;
@@ -769,7 +768,7 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         ret = nouveau_page_flip_emit(chan, old_bo, new_bo, s, &fence);
         if (ret)
                 goto fail_unreserve;
-        mutex_unlock(&chan->cli->mutex);
+        mutex_unlock(&cli->mutex);
 
         /* Update the crtc struct and cleanup */
         crtc->primary->fb = fb;
@@ -785,7 +784,7 @@ fail_unreserve:
         drm_vblank_put(dev, nouveau_crtc(crtc)->index);
         ttm_bo_unreserve(&old_bo->bo);
 fail_unpin:
-        mutex_unlock(&chan->cli->mutex);
+        mutex_unlock(&cli->mutex);
         if (old_bo != new_bo)
                 nouveau_bo_unpin(new_bo);
 fail_free:
@@ -815,7 +814,7 @@ nouveau_finish_page_flip(struct nouveau_channel *chan,
         s = list_first_entry(&fctx->flip, struct nouveau_page_flip_state, head);
         if (s->event) {
                 /* Vblank timestamps/counts are only correct on >= NV-50 */
-                if (nv_device(drm->device)->card_type >= NV_50)
+                if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
                         crtcid = s->crtc;
 
                 drm_send_vblank_event(dev, crtcid, s->event);
@@ -841,7 +840,7 @@ nouveau_flip_complete(void *data)
         struct nouveau_page_flip_state state;
 
         if (!nouveau_finish_page_flip(chan, &state)) {
-                if (nv_device(drm->device)->card_type < NV_50) {
+                if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
                         nv_set_crtc_base(drm->dev, state.crtc, state.offset +
                                          state.y * state.pitch +
                                          state.x * state.bpp / 8);
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h
index a71cf77e55b2..88ca177cb1c7 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.h
+++ b/drivers/gpu/drm/nouveau/nouveau_display.h
@@ -9,9 +9,11 @@ struct nouveau_framebuffer {
         struct drm_framebuffer base;
         struct nouveau_bo *nvbo;
         struct nouveau_vma vma;
-        u32 r_dma;
+        u32 r_handle;
         u32 r_format;
         u32 r_pitch;
+        struct nvif_object h_base[4];
+        struct nvif_object h_core;
 };
 
 static inline struct nouveau_framebuffer *
@@ -36,8 +38,10 @@ struct nouveau_display {
         int  (*init)(struct drm_device *);
         void (*fini)(struct drm_device *);
 
-        struct nouveau_object *core;
-        struct nouveau_eventh **vblank;
+        int  (*fb_ctor)(struct drm_framebuffer *);
+        void (*fb_dtor)(struct drm_framebuffer *);
+
+        struct nvif_object disp;
 
         struct drm_property *dithering_mode;
         struct drm_property *dithering_depth;
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.c b/drivers/gpu/drm/nouveau/nouveau_dma.c
index c177272152e2..8508603cc8c3 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.c
@@ -24,8 +24,6 @@
  *
  */
 
-#include <core/client.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 
@@ -54,9 +52,9 @@ READ_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout)
 {
         uint64_t val;
 
-        val = nv_ro32(chan->object, chan->user_get);
+        val = nvif_rd32(chan, chan->user_get);
         if (chan->user_get_hi)
-                val |= (uint64_t)nv_ro32(chan->object, chan->user_get_hi) << 32;
+                val |= (uint64_t)nvif_rd32(chan, chan->user_get_hi) << 32;
 
         /* reset counter as long as GET is still advancing, this is
          * to avoid misdetecting a GPU lockup if the GPU happens to
@@ -84,12 +82,13 @@ void
 nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo,
               int delta, int length)
 {
+        struct nouveau_cli *cli = (void *)nvif_client(&chan->device->base);
         struct nouveau_bo *pb = chan->push.buffer;
         struct nouveau_vma *vma;
         int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base;
         u64 offset;
 
-        vma = nouveau_bo_vma_find(bo, nv_client(chan->cli)->vm);
+        vma = nouveau_bo_vma_find(bo, cli->vm);
         BUG_ON(!vma);
         offset = vma->offset + delta;
 
@@ -104,7 +103,7 @@ nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo,
         /* Flush writes. */
         nouveau_bo_rd32(pb, 0);
 
-        nv_wo32(chan->object, 0x8c, chan->dma.ib_put);
+        nvif_wr32(chan, 0x8c, chan->dma.ib_put);
         chan->dma.ib_free--;
 }
 
@@ -114,7 +113,7 @@ nv50_dma_push_wait(struct nouveau_channel *chan, int count)
         uint32_t cnt = 0, prev_get = 0;
 
         while (chan->dma.ib_free < count) {
-                uint32_t get = nv_ro32(chan->object, 0x88);
+                uint32_t get = nvif_rd32(chan, 0x88);
                 if (get != prev_get) {
                         prev_get = get;
                         cnt = 0;
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.h b/drivers/gpu/drm/nouveau/nouveau_dma.h
index dc0e0c5cadb4..8da0a272c45a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.h
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.h
@@ -58,31 +58,14 @@ enum {
         FermiSw         = 5, /* DO NOT CHANGE (well.. 6/7 will work...) */
 };
 
-/* Object handles. */
+/* Object handles - for stuff that's doesn't use handle == oclass. */
 enum {
-        NvM2MF          = 0x80000001,
         NvDmaFB         = 0x80000002,
         NvDmaTT         = 0x80000003,
         NvNotify0       = 0x80000006,
-        Nv2D            = 0x80000007,
-        NvCtxSurf2D     = 0x80000008,
-        NvRop           = 0x80000009,
-        NvImagePatt     = 0x8000000a,
-        NvClipRect      = 0x8000000b,
-        NvGdiRect       = 0x8000000c,
-        NvImageBlit     = 0x8000000d,
-        NvSw            = 0x8000000e,
         NvSema          = 0x8000000f,
         NvEvoSema0      = 0x80000010,
         NvEvoSema1      = 0x80000011,
-        NvNotify1       = 0x80000012,
-
-        /* G80+ display objects */
-        NvEvoVRAM       = 0x01000000,
-        NvEvoFB16       = 0x01000001,
-        NvEvoFB32       = 0x01000002,
-        NvEvoVRAM_LP    = 0x01000003,
-        NvEvoSync       = 0xcafe0000
 };
 
 #define NV_MEMORY_TO_MEMORY_FORMAT                                    0x00000039
@@ -157,7 +140,7 @@ BEGIN_IMC0(struct nouveau_channel *chan, int subc, int mthd, u16 data)
 #define WRITE_PUT(val) do {                                                    \
         mb();                                                   \
         nouveau_bo_rd32(chan->push.buffer, 0);                                 \
-        nv_wo32(chan->object, chan->user_put, ((val) << 2) + chan->push.vma.offset);  \
+        nvif_wr32(chan, chan->user_put, ((val) << 2) + chan->push.vma.offset); \
 } while (0)
 
 static inline void
diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c
index 5675ffc175ae..c5137cccce7d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dp.c
@@ -30,11 +30,6 @@
 #include "nouveau_encoder.h"
 #include "nouveau_crtc.h"
 
-#include <core/class.h>
-
-#include <subdev/gpio.h>
-#include <subdev/i2c.h>
-
 static void
 nouveau_dp_probe_oui(struct drm_device *dev, struct nouveau_i2c_port *auxch,
                      u8 *dpcd)
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index c9428c943afb..250a5e88c751 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -27,21 +27,14 @@
 #include <linux/pci.h>
 #include <linux/pm_runtime.h>
 #include <linux/vga_switcheroo.h>
+
 #include "drmP.h"
 #include "drm_crtc_helper.h"
+
 #include <core/device.h>
-#include <core/client.h>
 #include <core/gpuobj.h>
-#include <core/class.h>
 #include <core/option.h>
 
-#include <engine/device.h>
-#include <engine/disp.h>
-#include <engine/fifo.h>
-#include <engine/software.h>
-
-#include <subdev/vm.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_ttm.h"
@@ -57,6 +50,7 @@
 #include "nouveau_fbcon.h"
 #include "nouveau_fence.h"
 #include "nouveau_debugfs.h"
+#include "nouveau_usif.h"
 
 MODULE_PARM_DESC(config, "option string to pass to driver core");
 static char *nouveau_config;
@@ -109,40 +103,37 @@ static int
 nouveau_cli_create(u64 name, const char *sname,
                    int size, void **pcli)
 {
-        struct nouveau_cli *cli;
-        int ret;
-
-        *pcli = NULL;
-        ret = nouveau_client_create_(sname, name, nouveau_config,
-                                     nouveau_debug, size, pcli);
-        cli = *pcli;
-        if (ret) {
-                if (cli)
-                        nouveau_client_destroy(&cli->base);
-                *pcli = NULL;
+        struct nouveau_cli *cli = *pcli = kzalloc(size, GFP_KERNEL);
+        if (cli) {
+                int ret = nvif_client_init(NULL, NULL, sname, name,
+                                           nouveau_config, nouveau_debug,
+                                          &cli->base);
+                if (ret == 0) {
+                        mutex_init(&cli->mutex);
+                        usif_client_init(cli);
+                }
                 return ret;
         }
-
-        mutex_init(&cli->mutex);
-        return 0;
+        return -ENOMEM;
 }
 
 static void
 nouveau_cli_destroy(struct nouveau_cli *cli)
 {
-        struct nouveau_object *client = nv_object(cli);
-        nouveau_vm_ref(NULL, &cli->base.vm, NULL);
-        nouveau_client_fini(&cli->base, false);
-        atomic_set(&client->refcount, 1);
-        nouveau_object_ref(NULL, &client);
+        nouveau_vm_ref(NULL, &nvkm_client(&cli->base)->vm, NULL);
+        nvif_client_fini(&cli->base);
+        usif_client_fini(cli);
 }
 
 static void
 nouveau_accel_fini(struct nouveau_drm *drm)
 {
-        nouveau_gpuobj_ref(NULL, &drm->notify);
         nouveau_channel_del(&drm->channel);
+        nvif_object_fini(&drm->ntfy);
+        nouveau_gpuobj_ref(NULL, &drm->notify);
+        nvif_object_fini(&drm->nvsw);
         nouveau_channel_del(&drm->cechan);
+        nvif_object_fini(&drm->ttm.copy);
         if (drm->fence)
                 nouveau_fence(drm)->dtor(drm);
 }
@@ -150,46 +141,71 @@ nouveau_accel_fini(struct nouveau_drm *drm)
 static void
 nouveau_accel_init(struct nouveau_drm *drm)
 {
-        struct nouveau_device *device = nv_device(drm->device);
-        struct nouveau_object *object;
+        struct nvif_device *device = &drm->device;
         u32 arg0, arg1;
-        int ret;
+        u32 sclass[16];
+        int ret, i;
 
-        if (nouveau_noaccel || !nouveau_fifo(device) /*XXX*/)
+        if (nouveau_noaccel)
                 return;
 
         /* initialise synchronisation routines */
-        if      (device->card_type < NV_10) ret = nv04_fence_create(drm);
-        else if (device->card_type < NV_11 ||
-                 device->chipset   <  0x17) ret = nv10_fence_create(drm);
-        else if (device->card_type < NV_50) ret = nv17_fence_create(drm);
-        else if (device->chipset   <  0x84) ret = nv50_fence_create(drm);
-        else if (device->card_type < NV_C0) ret = nv84_fence_create(drm);
-        else                                ret = nvc0_fence_create(drm);
+        /*XXX: this is crap, but the fence/channel stuff is a little
+         *     backwards in some places.  this will be fixed.
+         */
+        ret = nvif_object_sclass(&device->base, sclass, ARRAY_SIZE(sclass));
+        if (ret < 0)
+                return;
+
+        for (ret = -ENOSYS, i = 0; ret && i < ARRAY_SIZE(sclass); i++) {
+                switch (sclass[i]) {
+                case NV03_CHANNEL_DMA:
+                        ret = nv04_fence_create(drm);
+                        break;
+                case NV10_CHANNEL_DMA:
+                        ret = nv10_fence_create(drm);
+                        break;
+                case NV17_CHANNEL_DMA:
+                case NV40_CHANNEL_DMA:
+                        ret = nv17_fence_create(drm);
+                        break;
+                case NV50_CHANNEL_GPFIFO:
+                        ret = nv50_fence_create(drm);
+                        break;
+                case G82_CHANNEL_GPFIFO:
+                        ret = nv84_fence_create(drm);
+                        break;
+                case FERMI_CHANNEL_GPFIFO:
+                case KEPLER_CHANNEL_GPFIFO_A:
+                        ret = nvc0_fence_create(drm);
+                        break;
+                default:
+                        break;
+                }
+        }
+
         if (ret) {
                 NV_ERROR(drm, "failed to initialise sync subsystem, %d\n", ret);
                 nouveau_accel_fini(drm);
                 return;
         }
 
-        if (device->card_type >= NV_E0) {
-                ret = nouveau_channel_new(drm, &drm->client, NVDRM_DEVICE,
-                                          NVDRM_CHAN + 1,
-                                          NVE0_CHANNEL_IND_ENGINE_CE0 |
-                                          NVE0_CHANNEL_IND_ENGINE_CE1, 0,
-                                          &drm->cechan);
+        if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
+                ret = nouveau_channel_new(drm, &drm->device, NVDRM_CHAN + 1,
+                                          KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_CE0|
+                                          KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_CE1,
+                                          0, &drm->cechan);
                 if (ret)
                         NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
 
-                arg0 = NVE0_CHANNEL_IND_ENGINE_GR;
+                arg0 = KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_GR;
                 arg1 = 1;
         } else
-        if (device->chipset >= 0xa3 &&
-            device->chipset != 0xaa &&
-            device->chipset != 0xac) {
-                ret = nouveau_channel_new(drm, &drm->client, NVDRM_DEVICE,
-                                          NVDRM_CHAN + 1, NvDmaFB, NvDmaTT,
-                                          &drm->cechan);
+        if (device->info.chipset >= 0xa3 &&
+            device->info.chipset != 0xaa &&
+            device->info.chipset != 0xac) {
+                ret = nouveau_channel_new(drm, &drm->device, NVDRM_CHAN + 1,
+                                          NvDmaFB, NvDmaTT, &drm->cechan);
                 if (ret)
                         NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
 
@@ -200,30 +216,30 @@ nouveau_accel_init(struct nouveau_drm *drm)
                 arg1 = NvDmaTT;
         }
 
-        ret = nouveau_channel_new(drm, &drm->client, NVDRM_DEVICE, NVDRM_CHAN,
-                                  arg0, arg1, &drm->channel);
+        ret = nouveau_channel_new(drm, &drm->device, NVDRM_CHAN, arg0, arg1,
+                                 &drm->channel);
         if (ret) {
                 NV_ERROR(drm, "failed to create kernel channel, %d\n", ret);
                 nouveau_accel_fini(drm);
                 return;
         }
 
-        ret = nouveau_object_new(nv_object(drm), NVDRM_CHAN, NVDRM_NVSW,
-                                 nouveau_abi16_swclass(drm), NULL, 0, &object);
+        ret = nvif_object_init(drm->channel->object, NULL, NVDRM_NVSW,
+                               nouveau_abi16_swclass(drm), NULL, 0, &drm->nvsw);
         if (ret == 0) {
-                struct nouveau_software_chan *swch = (void *)object->parent;
+                struct nouveau_software_chan *swch;
                 ret = RING_SPACE(drm->channel, 2);
                 if (ret == 0) {
-                        if (device->card_type < NV_C0) {
+                        if (device->info.family < NV_DEVICE_INFO_V0_FERMI) {
                                 BEGIN_NV04(drm->channel, NvSubSw, 0, 1);
                                 OUT_RING  (drm->channel, NVDRM_NVSW);
                         } else
-                        if (device->card_type < NV_E0) {
+                        if (device->info.family < NV_DEVICE_INFO_V0_KEPLER) {
                                 BEGIN_NVC0(drm->channel, FermiSw, 0, 1);
                                 OUT_RING  (drm->channel, 0x001f0000);
                         }
                 }
-                swch = (void *)object->parent;
+                swch = (void *)nvkm_object(&drm->nvsw)->parent;
                 swch->flip = nouveau_flip_complete;
                 swch->flip_data = drm->channel;
         }
@@ -234,24 +250,24 @@ nouveau_accel_init(struct nouveau_drm *drm)
                 return;
         }
 
-        if (device->card_type < NV_C0) {
-                ret = nouveau_gpuobj_new(drm->device, NULL, 32, 0, 0,
-                                        &drm->notify);
+        if (device->info.family < NV_DEVICE_INFO_V0_FERMI) {
+                ret = nouveau_gpuobj_new(nvkm_object(&drm->device), NULL, 32,
+                                         0, 0, &drm->notify);
                 if (ret) {
                         NV_ERROR(drm, "failed to allocate notifier, %d\n", ret);
                         nouveau_accel_fini(drm);
                         return;
                 }
 
-                ret = nouveau_object_new(nv_object(drm),
-                                         drm->channel->handle, NvNotify0,
-                                         0x003d, &(struct nv_dma_class) {
-                                                .flags = NV_DMA_TARGET_VRAM |
-                                                         NV_DMA_ACCESS_RDWR,
+                ret = nvif_object_init(drm->channel->object, NULL, NvNotify0,
+                                       NV_DMA_IN_MEMORY,
+                                       &(struct nv_dma_v0) {
+                                                .target = NV_DMA_V0_TARGET_VRAM,
+                                                .access = NV_DMA_V0_ACCESS_RDWR,
                                                 .start = drm->notify->addr,
                                                 .limit = drm->notify->addr + 31
-                                                }, sizeof(struct nv_dma_class),
-                                         &object);
+                                       }, sizeof(struct nv_dma_v0),
+                                       &drm->ntfy);
                 if (ret) {
                         nouveau_accel_fini(drm);
                         return;
@@ -294,7 +310,8 @@ static int nouveau_drm_probe(struct pci_dev *pdev,
 #ifdef CONFIG_X86
         boot = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
 #endif
-        remove_conflicting_framebuffers(aper, "nouveaufb", boot);
+        if (nouveau_modeset != 2)
+                remove_conflicting_framebuffers(aper, "nouveaufb", boot);
         kfree(aper);
 
         ret = nouveau_device_create(pdev, NOUVEAU_BUS_PCI,
@@ -348,7 +365,6 @@ static int
 nouveau_drm_load(struct drm_device *dev, unsigned long flags)
 {
         struct pci_dev *pdev = dev->pdev;
-        struct nouveau_device *device;
         struct nouveau_drm *drm;
         int ret;
 
@@ -359,7 +375,8 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags)
 
         dev->dev_private = drm;
         drm->dev = dev;
-        nouveau_client(drm)->debug = nouveau_dbgopt(nouveau_debug, "DRM");
+        nvkm_client(&drm->client.base)->debug =
+                nouveau_dbgopt(nouveau_debug, "DRM");
 
         INIT_LIST_HEAD(&drm->clients);
         spin_lock_init(&drm->tile.lock);
@@ -370,33 +387,34 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags)
          * (possibly) execute vbios init tables (see nouveau_agp.h)
          */
         if (pdev && drm_pci_device_is_agp(dev) && dev->agp) {
+                const u64 enables = NV_DEVICE_V0_DISABLE_IDENTIFY |
+                                    NV_DEVICE_V0_DISABLE_MMIO;
                 /* dummy device object, doesn't init anything, but allows
                  * agp code access to registers
                  */
-                ret = nouveau_object_new(nv_object(drm), NVDRM_CLIENT,
-                                         NVDRM_DEVICE, 0x0080,
-                                         &(struct nv_device_class) {
+                ret = nvif_device_init(&drm->client.base.base, NULL,
+                                       NVDRM_DEVICE, NV_DEVICE,
+                                       &(struct nv_device_v0) {
                                                 .device = ~0,
-                                                .disable =
-                                                 ~(NV_DEVICE_DISABLE_MMIO |
-                                                   NV_DEVICE_DISABLE_IDENTIFY),
+                                                .disable = ~enables,
                                                 .debug0 = ~0,
-                                         }, sizeof(struct nv_device_class),
-                                         &drm->device);
+                                       }, sizeof(struct nv_device_v0),
+                                       &drm->device);
                 if (ret)
                         goto fail_device;
 
                 nouveau_agp_reset(drm);
-                nouveau_object_del(nv_object(drm), NVDRM_CLIENT, NVDRM_DEVICE);
+                nvif_device_fini(&drm->device);
         }
 
-        ret = nouveau_object_new(nv_object(drm), NVDRM_CLIENT, NVDRM_DEVICE,
-                                 0x0080, &(struct nv_device_class) {
+        ret = nvif_device_init(&drm->client.base.base, NULL, NVDRM_DEVICE,
+                               NV_DEVICE,
+                               &(struct nv_device_v0) {
                                         .device = ~0,
                                         .disable = 0,
                                         .debug0 = 0,
-                                 }, sizeof(struct nv_device_class),
-                                 &drm->device);
+                               }, sizeof(struct nv_device_v0),
+                               &drm->device);
         if (ret)
                 goto fail_device;
 
@@ -406,18 +424,19 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags)
          * nosnoop capability.  hopefully won't cause issues until a
          * better fix is found - assuming there is one...
          */
-        device = nv_device(drm->device);
-        if (nv_device(drm->device)->chipset == 0xc1)
-                nv_mask(device, 0x00088080, 0x00000800, 0x00000000);
+        if (drm->device.info.chipset == 0xc1)
+                nvif_mask(&drm->device, 0x00088080, 0x00000800, 0x00000000);
 
         nouveau_vga_init(drm);
         nouveau_agp_init(drm);
 
-        if (device->card_type >= NV_50) {
-                ret = nouveau_vm_new(nv_device(drm->device), 0, (1ULL << 40),
-                                     0x1000, &drm->client.base.vm);
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
+                ret = nouveau_vm_new(nvkm_device(&drm->device), 0, (1ULL << 40),
+                                     0x1000, &drm->client.vm);
                 if (ret)
                         goto fail_device;
+
+                nvkm_client(&drm->client.base)->vm = drm->client.vm;
         }
 
         ret = nouveau_ttm_init(drm);
@@ -463,6 +482,7 @@ fail_ttm:
         nouveau_agp_fini(drm);
         nouveau_vga_fini(drm);
 fail_device:
+        nvif_device_fini(&drm->device);
         nouveau_cli_destroy(&drm->client);
         return ret;
 }
@@ -488,26 +508,37 @@ nouveau_drm_unload(struct drm_device *dev)
         nouveau_agp_fini(drm);
         nouveau_vga_fini(drm);
 
+        nvif_device_fini(&drm->device);
         if (drm->hdmi_device)
                 pci_dev_put(drm->hdmi_device);
         nouveau_cli_destroy(&drm->client);
         return 0;
 }
 
-static void
-nouveau_drm_remove(struct pci_dev *pdev)
+void
+nouveau_drm_device_remove(struct drm_device *dev)
 {
-        struct drm_device *dev = pci_get_drvdata(pdev);
         struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nouveau_client *client;
         struct nouveau_object *device;
 
         dev->irq_enabled = false;
-        device = drm->client.base.device;
+        client = nvkm_client(&drm->client.base);
+        device = client->device;
         drm_put_dev(dev);
 
         nouveau_object_ref(NULL, &device);
         nouveau_object_debug();
 }
+EXPORT_SYMBOL(nouveau_drm_device_remove);
+
+static void
+nouveau_drm_remove(struct pci_dev *pdev)
+{
+        struct drm_device *dev = pci_get_drvdata(pdev);
+
+        nouveau_drm_device_remove(dev);
+}
 
 static int
 nouveau_do_suspend(struct drm_device *dev, bool runtime)
@@ -548,13 +579,13 @@ nouveau_do_suspend(struct drm_device *dev, bool runtime)
         }
 
         list_for_each_entry(cli, &drm->clients, head) {
-                ret = nouveau_client_fini(&cli->base, true);
+                ret = nvif_client_suspend(&cli->base);
                 if (ret)
                         goto fail_client;
         }
 
         NV_INFO(drm, "suspending kernel object tree...\n");
-        ret = nouveau_client_fini(&drm->client.base, true);
+        ret = nvif_client_suspend(&drm->client.base);
         if (ret)
                 goto fail_client;
 
@@ -563,7 +594,7 @@ nouveau_do_suspend(struct drm_device *dev, bool runtime)
 
 fail_client:
         list_for_each_entry_continue_reverse(cli, &drm->clients, head) {
-                nouveau_client_init(&cli->base);
+                nvif_client_resume(&cli->base);
         }
 
         if (drm->fence && nouveau_fence(drm)->resume)
@@ -611,7 +642,7 @@ nouveau_do_resume(struct drm_device *dev)
         nouveau_agp_reset(drm);
 
         NV_INFO(drm, "resuming kernel object tree...\n");
-        nouveau_client_init(&drm->client.base);
+        nvif_client_resume(&drm->client.base);
         nouveau_agp_init(drm);
 
         NV_INFO(drm, "resuming client object trees...\n");
@@ -619,7 +650,7 @@ nouveau_do_resume(struct drm_device *dev)
                 nouveau_fence(drm)->resume(drm);
 
         list_for_each_entry(cli, &drm->clients, head) {
-                nouveau_client_init(&cli->base);
+                nvif_client_resume(&cli->base);
         }
 
         nouveau_run_vbios_init(dev);
@@ -715,13 +746,17 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv)
         if (ret)
                 goto out_suspend;
 
-        if (nv_device(drm->device)->card_type >= NV_50) {
-                ret = nouveau_vm_new(nv_device(drm->device), 0, (1ULL << 40),
-                                     0x1000, &cli->base.vm);
+        cli->base.super = false;
+
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
+                ret = nouveau_vm_new(nvkm_device(&drm->device), 0, (1ULL << 40),
+                                     0x1000, &cli->vm);
                 if (ret) {
                         nouveau_cli_destroy(cli);
                         goto out_suspend;
                 }
+
+                nvkm_client(&cli->base)->vm = cli->vm;
         }
 
         fpriv->driver_priv = cli;
@@ -779,24 +814,31 @@ nouveau_ioctls[] = {
         DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
 };
 
-long nouveau_drm_ioctl(struct file *filp,
-                       unsigned int cmd, unsigned long arg)
+long
+nouveau_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
-        struct drm_file *file_priv = filp->private_data;
-        struct drm_device *dev;
+        struct drm_file *filp = file->private_data;
+        struct drm_device *dev = filp->minor->dev;
         long ret;
-        dev = file_priv->minor->dev;
 
         ret = pm_runtime_get_sync(dev->dev);
         if (ret < 0 && ret != -EACCES)
                 return ret;
 
-        ret = drm_ioctl(filp, cmd, arg);
+        switch (_IOC_NR(cmd) - DRM_COMMAND_BASE) {
+        case DRM_NOUVEAU_NVIF:
+                ret = usif_ioctl(filp, (void __user *)arg, _IOC_SIZE(cmd));
+                break;
+        default:
+                ret = drm_ioctl(file, cmd, arg);
+                break;
+        }
 
         pm_runtime_mark_last_busy(dev->dev);
         pm_runtime_put_autosuspend(dev->dev);
         return ret;
 }
+
 static const struct file_operations
 nouveau_driver_fops = {
         .owner = THIS_MODULE,
@@ -921,7 +963,7 @@ static int nouveau_pmops_runtime_resume(struct device *dev)
 {
         struct pci_dev *pdev = to_pci_dev(dev);
         struct drm_device *drm_dev = pci_get_drvdata(pdev);
-        struct nouveau_device *device = nouveau_dev(drm_dev);
+        struct nvif_device *device = &nouveau_drm(drm_dev)->device;
         int ret;
 
         if (nouveau_runtime_pm == 0)
@@ -937,7 +979,7 @@ static int nouveau_pmops_runtime_resume(struct device *dev)
         ret = nouveau_do_resume(drm_dev);
         drm_kms_helper_poll_enable(drm_dev);
         /* do magic */
-        nv_mask(device, 0x88488, (1 << 25), (1 << 25));
+        nvif_mask(device, 0x88488, (1 << 25), (1 << 25));
         vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_ON);
         drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
         nv_debug_level(NORMAL);
@@ -1005,24 +1047,41 @@ nouveau_drm_pci_driver = {
         .driver.pm = &nouveau_pm_ops,
 };
 
-int nouveau_drm_platform_probe(struct platform_device *pdev)
+struct drm_device *
+nouveau_platform_device_create_(struct platform_device *pdev, int size,
+                                void **pobject)
 {
-        struct nouveau_device *device;
-        int ret;
+        struct drm_device *drm;
+        int err;
 
-        ret = nouveau_device_create(pdev, NOUVEAU_BUS_PLATFORM,
+        err = nouveau_device_create_(pdev, NOUVEAU_BUS_PLATFORM,
                                     nouveau_platform_name(pdev),
                                     dev_name(&pdev->dev), nouveau_config,
-                                    nouveau_debug, &device);
-
-        ret = drm_platform_init(&driver, pdev);
-        if (ret) {
-                nouveau_object_ref(NULL, (struct nouveau_object **)&device);
-                return ret;
+                                    nouveau_debug, size, pobject);
+        if (err)
+                return ERR_PTR(err);
+
+        drm = drm_dev_alloc(&driver, &pdev->dev);
+        if (!drm) {
+                err = -ENOMEM;
+                goto err_free;
         }
 
-        return ret;
+        err = drm_dev_set_unique(drm, "%s", dev_name(&pdev->dev));
+        if (err < 0)
+                goto err_free;
+
+        drm->platformdev = pdev;
+        platform_set_drvdata(pdev, drm);
+
+        return drm;
+
+err_free:
+        nouveau_object_ref(NULL, (struct nouveau_object **)pobject);
+
+        return ERR_PTR(err);
 }
+EXPORT_SYMBOL(nouveau_platform_device_create_);
 
 static int __init
 nouveau_drm_init(void)
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.h b/drivers/gpu/drm/nouveau/nouveau_drm.h
index 7efbafaf7c1d..b02b02452c85 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.h
@@ -9,8 +9,8 @@
 #define DRIVER_DATE             "20120801"
 
 #define DRIVER_MAJOR            1
-#define DRIVER_MINOR            1
-#define DRIVER_PATCHLEVEL       1
+#define DRIVER_MINOR            2
+#define DRIVER_PATCHLEVEL       0
 
 /*
  * 1.1.1:
@@ -21,15 +21,17 @@
  *        to control registers on the MPs to enable performance counters,
  *        and to control the warp error enable mask (OpenGL requires out of
  *        bounds access to local memory to be silently ignored / return 0).
+ * 1.1.2:
+ *      - fixes multiple bugs in flip completion events and timestamping
+ * 1.2.0:
+ *      - object api exposed to userspace
+ *      - fermi,kepler,maxwell zbc
  */
 
-#include <core/client.h>
-#include <core/event.h>
-
-#include <subdev/vm.h>
+#include <nvif/client.h>
+#include <nvif/device.h>
 
 #include <drmP.h>
-#include <drm/nouveau_drm.h>
 
 #include <drm/ttm/ttm_bo_api.h>
 #include <drm/ttm/ttm_bo_driver.h>
@@ -38,7 +40,10 @@
 #include <drm/ttm/ttm_module.h>
 #include <drm/ttm/ttm_page_alloc.h>
 
+#include "uapi/drm/nouveau_drm.h"
+
 struct nouveau_channel;
+struct platform_device;
 
 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
 
@@ -50,6 +55,17 @@ struct nouveau_drm_tile {
         bool used;
 };
 
+enum nouveau_drm_object_route {
+        NVDRM_OBJECT_NVIF = 0,
+        NVDRM_OBJECT_USIF,
+        NVDRM_OBJECT_ABI16,
+};
+
+enum nouveau_drm_notify_route {
+        NVDRM_NOTIFY_NVIF = 0,
+        NVDRM_NOTIFY_USIF
+};
+
 enum nouveau_drm_handle {
         NVDRM_CLIENT  = 0xffffffff,
         NVDRM_DEVICE  = 0xdddddddd,
@@ -61,10 +77,13 @@ enum nouveau_drm_handle {
 };
 
 struct nouveau_cli {
-        struct nouveau_client base;
+        struct nvif_client base;
+        struct nouveau_vm *vm; /*XXX*/
         struct list_head head;
         struct mutex mutex;
         void *abi16;
+        struct list_head objects;
+        struct list_head notifys;
 };
 
 static inline struct nouveau_cli *
@@ -73,13 +92,16 @@ nouveau_cli(struct drm_file *fpriv)
         return fpriv ? fpriv->driver_priv : NULL;
 }
 
+#include <nvif/object.h>
+#include <nvif/device.h>
+
 extern int nouveau_runtime_pm;
 
 struct nouveau_drm {
         struct nouveau_cli client;
         struct drm_device *dev;
 
-        struct nouveau_object *device;
+        struct nvif_device device;
         struct list_head clients;
 
         struct {
@@ -102,6 +124,7 @@ struct nouveau_drm {
                             struct ttm_buffer_object *,
                             struct ttm_mem_reg *, struct ttm_mem_reg *);
                 struct nouveau_channel *chan;
+                struct nvif_object copy;
                 int mtrr;
         } ttm;
 
@@ -119,6 +142,8 @@ struct nouveau_drm {
         struct nouveau_channel *channel;
         struct nouveau_gpuobj *notify;
         struct nouveau_fbdev *fbcon;
+        struct nvif_object nvsw;
+        struct nvif_object ntfy;
 
         /* nv10-nv40 tiling regions */
         struct {
@@ -148,20 +173,25 @@ nouveau_drm(struct drm_device *dev)
         return dev->dev_private;
 }
 
-static inline struct nouveau_device *
-nouveau_dev(struct drm_device *dev)
-{
-        return nv_device(nouveau_drm(dev)->device);
-}
-
 int nouveau_pmops_suspend(struct device *);
 int nouveau_pmops_resume(struct device *);
 
-#define NV_FATAL(cli, fmt, args...) nv_fatal((cli), fmt, ##args)
-#define NV_ERROR(cli, fmt, args...) nv_error((cli), fmt, ##args)
-#define NV_WARN(cli, fmt, args...) nv_warn((cli), fmt, ##args)
-#define NV_INFO(cli, fmt, args...) nv_info((cli), fmt, ##args)
-#define NV_DEBUG(cli, fmt, args...) nv_debug((cli), fmt, ##args)
+#define nouveau_platform_device_create(p, u)                                   \
+        nouveau_platform_device_create_(p, sizeof(**u), (void **)u)
+struct drm_device *
+nouveau_platform_device_create_(struct platform_device *pdev,
+                                int size, void **pobject);
+void nouveau_drm_device_remove(struct drm_device *dev);
+
+#define NV_PRINTK(l,c,f,a...) do {                                             \
+        struct nouveau_cli *_cli = (c);                                        \
+        nv_##l(_cli->base.base.priv, f, ##a);                                  \
+} while(0)
+#define NV_FATAL(drm,f,a...) NV_PRINTK(fatal, &(drm)->client, f, ##a)
+#define NV_ERROR(drm,f,a...) NV_PRINTK(error, &(drm)->client, f, ##a)
+#define NV_WARN(drm,f,a...) NV_PRINTK(warn, &(drm)->client, f, ##a)
+#define NV_INFO(drm,f,a...) NV_PRINTK(info, &(drm)->client, f, ##a)
+#define NV_DEBUG(drm,f,a...) NV_PRINTK(debug, &(drm)->client, f, ##a)
 
 extern int nouveau_modeset;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 758c11cb9a9a..ebfe3180109e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -51,11 +51,6 @@
 
 #include "nouveau_crtc.h"
 
-#include <core/client.h>
-#include <core/device.h>
-
-#include <subdev/fb.h>
-
 MODULE_PARM_DESC(nofbaccel, "Disable fbcon acceleration");
 static int nouveau_nofbaccel = 0;
 module_param_named(nofbaccel, nouveau_nofbaccel, int, 0400);
@@ -65,7 +60,7 @@ nouveau_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
         struct nouveau_fbdev *fbcon = info->par;
         struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         int ret;
 
         if (info->state != FBINFO_STATE_RUNNING)
@@ -74,10 +69,10 @@ nouveau_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
         ret = -ENODEV;
         if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
             mutex_trylock(&drm->client.mutex)) {
-                if (device->card_type < NV_50)
+                if (device->info.family < NV_DEVICE_INFO_V0_TESLA)
                         ret = nv04_fbcon_fillrect(info, rect);
                 else
-                if (device->card_type < NV_C0)
+                if (device->info.family < NV_DEVICE_INFO_V0_FERMI)
                         ret = nv50_fbcon_fillrect(info, rect);
                 else
                         ret = nvc0_fbcon_fillrect(info, rect);
@@ -97,7 +92,7 @@ nouveau_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *image)
 {
         struct nouveau_fbdev *fbcon = info->par;
         struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         int ret;
 
         if (info->state != FBINFO_STATE_RUNNING)
@@ -106,10 +101,10 @@ nouveau_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *image)
         ret = -ENODEV;
         if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
             mutex_trylock(&drm->client.mutex)) {
-                if (device->card_type < NV_50)
+                if (device->info.family < NV_DEVICE_INFO_V0_TESLA)
                         ret = nv04_fbcon_copyarea(info, image);
                 else
-                if (device->card_type < NV_C0)
+                if (device->info.family < NV_DEVICE_INFO_V0_FERMI)
                         ret = nv50_fbcon_copyarea(info, image);
                 else
                         ret = nvc0_fbcon_copyarea(info, image);
@@ -129,7 +124,7 @@ nouveau_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 {
         struct nouveau_fbdev *fbcon = info->par;
         struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         int ret;
 
         if (info->state != FBINFO_STATE_RUNNING)
@@ -138,10 +133,10 @@ nouveau_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
         ret = -ENODEV;
         if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
             mutex_trylock(&drm->client.mutex)) {
-                if (device->card_type < NV_50)
+                if (device->info.family < NV_DEVICE_INFO_V0_TESLA)
                         ret = nv04_fbcon_imageblit(info, image);
                 else
-                if (device->card_type < NV_C0)
+                if (device->info.family < NV_DEVICE_INFO_V0_FERMI)
                         ret = nv50_fbcon_imageblit(info, image);
                 else
                         ret = nvc0_fbcon_imageblit(info, image);
@@ -212,6 +207,65 @@ static struct fb_ops nouveau_fbcon_sw_ops = {
         .fb_debug_leave = drm_fb_helper_debug_leave,
 };
 
+void
+nouveau_fbcon_accel_save_disable(struct drm_device *dev)
+{
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        if (drm->fbcon) {
+                drm->fbcon->saved_flags = drm->fbcon->helper.fbdev->flags;
+                drm->fbcon->helper.fbdev->flags |= FBINFO_HWACCEL_DISABLED;
+        }
+}
+
+void
+nouveau_fbcon_accel_restore(struct drm_device *dev)
+{
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        if (drm->fbcon) {
+                drm->fbcon->helper.fbdev->flags = drm->fbcon->saved_flags;
+        }
+}
+
+void
+nouveau_fbcon_accel_fini(struct drm_device *dev)
+{
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nouveau_fbdev *fbcon = drm->fbcon;
+        if (fbcon && drm->channel) {
+                console_lock();
+                fbcon->helper.fbdev->flags |= FBINFO_HWACCEL_DISABLED;
+                console_unlock();
+                nouveau_channel_idle(drm->channel);
+                nvif_object_fini(&fbcon->twod);
+                nvif_object_fini(&fbcon->blit);
+                nvif_object_fini(&fbcon->gdi);
+                nvif_object_fini(&fbcon->patt);
+                nvif_object_fini(&fbcon->rop);
+                nvif_object_fini(&fbcon->clip);
+                nvif_object_fini(&fbcon->surf2d);
+        }
+}
+
+void
+nouveau_fbcon_accel_init(struct drm_device *dev)
+{
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nouveau_fbdev *fbcon = drm->fbcon;
+        struct fb_info *info = fbcon->helper.fbdev;
+        int ret;
+
+        if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA)
+                ret = nv04_fbcon_accel_init(info);
+        else
+        if (drm->device.info.family < NV_DEVICE_INFO_V0_FERMI)
+                ret = nv50_fbcon_accel_init(info);
+        else
+                ret = nvc0_fbcon_accel_init(info);
+
+        if (ret == 0)
+                info->fbops = &nouveau_fbcon_ops;
+}
+
 static void nouveau_fbcon_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
                                     u16 blue, int regno)
 {
@@ -257,7 +311,7 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
         struct nouveau_fbdev *fbcon = (struct nouveau_fbdev *)helper;
         struct drm_device *dev = fbcon->dev;
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         struct fb_info *info;
         struct drm_framebuffer *fb;
         struct nouveau_framebuffer *nouveau_fb;
@@ -299,8 +353,8 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
         }
 
         chan = nouveau_nofbaccel ? NULL : drm->channel;
-        if (chan && device->card_type >= NV_50) {
-                ret = nouveau_bo_vma_add(nvbo, nv_client(chan->cli)->vm,
+        if (chan && device->info.family >= NV_DEVICE_INFO_V0_TESLA) {
+                ret = nouveau_bo_vma_add(nvbo, drm->client.vm,
                                         &fbcon->nouveau_fb.vma);
                 if (ret) {
                         NV_ERROR(drm, "failed to map fb into chan: %d\n", ret);
@@ -357,20 +411,8 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
 
         mutex_unlock(&dev->struct_mutex);
 
-        if (chan) {
-                ret = -ENODEV;
-                if (device->card_type < NV_50)
-                        ret = nv04_fbcon_accel_init(info);
-                else
-                if (device->card_type < NV_C0)
-                        ret = nv50_fbcon_accel_init(info);
-                else
-                        ret = nvc0_fbcon_accel_init(info);
-
-                if (ret == 0)
-                        info->fbops = &nouveau_fbcon_ops;
-        }
-
+        if (chan)
+                nouveau_fbcon_accel_init(dev);
         nouveau_fbcon_zfill(dev, fbcon);
 
         /* To allow resizeing without swapping buffers */
@@ -449,7 +491,6 @@ int
 nouveau_fbcon_init(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_fb *pfb = nouveau_fb(drm->device);
         struct nouveau_fbdev *fbcon;
         int preferred_bpp;
         int ret;
@@ -476,10 +517,10 @@ nouveau_fbcon_init(struct drm_device *dev)
 
         drm_fb_helper_single_add_all_connectors(&fbcon->helper);
 
-        if (pfb->ram->size <= 32 * 1024 * 1024)
+        if (drm->device.info.ram_size <= 32 * 1024 * 1024)
                 preferred_bpp = 8;
         else
-        if (pfb->ram->size <= 64 * 1024 * 1024)
+        if (drm->device.info.ram_size <= 64 * 1024 * 1024)
                 preferred_bpp = 16;
         else
                 preferred_bpp = 32;
@@ -499,43 +540,25 @@ nouveau_fbcon_fini(struct drm_device *dev)
         if (!drm->fbcon)
                 return;
 
+        nouveau_fbcon_accel_fini(dev);
         nouveau_fbcon_destroy(dev, drm->fbcon);
         kfree(drm->fbcon);
         drm->fbcon = NULL;
 }
 
 void
-nouveau_fbcon_save_disable_accel(struct drm_device *dev)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        if (drm->fbcon) {
-                drm->fbcon->saved_flags = drm->fbcon->helper.fbdev->flags;
-                drm->fbcon->helper.fbdev->flags |= FBINFO_HWACCEL_DISABLED;
-        }
-}
-
-void
-nouveau_fbcon_restore_accel(struct drm_device *dev)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        if (drm->fbcon) {
-                drm->fbcon->helper.fbdev->flags = drm->fbcon->saved_flags;
-        }
-}
-
-void
 nouveau_fbcon_set_suspend(struct drm_device *dev, int state)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
         if (drm->fbcon) {
                 console_lock();
-                if (state == 1)
-                        nouveau_fbcon_save_disable_accel(dev);
-                fb_set_suspend(drm->fbcon->helper.fbdev, state);
                 if (state == 0) {
-                        nouveau_fbcon_restore_accel(dev);
+                        nouveau_fbcon_accel_restore(dev);
                         nouveau_fbcon_zfill(dev, drm->fbcon);
                 }
+                fb_set_suspend(drm->fbcon->helper.fbdev, state);
+                if (state == 1)
+                        nouveau_fbcon_accel_save_disable(dev);
                 console_unlock();
         }
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.h b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
index fcff797d2084..34658cfa8f5d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
@@ -37,6 +37,13 @@ struct nouveau_fbdev {
         struct list_head fbdev_list;
         struct drm_device *dev;
         unsigned int saved_flags;
+        struct nvif_object surf2d;
+        struct nvif_object clip;
+        struct nvif_object rop;
+        struct nvif_object patt;
+        struct nvif_object gdi;
+        struct nvif_object blit;
+        struct nvif_object twod;
 };
 
 void nouveau_fbcon_restore(void);
@@ -61,8 +68,8 @@ void nouveau_fbcon_gpu_lockup(struct fb_info *info);
 int nouveau_fbcon_init(struct drm_device *dev);
 void nouveau_fbcon_fini(struct drm_device *dev);
 void nouveau_fbcon_set_suspend(struct drm_device *dev, int state);
-void nouveau_fbcon_save_disable_accel(struct drm_device *dev);
-void nouveau_fbcon_restore_accel(struct drm_device *dev);
+void nouveau_fbcon_accel_save_disable(struct drm_device *dev);
+void nouveau_fbcon_accel_restore(struct drm_device *dev);
 
 void nouveau_fbcon_output_poll_changed(struct drm_device *dev);
 #endif /* __NV50_FBCON_H__ */
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index ab5ea3b0d666..0a93114158cd 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -29,12 +29,13 @@
 #include <linux/ktime.h>
 #include <linux/hrtimer.h>
 
+#include <nvif/notify.h>
+#include <nvif/event.h>
+
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_fence.h"
 
-#include <engine/fifo.h>
-
 struct fence_work {
         struct work_struct base;
         struct list_head head;
@@ -165,12 +166,18 @@ nouveau_fence_done(struct nouveau_fence *fence)
         return !fence->channel;
 }
 
+struct nouveau_fence_wait {
+        struct nouveau_fence_priv *priv;
+        struct nvif_notify notify;
+};
+
 static int
-nouveau_fence_wait_uevent_handler(void *data, u32 type, int index)
+nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
 {
-        struct nouveau_fence_priv *priv = data;
-        wake_up_all(&priv->waiting);
-        return NVKM_EVENT_KEEP;
+        struct nouveau_fence_wait *wait =
+                container_of(notify, typeof(*wait), notify);
+        wake_up_all(&wait->priv->waiting);
+        return NVIF_NOTIFY_KEEP;
 }
 
 static int
@@ -178,18 +185,22 @@ nouveau_fence_wait_uevent(struct nouveau_fence *fence, bool intr)
 
 {
         struct nouveau_channel *chan = fence->channel;
-        struct nouveau_fifo *pfifo = nouveau_fifo(chan->drm->device);
         struct nouveau_fence_priv *priv = chan->drm->fence;
-        struct nouveau_eventh *handler;
+        struct nouveau_fence_wait wait = { .priv = priv };
         int ret = 0;
 
-        ret = nouveau_event_new(pfifo->uevent, 1, 0,
-                                nouveau_fence_wait_uevent_handler,
-                                priv, &handler);
+        ret = nvif_notify_init(chan->object, NULL,
+                               nouveau_fence_wait_uevent_handler, false,
+                               G82_CHANNEL_DMA_V0_NTFY_UEVENT,
+                               &(struct nvif_notify_uevent_req) {
+                               },
+                               sizeof(struct nvif_notify_uevent_req),
+                               sizeof(struct nvif_notify_uevent_rep),
+                               &wait.notify);
         if (ret)
                 return ret;
 
-        nouveau_event_get(handler);
+        nvif_notify_get(&wait.notify);
 
         if (fence->timeout) {
                 unsigned long timeout = fence->timeout - jiffies;
@@ -221,7 +232,7 @@ nouveau_fence_wait_uevent(struct nouveau_fence *fence, bool intr)
                 }
         }
 
-        nouveau_event_ref(NULL, &handler);
+        nvif_notify_fini(&wait.notify);
         if (unlikely(ret < 0))
                 return ret;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index df9d451afdcd..292a677bfed4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -24,8 +24,6 @@
  *
  */
 
-#include <subdev/fb.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_fence.h"
@@ -58,14 +56,14 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
         struct nouveau_vma *vma;
         int ret;
 
-        if (!cli->base.vm)
+        if (!cli->vm)
                 return 0;
 
         ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL);
         if (ret)
                 return ret;
 
-        vma = nouveau_bo_vma_find(nvbo, cli->base.vm);
+        vma = nouveau_bo_vma_find(nvbo, cli->vm);
         if (!vma) {
                 vma = kzalloc(sizeof(*vma), GFP_KERNEL);
                 if (!vma) {
@@ -73,7 +71,7 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
                         goto out;
                 }
 
-                ret = nouveau_bo_vma_add(nvbo, cli->base.vm, vma);
+                ret = nouveau_bo_vma_add(nvbo, cli->vm, vma);
                 if (ret) {
                         kfree(vma);
                         goto out;
@@ -129,14 +127,14 @@ nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
         struct nouveau_vma *vma;
         int ret;
 
-        if (!cli->base.vm)
+        if (!cli->vm)
                 return;
 
         ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL);
         if (ret)
                 return;
 
-        vma = nouveau_bo_vma_find(nvbo, cli->base.vm);
+        vma = nouveau_bo_vma_find(nvbo, cli->vm);
         if (vma) {
                 if (--vma->refcount == 0)
                         nouveau_gem_object_unmap(nvbo, vma);
@@ -173,7 +171,7 @@ nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain,
          */
         nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
                               NOUVEAU_GEM_DOMAIN_GART;
-        if (nv_device(drm->device)->card_type >= NV_50)
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
                 nvbo->valid_domains &= domain;
 
         /* Initialize the embedded gem-object. We return a single gem-reference
@@ -202,8 +200,8 @@ nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
                 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
 
         rep->offset = nvbo->bo.offset;
-        if (cli->base.vm) {
-                vma = nouveau_bo_vma_find(nvbo, cli->base.vm);
+        if (cli->vm) {
+                vma = nouveau_bo_vma_find(nvbo, cli->vm);
                 if (!vma)
                         return -EINVAL;
 
@@ -223,13 +221,13 @@ nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_cli *cli = nouveau_cli(file_priv);
-        struct nouveau_fb *pfb = nouveau_fb(drm->device);
+        struct nouveau_fb *pfb = nvkm_fb(&drm->device);
         struct drm_nouveau_gem_new *req = data;
         struct nouveau_bo *nvbo = NULL;
         int ret = 0;
 
         if (!pfb->memtype_valid(pfb, req->info.tile_flags)) {
-                NV_ERROR(cli, "bad page flags: 0x%08x\n", req->info.tile_flags);
+                NV_PRINTK(error, cli, "bad page flags: 0x%08x\n", req->info.tile_flags);
                 return -EINVAL;
         }
 
@@ -350,7 +348,7 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
         ww_acquire_init(&op->ticket, &reservation_ww_class);
 retry:
         if (++trycnt > 100000) {
-                NV_ERROR(cli, "%s failed and gave up.\n", __func__);
+                NV_PRINTK(error, cli, "%s failed and gave up.\n", __func__);
                 return -EINVAL;
         }
 
@@ -361,7 +359,7 @@ retry:
 
                 gem = drm_gem_object_lookup(dev, file_priv, b->handle);
                 if (!gem) {
-                        NV_ERROR(cli, "Unknown handle 0x%08x\n", b->handle);
+                        NV_PRINTK(error, cli, "Unknown handle 0x%08x\n", b->handle);
                         ww_acquire_done(&op->ticket);
                         validate_fini(op, NULL);
                         return -ENOENT;
@@ -374,7 +372,7 @@ retry:
                 }
 
                 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
-                        NV_ERROR(cli, "multiple instances of buffer %d on "
+                        NV_PRINTK(error, cli, "multiple instances of buffer %d on "
                                       "validation list\n", b->handle);
                         drm_gem_object_unreference_unlocked(gem);
                         ww_acquire_done(&op->ticket);
@@ -396,7 +394,7 @@ retry:
                                 ww_acquire_fini(&op->ticket);
                                 drm_gem_object_unreference_unlocked(gem);
                                 if (ret != -ERESTARTSYS)
-                                        NV_ERROR(cli, "fail reserve\n");
+                                        NV_PRINTK(error, cli, "fail reserve\n");
                                 return ret;
                         }
                 }
@@ -414,7 +412,7 @@ retry:
                 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
                         list_add_tail(&nvbo->entry, &op->gart_list);
                 else {
-                        NV_ERROR(cli, "invalid valid domains: 0x%08x\n",
+                        NV_PRINTK(error, cli, "invalid valid domains: 0x%08x\n",
                                  b->valid_domains);
                         list_add_tail(&nvbo->entry, &op->both_list);
                         ww_acquire_done(&op->ticket);
@@ -465,24 +463,24 @@ validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli,
                                              b->write_domains,
                                              b->valid_domains);
                 if (unlikely(ret)) {
-                        NV_ERROR(cli, "fail set_domain\n");
+                        NV_PRINTK(error, cli, "fail set_domain\n");
                         return ret;
                 }
 
                 ret = nouveau_bo_validate(nvbo, true, false);
                 if (unlikely(ret)) {
                         if (ret != -ERESTARTSYS)
-                                NV_ERROR(cli, "fail ttm_validate\n");
+                                NV_PRINTK(error, cli, "fail ttm_validate\n");
                         return ret;
                 }
 
                 ret = validate_sync(chan, nvbo);
                 if (unlikely(ret)) {
-                        NV_ERROR(cli, "fail post-validate sync\n");
+                        NV_PRINTK(error, cli, "fail post-validate sync\n");
                         return ret;
                 }
 
-                if (nv_device(drm->device)->card_type < NV_50) {
+                if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
                         if (nvbo->bo.offset == b->presumed.offset &&
                             ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
                               b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
@@ -527,14 +525,14 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
         ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
         if (unlikely(ret)) {
                 if (ret != -ERESTARTSYS)
-                        NV_ERROR(cli, "validate_init\n");
+                        NV_PRINTK(error, cli, "validate_init\n");
                 return ret;
         }
 
         ret = validate_list(chan, cli, &op->vram_list, pbbo, user_buffers);
         if (unlikely(ret < 0)) {
                 if (ret != -ERESTARTSYS)
-                        NV_ERROR(cli, "validate vram_list\n");
+                        NV_PRINTK(error, cli, "validate vram_list\n");
                 validate_fini(op, NULL);
                 return ret;
         }
@@ -543,7 +541,7 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
         ret = validate_list(chan, cli, &op->gart_list, pbbo, user_buffers);
         if (unlikely(ret < 0)) {
                 if (ret != -ERESTARTSYS)
-                        NV_ERROR(cli, "validate gart_list\n");
+                        NV_PRINTK(error, cli, "validate gart_list\n");
                 validate_fini(op, NULL);
                 return ret;
         }
@@ -552,7 +550,7 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
         ret = validate_list(chan, cli, &op->both_list, pbbo, user_buffers);
         if (unlikely(ret < 0)) {
                 if (ret != -ERESTARTSYS)
-                        NV_ERROR(cli, "validate both_list\n");
+                        NV_PRINTK(error, cli, "validate both_list\n");
                 validate_fini(op, NULL);
                 return ret;
         }
@@ -613,7 +611,7 @@ nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
                 uint32_t data;
 
                 if (unlikely(r->bo_index > req->nr_buffers)) {
-                        NV_ERROR(cli, "reloc bo index invalid\n");
+                        NV_PRINTK(error, cli, "reloc bo index invalid\n");
                         ret = -EINVAL;
                         break;
                 }
@@ -623,7 +621,7 @@ nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
                         continue;
 
                 if (unlikely(r->reloc_bo_index > req->nr_buffers)) {
-                        NV_ERROR(cli, "reloc container bo index invalid\n");
+                        NV_PRINTK(error, cli, "reloc container bo index invalid\n");
                         ret = -EINVAL;
                         break;
                 }
@@ -631,7 +629,7 @@ nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
 
                 if (unlikely(r->reloc_bo_offset + 4 >
                              nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
-                        NV_ERROR(cli, "reloc outside of bo\n");
+                        NV_PRINTK(error, cli, "reloc outside of bo\n");
                         ret = -EINVAL;
                         break;
                 }
@@ -640,7 +638,7 @@ nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
                         ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
                                           &nvbo->kmap);
                         if (ret) {
-                                NV_ERROR(cli, "failed kmap for reloc\n");
+                                NV_PRINTK(error, cli, "failed kmap for reloc\n");
                                 break;
                         }
                         nvbo->validate_mapped = true;
@@ -665,7 +663,7 @@ nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
                 ret = ttm_bo_wait(&nvbo->bo, false, false, false);
                 spin_unlock(&nvbo->bo.bdev->fence_lock);
                 if (ret) {
-                        NV_ERROR(cli, "reloc wait_idle failed: %d\n", ret);
+                        NV_PRINTK(error, cli, "reloc wait_idle failed: %d\n", ret);
                         break;
                 }
 
@@ -696,7 +694,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
                 return -ENOMEM;
 
         list_for_each_entry(temp, &abi16->channels, head) {
-                if (temp->chan->handle == (NVDRM_CHAN | req->channel)) {
+                if (temp->chan->object->handle == (NVDRM_CHAN | req->channel)) {
                         chan = temp->chan;
                         break;
                 }
@@ -711,19 +709,19 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
                 goto out_next;
 
         if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
-                NV_ERROR(cli, "pushbuf push count exceeds limit: %d max %d\n",
+                NV_PRINTK(error, cli, "pushbuf push count exceeds limit: %d max %d\n",
                          req->nr_push, NOUVEAU_GEM_MAX_PUSH);
                 return nouveau_abi16_put(abi16, -EINVAL);
         }
 
         if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
-                NV_ERROR(cli, "pushbuf bo count exceeds limit: %d max %d\n",
+                NV_PRINTK(error, cli, "pushbuf bo count exceeds limit: %d max %d\n",
                          req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
                 return nouveau_abi16_put(abi16, -EINVAL);
         }
 
         if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
-                NV_ERROR(cli, "pushbuf reloc count exceeds limit: %d max %d\n",
+                NV_PRINTK(error, cli, "pushbuf reloc count exceeds limit: %d max %d\n",
                          req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
                 return nouveau_abi16_put(abi16, -EINVAL);
         }
@@ -741,7 +739,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
         /* Ensure all push buffers are on validate list */
         for (i = 0; i < req->nr_push; i++) {
                 if (push[i].bo_index >= req->nr_buffers) {
-                        NV_ERROR(cli, "push %d buffer not in list\n", i);
+                        NV_PRINTK(error, cli, "push %d buffer not in list\n", i);
                         ret = -EINVAL;
                         goto out_prevalid;
                 }
@@ -752,7 +750,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
                                            req->nr_buffers, &op, &do_reloc);
         if (ret) {
                 if (ret != -ERESTARTSYS)
-                        NV_ERROR(cli, "validate: %d\n", ret);
+                        NV_PRINTK(error, cli, "validate: %d\n", ret);
                 goto out_prevalid;
         }
 
@@ -760,7 +758,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
         if (do_reloc) {
                 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo);
                 if (ret) {
-                        NV_ERROR(cli, "reloc apply: %d\n", ret);
+                        NV_PRINTK(error, cli, "reloc apply: %d\n", ret);
                         goto out;
                 }
         }
@@ -768,7 +766,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
         if (chan->dma.ib_max) {
                 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16);
                 if (ret) {
-                        NV_ERROR(cli, "nv50cal_space: %d\n", ret);
+                        NV_PRINTK(error, cli, "nv50cal_space: %d\n", ret);
                         goto out;
                 }
 
@@ -780,10 +778,10 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
                                       push[i].length);
                 }
         } else
-        if (nv_device(drm->device)->chipset >= 0x25) {
+        if (drm->device.info.chipset >= 0x25) {
                 ret = RING_SPACE(chan, req->nr_push * 2);
                 if (ret) {
-                        NV_ERROR(cli, "cal_space: %d\n", ret);
+                        NV_PRINTK(error, cli, "cal_space: %d\n", ret);
                         goto out;
                 }
 
@@ -797,7 +795,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
         } else {
                 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
                 if (ret) {
-                        NV_ERROR(cli, "jmp_space: %d\n", ret);
+                        NV_PRINTK(error, cli, "jmp_space: %d\n", ret);
                         goto out;
                 }
 
@@ -835,7 +833,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
 
         ret = nouveau_fence_new(chan, false, &fence);
         if (ret) {
-                NV_ERROR(cli, "error fencing pushbuf: %d\n", ret);
+                NV_PRINTK(error, cli, "error fencing pushbuf: %d\n", ret);
                 WIND_RING(chan);
                 goto out;
         }
@@ -853,7 +851,7 @@ out_next:
                 req->suffix0 = 0x00000000;
                 req->suffix1 = 0x00000000;
         } else
-        if (nv_device(drm->device)->chipset >= 0x25) {
+        if (drm->device.info.chipset >= 0x25) {
                 req->suffix0 = 0x00020000;
                 req->suffix1 = 0x00000000;
         } else {
diff --git a/drivers/gpu/drm/nouveau/nouveau_hwmon.c b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
index 19fd767bab10..afb36d66e78d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hwmon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
@@ -34,17 +34,13 @@
 #include "nouveau_drm.h"
 #include "nouveau_hwmon.h"
 
-#include <subdev/gpio.h>
-#include <subdev/timer.h>
-#include <subdev/therm.h>
-
 #if defined(CONFIG_HWMON) || (defined(MODULE) && defined(CONFIG_HWMON_MODULE))
 static ssize_t
 nouveau_hwmon_show_temp(struct device *d, struct device_attribute *a, char *buf)
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         int temp = therm->temp_get(therm);
 
         if (temp < 0)
@@ -70,7 +66,7 @@ nouveau_hwmon_temp1_auto_point1_temp(struct device *d,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
 
         return snprintf(buf, PAGE_SIZE, "%d\n",
               therm->attr_get(therm, NOUVEAU_THERM_ATTR_THRS_FAN_BOOST) * 1000);
@@ -82,7 +78,7 @@ nouveau_hwmon_set_temp1_auto_point1_temp(struct device *d,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         long value;
 
         if (kstrtol(buf, 10, &value) == -EINVAL)
@@ -103,7 +99,7 @@ nouveau_hwmon_temp1_auto_point1_temp_hyst(struct device *d,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
 
         return snprintf(buf, PAGE_SIZE, "%d\n",
          therm->attr_get(therm, NOUVEAU_THERM_ATTR_THRS_FAN_BOOST_HYST) * 1000);
@@ -115,7 +111,7 @@ nouveau_hwmon_set_temp1_auto_point1_temp_hyst(struct device *d,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         long value;
 
         if (kstrtol(buf, 10, &value) == -EINVAL)
@@ -135,7 +131,7 @@ nouveau_hwmon_max_temp(struct device *d, struct device_attribute *a, char *buf)
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
 
         return snprintf(buf, PAGE_SIZE, "%d\n",
                therm->attr_get(therm, NOUVEAU_THERM_ATTR_THRS_DOWN_CLK) * 1000);
@@ -146,7 +142,7 @@ nouveau_hwmon_set_max_temp(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         long value;
 
         if (kstrtol(buf, 10, &value) == -EINVAL)
@@ -166,7 +162,7 @@ nouveau_hwmon_max_temp_hyst(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
 
         return snprintf(buf, PAGE_SIZE, "%d\n",
           therm->attr_get(therm, NOUVEAU_THERM_ATTR_THRS_DOWN_CLK_HYST) * 1000);
@@ -177,7 +173,7 @@ nouveau_hwmon_set_max_temp_hyst(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         long value;
 
         if (kstrtol(buf, 10, &value) == -EINVAL)
@@ -198,7 +194,7 @@ nouveau_hwmon_critical_temp(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
 
         return snprintf(buf, PAGE_SIZE, "%d\n",
                therm->attr_get(therm, NOUVEAU_THERM_ATTR_THRS_CRITICAL) * 1000);
@@ -210,7 +206,7 @@ nouveau_hwmon_set_critical_temp(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         long value;
 
         if (kstrtol(buf, 10, &value) == -EINVAL)
@@ -231,7 +227,7 @@ nouveau_hwmon_critical_temp_hyst(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
 
         return snprintf(buf, PAGE_SIZE, "%d\n",
           therm->attr_get(therm, NOUVEAU_THERM_ATTR_THRS_CRITICAL_HYST) * 1000);
@@ -244,7 +240,7 @@ nouveau_hwmon_set_critical_temp_hyst(struct device *d,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         long value;
 
         if (kstrtol(buf, 10, &value) == -EINVAL)
@@ -264,7 +260,7 @@ nouveau_hwmon_emergency_temp(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
 
         return snprintf(buf, PAGE_SIZE, "%d\n",
                therm->attr_get(therm, NOUVEAU_THERM_ATTR_THRS_SHUTDOWN) * 1000);
@@ -276,7 +272,7 @@ nouveau_hwmon_set_emergency_temp(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         long value;
 
         if (kstrtol(buf, 10, &value) == -EINVAL)
@@ -297,7 +293,7 @@ nouveau_hwmon_emergency_temp_hyst(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
 
         return snprintf(buf, PAGE_SIZE, "%d\n",
           therm->attr_get(therm, NOUVEAU_THERM_ATTR_THRS_SHUTDOWN_HYST) * 1000);
@@ -310,7 +306,7 @@ nouveau_hwmon_set_emergency_temp_hyst(struct device *d,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         long value;
 
         if (kstrtol(buf, 10, &value) == -EINVAL)
@@ -350,7 +346,7 @@ nouveau_hwmon_show_fan1_input(struct device *d, struct device_attribute *attr,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
 
         return snprintf(buf, PAGE_SIZE, "%d\n", therm->fan_sense(therm));
 }
@@ -363,7 +359,7 @@ nouveau_hwmon_get_pwm1_enable(struct device *d,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         int ret;
 
         ret = therm->attr_get(therm, NOUVEAU_THERM_ATTR_FAN_MODE);
@@ -379,7 +375,7 @@ nouveau_hwmon_set_pwm1_enable(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         long value;
         int ret;
 
@@ -402,7 +398,7 @@ nouveau_hwmon_get_pwm1(struct device *d, struct device_attribute *a, char *buf)
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         int ret;
 
         ret = therm->fan_get(therm);
@@ -418,7 +414,7 @@ nouveau_hwmon_set_pwm1(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         int ret = -ENODEV;
         long value;
 
@@ -442,7 +438,7 @@ nouveau_hwmon_get_pwm1_min(struct device *d,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         int ret;
 
         ret = therm->attr_get(therm, NOUVEAU_THERM_ATTR_FAN_MIN_DUTY);
@@ -458,7 +454,7 @@ nouveau_hwmon_set_pwm1_min(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         long value;
         int ret;
 
@@ -482,7 +478,7 @@ nouveau_hwmon_get_pwm1_max(struct device *d,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         int ret;
 
         ret = therm->attr_get(therm, NOUVEAU_THERM_ATTR_FAN_MAX_DUTY);
@@ -498,7 +494,7 @@ nouveau_hwmon_set_pwm1_max(struct device *d, struct device_attribute *a,
 {
         struct drm_device *dev = dev_get_drvdata(d);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         long value;
         int ret;
 
@@ -565,7 +561,7 @@ nouveau_hwmon_init(struct drm_device *dev)
 {
 #if defined(CONFIG_HWMON) || (defined(MODULE) && defined(CONFIG_HWMON_MODULE))
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_therm *therm = nvkm_therm(&drm->device);
         struct nouveau_hwmon *hwmon;
         struct device *hwmon_dev;
         int ret = 0;
diff --git a/drivers/gpu/drm/nouveau/nouveau_nvif.c b/drivers/gpu/drm/nouveau/nouveau_nvif.c
new file mode 100644
index 000000000000..47ca88623753
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_nvif.c
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+/*******************************************************************************
+ * NVIF client driver - NVKM directly linked
+ ******************************************************************************/
+
+#include <core/client.h>
+#include <core/notify.h>
+#include <core/ioctl.h>
+
+#include <nvif/client.h>
+#include <nvif/driver.h>
+#include <nvif/notify.h>
+#include <nvif/event.h>
+#include <nvif/ioctl.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_usif.h"
+
+static void
+nvkm_client_unmap(void *priv, void *ptr, u32 size)
+{
+        iounmap(ptr);
+}
+
+static void *
+nvkm_client_map(void *priv, u64 handle, u32 size)
+{
+        return ioremap(handle, size);
+}
+
+static int
+nvkm_client_ioctl(void *priv, bool super, void *data, u32 size, void **hack)
+{
+        return nvkm_ioctl(priv, super, data, size, hack);
+}
+
+static int
+nvkm_client_resume(void *priv)
+{
+        return nouveau_client_init(priv);
+}
+
+static int
+nvkm_client_suspend(void *priv)
+{
+        return nouveau_client_fini(priv, true);
+}
+
+static void
+nvkm_client_fini(void *priv)
+{
+        struct nouveau_object *client = priv;
+        nouveau_client_fini(nv_client(client), false);
+        atomic_set(&client->refcount, 1);
+        nouveau_object_ref(NULL, &client);
+}
+
+static int
+nvkm_client_ntfy(const void *header, u32 length, const void *data, u32 size)
+{
+        const union {
+                struct nvif_notify_req_v0 v0;
+        } *args = header;
+        u8 route;
+
+        if (length == sizeof(args->v0) && args->v0.version == 0) {
+                route = args->v0.route;
+        } else {
+                WARN_ON(1);
+                return NVKM_NOTIFY_DROP;
+        }
+
+        switch (route) {
+        case NVDRM_NOTIFY_NVIF:
+                return nvif_notify(header, length, data, size);
+        case NVDRM_NOTIFY_USIF:
+                return usif_notify(header, length, data, size);
+        default:
+                WARN_ON(1);
+                break;
+        }
+
+        return NVKM_NOTIFY_DROP;
+}
+
+static int
+nvkm_client_init(const char *name, u64 device, const char *cfg,
+                 const char *dbg, void **ppriv)
+{
+        struct nouveau_client *client;
+        int ret;
+
+        ret = nouveau_client_create(name, device, cfg, dbg, &client);
+        *ppriv = client;
+        if (ret)
+                return ret;
+
+        client->ntfy = nvkm_client_ntfy;
+        return 0;
+}
+
+const struct nvif_driver
+nvif_driver_nvkm = {
+        .name = "nvkm",
+        .init = nvkm_client_init,
+        .fini = nvkm_client_fini,
+        .suspend = nvkm_client_suspend,
+        .resume = nvkm_client_resume,
+        .ioctl = nvkm_client_ioctl,
+        .map = nvkm_client_map,
+        .unmap = nvkm_client_unmap,
+        .keep = false,
+};
diff --git a/drivers/gpu/drm/nouveau/nouveau_platform.c b/drivers/gpu/drm/nouveau/nouveau_platform.c
new file mode 100644
index 000000000000..0ffeb50d0088
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_platform.c
@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/reset.h>
+#include <linux/regulator/consumer.h>
+#include <soc/tegra/pmc.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_platform.h"
+
+static int nouveau_platform_power_up(struct nouveau_platform_gpu *gpu)
+{
+        int err;
+
+        err = regulator_enable(gpu->vdd);
+        if (err)
+                goto err_power;
+
+        err = clk_prepare_enable(gpu->clk);
+        if (err)
+                goto err_clk;
+        err = clk_prepare_enable(gpu->clk_pwr);
+        if (err)
+                goto err_clk_pwr;
+        clk_set_rate(gpu->clk_pwr, 204000000);
+        udelay(10);
+
+        reset_control_assert(gpu->rst);
+        udelay(10);
+
+        err = tegra_powergate_remove_clamping(TEGRA_POWERGATE_3D);
+        if (err)
+                goto err_clamp;
+        udelay(10);
+
+        reset_control_deassert(gpu->rst);
+        udelay(10);
+
+        return 0;
+
+err_clamp:
+        clk_disable_unprepare(gpu->clk_pwr);
+err_clk_pwr:
+        clk_disable_unprepare(gpu->clk);
+err_clk:
+        regulator_disable(gpu->vdd);
+err_power:
+        return err;
+}
+
+static int nouveau_platform_power_down(struct nouveau_platform_gpu *gpu)
+{
+        int err;
+
+        reset_control_assert(gpu->rst);
+        udelay(10);
+
+        clk_disable_unprepare(gpu->clk_pwr);
+        clk_disable_unprepare(gpu->clk);
+        udelay(10);
+
+        err = regulator_disable(gpu->vdd);
+        if (err)
+                return err;
+
+        return 0;
+}
+
+static int nouveau_platform_probe(struct platform_device *pdev)
+{
+        struct nouveau_platform_gpu *gpu;
+        struct nouveau_platform_device *device;
+        struct drm_device *drm;
+        int err;
+
+        gpu = devm_kzalloc(&pdev->dev, sizeof(*gpu), GFP_KERNEL);
+        if (!gpu)
+                return -ENOMEM;
+
+        gpu->vdd = devm_regulator_get(&pdev->dev, "vdd");
+        if (IS_ERR(gpu->vdd))
+                return PTR_ERR(gpu->vdd);
+
+        gpu->rst = devm_reset_control_get(&pdev->dev, "gpu");
+        if (IS_ERR(gpu->rst))
+                return PTR_ERR(gpu->rst);
+
+        gpu->clk = devm_clk_get(&pdev->dev, "gpu");
+        if (IS_ERR(gpu->clk))
+                return PTR_ERR(gpu->clk);
+
+        gpu->clk_pwr = devm_clk_get(&pdev->dev, "pwr");
+        if (IS_ERR(gpu->clk_pwr))
+                return PTR_ERR(gpu->clk_pwr);
+
+        err = nouveau_platform_power_up(gpu);
+        if (err)
+                return err;
+
+        drm = nouveau_platform_device_create(pdev, &device);
+        if (IS_ERR(drm)) {
+                err = PTR_ERR(drm);
+                goto power_down;
+        }
+
+        device->gpu = gpu;
+
+        err = drm_dev_register(drm, 0);
+        if (err < 0)
+                goto err_unref;
+
+        return 0;
+
+err_unref:
+        drm_dev_unref(drm);
+
+        return 0;
+
+power_down:
+        nouveau_platform_power_down(gpu);
+
+        return err;
+}
+
+static int nouveau_platform_remove(struct platform_device *pdev)
+{
+        struct drm_device *drm_dev = platform_get_drvdata(pdev);
+        struct nouveau_device *device = nouveau_dev(drm_dev);
+        struct nouveau_platform_gpu *gpu = nv_device_to_platform(device)->gpu;
+
+        nouveau_drm_device_remove(drm_dev);
+
+        return nouveau_platform_power_down(gpu);
+}
+
+#if IS_ENABLED(CONFIG_OF)
+static const struct of_device_id nouveau_platform_match[] = {
+        { .compatible = "nvidia,gk20a" },
+        { }
+};
+
+MODULE_DEVICE_TABLE(of, nouveau_platform_match);
+#endif
+
+struct platform_driver nouveau_platform_driver = {
+        .driver = {
+                .name = "nouveau",
+                .of_match_table = of_match_ptr(nouveau_platform_match),
+        },
+        .probe = nouveau_platform_probe,
+        .remove = nouveau_platform_remove,
+};
+
+module_platform_driver(nouveau_platform_driver);
+
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/nouveau/nouveau_platform.h b/drivers/gpu/drm/nouveau/nouveau_platform.h
new file mode 100644
index 000000000000..91f66504900e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_platform.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __NOUVEAU_PLATFORM_H__
+#define __NOUVEAU_PLATFORM_H__
+
+#include "core/device.h"
+
+struct reset_control;
+struct clk;
+struct regulator;
+
+struct nouveau_platform_gpu {
+        struct reset_control *rst;
+        struct clk *clk;
+        struct clk *clk_pwr;
+
+        struct regulator *vdd;
+};
+
+struct nouveau_platform_device {
+        struct nouveau_device device;
+
+        struct nouveau_platform_gpu *gpu;
+};
+
+#define nv_device_to_platform(d)                                               \
+        container_of(d, struct nouveau_platform_device, device)
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
index a4d22e5eb176..01707e7deaf5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
@@ -1,8 +1,6 @@
 #include <linux/pagemap.h>
 #include <linux/slab.h>
 
-#include <subdev/fb.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_ttm.h"
 
@@ -104,7 +102,7 @@ nouveau_sgdma_create_ttm(struct ttm_bo_device *bdev,
                 return NULL;
 
         nvbe->dev = drm->dev;
-        if (nv_device(drm->device)->card_type < NV_50)
+        if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA)
                 nvbe->ttm.ttm.func = &nv04_sgdma_backend;
         else
                 nvbe->ttm.ttm.func = &nv50_sgdma_backend;
diff --git a/drivers/gpu/drm/nouveau/nouveau_sysfs.c b/drivers/gpu/drm/nouveau/nouveau_sysfs.c
index 75dda2b07176..3c6962d15b26 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sysfs.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sysfs.c
@@ -22,10 +22,15 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
+#include <nvif/os.h>
+#include <nvif/class.h>
+#include <nvif/ioctl.h>
+
 #include "nouveau_sysfs.h"
 
-#include <core/object.h>
-#include <core/class.h>
+MODULE_PARM_DESC(pstate, "enable sysfs pstate file, which will be moved in the future");
+static int nouveau_pstate;
+module_param_named(pstate, nouveau_pstate, int, 0400);
 
 static inline struct drm_device *
 drm_device(struct device *d)
@@ -43,38 +48,42 @@ static ssize_t
 nouveau_sysfs_pstate_get(struct device *d, struct device_attribute *a, char *b)
 {
         struct nouveau_sysfs *sysfs = nouveau_sysfs(drm_device(d));
-        struct nv_control_pstate_info info;
+        struct nvif_control_pstate_info_v0 info = {};
         size_t cnt = PAGE_SIZE;
         char *buf = b;
         int ret, i;
 
-        ret = nv_exec(sysfs->ctrl, NV_CONTROL_PSTATE_INFO, &info, sizeof(info));
+        ret = nvif_mthd(&sysfs->ctrl, NVIF_CONTROL_PSTATE_INFO,
+                        &info, sizeof(info));
         if (ret)
                 return ret;
 
         for (i = 0; i < info.count + 1; i++) {
                 const s32 state = i < info.count ? i :
-                        NV_CONTROL_PSTATE_ATTR_STATE_CURRENT;
-                struct nv_control_pstate_attr attr = {
+                        NVIF_CONTROL_PSTATE_ATTR_V0_STATE_CURRENT;
+                struct nvif_control_pstate_attr_v0 attr = {
                         .state = state,
                         .index = 0,
                 };
 
-                ret = nv_exec(sysfs->ctrl, NV_CONTROL_PSTATE_ATTR,
-                             &attr, sizeof(attr));
+                ret = nvif_mthd(&sysfs->ctrl, NVIF_CONTROL_PSTATE_ATTR,
+                                &attr, sizeof(attr));
                 if (ret)
                         return ret;
 
                 if (i < info.count)
                         snappendf(buf, cnt, "%02x:", attr.state);
                 else
-                        snappendf(buf, cnt, "--:");
+                        snappendf(buf, cnt, "%s:", info.pwrsrc == 0 ? "DC" :
+                                                   info.pwrsrc == 1 ? "AC" :
+                                                   "--");
 
                 attr.index = 0;
                 do {
                         attr.state = state;
-                        ret = nv_exec(sysfs->ctrl, NV_CONTROL_PSTATE_ATTR,
-                                     &attr, sizeof(attr));
+                        ret = nvif_mthd(&sysfs->ctrl,
+                                        NVIF_CONTROL_PSTATE_ATTR,
+                                        &attr, sizeof(attr));
                         if (ret)
                                 return ret;
 
@@ -84,9 +93,20 @@ nouveau_sysfs_pstate_get(struct device *d, struct device_attribute *a, char *b)
                         snappendf(buf, cnt, " %s", attr.unit);
                 } while (attr.index);
 
-                if ((state >= 0 && info.pstate == state) ||
-                    (state <  0 && info.ustate < 0))
-                        snappendf(buf, cnt, " *");
+                if (state >= 0) {
+                        if (info.ustate_ac == state)
+                                snappendf(buf, cnt, " AC");
+                        if (info.ustate_dc == state)
+                                snappendf(buf, cnt, " DC");
+                        if (info.pstate == state)
+                                snappendf(buf, cnt, " *");
+                } else {
+                        if (info.ustate_ac < -1)
+                                snappendf(buf, cnt, " AC");
+                        if (info.ustate_dc < -1)
+                                snappendf(buf, cnt, " DC");
+                }
+
                 snappendf(buf, cnt, "\n");
         }
 
@@ -98,26 +118,36 @@ nouveau_sysfs_pstate_set(struct device *d, struct device_attribute *a,
                          const char *buf, size_t count)
 {
         struct nouveau_sysfs *sysfs = nouveau_sysfs(drm_device(d));
-        struct nv_control_pstate_user args;
+        struct nvif_control_pstate_user_v0 args = { .pwrsrc = -EINVAL };
         long value, ret;
         char *tmp;
 
         if ((tmp = strchr(buf, '\n')))
                 *tmp = '\0';
 
+        if (!strncasecmp(buf, "dc:", 3)) {
+                args.pwrsrc = 0;
+                buf += 3;
+        } else
+        if (!strncasecmp(buf, "ac:", 3)) {
+                args.pwrsrc = 1;
+                buf += 3;
+        }
+
         if (!strcasecmp(buf, "none"))
-                args.state = NV_CONTROL_PSTATE_USER_STATE_UNKNOWN;
+                args.ustate = NVIF_CONTROL_PSTATE_USER_V0_STATE_UNKNOWN;
         else
         if (!strcasecmp(buf, "auto"))
-                args.state = NV_CONTROL_PSTATE_USER_STATE_PERFMON;
+                args.ustate = NVIF_CONTROL_PSTATE_USER_V0_STATE_PERFMON;
         else {
                 ret = kstrtol(buf, 16, &value);
                 if (ret)
                         return ret;
-                args.state = value;
+                args.ustate = value;
         }
 
-        ret = nv_exec(sysfs->ctrl, NV_CONTROL_PSTATE_USER, &args, sizeof(args));
+        ret = nvif_mthd(&sysfs->ctrl, NVIF_CONTROL_PSTATE_USER,
+                        &args, sizeof(args));
         if (ret < 0)
                 return ret;
 
@@ -132,11 +162,11 @@ nouveau_sysfs_fini(struct drm_device *dev)
 {
         struct nouveau_sysfs *sysfs = nouveau_sysfs(dev);
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
 
-        if (sysfs->ctrl) {
-                device_remove_file(nv_device_base(device), &dev_attr_pstate);
-                nouveau_object_del(nv_object(drm), NVDRM_DEVICE, NVDRM_CONTROL);
+        if (sysfs && sysfs->ctrl.priv) {
+                device_remove_file(nv_device_base(nvkm_device(device)), &dev_attr_pstate);
+                nvif_object_fini(&sysfs->ctrl);
         }
 
         drm->sysfs = NULL;
@@ -147,18 +177,22 @@ int
 nouveau_sysfs_init(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nv_device(drm->device);
+        struct nvif_device *device = &drm->device;
         struct nouveau_sysfs *sysfs;
         int ret;
 
+        if (!nouveau_pstate)
+                return 0;
+
         sysfs = drm->sysfs = kzalloc(sizeof(*sysfs), GFP_KERNEL);
         if (!sysfs)
                 return -ENOMEM;
 
-        ret = nouveau_object_new(nv_object(drm), NVDRM_DEVICE, NVDRM_CONTROL,
-                                 NV_CONTROL_CLASS, NULL, 0, &sysfs->ctrl);
+        ret = nvif_object_init(nvif_object(device), NULL, NVDRM_CONTROL,
+                               NVIF_IOCTL_NEW_V0_CONTROL, NULL, 0,
+                              &sysfs->ctrl);
         if (ret == 0)
-                device_create_file(nv_device_base(device), &dev_attr_pstate);
+                device_create_file(nv_device_base(nvkm_device(device)), &dev_attr_pstate);
 
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_sysfs.h b/drivers/gpu/drm/nouveau/nouveau_sysfs.h
index 74b47f1e01ed..f973378160f8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sysfs.h
+++ b/drivers/gpu/drm/nouveau/nouveau_sysfs.h
@@ -4,7 +4,7 @@
 #include "nouveau_drm.h"
 
 struct nouveau_sysfs {
-        struct nouveau_object *ctrl;
+        struct nvif_object ctrl;
 };
 
 static inline struct nouveau_sysfs *
diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c
index 7e185c122750..53874b76b031 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ttm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c
@@ -24,10 +24,6 @@
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <subdev/fb.h>
-#include <subdev/vm.h>
-#include <subdev/instmem.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_ttm.h"
 #include "nouveau_gem.h"
@@ -36,7 +32,7 @@ static int
 nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)
 {
         struct nouveau_drm *drm = nouveau_bdev(man->bdev);
-        struct nouveau_fb *pfb = nouveau_fb(drm->device);
+        struct nouveau_fb *pfb = nvkm_fb(&drm->device);
         man->priv = pfb;
         return 0;
 }
@@ -67,7 +63,7 @@ nouveau_vram_manager_del(struct ttm_mem_type_manager *man,
                          struct ttm_mem_reg *mem)
 {
         struct nouveau_drm *drm = nouveau_bdev(man->bdev);
-        struct nouveau_fb *pfb = nouveau_fb(drm->device);
+        struct nouveau_fb *pfb = nvkm_fb(&drm->device);
         nouveau_mem_node_cleanup(mem->mm_node);
         pfb->ram->put(pfb, (struct nouveau_mem **)&mem->mm_node);
 }
@@ -80,7 +76,7 @@ nouveau_vram_manager_new(struct ttm_mem_type_manager *man,
                          struct ttm_mem_reg *mem)
 {
         struct nouveau_drm *drm = nouveau_bdev(man->bdev);
-        struct nouveau_fb *pfb = nouveau_fb(drm->device);
+        struct nouveau_fb *pfb = nvkm_fb(&drm->device);
         struct nouveau_bo *nvbo = nouveau_bo(bo);
         struct nouveau_mem *node;
         u32 size_nc = 0;
@@ -176,14 +172,13 @@ nouveau_gart_manager_new(struct ttm_mem_type_manager *man,
 
         node->page_shift = 12;
 
-        switch (nv_device(drm->device)->card_type) {
-        case NV_50:
-                if (nv_device(drm->device)->chipset != 0x50)
+        switch (drm->device.info.family) {
+        case NV_DEVICE_INFO_V0_TESLA:
+                if (drm->device.info.chipset != 0x50)
                         node->memtype = (nvbo->tile_flags & 0x7f00) >> 8;
                 break;
-        case NV_C0:
-        case NV_D0:
-        case NV_E0:
+        case NV_DEVICE_INFO_V0_FERMI:
+        case NV_DEVICE_INFO_V0_KEPLER:
                 node->memtype = (nvbo->tile_flags & 0xff00) >> 8;
                 break;
         default:
@@ -208,12 +203,13 @@ const struct ttm_mem_type_manager_func nouveau_gart_manager = {
         nouveau_gart_manager_debug
 };
 
+/*XXX*/
 #include <core/subdev/vm/nv04.h>
 static int
 nv04_gart_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)
 {
         struct nouveau_drm *drm = nouveau_bdev(man->bdev);
-        struct nouveau_vmmgr *vmm = nouveau_vmmgr(drm->device);
+        struct nouveau_vmmgr *vmm = nvkm_vmmgr(&drm->device);
         struct nv04_vmmgr_priv *priv = (void *)vmm;
         struct nouveau_vm *vm = NULL;
         nouveau_vm_ref(priv->vm, &vm, NULL);
@@ -357,12 +353,11 @@ int
 nouveau_ttm_init(struct nouveau_drm *drm)
 {
         struct drm_device *dev = drm->dev;
-        struct nouveau_device *device = nv_device(drm->device);
         u32 bits;
         int ret;
 
-        bits = nouveau_vmmgr(drm->device)->dma_bits;
-        if (nv_device_is_pci(device)) {
+        bits = nvkm_vmmgr(&drm->device)->dma_bits;
+        if (nv_device_is_pci(nvkm_device(&drm->device))) {
                 if (drm->agp.stat == ENABLED ||
                      !pci_dma_supported(dev->pdev, DMA_BIT_MASK(bits)))
                         bits = 32;
@@ -394,8 +389,7 @@ nouveau_ttm_init(struct nouveau_drm *drm)
         }
 
         /* VRAM init */
-        drm->gem.vram_available  = nouveau_fb(drm->device)->ram->size;
-        drm->gem.vram_available -= nouveau_instmem(drm->device)->reserved;
+        drm->gem.vram_available = drm->device.info.ram_user;
 
         ret = ttm_bo_init_mm(&drm->ttm.bdev, TTM_PL_VRAM,
                               drm->gem.vram_available >> PAGE_SHIFT);
@@ -404,12 +398,12 @@ nouveau_ttm_init(struct nouveau_drm *drm)
                 return ret;
         }
 
-        drm->ttm.mtrr = arch_phys_wc_add(nv_device_resource_start(device, 1),
-                                         nv_device_resource_len(device, 1));
+        drm->ttm.mtrr = arch_phys_wc_add(nv_device_resource_start(nvkm_device(&drm->device), 1),
+                                         nv_device_resource_len(nvkm_device(&drm->device), 1));
 
         /* GART init */
         if (drm->agp.stat != ENABLED) {
-                drm->gem.gart_available = nouveau_vmmgr(drm->device)->limit;
+                drm->gem.gart_available = nvkm_vmmgr(&drm->device)->limit;
         } else {
                 drm->gem.gart_available = drm->agp.size;
         }
diff --git a/drivers/gpu/drm/nouveau/nouveau_usif.c b/drivers/gpu/drm/nouveau/nouveau_usif.c
new file mode 100644
index 000000000000..cb1182d7e80e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_usif.c
@@ -0,0 +1,384 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include "nouveau_drm.h"
+#include "nouveau_usif.h"
+
+#include <nvif/notify.h>
+#include <nvif/unpack.h>
+#include <nvif/client.h>
+#include <nvif/event.h>
+#include <nvif/ioctl.h>
+
+struct usif_notify_p {
+        struct drm_pending_event base;
+        struct {
+                struct drm_event base;
+                u8 data[];
+        } e;
+};
+
+struct usif_notify {
+        struct list_head head;
+        atomic_t enabled;
+        u32 handle;
+        u16 reply;
+        u8  route;
+        u64 token;
+        struct usif_notify_p *p;
+};
+
+static inline struct usif_notify *
+usif_notify_find(struct drm_file *filp, u32 handle)
+{
+        struct nouveau_cli *cli = nouveau_cli(filp);
+        struct usif_notify *ntfy;
+        list_for_each_entry(ntfy, &cli->notifys, head) {
+                if (ntfy->handle == handle)
+                        return ntfy;
+        }
+        return NULL;
+}
+
+static inline void
+usif_notify_dtor(struct usif_notify *ntfy)
+{
+        list_del(&ntfy->head);
+        kfree(ntfy);
+}
+
+int
+usif_notify(const void *header, u32 length, const void *data, u32 size)
+{
+        struct usif_notify *ntfy = NULL;
+        const union {
+                struct nvif_notify_rep_v0 v0;
+        } *rep = header;
+        struct drm_device *dev;
+        struct drm_file *filp;
+        unsigned long flags;
+
+        if (length == sizeof(rep->v0) && rep->v0.version == 0) {
+                if (WARN_ON(!(ntfy = (void *)(unsigned long)rep->v0.token)))
+                        return NVIF_NOTIFY_DROP;
+                BUG_ON(rep->v0.route != NVDRM_NOTIFY_USIF);
+        } else
+        if (WARN_ON(1))
+                return NVIF_NOTIFY_DROP;
+
+        if (WARN_ON(!ntfy->p || ntfy->reply != (length + size)))
+                return NVIF_NOTIFY_DROP;
+        filp = ntfy->p->base.file_priv;
+        dev = filp->minor->dev;
+
+        memcpy(&ntfy->p->e.data[0], header, length);
+        memcpy(&ntfy->p->e.data[length], data, size);
+        switch (rep->v0.version) {
+        case 0: {
+                struct nvif_notify_rep_v0 *rep = (void *)ntfy->p->e.data;
+                rep->route = ntfy->route;
+                rep->token = ntfy->token;
+        }
+                break;
+        default:
+                BUG_ON(1);
+                break;
+        }
+
+        spin_lock_irqsave(&dev->event_lock, flags);
+        if (!WARN_ON(filp->event_space < ntfy->p->e.base.length)) {
+                list_add_tail(&ntfy->p->base.link, &filp->event_list);
+                filp->event_space -= ntfy->p->e.base.length;
+        }
+        wake_up_interruptible(&filp->event_wait);
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+        atomic_set(&ntfy->enabled, 0);
+        return NVIF_NOTIFY_DROP;
+}
+
+static int
+usif_notify_new(struct drm_file *f, void *data, u32 size, void *argv, u32 argc)
+{
+        struct nouveau_cli *cli = nouveau_cli(f);
+        struct nvif_client *client = &cli->base;
+        union {
+                struct nvif_ioctl_ntfy_new_v0 v0;
+        } *args = data;
+        union {
+                struct nvif_notify_req_v0 v0;
+        } *req;
+        struct usif_notify *ntfy;
+        int ret;
+
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                if (usif_notify_find(f, args->v0.index))
+                        return -EEXIST;
+        } else
+                return ret;
+        req = data;
+
+        if (!(ntfy = kmalloc(sizeof(*ntfy), GFP_KERNEL)))
+                return -ENOMEM;
+        atomic_set(&ntfy->enabled, 0);
+
+        if (nvif_unpack(req->v0, 0, 0, true)) {
+                ntfy->reply = sizeof(struct nvif_notify_rep_v0) + req->v0.reply;
+                ntfy->route = req->v0.route;
+                ntfy->token = req->v0.token;
+                req->v0.route = NVDRM_NOTIFY_USIF;
+                req->v0.token = (unsigned long)(void *)ntfy;
+                ret = nvif_client_ioctl(client, argv, argc);
+                req->v0.token = ntfy->token;
+                req->v0.route = ntfy->route;
+                ntfy->handle = args->v0.index;
+        }
+
+        if (ret == 0)
+                list_add(&ntfy->head, &cli->notifys);
+        if (ret)
+                kfree(ntfy);
+        return ret;
+}
+
+static int
+usif_notify_del(struct drm_file *f, void *data, u32 size, void *argv, u32 argc)
+{
+        struct nouveau_cli *cli = nouveau_cli(f);
+        struct nvif_client *client = &cli->base;
+        union {
+                struct nvif_ioctl_ntfy_del_v0 v0;
+        } *args = data;
+        struct usif_notify *ntfy;
+        int ret;
+
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                if (!(ntfy = usif_notify_find(f, args->v0.index)))
+                        return -ENOENT;
+        } else
+                return ret;
+
+        ret = nvif_client_ioctl(client, argv, argc);
+        if (ret == 0)
+                usif_notify_dtor(ntfy);
+        return ret;
+}
+
+static int
+usif_notify_get(struct drm_file *f, void *data, u32 size, void *argv, u32 argc)
+{
+        struct nouveau_cli *cli = nouveau_cli(f);
+        struct nvif_client *client = &cli->base;
+        union {
+                struct nvif_ioctl_ntfy_del_v0 v0;
+        } *args = data;
+        struct usif_notify *ntfy;
+        int ret;
+
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                if (!(ntfy = usif_notify_find(f, args->v0.index)))
+                        return -ENOENT;
+        } else
+                return ret;
+
+        if (atomic_xchg(&ntfy->enabled, 1))
+                return 0;
+
+        ntfy->p = kmalloc(sizeof(*ntfy->p) + ntfy->reply, GFP_KERNEL);
+        if (ret = -ENOMEM, !ntfy->p)
+                goto done;
+        ntfy->p->base.event = &ntfy->p->e.base;
+        ntfy->p->base.file_priv = f;
+        ntfy->p->base.pid = current->pid;
+        ntfy->p->base.destroy =(void(*)(struct drm_pending_event *))kfree;
+        ntfy->p->e.base.type = DRM_NOUVEAU_EVENT_NVIF;
+        ntfy->p->e.base.length = sizeof(ntfy->p->e.base) + ntfy->reply;
+
+        ret = nvif_client_ioctl(client, argv, argc);
+done:
+        if (ret) {
+                atomic_set(&ntfy->enabled, 0);
+                kfree(ntfy->p);
+        }
+        return ret;
+}
+
+static int
+usif_notify_put(struct drm_file *f, void *data, u32 size, void *argv, u32 argc)
+{
+        struct nouveau_cli *cli = nouveau_cli(f);
+        struct nvif_client *client = &cli->base;
+        union {
+                struct nvif_ioctl_ntfy_put_v0 v0;
+        } *args = data;
+        struct usif_notify *ntfy;
+        int ret;
+
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                if (!(ntfy = usif_notify_find(f, args->v0.index)))
+                        return -ENOENT;
+        } else
+                return ret;
+
+        ret = nvif_client_ioctl(client, argv, argc);
+        if (ret == 0 && atomic_xchg(&ntfy->enabled, 0))
+                kfree(ntfy->p);
+        return ret;
+}
+
+struct usif_object {
+        struct list_head head;
+        struct list_head ntfy;
+        u8  route;
+        u64 token;
+};
+
+static void
+usif_object_dtor(struct usif_object *object)
+{
+        list_del(&object->head);
+        kfree(object);
+}
+
+static int
+usif_object_new(struct drm_file *f, void *data, u32 size, void *argv, u32 argc)
+{
+        struct nouveau_cli *cli = nouveau_cli(f);
+        struct nvif_client *client = &cli->base;
+        union {
+                struct nvif_ioctl_new_v0 v0;
+        } *args = data;
+        struct usif_object *object;
+        int ret;
+
+        if (!(object = kmalloc(sizeof(*object), GFP_KERNEL)))
+                return -ENOMEM;
+        list_add(&object->head, &cli->objects);
+
+        if (nvif_unpack(args->v0, 0, 0, true)) {
+                object->route = args->v0.route;
+                object->token = args->v0.token;
+                args->v0.route = NVDRM_OBJECT_USIF;
+                args->v0.token = (unsigned long)(void *)object;
+                ret = nvif_client_ioctl(client, argv, argc);
+                args->v0.token = object->token;
+                args->v0.route = object->route;
+        }
+
+        if (ret)
+                usif_object_dtor(object);
+        return ret;
+}
+
+int
+usif_ioctl(struct drm_file *filp, void __user *user, u32 argc)
+{
+        struct nouveau_cli *cli = nouveau_cli(filp);
+        struct nvif_client *client = &cli->base;
+        void *data = kmalloc(argc, GFP_KERNEL);
+        u32   size = argc;
+        union {
+                struct nvif_ioctl_v0 v0;
+        } *argv = data;
+        struct usif_object *object;
+        u8 owner;
+        int ret;
+
+        if (ret = -ENOMEM, !argv)
+                goto done;
+        if (ret = -EFAULT, copy_from_user(argv, user, size))
+                goto done;
+
+        if (nvif_unpack(argv->v0, 0, 0, true)) {
+                /* block access to objects not created via this interface */
+                owner = argv->v0.owner;
+                argv->v0.owner = NVDRM_OBJECT_USIF;
+        } else
+                goto done;
+
+        mutex_lock(&cli->mutex);
+        switch (argv->v0.type) {
+        case NVIF_IOCTL_V0_NEW:
+                /* ... except if we're creating children */
+                argv->v0.owner = NVIF_IOCTL_V0_OWNER_ANY;
+                ret = usif_object_new(filp, data, size, argv, argc);
+                break;
+        case NVIF_IOCTL_V0_NTFY_NEW:
+                ret = usif_notify_new(filp, data, size, argv, argc);
+                break;
+        case NVIF_IOCTL_V0_NTFY_DEL:
+                ret = usif_notify_del(filp, data, size, argv, argc);
+                break;
+        case NVIF_IOCTL_V0_NTFY_GET:
+                ret = usif_notify_get(filp, data, size, argv, argc);
+                break;
+        case NVIF_IOCTL_V0_NTFY_PUT:
+                ret = usif_notify_put(filp, data, size, argv, argc);
+                break;
+        default:
+                ret = nvif_client_ioctl(client, argv, argc);
+                break;
+        }
+        if (argv->v0.route == NVDRM_OBJECT_USIF) {
+                object = (void *)(unsigned long)argv->v0.token;
+                argv->v0.route = object->route;
+                argv->v0.token = object->token;
+                if (ret == 0 && argv->v0.type == NVIF_IOCTL_V0_DEL) {
+                        list_del(&object->head);
+                        kfree(object);
+                }
+        } else {
+                argv->v0.route = NVIF_IOCTL_V0_ROUTE_HIDDEN;
+                argv->v0.token = 0;
+        }
+        argv->v0.owner = owner;
+        mutex_unlock(&cli->mutex);
+
+        if (copy_to_user(user, argv, argc))
+                ret = -EFAULT;
+done:
+        kfree(argv);
+        return ret;
+}
+
+void
+usif_client_fini(struct nouveau_cli *cli)
+{
+        struct usif_object *object, *otemp;
+        struct usif_notify *notify, *ntemp;
+
+        list_for_each_entry_safe(notify, ntemp, &cli->notifys, head) {
+                usif_notify_dtor(notify);
+        }
+
+        list_for_each_entry_safe(object, otemp, &cli->objects, head) {
+                usif_object_dtor(object);
+        }
+}
+
+void
+usif_client_init(struct nouveau_cli *cli)
+{
+        INIT_LIST_HEAD(&cli->objects);
+        INIT_LIST_HEAD(&cli->notifys);
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_usif.h b/drivers/gpu/drm/nouveau/nouveau_usif.h
new file mode 100644
index 000000000000..c037e3ae8c70
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_usif.h
@@ -0,0 +1,9 @@
+#ifndef __NOUVEAU_USIF_H__
+#define __NOUVEAU_USIF_H__
+
+void usif_client_init(struct nouveau_cli *);
+void usif_client_fini(struct nouveau_cli *);
+int  usif_ioctl(struct drm_file *, void __user *, u32);
+int  usif_notify(const void *, u32, const void *, u32);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_vga.c b/drivers/gpu/drm/nouveau/nouveau_vga.c
index 4f4c3fec6916..18d55d447248 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vga.c
+++ b/drivers/gpu/drm/nouveau/nouveau_vga.c
@@ -12,14 +12,16 @@
 static unsigned int
 nouveau_vga_set_decode(void *priv, bool state)
 {
-        struct nouveau_device *device = nouveau_dev(priv);
+        struct nvif_device *device = &nouveau_drm(priv)->device;
 
-        if (device->card_type == NV_40 && device->chipset >= 0x4c)
-                nv_wr32(device, 0x088060, state);
-        else if (device->chipset >= 0x40)
-                nv_wr32(device, 0x088054, state);
+        if (device->info.family == NV_DEVICE_INFO_V0_CURIE &&
+            device->info.chipset >= 0x4c)
+                nvif_wr32(device, 0x088060, state);
         else
-                nv_wr32(device, 0x001854, state);
+        if (device->info.chipset >= 0x40)
+                nvif_wr32(device, 0x088054, state);
+        else
+                nvif_wr32(device, 0x001854, state);
 
         if (state)
                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
diff --git a/drivers/gpu/drm/nouveau/nv04_fbcon.c b/drivers/gpu/drm/nouveau/nv04_fbcon.c
index 8fe32bbed99a..4ef602c5469d 100644
--- a/drivers/gpu/drm/nouveau/nv04_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv04_fbcon.c
@@ -22,8 +22,6 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-#include <core/object.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_fbcon.h"
@@ -141,8 +139,7 @@ nv04_fbcon_accel_init(struct fb_info *info)
         struct drm_device *dev = nfbdev->dev;
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_channel *chan = drm->channel;
-        struct nouveau_device *device = nv_device(drm->device);
-        struct nouveau_object *object;
+        struct nvif_device *device = &drm->device;
         int surface_fmt, pattern_fmt, rect_fmt;
         int ret;
 
@@ -174,35 +171,35 @@ nv04_fbcon_accel_init(struct fb_info *info)
                 return -EINVAL;
         }
 
-        ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvCtxSurf2D,
-                                 device->card_type >= NV_10 ? 0x0062 : 0x0042,
-                                 NULL, 0, &object);
+        ret = nvif_object_init(chan->object, NULL, 0x0062,
+                               device->info.family >= NV_DEVICE_INFO_V0_CELSIUS ?
+                               0x0062 : 0x0042, NULL, 0, &nfbdev->surf2d);
         if (ret)
                 return ret;
 
-        ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvClipRect,
-                                 0x0019, NULL, 0, &object);
+        ret = nvif_object_init(chan->object, NULL, 0x0019, 0x0019, NULL, 0,
+                               &nfbdev->clip);
         if (ret)
                 return ret;
 
-        ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvRop,
-                                 0x0043, NULL, 0, &object);
+        ret = nvif_object_init(chan->object, NULL, 0x0043, 0x0043, NULL, 0,
+                               &nfbdev->rop);
         if (ret)
                 return ret;
 
-        ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvImagePatt,
-                                 0x0044, NULL, 0, &object);
+        ret = nvif_object_init(chan->object, NULL, 0x0044, 0x0044, NULL, 0,
+                               &nfbdev->patt);
         if (ret)
                 return ret;
 
-        ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvGdiRect,
-                                 0x004a, NULL, 0, &object);
+        ret = nvif_object_init(chan->object, NULL, 0x004a, 0x004a, NULL, 0,
+                               &nfbdev->gdi);
         if (ret)
                 return ret;
 
-        ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvImageBlit,
-                                 device->chipset >= 0x11 ? 0x009f : 0x005f,
-                                 NULL, 0, &object);
+        ret = nvif_object_init(chan->object, NULL, 0x005f,
+                               device->info.chipset >= 0x11 ? 0x009f : 0x005f,
+                               NULL, 0, &nfbdev->blit);
         if (ret)
                 return ret;
 
@@ -212,10 +209,10 @@ nv04_fbcon_accel_init(struct fb_info *info)
         }
 
         BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0000, 1);
-        OUT_RING(chan, NvCtxSurf2D);
+        OUT_RING(chan, nfbdev->surf2d.handle);
         BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0184, 2);
-        OUT_RING(chan, NvDmaFB);
-        OUT_RING(chan, NvDmaFB);
+        OUT_RING(chan, chan->vram.handle);
+        OUT_RING(chan, chan->vram.handle);
         BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0300, 4);
         OUT_RING(chan, surface_fmt);
         OUT_RING(chan, info->fix.line_length | (info->fix.line_length << 16));
@@ -223,12 +220,12 @@ nv04_fbcon_accel_init(struct fb_info *info)
         OUT_RING(chan, info->fix.smem_start - dev->mode_config.fb_base);
 
         BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0000, 1);
-        OUT_RING(chan, NvRop);
+        OUT_RING(chan, nfbdev->rop.handle);
         BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0300, 1);
         OUT_RING(chan, 0x55);
 
         BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0000, 1);
-        OUT_RING(chan, NvImagePatt);
+        OUT_RING(chan, nfbdev->patt.handle);
         BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0300, 8);
         OUT_RING(chan, pattern_fmt);
 #ifdef __BIG_ENDIAN
@@ -244,18 +241,18 @@ nv04_fbcon_accel_init(struct fb_info *info)
         OUT_RING(chan, ~0);
 
         BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0000, 1);
-        OUT_RING(chan, NvClipRect);
+        OUT_RING(chan, nfbdev->clip.handle);
         BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0300, 2);
         OUT_RING(chan, 0);
         OUT_RING(chan, (info->var.yres_virtual << 16) | info->var.xres_virtual);
 
         BEGIN_NV04(chan, NvSubImageBlit, 0x0000, 1);
-        OUT_RING(chan, NvImageBlit);
+        OUT_RING(chan, nfbdev->blit.handle);
         BEGIN_NV04(chan, NvSubImageBlit, 0x019c, 1);
-        OUT_RING(chan, NvCtxSurf2D);
+        OUT_RING(chan, nfbdev->surf2d.handle);
         BEGIN_NV04(chan, NvSubImageBlit, 0x02fc, 1);
         OUT_RING(chan, 3);
-        if (device->chipset >= 0x11 /*XXX: oclass == 0x009f*/) {
+        if (device->info.chipset >= 0x11 /*XXX: oclass == 0x009f*/) {
                 BEGIN_NV04(chan, NvSubImageBlit, 0x0120, 3);
                 OUT_RING(chan, 0);
                 OUT_RING(chan, 1);
@@ -263,12 +260,12 @@ nv04_fbcon_accel_init(struct fb_info *info)
         }
 
         BEGIN_NV04(chan, NvSubGdiRect, 0x0000, 1);
-        OUT_RING(chan, NvGdiRect);
+        OUT_RING(chan, nfbdev->gdi.handle);
         BEGIN_NV04(chan, NvSubGdiRect, 0x0198, 1);
-        OUT_RING(chan, NvCtxSurf2D);
+        OUT_RING(chan, nfbdev->surf2d.handle);
         BEGIN_NV04(chan, NvSubGdiRect, 0x0188, 2);
-        OUT_RING(chan, NvImagePatt);
-        OUT_RING(chan, NvRop);
+        OUT_RING(chan, nfbdev->patt.handle);
+        OUT_RING(chan, nfbdev->rop.handle);
         BEGIN_NV04(chan, NvSubGdiRect, 0x0304, 1);
         OUT_RING(chan, 1);
         BEGIN_NV04(chan, NvSubGdiRect, 0x0300, 1);
diff --git a/drivers/gpu/drm/nouveau/nv04_fence.c b/drivers/gpu/drm/nouveau/nv04_fence.c
index 94eadd1dd10a..239c2c5a9615 100644
--- a/drivers/gpu/drm/nouveau/nv04_fence.c
+++ b/drivers/gpu/drm/nouveau/nv04_fence.c
@@ -22,8 +22,6 @@
  * Authors: Ben Skeggs
  */
 
-#include <engine/fifo.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_fence.h"
@@ -59,7 +57,7 @@ nv04_fence_sync(struct nouveau_fence *fence,
 static u32
 nv04_fence_read(struct nouveau_channel *chan)
 {
-        struct nouveau_fifo_chan *fifo = (void *)chan->object;
+        struct nouveau_fifo_chan *fifo = nvkm_fifo_chan(chan);;
         return atomic_read(&fifo->refcnt);
 }
 
diff --git a/drivers/gpu/drm/nouveau/nv10_fence.c b/drivers/gpu/drm/nouveau/nv10_fence.c
index 06f434f03fba..4faaf0acf5d7 100644
--- a/drivers/gpu/drm/nouveau/nv10_fence.c
+++ b/drivers/gpu/drm/nouveau/nv10_fence.c
@@ -22,9 +22,6 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include <core/object.h>
-#include <core/class.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nv10_fence.h"
@@ -53,14 +50,18 @@ nv10_fence_sync(struct nouveau_fence *fence,
 u32
 nv10_fence_read(struct nouveau_channel *chan)
 {
-        return nv_ro32(chan->object, 0x0048);
+        return nvif_rd32(chan, 0x0048);
 }
 
 void
 nv10_fence_context_del(struct nouveau_channel *chan)
 {
         struct nv10_fence_chan *fctx = chan->fence;
+        int i;
         nouveau_fence_context_del(&fctx->base);
+        for (i = 0; i < ARRAY_SIZE(fctx->head); i++)
+                nvif_object_fini(&fctx->head[i]);
+        nvif_object_fini(&fctx->sema);
         chan->fence = NULL;
         kfree(fctx);
 }
diff --git a/drivers/gpu/drm/nouveau/nv10_fence.h b/drivers/gpu/drm/nouveau/nv10_fence.h
index e5d9204826c2..a87259f3983a 100644
--- a/drivers/gpu/drm/nouveau/nv10_fence.h
+++ b/drivers/gpu/drm/nouveau/nv10_fence.h
@@ -1,12 +1,13 @@
 #ifndef __NV10_FENCE_H_
 #define __NV10_FENCE_H_
 
-#include <core/os.h>
 #include "nouveau_fence.h"
 #include "nouveau_bo.h"
 
 struct nv10_fence_chan {
         struct nouveau_fence_chan base;
+        struct nvif_object sema;
+        struct nvif_object head[4];
 };
 
 struct nv10_fence_priv {
diff --git a/drivers/gpu/drm/nouveau/nv17_fence.c b/drivers/gpu/drm/nouveau/nv17_fence.c
index 22aa9963ea6f..ca907479f92f 100644
--- a/drivers/gpu/drm/nouveau/nv17_fence.c
+++ b/drivers/gpu/drm/nouveau/nv17_fence.c
@@ -22,8 +22,8 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include <core/object.h>
-#include <core/class.h>
+#include <nvif/os.h>
+#include <nvif/class.h>
 
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
@@ -33,11 +33,13 @@ int
 nv17_fence_sync(struct nouveau_fence *fence,
                 struct nouveau_channel *prev, struct nouveau_channel *chan)
 {
+        struct nouveau_cli *cli = (void *)nvif_client(&prev->device->base);
         struct nv10_fence_priv *priv = chan->drm->fence;
+        struct nv10_fence_chan *fctx = chan->fence;
         u32 value;
         int ret;
 
-        if (!mutex_trylock(&prev->cli->mutex))
+        if (!mutex_trylock(&cli->mutex))
                 return -EBUSY;
 
         spin_lock(&priv->lock);
@@ -48,7 +50,7 @@ nv17_fence_sync(struct nouveau_fence *fence,
         ret = RING_SPACE(prev, 5);
         if (!ret) {
                 BEGIN_NV04(prev, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 4);
-                OUT_RING  (prev, NvSema);
+                OUT_RING  (prev, fctx->sema.handle);
                 OUT_RING  (prev, 0);
                 OUT_RING  (prev, value + 0);
                 OUT_RING  (prev, value + 1);
@@ -57,14 +59,14 @@ nv17_fence_sync(struct nouveau_fence *fence,
 
         if (!ret && !(ret = RING_SPACE(chan, 5))) {
                 BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 4);
-                OUT_RING  (chan, NvSema);
+                OUT_RING  (chan, fctx->sema.handle);
                 OUT_RING  (chan, 0);
                 OUT_RING  (chan, value + 1);
                 OUT_RING  (chan, value + 2);
                 FIRE_RING (chan);
         }
 
-        mutex_unlock(&prev->cli->mutex);
+        mutex_unlock(&cli->mutex);
         return 0;
 }
 
@@ -74,7 +76,6 @@ nv17_fence_context_new(struct nouveau_channel *chan)
         struct nv10_fence_priv *priv = chan->drm->fence;
         struct nv10_fence_chan *fctx;
         struct ttm_mem_reg *mem = &priv->bo->bo.mem;
-        struct nouveau_object *object;
         u32 start = mem->start * PAGE_SIZE;
         u32 limit = start + mem->size - 1;
         int ret = 0;
@@ -88,15 +89,14 @@ nv17_fence_context_new(struct nouveau_channel *chan)
         fctx->base.read = nv10_fence_read;
         fctx->base.sync = nv17_fence_sync;
 
-        ret = nouveau_object_new(nv_object(chan->cli), chan->handle,
-                                 NvSema, 0x0002,
-                                 &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
+        ret = nvif_object_init(chan->object, NULL, NvSema, NV_DMA_FROM_MEMORY,
+                               &(struct nv_dma_v0) {
+                                        .target = NV_DMA_V0_TARGET_VRAM,
+                                        .access = NV_DMA_V0_ACCESS_RDWR,
                                         .start = start,
                                         .limit = limit,
-                                 }, sizeof(struct nv_dma_class),
-                                 &object);
+                               }, sizeof(struct nv_dma_v0),
+                               &fctx->sema);
         if (ret)
                 nv10_fence_context_del(chan);
         return ret;
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index 4c534b7b04da..03949eaa629f 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -28,6 +28,8 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_dp_helper.h>
 
+#include <nvif/class.h>
+
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_gem.h"
@@ -37,15 +39,6 @@
 #include "nouveau_fence.h"
 #include "nv50_display.h"
 
-#include <core/client.h>
-#include <core/gpuobj.h>
-#include <core/class.h>
-
-#include <subdev/timer.h>
-#include <subdev/bar.h>
-#include <subdev/fb.h>
-#include <subdev/i2c.h>
-
 #define EVO_DMA_NR 9
 
 #define EVO_MASTER  (0x00)
@@ -60,45 +53,34 @@
 #define EVO_FLIP_SEM0(c)  EVO_SYNC((c) + 1, 0x00)
 #define EVO_FLIP_SEM1(c)  EVO_SYNC((c) + 1, 0x10)
 
-#define EVO_CORE_HANDLE      (0xd1500000)
-#define EVO_CHAN_HANDLE(t,i) (0xd15c0000 | (((t) & 0x00ff) << 8) | (i))
-#define EVO_CHAN_OCLASS(t,c) ((nv_hclass(c) & 0xff00) | ((t) & 0x00ff))
-#define EVO_PUSH_HANDLE(t,i) (0xd15b0000 | (i) |                               \
-                              (((NV50_DISP_##t##_CLASS) & 0x00ff) << 8))
-
 /******************************************************************************
  * EVO channel
  *****************************************************************************/
 
 struct nv50_chan {
-        struct nouveau_object *user;
-        u32 handle;
+        struct nvif_object user;
 };
 
 static int
-nv50_chan_create(struct nouveau_object *core, u32 bclass, u8 head,
+nv50_chan_create(struct nvif_object *disp, const u32 *oclass, u8 head,
                  void *data, u32 size, struct nv50_chan *chan)
 {
-        struct nouveau_object *client = nv_pclass(core, NV_CLIENT_CLASS);
-        const u32 oclass = EVO_CHAN_OCLASS(bclass, core);
-        const u32 handle = EVO_CHAN_HANDLE(bclass, head);
-        int ret;
-
-        ret = nouveau_object_new(client, EVO_CORE_HANDLE, handle,
-                                 oclass, data, size, &chan->user);
-        if (ret)
-                return ret;
-
-        chan->handle = handle;
-        return 0;
+        while (oclass[0]) {
+                int ret = nvif_object_init(disp, NULL, (oclass[0] << 16) | head,
+                                           oclass[0], data, size,
+                                          &chan->user);
+                if (oclass++, ret == 0) {
+                        nvif_object_map(&chan->user);
+                        return ret;
+                }
+        }
+        return -ENOSYS;
 }
 
 static void
-nv50_chan_destroy(struct nouveau_object *core, struct nv50_chan *chan)
+nv50_chan_destroy(struct nv50_chan *chan)
 {
-        struct nouveau_object *client = nv_pclass(core, NV_CLIENT_CLASS);
-        if (chan->handle)
-                nouveau_object_del(client, EVO_CORE_HANDLE, chan->handle);
+        nvif_object_fini(&chan->user);
 }
 
 /******************************************************************************
@@ -110,16 +92,70 @@ struct nv50_pioc {
 };
 
 static void
-nv50_pioc_destroy(struct nouveau_object *core, struct nv50_pioc *pioc)
+nv50_pioc_destroy(struct nv50_pioc *pioc)
 {
-        nv50_chan_destroy(core, &pioc->base);
+        nv50_chan_destroy(&pioc->base);
 }
 
 static int
-nv50_pioc_create(struct nouveau_object *core, u32 bclass, u8 head,
+nv50_pioc_create(struct nvif_object *disp, const u32 *oclass, u8 head,
                  void *data, u32 size, struct nv50_pioc *pioc)
 {
-        return nv50_chan_create(core, bclass, head, data, size, &pioc->base);
+        return nv50_chan_create(disp, oclass, head, data, size, &pioc->base);
+}
+
+/******************************************************************************
+ * Cursor Immediate
+ *****************************************************************************/
+
+struct nv50_curs {
+        struct nv50_pioc base;
+};
+
+static int
+nv50_curs_create(struct nvif_object *disp, int head, struct nv50_curs *curs)
+{
+        struct nv50_disp_cursor_v0 args = {
+                .head = head,
+        };
+        static const u32 oclass[] = {
+                GK104_DISP_CURSOR,
+                GF110_DISP_CURSOR,
+                GT214_DISP_CURSOR,
+                G82_DISP_CURSOR,
+                NV50_DISP_CURSOR,
+                0
+        };
+
+        return nv50_pioc_create(disp, oclass, head, &args, sizeof(args),
+                               &curs->base);
+}
+
+/******************************************************************************
+ * Overlay Immediate
+ *****************************************************************************/
+
+struct nv50_oimm {
+        struct nv50_pioc base;
+};
+
+static int
+nv50_oimm_create(struct nvif_object *disp, int head, struct nv50_oimm *oimm)
+{
+        struct nv50_disp_cursor_v0 args = {
+                .head = head,
+        };
+        static const u32 oclass[] = {
+                GK104_DISP_OVERLAY,
+                GF110_DISP_OVERLAY,
+                GT214_DISP_OVERLAY,
+                G82_DISP_OVERLAY,
+                NV50_DISP_OVERLAY,
+                0
+        };
+
+        return nv50_pioc_create(disp, oclass, head, &args, sizeof(args),
+                               &oimm->base);
 }
 
 /******************************************************************************
@@ -131,6 +167,9 @@ struct nv50_dmac {
         dma_addr_t handle;
         u32 *ptr;
 
+        struct nvif_object sync;
+        struct nvif_object vram;
+
         /* Protects against concurrent pushbuf access to this channel, lock is
          * grabbed by evo_wait (if the pushbuf reservation is successful) and
          * dropped again by evo_kick. */
@@ -138,207 +177,113 @@ struct nv50_dmac {
 };
 
 static void
-nv50_dmac_destroy(struct nouveau_object *core, struct nv50_dmac *dmac)
+nv50_dmac_destroy(struct nv50_dmac *dmac, struct nvif_object *disp)
 {
+        nvif_object_fini(&dmac->vram);
+        nvif_object_fini(&dmac->sync);
+
+        nv50_chan_destroy(&dmac->base);
+
         if (dmac->ptr) {
-                struct pci_dev *pdev = nv_device(core)->pdev;
+                struct pci_dev *pdev = nvkm_device(nvif_device(disp))->pdev;
                 pci_free_consistent(pdev, PAGE_SIZE, dmac->ptr, dmac->handle);
         }
-
-        nv50_chan_destroy(core, &dmac->base);
-}
-
-static int
-nv50_dmac_create_fbdma(struct nouveau_object *core, u32 parent)
-{
-        struct nouveau_fb *pfb = nouveau_fb(core);
-        struct nouveau_object *client = nv_pclass(core, NV_CLIENT_CLASS);
-        struct nouveau_object *object;
-        int ret = nouveau_object_new(client, parent, NvEvoVRAM_LP,
-                                     NV_DMA_IN_MEMORY_CLASS,
-                                     &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
-                                        .start = 0,
-                                        .limit = pfb->ram->size - 1,
-                                        .conf0 = NV50_DMA_CONF0_ENABLE |
-                                                 NV50_DMA_CONF0_PART_256,
-                                     }, sizeof(struct nv_dma_class), &object);
-        if (ret)
-                return ret;
-
-        ret = nouveau_object_new(client, parent, NvEvoFB16,
-                                 NV_DMA_IN_MEMORY_CLASS,
-                                 &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
-                                        .start = 0,
-                                        .limit = pfb->ram->size - 1,
-                                        .conf0 = NV50_DMA_CONF0_ENABLE | 0x70 |
-                                                 NV50_DMA_CONF0_PART_256,
-                                 }, sizeof(struct nv_dma_class), &object);
-        if (ret)
-                return ret;
-
-        ret = nouveau_object_new(client, parent, NvEvoFB32,
-                                 NV_DMA_IN_MEMORY_CLASS,
-                                 &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
-                                        .start = 0,
-                                        .limit = pfb->ram->size - 1,
-                                        .conf0 = NV50_DMA_CONF0_ENABLE | 0x7a |
-                                                 NV50_DMA_CONF0_PART_256,
-                                 }, sizeof(struct nv_dma_class), &object);
-        return ret;
-}
-
-static int
-nvc0_dmac_create_fbdma(struct nouveau_object *core, u32 parent)
-{
-        struct nouveau_fb *pfb = nouveau_fb(core);
-        struct nouveau_object *client = nv_pclass(core, NV_CLIENT_CLASS);
-        struct nouveau_object *object;
-        int ret = nouveau_object_new(client, parent, NvEvoVRAM_LP,
-                                     NV_DMA_IN_MEMORY_CLASS,
-                                     &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
-                                        .start = 0,
-                                        .limit = pfb->ram->size - 1,
-                                        .conf0 = NVC0_DMA_CONF0_ENABLE,
-                                     }, sizeof(struct nv_dma_class), &object);
-        if (ret)
-                return ret;
-
-        ret = nouveau_object_new(client, parent, NvEvoFB16,
-                                 NV_DMA_IN_MEMORY_CLASS,
-                                 &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
-                                        .start = 0,
-                                        .limit = pfb->ram->size - 1,
-                                        .conf0 = NVC0_DMA_CONF0_ENABLE | 0xfe,
-                                 }, sizeof(struct nv_dma_class), &object);
-        if (ret)
-                return ret;
-
-        ret = nouveau_object_new(client, parent, NvEvoFB32,
-                                 NV_DMA_IN_MEMORY_CLASS,
-                                 &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
-                                        .start = 0,
-                                        .limit = pfb->ram->size - 1,
-                                        .conf0 = NVC0_DMA_CONF0_ENABLE | 0xfe,
-                                 }, sizeof(struct nv_dma_class), &object);
-        return ret;
-}
-
-static int
-nvd0_dmac_create_fbdma(struct nouveau_object *core, u32 parent)
-{
-        struct nouveau_fb *pfb = nouveau_fb(core);
-        struct nouveau_object *client = nv_pclass(core, NV_CLIENT_CLASS);
-        struct nouveau_object *object;
-        int ret = nouveau_object_new(client, parent, NvEvoVRAM_LP,
-                                     NV_DMA_IN_MEMORY_CLASS,
-                                     &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
-                                        .start = 0,
-                                        .limit = pfb->ram->size - 1,
-                                        .conf0 = NVD0_DMA_CONF0_ENABLE |
-                                                 NVD0_DMA_CONF0_PAGE_LP,
-                                     }, sizeof(struct nv_dma_class), &object);
-        if (ret)
-                return ret;
-
-        ret = nouveau_object_new(client, parent, NvEvoFB32,
-                                 NV_DMA_IN_MEMORY_CLASS,
-                                 &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
-                                        .start = 0,
-                                        .limit = pfb->ram->size - 1,
-                                        .conf0 = NVD0_DMA_CONF0_ENABLE | 0xfe |
-                                                 NVD0_DMA_CONF0_PAGE_LP,
-                                 }, sizeof(struct nv_dma_class), &object);
-        return ret;
 }
 
 static int
-nv50_dmac_create(struct nouveau_object *core, u32 bclass, u8 head,
+nv50_dmac_create(struct nvif_object *disp, const u32 *oclass, u8 head,
                  void *data, u32 size, u64 syncbuf,
                  struct nv50_dmac *dmac)
 {
-        struct nouveau_fb *pfb = nouveau_fb(core);
-        struct nouveau_object *client = nv_pclass(core, NV_CLIENT_CLASS);
-        struct nouveau_object *object;
-        u32 pushbuf = *(u32 *)data;
+        struct nvif_device *device = nvif_device(disp);
+        struct nv50_disp_core_channel_dma_v0 *args = data;
+        struct nvif_object pushbuf;
         int ret;
 
         mutex_init(&dmac->lock);
 
-        dmac->ptr = pci_alloc_consistent(nv_device(core)->pdev, PAGE_SIZE,
-                                        &dmac->handle);
+        dmac->ptr = pci_alloc_consistent(nvkm_device(device)->pdev,
+                                         PAGE_SIZE, &dmac->handle);
         if (!dmac->ptr)
                 return -ENOMEM;
 
-        ret = nouveau_object_new(client, NVDRM_DEVICE, pushbuf,
-                                 NV_DMA_FROM_MEMORY_CLASS,
-                                 &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_PCI_US |
-                                                 NV_DMA_ACCESS_RD,
+        ret = nvif_object_init(nvif_object(device), NULL,
+                               args->pushbuf, NV_DMA_FROM_MEMORY,
+                               &(struct nv_dma_v0) {
+                                        .target = NV_DMA_V0_TARGET_PCI_US,
+                                        .access = NV_DMA_V0_ACCESS_RD,
                                         .start = dmac->handle + 0x0000,
                                         .limit = dmac->handle + 0x0fff,
-                                 }, sizeof(struct nv_dma_class), &object);
+                               }, sizeof(struct nv_dma_v0), &pushbuf);
         if (ret)
                 return ret;
 
-        ret = nv50_chan_create(core, bclass, head, data, size, &dmac->base);
+        ret = nv50_chan_create(disp, oclass, head, data, size, &dmac->base);
+        nvif_object_fini(&pushbuf);
         if (ret)
                 return ret;
 
-        ret = nouveau_object_new(client, dmac->base.handle, NvEvoSync,
-                                 NV_DMA_IN_MEMORY_CLASS,
-                                 &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
+        ret = nvif_object_init(&dmac->base.user, NULL, 0xf0000000,
+                               NV_DMA_IN_MEMORY,
+                               &(struct nv_dma_v0) {
+                                        .target = NV_DMA_V0_TARGET_VRAM,
+                                        .access = NV_DMA_V0_ACCESS_RDWR,
                                         .start = syncbuf + 0x0000,
                                         .limit = syncbuf + 0x0fff,
-                                 }, sizeof(struct nv_dma_class), &object);
+                               }, sizeof(struct nv_dma_v0),
+                               &dmac->sync);
         if (ret)
                 return ret;
 
-        ret = nouveau_object_new(client, dmac->base.handle, NvEvoVRAM,
-                                 NV_DMA_IN_MEMORY_CLASS,
-                                 &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
+        ret = nvif_object_init(&dmac->base.user, NULL, 0xf0000001,
+                               NV_DMA_IN_MEMORY,
+                               &(struct nv_dma_v0) {
+                                        .target = NV_DMA_V0_TARGET_VRAM,
+                                        .access = NV_DMA_V0_ACCESS_RDWR,
                                         .start = 0,
-                                        .limit = pfb->ram->size - 1,
-                                 }, sizeof(struct nv_dma_class), &object);
+                                        .limit = device->info.ram_user - 1,
+                               }, sizeof(struct nv_dma_v0),
+                               &dmac->vram);
         if (ret)
                 return ret;
 
-        if (nv_device(core)->card_type < NV_C0)
-                ret = nv50_dmac_create_fbdma(core, dmac->base.handle);
-        else
-        if (nv_device(core)->card_type < NV_D0)
-                ret = nvc0_dmac_create_fbdma(core, dmac->base.handle);
-        else
-                ret = nvd0_dmac_create_fbdma(core, dmac->base.handle);
         return ret;
 }
 
+/******************************************************************************
+ * Core
+ *****************************************************************************/
+
 struct nv50_mast {
         struct nv50_dmac base;
 };
 
-struct nv50_curs {
-        struct nv50_pioc base;
-};
+static int
+nv50_core_create(struct nvif_object *disp, u64 syncbuf, struct nv50_mast *core)
+{
+        struct nv50_disp_core_channel_dma_v0 args = {
+                .pushbuf = 0xb0007d00,
+        };
+        static const u32 oclass[] = {
+                GM107_DISP_CORE_CHANNEL_DMA,
+                GK110_DISP_CORE_CHANNEL_DMA,
+                GK104_DISP_CORE_CHANNEL_DMA,
+                GF110_DISP_CORE_CHANNEL_DMA,
+                GT214_DISP_CORE_CHANNEL_DMA,
+                GT206_DISP_CORE_CHANNEL_DMA,
+                GT200_DISP_CORE_CHANNEL_DMA,
+                G82_DISP_CORE_CHANNEL_DMA,
+                NV50_DISP_CORE_CHANNEL_DMA,
+                0
+        };
+
+        return nv50_dmac_create(disp, oclass, 0, &args, sizeof(args), syncbuf,
+                               &core->base);
+}
+
+/******************************************************************************
+ * Base
+ *****************************************************************************/
 
 struct nv50_sync {
         struct nv50_dmac base;
@@ -346,13 +291,58 @@ struct nv50_sync {
         u32 data;
 };
 
+static int
+nv50_base_create(struct nvif_object *disp, int head, u64 syncbuf,
+                 struct nv50_sync *base)
+{
+        struct nv50_disp_base_channel_dma_v0 args = {
+                .pushbuf = 0xb0007c00 | head,
+                .head = head,
+        };
+        static const u32 oclass[] = {
+                GK110_DISP_BASE_CHANNEL_DMA,
+                GK104_DISP_BASE_CHANNEL_DMA,
+                GF110_DISP_BASE_CHANNEL_DMA,
+                GT214_DISP_BASE_CHANNEL_DMA,
+                GT200_DISP_BASE_CHANNEL_DMA,
+                G82_DISP_BASE_CHANNEL_DMA,
+                NV50_DISP_BASE_CHANNEL_DMA,
+                0
+        };
+
+        return nv50_dmac_create(disp, oclass, head, &args, sizeof(args),
+                                syncbuf, &base->base);
+}
+
+/******************************************************************************
+ * Overlay
+ *****************************************************************************/
+
 struct nv50_ovly {
         struct nv50_dmac base;
 };
 
-struct nv50_oimm {
-        struct nv50_pioc base;
-};
+static int
+nv50_ovly_create(struct nvif_object *disp, int head, u64 syncbuf,
+                 struct nv50_ovly *ovly)
+{
+        struct nv50_disp_overlay_channel_dma_v0 args = {
+                .pushbuf = 0xb0007e00 | head,
+                .head = head,
+        };
+        static const u32 oclass[] = {
+                GK104_DISP_OVERLAY_CONTROL_DMA,
+                GF110_DISP_OVERLAY_CONTROL_DMA,
+                GT214_DISP_OVERLAY_CHANNEL_DMA,
+                GT200_DISP_OVERLAY_CHANNEL_DMA,
+                G82_DISP_OVERLAY_CHANNEL_DMA,
+                NV50_DISP_OVERLAY_CHANNEL_DMA,
+                0
+        };
+
+        return nv50_dmac_create(disp, oclass, head, &args, sizeof(args),
+                                syncbuf, &ovly->base);
+}
 
 struct nv50_head {
         struct nouveau_crtc base;
@@ -369,13 +359,19 @@ struct nv50_head {
 #define nv50_ovly(c) (&nv50_head(c)->ovly)
 #define nv50_oimm(c) (&nv50_head(c)->oimm)
 #define nv50_chan(c) (&(c)->base.base)
-#define nv50_vers(c) nv_mclass(nv50_chan(c)->user)
+#define nv50_vers(c) nv50_chan(c)->user.oclass
+
+struct nv50_fbdma {
+        struct list_head head;
+        struct nvif_object core;
+        struct nvif_object base[4];
+};
 
 struct nv50_disp {
-        struct nouveau_object *core;
+        struct nvif_object *disp;
         struct nv50_mast mast;
 
-        u32 modeset;
+        struct list_head fbdma;
 
         struct nouveau_bo *sync;
 };
@@ -401,16 +397,16 @@ static u32 *
 evo_wait(void *evoc, int nr)
 {
         struct nv50_dmac *dmac = evoc;
-        u32 put = nv_ro32(dmac->base.user, 0x0000) / 4;
+        u32 put = nvif_rd32(&dmac->base.user, 0x0000) / 4;
 
         mutex_lock(&dmac->lock);
         if (put + nr >= (PAGE_SIZE / 4) - 8) {
                 dmac->ptr[put] = 0x20000000;
 
-                nv_wo32(dmac->base.user, 0x0000, 0x00000000);
-                if (!nv_wait(dmac->base.user, 0x0004, ~0, 0x00000000)) {
+                nvif_wr32(&dmac->base.user, 0x0000, 0x00000000);
+                if (!nvkm_wait(&dmac->base.user, 0x0004, ~0, 0x00000000)) {
                         mutex_unlock(&dmac->lock);
-                        NV_ERROR(dmac->base.user, "channel stalled\n");
+                        nv_error(nvkm_object(&dmac->base.user), "channel stalled\n");
                         return NULL;
                 }
 
@@ -424,7 +420,7 @@ static void
 evo_kick(u32 *push, void *evoc)
 {
         struct nv50_dmac *dmac = evoc;
-        nv_wo32(dmac->base.user, 0x0000, (push - dmac->ptr) << 2);
+        nvif_wr32(&dmac->base.user, 0x0000, (push - dmac->ptr) << 2);
         mutex_unlock(&dmac->lock);
 }
 
@@ -443,7 +439,7 @@ evo_sync_wait(void *data)
 static int
 evo_sync(struct drm_device *dev)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         struct nv50_disp *disp = nv50_disp(dev);
         struct nv50_mast *mast = nv50_mast(dev);
         u32 *push = evo_wait(mast, 8);
@@ -455,7 +451,7 @@ evo_sync(struct drm_device *dev)
                 evo_data(push, 0x00000000);
                 evo_data(push, 0x00000000);
                 evo_kick(push, mast);
-                if (nv_wait_cb(device, evo_sync_wait, disp->sync))
+                if (nv_wait_cb(nvkm_device(device), evo_sync_wait, disp->sync))
                         return 0;
         }
 
@@ -490,7 +486,7 @@ nv50_display_flip_wait(void *data)
 void
 nv50_display_flip_stop(struct drm_crtc *crtc)
 {
-        struct nouveau_device *device = nouveau_dev(crtc->dev);
+        struct nvif_device *device = &nouveau_drm(crtc->dev)->device;
         struct nv50_display_flip flip = {
                 .disp = nv50_disp(crtc->dev),
                 .chan = nv50_sync(crtc),
@@ -510,7 +506,7 @@ nv50_display_flip_stop(struct drm_crtc *crtc)
                 evo_kick(push, flip.chan);
         }
 
-        nv_wait_cb(device, nv50_display_flip_wait, &flip);
+        nv_wait_cb(nvkm_device(device), nv50_display_flip_wait, &flip);
 }
 
 int
@@ -534,7 +530,7 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         if (unlikely(push == NULL))
                 return -EBUSY;
 
-        if (chan && nv_mclass(chan->object) < NV84_CHANNEL_IND_CLASS) {
+        if (chan && chan->object->oclass < G82_CHANNEL_GPFIFO) {
                 ret = RING_SPACE(chan, 8);
                 if (ret)
                         return ret;
@@ -548,14 +544,14 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                 OUT_RING  (chan, sync->addr);
                 OUT_RING  (chan, sync->data);
         } else
-        if (chan && nv_mclass(chan->object) < NVC0_CHANNEL_IND_CLASS) {
+        if (chan && chan->object->oclass < FERMI_CHANNEL_GPFIFO) {
                 u64 addr = nv84_fence_crtc(chan, nv_crtc->index) + sync->addr;
                 ret = RING_SPACE(chan, 12);
                 if (ret)
                         return ret;
 
                 BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
-                OUT_RING  (chan, chan->vram);
+                OUT_RING  (chan, chan->vram.handle);
                 BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
                 OUT_RING  (chan, upper_32_bits(addr ^ 0x10));
                 OUT_RING  (chan, lower_32_bits(addr ^ 0x10));
@@ -606,16 +602,16 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         evo_data(push, sync->addr);
         evo_data(push, sync->data++);
         evo_data(push, sync->data);
-        evo_data(push, NvEvoSync);
+        evo_data(push, sync->base.sync.handle);
         evo_mthd(push, 0x00a0, 2);
         evo_data(push, 0x00000000);
         evo_data(push, 0x00000000);
         evo_mthd(push, 0x00c0, 1);
-        evo_data(push, nv_fb->r_dma);
+        evo_data(push, nv_fb->r_handle);
         evo_mthd(push, 0x0110, 2);
         evo_data(push, 0x00000000);
         evo_data(push, 0x00000000);
-        if (nv50_vers(sync) < NVD0_DISP_SYNC_CLASS) {
+        if (nv50_vers(sync) < GF110_DISP_BASE_CHANNEL_DMA) {
                 evo_mthd(push, 0x0800, 5);
                 evo_data(push, nv_fb->nvbo->bo.offset >> 8);
                 evo_data(push, 0);
@@ -667,11 +663,11 @@ nv50_crtc_set_dither(struct nouveau_crtc *nv_crtc, bool update)
 
         push = evo_wait(mast, 4);
         if (push) {
-                if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x08a0 + (nv_crtc->index * 0x0400), 1);
                         evo_data(push, mode);
                 } else
-                if (nv50_vers(mast) < NVE0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GK104_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0490 + (nv_crtc->index * 0x0300), 1);
                         evo_data(push, mode);
                 } else {
@@ -762,7 +758,7 @@ nv50_crtc_set_scale(struct nouveau_crtc *nv_crtc, bool update)
 
         push = evo_wait(mast, 8);
         if (push) {
-                if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                         /*XXX: SCALE_CTRL_ACTIVE??? */
                         evo_mthd(push, 0x08d8 + (nv_crtc->index * 0x400), 2);
                         evo_data(push, (oY << 16) | oX);
@@ -807,7 +803,7 @@ nv50_crtc_set_color_vibrance(struct nouveau_crtc *nv_crtc, bool update)
 
         push = evo_wait(mast, 16);
         if (push) {
-                if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x08a8 + (nv_crtc->index * 0x400), 1);
                         evo_data(push, (hue << 20) | (vib << 8));
                 } else {
@@ -835,7 +831,7 @@ nv50_crtc_set_image(struct nouveau_crtc *nv_crtc, struct drm_framebuffer *fb,
 
         push = evo_wait(mast, 16);
         if (push) {
-                if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0860 + (nv_crtc->index * 0x400), 1);
                         evo_data(push, nvfb->nvbo->bo.offset >> 8);
                         evo_mthd(push, 0x0868 + (nv_crtc->index * 0x400), 3);
@@ -844,9 +840,9 @@ nv50_crtc_set_image(struct nouveau_crtc *nv_crtc, struct drm_framebuffer *fb,
                         evo_data(push, nvfb->r_format);
                         evo_mthd(push, 0x08c0 + (nv_crtc->index * 0x400), 1);
                         evo_data(push, (y << 16) | x);
-                        if (nv50_vers(mast) > NV50_DISP_MAST_CLASS) {
+                        if (nv50_vers(mast) > NV50_DISP_CORE_CHANNEL_DMA) {
                                 evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1);
-                                evo_data(push, nvfb->r_dma);
+                                evo_data(push, nvfb->r_handle);
                         }
                 } else {
                         evo_mthd(push, 0x0460 + (nv_crtc->index * 0x300), 1);
@@ -855,7 +851,7 @@ nv50_crtc_set_image(struct nouveau_crtc *nv_crtc, struct drm_framebuffer *fb,
                         evo_data(push, (fb->height << 16) | fb->width);
                         evo_data(push, nvfb->r_pitch);
                         evo_data(push, nvfb->r_format);
-                        evo_data(push, nvfb->r_dma);
+                        evo_data(push, nvfb->r_handle);
                         evo_mthd(push, 0x04b0 + (nv_crtc->index * 0x300), 1);
                         evo_data(push, (y << 16) | x);
                 }
@@ -867,7 +863,7 @@ nv50_crtc_set_image(struct nouveau_crtc *nv_crtc, struct drm_framebuffer *fb,
                 evo_kick(push, mast);
         }
 
-        nv_crtc->fb.tile_flags = nvfb->r_dma;
+        nv_crtc->fb.handle = nvfb->r_handle;
         return 0;
 }
 
@@ -877,23 +873,23 @@ nv50_crtc_cursor_show(struct nouveau_crtc *nv_crtc)
         struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
         u32 *push = evo_wait(mast, 16);
         if (push) {
-                if (nv50_vers(mast) < NV84_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 2);
                         evo_data(push, 0x85000000);
                         evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8);
                 } else
-                if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 2);
                         evo_data(push, 0x85000000);
                         evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8);
                         evo_mthd(push, 0x089c + (nv_crtc->index * 0x400), 1);
-                        evo_data(push, NvEvoVRAM);
+                        evo_data(push, mast->base.vram.handle);
                 } else {
                         evo_mthd(push, 0x0480 + (nv_crtc->index * 0x300), 2);
                         evo_data(push, 0x85000000);
                         evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8);
                         evo_mthd(push, 0x048c + (nv_crtc->index * 0x300), 1);
-                        evo_data(push, NvEvoVRAM);
+                        evo_data(push, mast->base.vram.handle);
                 }
                 evo_kick(push, mast);
         }
@@ -905,11 +901,11 @@ nv50_crtc_cursor_hide(struct nouveau_crtc *nv_crtc)
         struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
         u32 *push = evo_wait(mast, 16);
         if (push) {
-                if (nv50_vers(mast) < NV84_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 1);
                         evo_data(push, 0x05000000);
                 } else
-                if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 1);
                         evo_data(push, 0x05000000);
                         evo_mthd(push, 0x089c + (nv_crtc->index * 0x400), 1);
@@ -960,13 +956,13 @@ nv50_crtc_prepare(struct drm_crtc *crtc)
 
         push = evo_wait(mast, 6);
         if (push) {
-                if (nv50_vers(mast) < NV84_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1);
                         evo_data(push, 0x00000000);
                         evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 1);
                         evo_data(push, 0x40000000);
                 } else
-                if (nv50_vers(mast) <  NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) <  GF110_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1);
                         evo_data(push, 0x00000000);
                         evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 1);
@@ -997,31 +993,31 @@ nv50_crtc_commit(struct drm_crtc *crtc)
 
         push = evo_wait(mast, 32);
         if (push) {
-                if (nv50_vers(mast) < NV84_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1);
-                        evo_data(push, NvEvoVRAM_LP);
+                        evo_data(push, nv_crtc->fb.handle);
                         evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 2);
                         evo_data(push, 0xc0000000);
                         evo_data(push, nv_crtc->lut.nvbo->bo.offset >> 8);
                 } else
-                if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1);
-                        evo_data(push, nv_crtc->fb.tile_flags);
+                        evo_data(push, nv_crtc->fb.handle);
                         evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 2);
                         evo_data(push, 0xc0000000);
                         evo_data(push, nv_crtc->lut.nvbo->bo.offset >> 8);
                         evo_mthd(push, 0x085c + (nv_crtc->index * 0x400), 1);
-                        evo_data(push, NvEvoVRAM);
+                        evo_data(push, mast->base.vram.handle);
                 } else {
                         evo_mthd(push, 0x0474 + (nv_crtc->index * 0x300), 1);
-                        evo_data(push, nv_crtc->fb.tile_flags);
+                        evo_data(push, nv_crtc->fb.handle);
                         evo_mthd(push, 0x0440 + (nv_crtc->index * 0x300), 4);
                         evo_data(push, 0x83000000);
                         evo_data(push, nv_crtc->lut.nvbo->bo.offset >> 8);
                         evo_data(push, 0x00000000);
                         evo_data(push, 0x00000000);
                         evo_mthd(push, 0x045c + (nv_crtc->index * 0x300), 1);
-                        evo_data(push, NvEvoVRAM);
+                        evo_data(push, mast->base.vram.handle);
                         evo_mthd(push, 0x0430 + (nv_crtc->index * 0x300), 1);
                         evo_data(push, 0xffffff00);
                 }
@@ -1099,7 +1095,7 @@ nv50_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *umode,
 
         push = evo_wait(mast, 64);
         if (push) {
-                if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0804 + (nv_crtc->index * 0x400), 2);
                         evo_data(push, 0x00800000 | mode->clock);
                         evo_data(push, (ilace == 2) ? 2 : 0);
@@ -1192,7 +1188,7 @@ nv50_crtc_lut_load(struct drm_crtc *crtc)
                 u16 g = nv_crtc->lut.g[i] >> 2;
                 u16 b = nv_crtc->lut.b[i] >> 2;
 
-                if (nv_mclass(disp->core) < NVD0_DISP_CLASS) {
+                if (disp->disp->oclass < GF110_DISP) {
                         writew(r + 0x0000, lut + (i * 0x08) + 0);
                         writew(g + 0x0000, lut + (i * 0x08) + 2);
                         writew(b + 0x0000, lut + (i * 0x08) + 4);
@@ -1259,8 +1255,8 @@ nv50_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
 {
         struct nv50_curs *curs = nv50_curs(crtc);
         struct nv50_chan *chan = nv50_chan(curs);
-        nv_wo32(chan->user, 0x0084, (y << 16) | (x & 0xffff));
-        nv_wo32(chan->user, 0x0080, 0x00000000);
+        nvif_wr32(&chan->user, 0x0084, (y << 16) | (x & 0xffff));
+        nvif_wr32(&chan->user, 0x0080, 0x00000000);
         return 0;
 }
 
@@ -1287,11 +1283,16 @@ nv50_crtc_destroy(struct drm_crtc *crtc)
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
         struct nv50_disp *disp = nv50_disp(crtc->dev);
         struct nv50_head *head = nv50_head(crtc);
+        struct nv50_fbdma *fbdma;
+
+        list_for_each_entry(fbdma, &disp->fbdma, head) {
+                nvif_object_fini(&fbdma->base[nv_crtc->index]);
+        }
 
-        nv50_dmac_destroy(disp->core, &head->ovly.base);
-        nv50_pioc_destroy(disp->core, &head->oimm.base);
-        nv50_dmac_destroy(disp->core, &head->sync.base);
-        nv50_pioc_destroy(disp->core, &head->curs.base);
+        nv50_dmac_destroy(&head->ovly.base, disp->disp);
+        nv50_pioc_destroy(&head->oimm.base);
+        nv50_dmac_destroy(&head->sync.base, disp->disp);
+        nv50_pioc_destroy(&head->curs.base);
 
         /*XXX: this shouldn't be necessary, but the core doesn't call
          *     disconnect() during the cleanup paths
@@ -1346,7 +1347,7 @@ nv50_cursor_set_offset(struct nouveau_crtc *nv_crtc, uint32_t offset)
 }
 
 static int
-nv50_crtc_create(struct drm_device *dev, struct nouveau_object *core, int index)
+nv50_crtc_create(struct drm_device *dev, int index)
 {
         struct nv50_disp *disp = nv50_disp(dev);
         struct nv50_head *head;
@@ -1395,11 +1396,7 @@ nv50_crtc_create(struct drm_device *dev, struct nouveau_object *core, int index)
         nv50_crtc_lut_load(crtc);
 
         /* allocate cursor resources */
-        ret = nv50_pioc_create(disp->core, NV50_DISP_CURS_CLASS, index,
-                              &(struct nv50_display_curs_class) {
-                                        .head = index,
-                              }, sizeof(struct nv50_display_curs_class),
-                              &head->curs.base);
+        ret = nv50_curs_create(disp->disp, index, &head->curs);
         if (ret)
                 goto out;
 
@@ -1420,12 +1417,8 @@ nv50_crtc_create(struct drm_device *dev, struct nouveau_object *core, int index)
                 goto out;
 
         /* allocate page flip / sync resources */
-        ret = nv50_dmac_create(disp->core, NV50_DISP_SYNC_CLASS, index,
-                              &(struct nv50_display_sync_class) {
-                                        .pushbuf = EVO_PUSH_HANDLE(SYNC, index),
-                                        .head = index,
-                              }, sizeof(struct nv50_display_sync_class),
-                              disp->sync->bo.offset, &head->sync.base);
+        ret = nv50_base_create(disp->disp, index, disp->sync->bo.offset,
+                              &head->sync);
         if (ret)
                 goto out;
 
@@ -1433,20 +1426,12 @@ nv50_crtc_create(struct drm_device *dev, struct nouveau_object *core, int index)
         head->sync.data = 0x00000000;
 
         /* allocate overlay resources */
-        ret = nv50_pioc_create(disp->core, NV50_DISP_OIMM_CLASS, index,
-                              &(struct nv50_display_oimm_class) {
-                                        .head = index,
-                              }, sizeof(struct nv50_display_oimm_class),
-                              &head->oimm.base);
+        ret = nv50_oimm_create(disp->disp, index, &head->oimm);
         if (ret)
                 goto out;
 
-        ret = nv50_dmac_create(disp->core, NV50_DISP_OVLY_CLASS, index,
-                              &(struct nv50_display_ovly_class) {
-                                        .pushbuf = EVO_PUSH_HANDLE(OVLY, index),
-                                        .head = index,
-                              }, sizeof(struct nv50_display_ovly_class),
-                              disp->sync->bo.offset, &head->ovly.base);
+        ret = nv50_ovly_create(disp->disp, index, disp->sync->bo.offset,
+                              &head->ovly);
         if (ret)
                 goto out;
 
@@ -1464,16 +1449,23 @@ nv50_dac_dpms(struct drm_encoder *encoder, int mode)
 {
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct nv50_disp *disp = nv50_disp(encoder->dev);
-        int or = nv_encoder->or;
-        u32 dpms_ctrl;
-
-        dpms_ctrl = 0x00000000;
-        if (mode == DRM_MODE_DPMS_STANDBY || mode == DRM_MODE_DPMS_OFF)
-                dpms_ctrl |= 0x00000001;
-        if (mode == DRM_MODE_DPMS_SUSPEND || mode == DRM_MODE_DPMS_OFF)
-                dpms_ctrl |= 0x00000004;
+        struct {
+                struct nv50_disp_mthd_v1 base;
+                struct nv50_disp_dac_pwr_v0 pwr;
+        } args = {
+                .base.version = 1,
+                .base.method = NV50_DISP_MTHD_V1_DAC_PWR,
+                .base.hasht  = nv_encoder->dcb->hasht,
+                .base.hashm  = nv_encoder->dcb->hashm,
+                .pwr.state = 1,
+                .pwr.data  = 1,
+                .pwr.vsync = (mode != DRM_MODE_DPMS_SUSPEND &&
+                              mode != DRM_MODE_DPMS_OFF),
+                .pwr.hsync = (mode != DRM_MODE_DPMS_STANDBY &&
+                              mode != DRM_MODE_DPMS_OFF),
+        };
 
-        nv_call(disp->core, NV50_DISP_DAC_PWR + or, dpms_ctrl);
+        nvif_mthd(disp->disp, 0, &args, sizeof(args));
 }
 
 static bool
@@ -1514,7 +1506,7 @@ nv50_dac_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 
         push = evo_wait(mast, 8);
         if (push) {
-                if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                         u32 syncs = 0x00000000;
 
                         if (mode->flags & DRM_MODE_FLAG_NHSYNC)
@@ -1563,7 +1555,7 @@ nv50_dac_disconnect(struct drm_encoder *encoder)
 
                 push = evo_wait(mast, 4);
                 if (push) {
-                        if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                        if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                                 evo_mthd(push, 0x0400 + (or * 0x080), 1);
                                 evo_data(push, 0x00000000);
                         } else {
@@ -1580,14 +1572,25 @@ nv50_dac_disconnect(struct drm_encoder *encoder)
 static enum drm_connector_status
 nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 {
+        struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct nv50_disp *disp = nv50_disp(encoder->dev);
-        int ret, or = nouveau_encoder(encoder)->or;
-        u32 load = nouveau_drm(encoder->dev)->vbios.dactestval;
-        if (load == 0)
-                load = 340;
+        struct {
+                struct nv50_disp_mthd_v1 base;
+                struct nv50_disp_dac_load_v0 load;
+        } args = {
+                .base.version = 1,
+                .base.method = NV50_DISP_MTHD_V1_DAC_LOAD,
+                .base.hasht  = nv_encoder->dcb->hasht,
+                .base.hashm  = nv_encoder->dcb->hashm,
+        };
+        int ret;
+
+        args.load.data = nouveau_drm(encoder->dev)->vbios.dactestval;
+        if (args.load.data == 0)
+                args.load.data = 340;
 
-        ret = nv_exec(disp->core, NV50_DISP_DAC_LOAD + or, &load, sizeof(load));
-        if (ret || !load)
+        ret = nvif_mthd(disp->disp, 0, &args, sizeof(args));
+        if (ret || !args.load.load)
                 return connector_status_disconnected;
 
         return connector_status_connected;
@@ -1619,7 +1622,7 @@ static int
 nv50_dac_create(struct drm_connector *connector, struct dcb_output *dcbe)
 {
         struct nouveau_drm *drm = nouveau_drm(connector->dev);
-        struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+        struct nouveau_i2c *i2c = nvkm_i2c(&drm->device);
         struct nouveau_encoder *nv_encoder;
         struct drm_encoder *encoder;
         int type = DRM_MODE_ENCODER_DAC;
@@ -1650,16 +1653,25 @@ nv50_audio_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode)
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct nouveau_connector *nv_connector;
         struct nv50_disp *disp = nv50_disp(encoder->dev);
+        struct {
+                struct nv50_disp_mthd_v1 base;
+                struct nv50_disp_sor_hda_eld_v0 eld;
+                u8 data[sizeof(nv_connector->base.eld)];
+        } args = {
+                .base.version = 1,
+                .base.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
+                .base.hasht   = nv_encoder->dcb->hasht,
+                .base.hashm   = nv_encoder->dcb->hashm,
+        };
 
         nv_connector = nouveau_encoder_connector_get(nv_encoder);
         if (!drm_detect_monitor_audio(nv_connector->edid))
                 return;
 
         drm_edid_to_eld(&nv_connector->base, nv_connector->edid);
+        memcpy(args.data, nv_connector->base.eld, sizeof(args.data));
 
-        nv_exec(disp->core, NVA3_DISP_SOR_HDA_ELD + nv_encoder->or,
-                            nv_connector->base.eld,
-                            nv_connector->base.eld[2] * 4);
+        nvif_mthd(disp->disp, 0, &args, sizeof(args));
 }
 
 static void
@@ -1667,8 +1679,17 @@ nv50_audio_disconnect(struct drm_encoder *encoder)
 {
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct nv50_disp *disp = nv50_disp(encoder->dev);
+        struct {
+                struct nv50_disp_mthd_v1 base;
+                struct nv50_disp_sor_hda_eld_v0 eld;
+        } args = {
+                .base.version = 1,
+                .base.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
+                .base.hasht   = nv_encoder->dcb->hasht,
+                .base.hashm   = nv_encoder->dcb->hashm,
+        };
 
-        nv_exec(disp->core, NVA3_DISP_SOR_HDA_ELD + nv_encoder->or, NULL, 0);
+        nvif_mthd(disp->disp, 0, &args, sizeof(args));
 }
 
 /******************************************************************************
@@ -1679,10 +1700,20 @@ nv50_hdmi_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode)
 {
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
-        struct nouveau_connector *nv_connector;
         struct nv50_disp *disp = nv50_disp(encoder->dev);
-        const u32 moff = (nv_crtc->index << 3) | nv_encoder->or;
-        u32 rekey = 56; /* binary driver, and tegra constant */
+        struct {
+                struct nv50_disp_mthd_v1 base;
+                struct nv50_disp_sor_hdmi_pwr_v0 pwr;
+        } args = {
+                .base.version = 1,
+                .base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
+                .base.hasht  = nv_encoder->dcb->hasht,
+                .base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
+                               (0x0100 << nv_crtc->index),
+                .pwr.state = 1,
+                .pwr.rekey = 56, /* binary driver, and tegra, constant */
+        };
+        struct nouveau_connector *nv_connector;
         u32 max_ac_packet;
 
         nv_connector = nouveau_encoder_connector_get(nv_encoder);
@@ -1690,14 +1721,11 @@ nv50_hdmi_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode)
                 return;
 
         max_ac_packet  = mode->htotal - mode->hdisplay;
-        max_ac_packet -= rekey;
+        max_ac_packet -= args.pwr.rekey;
         max_ac_packet -= 18; /* constant from tegra */
-        max_ac_packet /= 32;
-
-        nv_call(disp->core, NV84_DISP_SOR_HDMI_PWR + moff,
-                            NV84_DISP_SOR_HDMI_PWR_STATE_ON |
-                            (max_ac_packet << 16) | rekey);
+        args.pwr.max_ac_packet = max_ac_packet / 32;
 
+        nvif_mthd(disp->disp, 0, &args, sizeof(args));
         nv50_audio_mode_set(encoder, mode);
 }
 
@@ -1706,11 +1734,20 @@ nv50_hdmi_disconnect(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
 {
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct nv50_disp *disp = nv50_disp(encoder->dev);
-        const u32 moff = (nv_crtc->index << 3) | nv_encoder->or;
+        struct {
+                struct nv50_disp_mthd_v1 base;
+                struct nv50_disp_sor_hdmi_pwr_v0 pwr;
+        } args = {
+                .base.version = 1,
+                .base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
+                .base.hasht  = nv_encoder->dcb->hasht,
+                .base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
+                               (0x0100 << nv_crtc->index),
+        };
 
         nv50_audio_disconnect(encoder);
 
-        nv_call(disp->core, NV84_DISP_SOR_HDMI_PWR + moff, 0x00000000);
+        nvif_mthd(disp->disp, 0, &args, sizeof(args));
 }
 
 /******************************************************************************
@@ -1720,10 +1757,29 @@ static void
 nv50_sor_dpms(struct drm_encoder *encoder, int mode)
 {
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+        struct nv50_disp *disp = nv50_disp(encoder->dev);
+        struct {
+                struct nv50_disp_mthd_v1 base;
+                struct nv50_disp_sor_pwr_v0 pwr;
+        } args = {
+                .base.version = 1,
+                .base.method = NV50_DISP_MTHD_V1_SOR_PWR,
+                .base.hasht  = nv_encoder->dcb->hasht,
+                .base.hashm  = nv_encoder->dcb->hashm,
+                .pwr.state = mode == DRM_MODE_DPMS_ON,
+        };
+        struct {
+                struct nv50_disp_mthd_v1 base;
+                struct nv50_disp_sor_dp_pwr_v0 pwr;
+        } link = {
+                .base.version = 1,
+                .base.method = NV50_DISP_MTHD_V1_SOR_DP_PWR,
+                .base.hasht  = nv_encoder->dcb->hasht,
+                .base.hashm  = nv_encoder->dcb->hashm,
+                .pwr.state = mode == DRM_MODE_DPMS_ON,
+        };
         struct drm_device *dev = encoder->dev;
-        struct nv50_disp *disp = nv50_disp(dev);
         struct drm_encoder *partner;
-        u32 mthd;
 
         nv_encoder->last_dpms = mode;
 
@@ -1741,18 +1797,13 @@ nv50_sor_dpms(struct drm_encoder *encoder, int mode)
                 }
         }
 
-        mthd  = (ffs(nv_encoder->dcb->heads) - 1) << 3;
-        mthd |= (ffs(nv_encoder->dcb->sorconf.link) - 1) << 2;
-        mthd |= nv_encoder->or;
-
         if (nv_encoder->dcb->type == DCB_OUTPUT_DP) {
-                nv_call(disp->core, NV50_DISP_SOR_PWR | mthd, 1);
-                mthd |= NV94_DISP_SOR_DP_PWR;
+                args.pwr.state = 1;
+                nvif_mthd(disp->disp, 0, &args, sizeof(args));
+                nvif_mthd(disp->disp, 0, &link, sizeof(link));
         } else {
-                mthd |= NV50_DISP_SOR_PWR;
+                nvif_mthd(disp->disp, 0, &args, sizeof(args));
         }
-
-        nv_call(disp->core, mthd, (mode == DRM_MODE_DPMS_ON));
 }
 
 static bool
@@ -1781,7 +1832,7 @@ nv50_sor_ctrl(struct nouveau_encoder *nv_encoder, u32 mask, u32 data)
         struct nv50_mast *mast = nv50_mast(nv_encoder->base.base.dev);
         u32 temp = (nv_encoder->ctrl & ~mask) | (data & mask), *push;
         if (temp != nv_encoder->ctrl && (push = evo_wait(mast, 2))) {
-                if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                         evo_mthd(push, 0x0600 + (nv_encoder->or * 0x40), 1);
                         evo_data(push, (nv_encoder->ctrl = temp));
                 } else {
@@ -1817,15 +1868,24 @@ static void
 nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
                   struct drm_display_mode *mode)
 {
+        struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+        struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
+        struct {
+                struct nv50_disp_mthd_v1 base;
+                struct nv50_disp_sor_lvds_script_v0 lvds;
+        } lvds = {
+                .base.version = 1,
+                .base.method  = NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT,
+                .base.hasht   = nv_encoder->dcb->hasht,
+                .base.hashm   = nv_encoder->dcb->hashm,
+        };
         struct nv50_disp *disp = nv50_disp(encoder->dev);
         struct nv50_mast *mast = nv50_mast(encoder->dev);
         struct drm_device *dev = encoder->dev;
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-        struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
         struct nouveau_connector *nv_connector;
         struct nvbios *bios = &drm->vbios;
-        u32 lvds = 0, mask, ctrl;
+        u32 mask, ctrl;
         u8 owner = 1 << nv_crtc->index;
         u8 proto = 0xf;
         u8 depth = 0x0;
@@ -1851,31 +1911,31 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 
                 if (bios->fp_no_ddc) {
                         if (bios->fp.dual_link)
-                                lvds |= 0x0100;
+                                lvds.lvds.script |= 0x0100;
                         if (bios->fp.if_is_24bit)
-                                lvds |= 0x0200;
+                                lvds.lvds.script |= 0x0200;
                 } else {
                         if (nv_connector->type == DCB_CONNECTOR_LVDS_SPWG) {
                                 if (((u8 *)nv_connector->edid)[121] == 2)
-                                        lvds |= 0x0100;
+                                        lvds.lvds.script |= 0x0100;
                         } else
                         if (mode->clock >= bios->fp.duallink_transition_clk) {
-                                lvds |= 0x0100;
+                                lvds.lvds.script |= 0x0100;
                         }
 
-                        if (lvds & 0x0100) {
+                        if (lvds.lvds.script & 0x0100) {
                                 if (bios->fp.strapless_is_24bit & 2)
-                                        lvds |= 0x0200;
+                                        lvds.lvds.script |= 0x0200;
                         } else {
                                 if (bios->fp.strapless_is_24bit & 1)
-                                        lvds |= 0x0200;
+                                        lvds.lvds.script |= 0x0200;
                         }
 
                         if (nv_connector->base.display_info.bpc == 8)
-                                lvds |= 0x0200;
+                                lvds.lvds.script |= 0x0200;
                 }
 
-                nv_call(disp->core, NV50_DISP_SOR_LVDS_SCRIPT + nv_encoder->or, lvds);
+                nvif_mthd(disp->disp, 0, &lvds, sizeof(lvds));
                 break;
         case DCB_OUTPUT_DP:
                 if (nv_connector->base.display_info.bpc == 6) {
@@ -1902,7 +1962,7 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 
         nv50_sor_dpms(&nv_encoder->base.base, DRM_MODE_DPMS_ON);
 
-        if (nv50_vers(mast) >= NVD0_DISP_CLASS) {
+        if (nv50_vers(mast) >= GF110_DISP) {
                 u32 *push = evo_wait(mast, 3);
                 if (push) {
                         u32 magic = 0x31ec6000 | (nv_crtc->index << 25);
@@ -1961,7 +2021,7 @@ static int
 nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe)
 {
         struct nouveau_drm *drm = nouveau_drm(connector->dev);
-        struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+        struct nouveau_i2c *i2c = nvkm_i2c(&drm->device);
         struct nouveau_encoder *nv_encoder;
         struct drm_encoder *encoder;
         int type;
@@ -2002,9 +2062,19 @@ nv50_pior_dpms(struct drm_encoder *encoder, int mode)
 {
         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
         struct nv50_disp *disp = nv50_disp(encoder->dev);
-        u32 mthd = (nv_encoder->dcb->type << 12) | nv_encoder->or;
-        u32 ctrl = (mode == DRM_MODE_DPMS_ON);
-        nv_call(disp->core, NV50_DISP_PIOR_PWR + mthd, ctrl);
+        struct {
+                struct nv50_disp_mthd_v1 base;
+                struct nv50_disp_pior_pwr_v0 pwr;
+        } args = {
+                .base.version = 1,
+                .base.method = NV50_DISP_MTHD_V1_PIOR_PWR,
+                .base.hasht  = nv_encoder->dcb->hasht,
+                .base.hashm  = nv_encoder->dcb->hashm,
+                .pwr.state = mode == DRM_MODE_DPMS_ON,
+                .pwr.type = nv_encoder->dcb->type,
+        };
+
+        nvif_mthd(disp->disp, 0, &args, sizeof(args));
 }
 
 static bool
@@ -2067,7 +2137,7 @@ nv50_pior_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 
         push = evo_wait(mast, 8);
         if (push) {
-                if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                         u32 ctrl = (depth << 16) | (proto << 8) | owner;
                         if (mode->flags & DRM_MODE_FLAG_NHSYNC)
                                 ctrl |= 0x00001000;
@@ -2096,7 +2166,7 @@ nv50_pior_disconnect(struct drm_encoder *encoder)
 
                 push = evo_wait(mast, 4);
                 if (push) {
-                        if (nv50_vers(mast) < NVD0_DISP_MAST_CLASS) {
+                        if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                                 evo_mthd(push, 0x0700 + (or * 0x040), 1);
                                 evo_data(push, 0x00000000);
                         }
@@ -2132,7 +2202,7 @@ static int
 nv50_pior_create(struct drm_connector *connector, struct dcb_output *dcbe)
 {
         struct nouveau_drm *drm = nouveau_drm(connector->dev);
-        struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+        struct nouveau_i2c *i2c = nvkm_i2c(&drm->device);
         struct nouveau_i2c_port *ddc = NULL;
         struct nouveau_encoder *nv_encoder;
         struct drm_encoder *encoder;
@@ -2169,8 +2239,151 @@ nv50_pior_create(struct drm_connector *connector, struct dcb_output *dcbe)
 }
 
 /******************************************************************************
+ * Framebuffer
+ *****************************************************************************/
+
+static void
+nv50_fbdma_fini(struct nv50_fbdma *fbdma)
+{
+        int i;
+        for (i = 0; i < ARRAY_SIZE(fbdma->base); i++)
+                nvif_object_fini(&fbdma->base[i]);
+        nvif_object_fini(&fbdma->core);
+        list_del(&fbdma->head);
+        kfree(fbdma);
+}
+
+static int
+nv50_fbdma_init(struct drm_device *dev, u32 name, u64 offset, u64 length, u8 kind)
+{
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nv50_disp *disp = nv50_disp(dev);
+        struct nv50_mast *mast = nv50_mast(dev);
+        struct __attribute__ ((packed)) {
+                struct nv_dma_v0 base;
+                union {
+                        struct nv50_dma_v0 nv50;
+                        struct gf100_dma_v0 gf100;
+                        struct gf110_dma_v0 gf110;
+                };
+        } args = {};
+        struct nv50_fbdma *fbdma;
+        struct drm_crtc *crtc;
+        u32 size = sizeof(args.base);
+        int ret;
+
+        list_for_each_entry(fbdma, &disp->fbdma, head) {
+                if (fbdma->core.handle == name)
+                        return 0;
+        }
+
+        fbdma = kzalloc(sizeof(*fbdma), GFP_KERNEL);
+        if (!fbdma)
+                return -ENOMEM;
+        list_add(&fbdma->head, &disp->fbdma);
+
+        args.base.target = NV_DMA_V0_TARGET_VRAM;
+        args.base.access = NV_DMA_V0_ACCESS_RDWR;
+        args.base.start = offset;
+        args.base.limit = offset + length - 1;
+
+        if (drm->device.info.chipset < 0x80) {
+                args.nv50.part = NV50_DMA_V0_PART_256;
+                size += sizeof(args.nv50);
+        } else
+        if (drm->device.info.chipset < 0xc0) {
+                args.nv50.part = NV50_DMA_V0_PART_256;
+                args.nv50.kind = kind;
+                size += sizeof(args.nv50);
+        } else
+        if (drm->device.info.chipset < 0xd0) {
+                args.gf100.kind = kind;
+                size += sizeof(args.gf100);
+        } else {
+                args.gf110.page = GF110_DMA_V0_PAGE_LP;
+                args.gf110.kind = kind;
+                size += sizeof(args.gf110);
+        }
+
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                struct nv50_head *head = nv50_head(crtc);
+                int ret = nvif_object_init(&head->sync.base.base.user, NULL,
+                                            name, NV_DMA_IN_MEMORY, &args, size,
+                                           &fbdma->base[head->base.index]);
+                if (ret) {
+                        nv50_fbdma_fini(fbdma);
+                        return ret;
+                }
+        }
+
+        ret = nvif_object_init(&mast->base.base.user, NULL, name,
+                                NV_DMA_IN_MEMORY, &args, size,
+                               &fbdma->core);
+        if (ret) {
+                nv50_fbdma_fini(fbdma);
+                return ret;
+        }
+
+        return 0;
+}
+
+static void
+nv50_fb_dtor(struct drm_framebuffer *fb)
+{
+}
+
+static int
+nv50_fb_ctor(struct drm_framebuffer *fb)
+{
+        struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb);
+        struct nouveau_drm *drm = nouveau_drm(fb->dev);
+        struct nouveau_bo *nvbo = nv_fb->nvbo;
+        struct nv50_disp *disp = nv50_disp(fb->dev);
+        u8 kind = nouveau_bo_tile_layout(nvbo) >> 8;
+        u8 tile = nvbo->tile_mode;
+
+        if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG) {
+                NV_ERROR(drm, "framebuffer requires contiguous bo\n");
+                return -EINVAL;
+        }
+
+        if (drm->device.info.chipset >= 0xc0)
+                tile >>= 4; /* yep.. */
+
+        switch (fb->depth) {
+        case  8: nv_fb->r_format = 0x1e00; break;
+        case 15: nv_fb->r_format = 0xe900; break;
+        case 16: nv_fb->r_format = 0xe800; break;
+        case 24:
+        case 32: nv_fb->r_format = 0xcf00; break;
+        case 30: nv_fb->r_format = 0xd100; break;
+        default:
+                 NV_ERROR(drm, "unknown depth %d\n", fb->depth);
+                 return -EINVAL;
+        }
+
+        if (disp->disp->oclass < G82_DISP) {
+                nv_fb->r_pitch   = kind ? (((fb->pitches[0] / 4) << 4) | tile) :
+                                            (fb->pitches[0] | 0x00100000);
+                nv_fb->r_format |= kind << 16;
+        } else
+        if (disp->disp->oclass < GF110_DISP) {
+                nv_fb->r_pitch  = kind ? (((fb->pitches[0] / 4) << 4) | tile) :
+                                           (fb->pitches[0] | 0x00100000);
+        } else {
+                nv_fb->r_pitch  = kind ? (((fb->pitches[0] / 4) << 4) | tile) :
+                                           (fb->pitches[0] | 0x01000000);
+        }
+        nv_fb->r_handle = 0xffff0000 | kind;
+
+        return nv50_fbdma_init(fb->dev, nv_fb->r_handle, 0,
+                               drm->device.info.ram_user, kind);
+}
+
+/******************************************************************************
  * Init
  *****************************************************************************/
+
 void
 nv50_display_fini(struct drm_device *dev)
 {
@@ -2193,7 +2406,7 @@ nv50_display_init(struct drm_device *dev)
         }
 
         evo_mthd(push, 0x0088, 1);
-        evo_data(push, NvEvoSync);
+        evo_data(push, nv50_mast(dev)->base.sync.handle);
         evo_kick(push, nv50_mast(dev));
         return 0;
 }
@@ -2202,8 +2415,13 @@ void
 nv50_display_destroy(struct drm_device *dev)
 {
         struct nv50_disp *disp = nv50_disp(dev);
+        struct nv50_fbdma *fbdma, *fbtmp;
+
+        list_for_each_entry_safe(fbdma, fbtmp, &disp->fbdma, head) {
+                nv50_fbdma_fini(fbdma);
+        }
 
-        nv50_dmac_destroy(disp->core, &disp->mast.base);
+        nv50_dmac_destroy(&disp->mast.base, disp->disp);
 
         nouveau_bo_unmap(disp->sync);
         if (disp->sync)
@@ -2217,7 +2435,7 @@ nv50_display_destroy(struct drm_device *dev)
 int
 nv50_display_create(struct drm_device *dev)
 {
-        struct nouveau_device *device = nouveau_dev(dev);
+        struct nvif_device *device = &nouveau_drm(dev)->device;
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct dcb_table *dcb = &drm->vbios.dcb;
         struct drm_connector *connector, *tmp;
@@ -2228,12 +2446,15 @@ nv50_display_create(struct drm_device *dev)
         disp = kzalloc(sizeof(*disp), GFP_KERNEL);
         if (!disp)
                 return -ENOMEM;
+        INIT_LIST_HEAD(&disp->fbdma);
 
         nouveau_display(dev)->priv = disp;
         nouveau_display(dev)->dtor = nv50_display_destroy;
         nouveau_display(dev)->init = nv50_display_init;
         nouveau_display(dev)->fini = nv50_display_fini;
-        disp->core = nouveau_display(dev)->core;
+        nouveau_display(dev)->fb_ctor = nv50_fb_ctor;
+        nouveau_display(dev)->fb_dtor = nv50_fb_dtor;
+        disp->disp = &nouveau_display(dev)->disp;
 
         /* small shared memory area we use for notifiers and semaphores */
         ret = nouveau_bo_new(dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
@@ -2253,22 +2474,19 @@ nv50_display_create(struct drm_device *dev)
                 goto out;
 
         /* allocate master evo channel */
-        ret = nv50_dmac_create(disp->core, NV50_DISP_MAST_CLASS, 0,
-                              &(struct nv50_display_mast_class) {
-                                        .pushbuf = EVO_PUSH_HANDLE(MAST, 0),
-                              }, sizeof(struct nv50_display_mast_class),
-                              disp->sync->bo.offset, &disp->mast.base);
+        ret = nv50_core_create(disp->disp, disp->sync->bo.offset,
+                              &disp->mast);
         if (ret)
                 goto out;
 
         /* create crtc objects to represent the hw heads */
-        if (nv_mclass(disp->core) >= NVD0_DISP_CLASS)
-                crtcs = nv_rd32(device, 0x022448);
+        if (disp->disp->oclass >= GF110_DISP)
+                crtcs = nvif_rd32(device, 0x022448);
         else
                 crtcs = 2;
 
         for (i = 0; i < crtcs; i++) {
-                ret = nv50_crtc_create(dev, disp->core, i);
+                ret = nv50_crtc_create(dev, i);
                 if (ret)
                         goto out;
         }
diff --git a/drivers/gpu/drm/nouveau/nv50_fbcon.c b/drivers/gpu/drm/nouveau/nv50_fbcon.c
index 52068a0910dc..394c89abcc97 100644
--- a/drivers/gpu/drm/nouveau/nv50_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv50_fbcon.c
@@ -154,7 +154,6 @@ nv50_fbcon_accel_init(struct fb_info *info)
         struct drm_device *dev = nfbdev->dev;
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_channel *chan = drm->channel;
-        struct nouveau_object *object;
         int ret, format;
 
         switch (info->var.bits_per_pixel) {
@@ -184,8 +183,8 @@ nv50_fbcon_accel_init(struct fb_info *info)
                 return -EINVAL;
         }
 
-        ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, Nv2D,
-                                 0x502d, NULL, 0, &object);
+        ret = nvif_object_init(chan->object, NULL, 0x502d, 0x502d, NULL, 0,
+                               &nfbdev->twod);
         if (ret)
                 return ret;
 
@@ -196,11 +195,11 @@ nv50_fbcon_accel_init(struct fb_info *info)
         }
 
         BEGIN_NV04(chan, NvSub2D, 0x0000, 1);
-        OUT_RING(chan, Nv2D);
+        OUT_RING(chan, nfbdev->twod.handle);
         BEGIN_NV04(chan, NvSub2D, 0x0184, 3);
-        OUT_RING(chan, NvDmaFB);
-        OUT_RING(chan, NvDmaFB);
-        OUT_RING(chan, NvDmaFB);
+        OUT_RING(chan, chan->vram.handle);
+        OUT_RING(chan, chan->vram.handle);
+        OUT_RING(chan, chan->vram.handle);
         BEGIN_NV04(chan, NvSub2D, 0x0290, 1);
         OUT_RING(chan, 0);
         BEGIN_NV04(chan, NvSub2D, 0x0888, 1);
diff --git a/drivers/gpu/drm/nouveau/nv50_fence.c b/drivers/gpu/drm/nouveau/nv50_fence.c
index 0ee363840035..195cf51a7c31 100644
--- a/drivers/gpu/drm/nouveau/nv50_fence.c
+++ b/drivers/gpu/drm/nouveau/nv50_fence.c
@@ -22,8 +22,8 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include <core/object.h>
-#include <core/class.h>
+#include <nvif/os.h>
+#include <nvif/class.h>
 
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
@@ -38,7 +38,6 @@ nv50_fence_context_new(struct nouveau_channel *chan)
         struct nv10_fence_priv *priv = chan->drm->fence;
         struct nv10_fence_chan *fctx;
         struct ttm_mem_reg *mem = &priv->bo->bo.mem;
-        struct nouveau_object *object;
         u32 start = mem->start * PAGE_SIZE;
         u32 limit = start + mem->size - 1;
         int ret, i;
@@ -52,15 +51,14 @@ nv50_fence_context_new(struct nouveau_channel *chan)
         fctx->base.read = nv10_fence_read;
         fctx->base.sync = nv17_fence_sync;
 
-        ret = nouveau_object_new(nv_object(chan->cli), chan->handle,
-                                 NvSema, 0x003d,
-                                 &(struct nv_dma_class) {
-                                        .flags = NV_DMA_TARGET_VRAM |
-                                                 NV_DMA_ACCESS_RDWR,
+        ret = nvif_object_init(chan->object, NULL, NvSema, NV_DMA_IN_MEMORY,
+                               &(struct nv_dma_v0) {
+                                        .target = NV_DMA_V0_TARGET_VRAM,
+                                        .access = NV_DMA_V0_ACCESS_RDWR,
                                         .start = start,
                                         .limit = limit,
-                                 }, sizeof(struct nv_dma_class),
-                                 &object);
+                               }, sizeof(struct nv_dma_v0),
+                               &fctx->sema);
 
         /* dma objects for display sync channel semaphore blocks */
         for (i = 0; !ret && i < dev->mode_config.num_crtc; i++) {
@@ -68,15 +66,14 @@ nv50_fence_context_new(struct nouveau_channel *chan)
                 u32 start = bo->bo.mem.start * PAGE_SIZE;
                 u32 limit = start + bo->bo.mem.size - 1;
 
-                ret = nouveau_object_new(nv_object(chan->cli), chan->handle,
-                                         NvEvoSema0 + i, 0x003d,
-                                         &(struct nv_dma_class) {
-                                                .flags = NV_DMA_TARGET_VRAM |
-                                                         NV_DMA_ACCESS_RDWR,
+                ret = nvif_object_init(chan->object, NULL, NvEvoSema0 + i,
+                                       NV_DMA_IN_MEMORY, &(struct nv_dma_v0) {
+                                                .target = NV_DMA_V0_TARGET_VRAM,
+                                                .access = NV_DMA_V0_ACCESS_RDWR,
                                                 .start = start,
                                                 .limit = limit,
-                                         }, sizeof(struct nv_dma_class),
-                                         &object);
+                                       }, sizeof(struct nv_dma_v0),
+                                       &fctx->head[i]);
         }
 
         if (ret)
diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c
index 9fd475c89820..933a779c93ab 100644
--- a/drivers/gpu/drm/nouveau/nv84_fence.c
+++ b/drivers/gpu/drm/nouveau/nv84_fence.c
@@ -22,12 +22,6 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/object.h>
-#include <core/client.h>
-#include <core/class.h>
-
-#include <engine/fifo.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_fence.h"
@@ -47,7 +41,7 @@ nv84_fence_emit32(struct nouveau_channel *chan, u64 virtual, u32 sequence)
         int ret = RING_SPACE(chan, 8);
         if (ret == 0) {
                 BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
-                OUT_RING  (chan, chan->vram);
+                OUT_RING  (chan, chan->vram.handle);
                 BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 5);
                 OUT_RING  (chan, upper_32_bits(virtual));
                 OUT_RING  (chan, lower_32_bits(virtual));
@@ -65,7 +59,7 @@ nv84_fence_sync32(struct nouveau_channel *chan, u64 virtual, u32 sequence)
         int ret = RING_SPACE(chan, 7);
         if (ret == 0) {
                 BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
-                OUT_RING  (chan, chan->vram);
+                OUT_RING  (chan, chan->vram.handle);
                 BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
                 OUT_RING  (chan, upper_32_bits(virtual));
                 OUT_RING  (chan, lower_32_bits(virtual));
@@ -81,8 +75,7 @@ nv84_fence_emit(struct nouveau_fence *fence)
 {
         struct nouveau_channel *chan = fence->channel;
         struct nv84_fence_chan *fctx = chan->fence;
-        struct nouveau_fifo_chan *fifo = (void *)chan->object;
-        u64 addr = fifo->chid * 16;
+        u64 addr = chan->chid * 16;
 
         if (fence->sysmem)
                 addr += fctx->vma_gart.offset;
@@ -97,8 +90,7 @@ nv84_fence_sync(struct nouveau_fence *fence,
                 struct nouveau_channel *prev, struct nouveau_channel *chan)
 {
         struct nv84_fence_chan *fctx = chan->fence;
-        struct nouveau_fifo_chan *fifo = (void *)prev->object;
-        u64 addr = fifo->chid * 16;
+        u64 addr = prev->chid * 16;
 
         if (fence->sysmem)
                 addr += fctx->vma_gart.offset;
@@ -111,9 +103,8 @@ nv84_fence_sync(struct nouveau_fence *fence,
 static u32
 nv84_fence_read(struct nouveau_channel *chan)
 {
-        struct nouveau_fifo_chan *fifo = (void *)chan->object;
         struct nv84_fence_priv *priv = chan->drm->fence;
-        return nouveau_bo_rd32(priv->bo, fifo->chid * 16/4);
+        return nouveau_bo_rd32(priv->bo, chan->chid * 16/4);
 }
 
 static void
@@ -139,8 +130,7 @@ nv84_fence_context_del(struct nouveau_channel *chan)
 int
 nv84_fence_context_new(struct nouveau_channel *chan)
 {
-        struct nouveau_fifo_chan *fifo = (void *)chan->object;
-        struct nouveau_client *client = nouveau_client(fifo);
+        struct nouveau_cli *cli = (void *)nvif_client(&chan->device->base);
         struct nv84_fence_priv *priv = chan->drm->fence;
         struct nv84_fence_chan *fctx;
         int ret, i;
@@ -156,19 +146,19 @@ nv84_fence_context_new(struct nouveau_channel *chan)
         fctx->base.emit32 = nv84_fence_emit32;
         fctx->base.sync32 = nv84_fence_sync32;
 
-        ret = nouveau_bo_vma_add(priv->bo, client->vm, &fctx->vma);
+        ret = nouveau_bo_vma_add(priv->bo, cli->vm, &fctx->vma);
         if (ret == 0) {
-                ret = nouveau_bo_vma_add(priv->bo_gart, client->vm,
+                ret = nouveau_bo_vma_add(priv->bo_gart, cli->vm,
                                         &fctx->vma_gart);
         }
 
         /* map display semaphore buffers into channel's vm */
         for (i = 0; !ret && i < chan->drm->dev->mode_config.num_crtc; i++) {
                 struct nouveau_bo *bo = nv50_display_crtc_sema(chan->drm->dev, i);
-                ret = nouveau_bo_vma_add(bo, client->vm, &fctx->dispc_vma[i]);
+                ret = nouveau_bo_vma_add(bo, cli->vm, &fctx->dispc_vma[i]);
         }
 
-        nouveau_bo_wr32(priv->bo, fifo->chid * 16/4, 0x00000000);
+        nouveau_bo_wr32(priv->bo, chan->chid * 16/4, 0x00000000);
 
         if (ret)
                 nv84_fence_context_del(chan);
@@ -178,7 +168,7 @@ nv84_fence_context_new(struct nouveau_channel *chan)
 static bool
 nv84_fence_suspend(struct nouveau_drm *drm)
 {
-        struct nouveau_fifo *pfifo = nouveau_fifo(drm->device);
+        struct nouveau_fifo *pfifo = nvkm_fifo(&drm->device);
         struct nv84_fence_priv *priv = drm->fence;
         int i;
 
@@ -194,7 +184,7 @@ nv84_fence_suspend(struct nouveau_drm *drm)
 static void
 nv84_fence_resume(struct nouveau_drm *drm)
 {
-        struct nouveau_fifo *pfifo = nouveau_fifo(drm->device);
+        struct nouveau_fifo *pfifo = nvkm_fifo(&drm->device);
         struct nv84_fence_priv *priv = drm->fence;
         int i;
 
@@ -225,7 +215,7 @@ nv84_fence_destroy(struct nouveau_drm *drm)
 int
 nv84_fence_create(struct nouveau_drm *drm)
 {
-        struct nouveau_fifo *pfifo = nouveau_fifo(drm->device);
+        struct nouveau_fifo *pfifo = nvkm_fifo(&drm->device);
         struct nv84_fence_priv *priv;
         int ret;
 
diff --git a/drivers/gpu/drm/nouveau/nvc0_fbcon.c b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
index 9dcd30f3e1e0..61246677e8dc 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
@@ -154,11 +154,10 @@ nvc0_fbcon_accel_init(struct fb_info *info)
         struct nouveau_framebuffer *fb = &nfbdev->nouveau_fb;
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_channel *chan = drm->channel;
-        struct nouveau_object *object;
         int ret, format;
 
-        ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, Nv2D,
-                                 0x902d, NULL, 0, &object);
+        ret = nvif_object_init(chan->object, NULL, 0x902d, 0x902d, NULL, 0,
+                               &nfbdev->twod);
         if (ret)
                 return ret;
 
@@ -197,7 +196,7 @@ nvc0_fbcon_accel_init(struct fb_info *info)
         }
 
         BEGIN_NVC0(chan, NvSub2D, 0x0000, 1);
-        OUT_RING  (chan, 0x0000902d);
+        OUT_RING  (chan, nfbdev->twod.handle);
         BEGIN_NVC0(chan, NvSub2D, 0x0290, 1);
         OUT_RING  (chan, 0);
         BEGIN_NVC0(chan, NvSub2D, 0x0888, 1);
diff --git a/drivers/gpu/drm/nouveau/nvc0_fence.c b/drivers/gpu/drm/nouveau/nvc0_fence.c
index 9566267fbc42..becf19abda2d 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fence.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fence.c
@@ -22,12 +22,6 @@
  * Authors: Ben Skeggs
  */
 
-#include <core/object.h>
-#include <core/client.h>
-#include <core/class.h>
-
-#include <engine/fifo.h>
-
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_fence.h"
diff --git a/drivers/gpu/drm/nouveau/nvif/class.h b/drivers/gpu/drm/nouveau/nvif/class.h
new file mode 100644
index 000000000000..cc81e0e5fd30
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/class.h
@@ -0,0 +1,558 @@
+#ifndef __NVIF_CLASS_H__
+#define __NVIF_CLASS_H__
+
+/*******************************************************************************
+ * class identifiers
+ ******************************************************************************/
+
+/* the below match nvidia-assigned (either in hw, or sw) class numbers */
+#define NV_DEVICE                                                    0x00000080
+
+#define NV_DMA_FROM_MEMORY                                           0x00000002
+#define NV_DMA_TO_MEMORY                                             0x00000003
+#define NV_DMA_IN_MEMORY                                             0x0000003d
+
+#define NV04_DISP                                                    0x00000046
+
+#define NV03_CHANNEL_DMA                                             0x0000006b
+#define NV10_CHANNEL_DMA                                             0x0000006e
+#define NV17_CHANNEL_DMA                                             0x0000176e
+#define NV40_CHANNEL_DMA                                             0x0000406e
+#define NV50_CHANNEL_DMA                                             0x0000506e
+#define G82_CHANNEL_DMA                                              0x0000826e
+
+#define NV50_CHANNEL_GPFIFO                                          0x0000506f
+#define G82_CHANNEL_GPFIFO                                           0x0000826f
+#define FERMI_CHANNEL_GPFIFO                                         0x0000906f
+#define KEPLER_CHANNEL_GPFIFO_A                                      0x0000a06f
+
+#define NV50_DISP                                                    0x00005070
+#define G82_DISP                                                     0x00008270
+#define GT200_DISP                                                   0x00008370
+#define GT214_DISP                                                   0x00008570
+#define GT206_DISP                                                   0x00008870
+#define GF110_DISP                                                   0x00009070
+#define GK104_DISP                                                   0x00009170
+#define GK110_DISP                                                   0x00009270
+#define GM107_DISP                                                   0x00009470
+
+#define NV50_DISP_CURSOR                                             0x0000507a
+#define G82_DISP_CURSOR                                              0x0000827a
+#define GT214_DISP_CURSOR                                            0x0000857a
+#define GF110_DISP_CURSOR                                            0x0000907a
+#define GK104_DISP_CURSOR                                            0x0000917a
+
+#define NV50_DISP_OVERLAY                                            0x0000507b
+#define G82_DISP_OVERLAY                                             0x0000827b
+#define GT214_DISP_OVERLAY                                           0x0000857b
+#define GF110_DISP_OVERLAY                                           0x0000907b
+#define GK104_DISP_OVERLAY                                           0x0000917b
+
+#define NV50_DISP_BASE_CHANNEL_DMA                                   0x0000507c
+#define G82_DISP_BASE_CHANNEL_DMA                                    0x0000827c
+#define GT200_DISP_BASE_CHANNEL_DMA                                  0x0000837c
+#define GT214_DISP_BASE_CHANNEL_DMA                                  0x0000857c
+#define GF110_DISP_BASE_CHANNEL_DMA                                  0x0000907c
+#define GK104_DISP_BASE_CHANNEL_DMA                                  0x0000917c
+#define GK110_DISP_BASE_CHANNEL_DMA                                  0x0000927c
+
+#define NV50_DISP_CORE_CHANNEL_DMA                                   0x0000507d
+#define G82_DISP_CORE_CHANNEL_DMA                                    0x0000827d
+#define GT200_DISP_CORE_CHANNEL_DMA                                  0x0000837d
+#define GT214_DISP_CORE_CHANNEL_DMA                                  0x0000857d
+#define GT206_DISP_CORE_CHANNEL_DMA                                  0x0000887d
+#define GF110_DISP_CORE_CHANNEL_DMA                                  0x0000907d
+#define GK104_DISP_CORE_CHANNEL_DMA                                  0x0000917d
+#define GK110_DISP_CORE_CHANNEL_DMA                                  0x0000927d
+#define GM107_DISP_CORE_CHANNEL_DMA                                  0x0000947d
+
+#define NV50_DISP_OVERLAY_CHANNEL_DMA                                0x0000507e
+#define G82_DISP_OVERLAY_CHANNEL_DMA                                 0x0000827e
+#define GT200_DISP_OVERLAY_CHANNEL_DMA                               0x0000837e
+#define GT214_DISP_OVERLAY_CHANNEL_DMA                               0x0000857e
+#define GF110_DISP_OVERLAY_CONTROL_DMA                               0x0000907e
+#define GK104_DISP_OVERLAY_CONTROL_DMA                               0x0000917e
+
+#define FERMI_A                                                      0x00009097
+#define FERMI_B                                                      0x00009197
+#define FERMI_C                                                      0x00009297
+
+#define KEPLER_A                                                     0x0000a097
+#define KEPLER_B                                                     0x0000a197
+#define KEPLER_C                                                     0x0000a297
+
+#define MAXWELL_A                                                    0x0000b097
+
+#define FERMI_COMPUTE_A                                              0x000090c0
+#define FERMI_COMPUTE_B                                              0x000091c0
+
+#define KEPLER_COMPUTE_A                                             0x0000a0c0
+#define KEPLER_COMPUTE_B                                             0x0000a1c0
+
+#define MAXWELL_COMPUTE_A                                            0x0000b0c0
+
+
+/*******************************************************************************
+ * client
+ ******************************************************************************/
+
+#define NV_CLIENT_DEVLIST                                                  0x00
+
+struct nv_client_devlist_v0 {
+        __u8  version;
+        __u8  count;
+        __u8  pad02[6];
+        __u64 device[];
+};
+
+
+/*******************************************************************************
+ * device
+ ******************************************************************************/
+
+struct nv_device_v0 {
+        __u8  version;
+        __u8  pad01[7];
+        __u64 device;   /* device identifier, ~0 for client default */
+#define NV_DEVICE_V0_DISABLE_IDENTIFY                     0x0000000000000001ULL
+#define NV_DEVICE_V0_DISABLE_MMIO                         0x0000000000000002ULL
+#define NV_DEVICE_V0_DISABLE_VBIOS                        0x0000000000000004ULL
+#define NV_DEVICE_V0_DISABLE_CORE                         0x0000000000000008ULL
+#define NV_DEVICE_V0_DISABLE_DISP                         0x0000000000010000ULL
+#define NV_DEVICE_V0_DISABLE_FIFO                         0x0000000000020000ULL
+#define NV_DEVICE_V0_DISABLE_GRAPH                        0x0000000100000000ULL
+#define NV_DEVICE_V0_DISABLE_MPEG                         0x0000000200000000ULL
+#define NV_DEVICE_V0_DISABLE_ME                           0x0000000400000000ULL
+#define NV_DEVICE_V0_DISABLE_VP                           0x0000000800000000ULL
+#define NV_DEVICE_V0_DISABLE_CRYPT                        0x0000001000000000ULL
+#define NV_DEVICE_V0_DISABLE_BSP                          0x0000002000000000ULL
+#define NV_DEVICE_V0_DISABLE_PPP                          0x0000004000000000ULL
+#define NV_DEVICE_V0_DISABLE_COPY0                        0x0000008000000000ULL
+#define NV_DEVICE_V0_DISABLE_COPY1                        0x0000010000000000ULL
+#define NV_DEVICE_V0_DISABLE_VIC                          0x0000020000000000ULL
+#define NV_DEVICE_V0_DISABLE_VENC                         0x0000040000000000ULL
+        __u64 disable;  /* disable particular subsystems */
+        __u64 debug0;   /* as above, but *internal* ids, and *NOT* ABI */
+};
+
+#define NV_DEVICE_V0_INFO                                                  0x00
+
+struct nv_device_info_v0 {
+        __u8  version;
+#define NV_DEVICE_INFO_V0_IGP                                              0x00
+#define NV_DEVICE_INFO_V0_PCI                                              0x01
+#define NV_DEVICE_INFO_V0_AGP                                              0x02
+#define NV_DEVICE_INFO_V0_PCIE                                             0x03
+#define NV_DEVICE_INFO_V0_SOC                                              0x04
+        __u8  platform;
+        __u16 chipset;  /* from NV_PMC_BOOT_0 */
+        __u8  revision; /* from NV_PMC_BOOT_0 */
+#define NV_DEVICE_INFO_V0_TNT                                              0x01
+#define NV_DEVICE_INFO_V0_CELSIUS                                          0x02
+#define NV_DEVICE_INFO_V0_KELVIN                                           0x03
+#define NV_DEVICE_INFO_V0_RANKINE                                          0x04
+#define NV_DEVICE_INFO_V0_CURIE                                            0x05
+#define NV_DEVICE_INFO_V0_TESLA                                            0x06
+#define NV_DEVICE_INFO_V0_FERMI                                            0x07
+#define NV_DEVICE_INFO_V0_KEPLER                                           0x08
+#define NV_DEVICE_INFO_V0_MAXWELL                                          0x09
+        __u8  family;
+        __u8  pad06[2];
+        __u64 ram_size;
+        __u64 ram_user;
+};
+
+
+/*******************************************************************************
+ * context dma
+ ******************************************************************************/
+
+struct nv_dma_v0 {
+        __u8  version;
+#define NV_DMA_V0_TARGET_VM                                                0x00
+#define NV_DMA_V0_TARGET_VRAM                                              0x01
+#define NV_DMA_V0_TARGET_PCI                                               0x02
+#define NV_DMA_V0_TARGET_PCI_US                                            0x03
+#define NV_DMA_V0_TARGET_AGP                                               0x04
+        __u8  target;
+#define NV_DMA_V0_ACCESS_VM                                                0x00
+#define NV_DMA_V0_ACCESS_RD                                                0x01
+#define NV_DMA_V0_ACCESS_WR                                                0x02
+#define NV_DMA_V0_ACCESS_RDWR                 (NV_DMA_V0_ACCESS_RD | NV_DMA_V0_ACCESS_WR)
+        __u8  access;
+        __u8  pad03[5];
+        __u64 start;
+        __u64 limit;
+        /* ... chipset-specific class data */
+};
+
+struct nv50_dma_v0 {
+        __u8  version;
+#define NV50_DMA_V0_PRIV_VM                                                0x00
+#define NV50_DMA_V0_PRIV_US                                                0x01
+#define NV50_DMA_V0_PRIV__S                                                0x02
+        __u8  priv;
+#define NV50_DMA_V0_PART_VM                                                0x00
+#define NV50_DMA_V0_PART_256                                               0x01
+#define NV50_DMA_V0_PART_1KB                                               0x02
+        __u8  part;
+#define NV50_DMA_V0_COMP_NONE                                              0x00
+#define NV50_DMA_V0_COMP_1                                                 0x01
+#define NV50_DMA_V0_COMP_2                                                 0x02
+#define NV50_DMA_V0_COMP_VM                                                0x03
+        __u8  comp;
+#define NV50_DMA_V0_KIND_PITCH                                             0x00
+#define NV50_DMA_V0_KIND_VM                                                0x7f
+        __u8  kind;
+        __u8  pad05[3];
+};
+
+struct gf100_dma_v0 {
+        __u8  version;
+#define GF100_DMA_V0_PRIV_VM                                               0x00
+#define GF100_DMA_V0_PRIV_US                                               0x01
+#define GF100_DMA_V0_PRIV__S                                               0x02
+        __u8  priv;
+#define GF100_DMA_V0_KIND_PITCH                                            0x00
+#define GF100_DMA_V0_KIND_VM                                               0xff
+        __u8  kind;
+        __u8  pad03[5];
+};
+
+struct gf110_dma_v0 {
+        __u8  version;
+#define GF110_DMA_V0_PAGE_LP                                               0x00
+#define GF110_DMA_V0_PAGE_SP                                               0x01
+        __u8  page;
+#define GF110_DMA_V0_KIND_PITCH                                            0x00
+#define GF110_DMA_V0_KIND_VM                                               0xff
+        __u8  kind;
+        __u8  pad03[5];
+};
+
+
+/*******************************************************************************
+ * perfmon
+ ******************************************************************************/
+
+struct nvif_perfctr_v0 {
+        __u8  version;
+        __u8  pad01[1];
+        __u16 logic_op;
+        __u8  pad04[4];
+        char  name[4][64];
+};
+
+#define NVIF_PERFCTR_V0_QUERY                                              0x00
+#define NVIF_PERFCTR_V0_SAMPLE                                             0x01
+#define NVIF_PERFCTR_V0_READ                                               0x02
+
+struct nvif_perfctr_query_v0 {
+        __u8  version;
+        __u8  pad01[3];
+        __u32 iter;
+        char  name[64];
+};
+
+struct nvif_perfctr_sample {
+};
+
+struct nvif_perfctr_read_v0 {
+        __u8  version;
+        __u8  pad01[7];
+        __u32 ctr;
+        __u32 clk;
+};
+
+
+/*******************************************************************************
+ * device control
+ ******************************************************************************/
+
+#define NVIF_CONTROL_PSTATE_INFO                                           0x00
+#define NVIF_CONTROL_PSTATE_ATTR                                           0x01
+#define NVIF_CONTROL_PSTATE_USER                                           0x02
+
+struct nvif_control_pstate_info_v0 {
+        __u8  version;
+        __u8  count; /* out: number of power states */
+#define NVIF_CONTROL_PSTATE_INFO_V0_USTATE_DISABLE                         (-1)
+#define NVIF_CONTROL_PSTATE_INFO_V0_USTATE_PERFMON                         (-2)
+        __s8  ustate_ac; /* out: target pstate index */
+        __s8  ustate_dc; /* out: target pstate index */
+        __s8  pwrsrc; /* out: current power source */
+#define NVIF_CONTROL_PSTATE_INFO_V0_PSTATE_UNKNOWN                         (-1)
+#define NVIF_CONTROL_PSTATE_INFO_V0_PSTATE_PERFMON                         (-2)
+        __s8  pstate; /* out: current pstate index */
+        __u8  pad06[2];
+};
+
+struct nvif_control_pstate_attr_v0 {
+        __u8  version;
+#define NVIF_CONTROL_PSTATE_ATTR_V0_STATE_CURRENT                          (-1)
+        __s8  state; /*  in: index of pstate to query
+                      * out: pstate identifier
+                      */
+        __u8  index; /*  in: index of attribute to query
+                      * out: index of next attribute, or 0 if no more
+                      */
+        __u8  pad03[5];
+        __u32 min;
+        __u32 max;
+        char  name[32];
+        char  unit[16];
+};
+
+struct nvif_control_pstate_user_v0 {
+        __u8  version;
+#define NVIF_CONTROL_PSTATE_USER_V0_STATE_UNKNOWN                          (-1)
+#define NVIF_CONTROL_PSTATE_USER_V0_STATE_PERFMON                          (-2)
+        __s8  ustate; /*  in: pstate identifier */
+        __s8  pwrsrc; /*  in: target power source */
+        __u8  pad03[5];
+};
+
+
+/*******************************************************************************
+ * DMA FIFO channels
+ ******************************************************************************/
+
+struct nv03_channel_dma_v0 {
+        __u8  version;
+        __u8  chid;
+        __u8  pad02[2];
+        __u32 pushbuf;
+        __u64 offset;
+};
+
+#define G82_CHANNEL_DMA_V0_NTFY_UEVENT                                     0x00
+
+/*******************************************************************************
+ * GPFIFO channels
+ ******************************************************************************/
+
+struct nv50_channel_gpfifo_v0 {
+        __u8  version;
+        __u8  chid;
+        __u8  pad01[6];
+        __u32 pushbuf;
+        __u32 ilength;
+        __u64 ioffset;
+};
+
+struct kepler_channel_gpfifo_a_v0 {
+        __u8  version;
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_GR                               0x01
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_VP                               0x02
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_PPP                              0x04
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_BSP                              0x08
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_CE0                              0x10
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_CE1                              0x20
+#define KEPLER_CHANNEL_GPFIFO_A_V0_ENGINE_ENC                              0x40
+        __u8  engine;
+        __u16 chid;
+        __u8  pad04[4];
+        __u32 pushbuf;
+        __u32 ilength;
+        __u64 ioffset;
+};
+
+/*******************************************************************************
+ * legacy display
+ ******************************************************************************/
+
+#define NV04_DISP_NTFY_VBLANK                                              0x00
+#define NV04_DISP_NTFY_CONN                                                0x01
+
+struct nv04_disp_mthd_v0 {
+        __u8  version;
+#define NV04_DISP_SCANOUTPOS                                               0x00
+        __u8  method;
+        __u8  head;
+        __u8  pad03[5];
+};
+
+struct nv04_disp_scanoutpos_v0 {
+        __u8  version;
+        __u8  pad01[7];
+        __s64 time[2];
+        __u16 vblanks;
+        __u16 vblanke;
+        __u16 vtotal;
+        __u16 vline;
+        __u16 hblanks;
+        __u16 hblanke;
+        __u16 htotal;
+        __u16 hline;
+};
+
+/*******************************************************************************
+ * display
+ ******************************************************************************/
+
+#define NV50_DISP_MTHD                                                     0x00
+
+struct nv50_disp_mthd_v0 {
+        __u8  version;
+#define NV50_DISP_SCANOUTPOS                                               0x00
+        __u8  method;
+        __u8  head;
+        __u8  pad03[5];
+};
+
+struct nv50_disp_mthd_v1 {
+        __u8  version;
+#define NV50_DISP_MTHD_V1_DAC_PWR                                          0x10
+#define NV50_DISP_MTHD_V1_DAC_LOAD                                         0x11
+#define NV50_DISP_MTHD_V1_SOR_PWR                                          0x20
+#define NV50_DISP_MTHD_V1_SOR_HDA_ELD                                      0x21
+#define NV50_DISP_MTHD_V1_SOR_HDMI_PWR                                     0x22
+#define NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT                                  0x23
+#define NV50_DISP_MTHD_V1_SOR_DP_PWR                                       0x24
+#define NV50_DISP_MTHD_V1_PIOR_PWR                                         0x30
+        __u8  method;
+        __u16 hasht;
+        __u16 hashm;
+        __u8  pad06[2];
+};
+
+struct nv50_disp_dac_pwr_v0 {
+        __u8  version;
+        __u8  state;
+        __u8  data;
+        __u8  vsync;
+        __u8  hsync;
+        __u8  pad05[3];
+};
+
+struct nv50_disp_dac_load_v0 {
+        __u8  version;
+        __u8  load;
+        __u16 data;
+        __u8  pad04[4];
+};
+
+struct nv50_disp_sor_pwr_v0 {
+        __u8  version;
+        __u8  state;
+        __u8  pad02[6];
+};
+
+struct nv50_disp_sor_hda_eld_v0 {
+        __u8  version;
+        __u8  pad01[7];
+        __u8  data[];
+};
+
+struct nv50_disp_sor_hdmi_pwr_v0 {
+        __u8  version;
+        __u8  state;
+        __u8  max_ac_packet;
+        __u8  rekey;
+        __u8  pad04[4];
+};
+
+struct nv50_disp_sor_lvds_script_v0 {
+        __u8  version;
+        __u8  pad01[1];
+        __u16 script;
+        __u8  pad04[4];
+};
+
+struct nv50_disp_sor_dp_pwr_v0 {
+        __u8  version;
+        __u8  state;
+        __u8  pad02[6];
+};
+
+struct nv50_disp_pior_pwr_v0 {
+        __u8  version;
+        __u8  state;
+        __u8  type;
+        __u8  pad03[5];
+};
+
+/* core */
+struct nv50_disp_core_channel_dma_v0 {
+        __u8  version;
+        __u8  pad01[3];
+        __u32 pushbuf;
+};
+
+/* cursor immediate */
+struct nv50_disp_cursor_v0 {
+        __u8  version;
+        __u8  head;
+        __u8  pad02[6];
+};
+
+/* base */
+struct nv50_disp_base_channel_dma_v0 {
+        __u8  version;
+        __u8  pad01[2];
+        __u8  head;
+        __u32 pushbuf;
+};
+
+/* overlay */
+struct nv50_disp_overlay_channel_dma_v0 {
+        __u8  version;
+        __u8  pad01[2];
+        __u8  head;
+        __u32 pushbuf;
+};
+
+/* overlay immediate */
+struct nv50_disp_overlay_v0 {
+        __u8  version;
+        __u8  head;
+        __u8  pad02[6];
+};
+
+
+/*******************************************************************************
+ * fermi
+ ******************************************************************************/
+
+#define FERMI_A_ZBC_COLOR                                                  0x00
+#define FERMI_A_ZBC_DEPTH                                                  0x01
+
+struct fermi_a_zbc_color_v0 {
+        __u8  version;
+#define FERMI_A_ZBC_COLOR_V0_FMT_ZERO                                      0x01
+#define FERMI_A_ZBC_COLOR_V0_FMT_UNORM_ONE                                 0x02
+#define FERMI_A_ZBC_COLOR_V0_FMT_RF32_GF32_BF32_AF32                       0x04
+#define FERMI_A_ZBC_COLOR_V0_FMT_R16_G16_B16_A16                           0x08
+#define FERMI_A_ZBC_COLOR_V0_FMT_RN16_GN16_BN16_AN16                       0x0c
+#define FERMI_A_ZBC_COLOR_V0_FMT_RS16_GS16_BS16_AS16                       0x10
+#define FERMI_A_ZBC_COLOR_V0_FMT_RU16_GU16_BU16_AU16                       0x14
+#define FERMI_A_ZBC_COLOR_V0_FMT_RF16_GF16_BF16_AF16                       0x16
+#define FERMI_A_ZBC_COLOR_V0_FMT_A8R8G8B8                                  0x18
+#define FERMI_A_ZBC_COLOR_V0_FMT_A8RL8GL8BL8                               0x1c
+#define FERMI_A_ZBC_COLOR_V0_FMT_A2B10G10R10                               0x20
+#define FERMI_A_ZBC_COLOR_V0_FMT_AU2BU10GU10RU10                           0x24
+#define FERMI_A_ZBC_COLOR_V0_FMT_A8B8G8R8                                  0x28
+#define FERMI_A_ZBC_COLOR_V0_FMT_A8BL8GL8RL8                               0x2c
+#define FERMI_A_ZBC_COLOR_V0_FMT_AN8BN8GN8RN8                              0x30
+#define FERMI_A_ZBC_COLOR_V0_FMT_AS8BS8GS8RS8                              0x34
+#define FERMI_A_ZBC_COLOR_V0_FMT_AU8BU8GU8RU8                              0x38
+#define FERMI_A_ZBC_COLOR_V0_FMT_A2R10G10B10                               0x3c
+#define FERMI_A_ZBC_COLOR_V0_FMT_BF10GF11RF11                              0x40
+        __u8  format;
+        __u8  index;
+        __u8  pad03[5];
+        __u32 ds[4];
+        __u32 l2[4];
+};
+
+struct fermi_a_zbc_depth_v0 {
+        __u8  version;
+#define FERMI_A_ZBC_DEPTH_V0_FMT_FP32                                      0x01
+        __u8  format;
+        __u8  index;
+        __u8  pad03[5];
+        __u32 ds;
+        __u32 l2;
+};
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvif/client.c b/drivers/gpu/drm/nouveau/nvif/client.c
new file mode 100644
index 000000000000..3c4df1fc26dc
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/client.c
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include "client.h"
+#include "driver.h"
+#include "ioctl.h"
+
+int
+nvif_client_ioctl(struct nvif_client *client, void *data, u32 size)
+{
+        return client->driver->ioctl(client->base.priv, client->super, data, size, NULL);
+}
+
+int
+nvif_client_suspend(struct nvif_client *client)
+{
+        return client->driver->suspend(client->base.priv);
+}
+
+int
+nvif_client_resume(struct nvif_client *client)
+{
+        return client->driver->resume(client->base.priv);
+}
+
+void
+nvif_client_fini(struct nvif_client *client)
+{
+        if (client->driver) {
+                client->driver->fini(client->base.priv);
+                client->driver = NULL;
+                client->base.parent = NULL;
+                nvif_object_fini(&client->base);
+        }
+}
+
+const struct nvif_driver *
+nvif_drivers[] = {
+#ifdef __KERNEL__
+        &nvif_driver_nvkm,
+#else
+        &nvif_driver_drm,
+        &nvif_driver_lib,
+#endif
+        NULL
+};
+
+int
+nvif_client_init(void (*dtor)(struct nvif_client *), const char *driver,
+                 const char *name, u64 device, const char *cfg, const char *dbg,
+                 struct nvif_client *client)
+{
+        int ret, i;
+
+        ret = nvif_object_init(NULL, (void*)dtor, 0, 0, NULL, 0, &client->base);
+        if (ret)
+                return ret;
+
+        client->base.parent = &client->base;
+        client->base.handle = ~0;
+        client->object = &client->base;
+        client->super = true;
+
+        for (i = 0, ret = -EINVAL; (client->driver = nvif_drivers[i]); i++) {
+                if (!driver || !strcmp(client->driver->name, driver)) {
+                        ret = client->driver->init(name, device, cfg, dbg,
+                                                  &client->base.priv);
+                        if (!ret || driver)
+                                break;
+                }
+        }
+
+        if (ret)
+                nvif_client_fini(client);
+        return ret;
+}
+
+static void
+nvif_client_del(struct nvif_client *client)
+{
+        nvif_client_fini(client);
+        kfree(client);
+}
+
+int
+nvif_client_new(const char *driver, const char *name, u64 device,
+                const char *cfg, const char *dbg,
+                struct nvif_client **pclient)
+{
+        struct nvif_client *client = kzalloc(sizeof(*client), GFP_KERNEL);
+        if (client) {
+                int ret = nvif_client_init(nvif_client_del, driver, name,
+                                           device, cfg, dbg, client);
+                if (ret) {
+                        kfree(client);
+                        client = NULL;
+                }
+                *pclient = client;
+                return ret;
+        }
+        return -ENOMEM;
+}
+
+void
+nvif_client_ref(struct nvif_client *client, struct nvif_client **pclient)
+{
+        nvif_object_ref(&client->base, (struct nvif_object **)pclient);
+}
diff --git a/drivers/gpu/drm/nouveau/nvif/client.h b/drivers/gpu/drm/nouveau/nvif/client.h
new file mode 100644
index 000000000000..28352f0882ec
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/client.h
@@ -0,0 +1,39 @@
+#ifndef __NVIF_CLIENT_H__
+#define __NVIF_CLIENT_H__
+
+#include "object.h"
+
+struct nvif_client {
+        struct nvif_object base;
+        struct nvif_object *object; /*XXX: hack for nvif_object() */
+        const struct nvif_driver *driver;
+        bool super;
+};
+
+static inline struct nvif_client *
+nvif_client(struct nvif_object *object)
+{
+        while (object && object->parent != object)
+                object = object->parent;
+        return (void *)object;
+}
+
+int  nvif_client_init(void (*dtor)(struct nvif_client *), const char *,
+                      const char *, u64, const char *, const char *,
+                      struct nvif_client *);
+void nvif_client_fini(struct nvif_client *);
+int  nvif_client_new(const char *, const char *, u64, const char *,
+                     const char *, struct nvif_client **);
+void nvif_client_ref(struct nvif_client *, struct nvif_client **);
+int  nvif_client_ioctl(struct nvif_client *, void *, u32);
+int  nvif_client_suspend(struct nvif_client *);
+int  nvif_client_resume(struct nvif_client *);
+
+/*XXX*/
+#include <core/client.h>
+#define nvkm_client(a) ({ \
+        struct nvif_client *_client = nvif_client(nvif_object(a)); \
+        nouveau_client(_client->base.priv); \
+})
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvif/device.c b/drivers/gpu/drm/nouveau/nvif/device.c
new file mode 100644
index 000000000000..f477579725e3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/device.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include "device.h"
+
+void
+nvif_device_fini(struct nvif_device *device)
+{
+        nvif_object_fini(&device->base);
+}
+
+int
+nvif_device_init(struct nvif_object *parent, void (*dtor)(struct nvif_device *),
+                 u32 handle, u32 oclass, void *data, u32 size,
+                 struct nvif_device *device)
+{
+        int ret = nvif_object_init(parent, (void *)dtor, handle, oclass,
+                                   data, size, &device->base);
+        if (ret == 0) {
+                device->object = &device->base;
+                device->info.version = 0;
+                ret = nvif_object_mthd(&device->base, NV_DEVICE_V0_INFO,
+                                       &device->info, sizeof(device->info));
+        }
+        return ret;
+}
+
+static void
+nvif_device_del(struct nvif_device *device)
+{
+        nvif_device_fini(device);
+        kfree(device);
+}
+
+int
+nvif_device_new(struct nvif_object *parent, u32 handle, u32 oclass,
+                void *data, u32 size, struct nvif_device **pdevice)
+{
+        struct nvif_device *device = kzalloc(sizeof(*device), GFP_KERNEL);
+        if (device) {
+                int ret = nvif_device_init(parent, nvif_device_del, handle,
+                                           oclass, data, size, device);
+                if (ret) {
+                        kfree(device);
+                        device = NULL;
+                }
+                *pdevice = device;
+                return ret;
+        }
+        return -ENOMEM;
+}
+
+void
+nvif_device_ref(struct nvif_device *device, struct nvif_device **pdevice)
+{
+        nvif_object_ref(&device->base, (struct nvif_object **)pdevice);
+}
diff --git a/drivers/gpu/drm/nouveau/nvif/device.h b/drivers/gpu/drm/nouveau/nvif/device.h
new file mode 100644
index 000000000000..43180f9fe630
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/device.h
@@ -0,0 +1,62 @@
+#ifndef __NVIF_DEVICE_H__
+#define __NVIF_DEVICE_H__
+
+#include "object.h"
+#include "class.h"
+
+struct nvif_device {
+        struct nvif_object base;
+        struct nvif_object *object; /*XXX: hack for nvif_object() */
+        struct nv_device_info_v0 info;
+};
+
+static inline struct nvif_device *
+nvif_device(struct nvif_object *object)
+{
+        while (object && object->oclass != 0x0080 /*XXX: NV_DEVICE_CLASS*/ )
+                object = object->parent;
+        return (void *)object;
+}
+
+int  nvif_device_init(struct nvif_object *, void (*dtor)(struct nvif_device *),
+                      u32 handle, u32 oclass, void *, u32,
+                      struct nvif_device *);
+void nvif_device_fini(struct nvif_device *);
+int  nvif_device_new(struct nvif_object *, u32 handle, u32 oclass,
+                     void *, u32, struct nvif_device **);
+void nvif_device_ref(struct nvif_device *, struct nvif_device **);
+
+/*XXX*/
+#include <subdev/bios.h>
+#include <subdev/fb.h>
+#include <subdev/vm.h>
+#include <subdev/bar.h>
+#include <subdev/gpio.h>
+#include <subdev/clock.h>
+#include <subdev/i2c.h>
+#include <subdev/timer.h>
+#include <subdev/therm.h>
+
+#define nvkm_device(a) nv_device(nvkm_object((a)))
+#define nvkm_bios(a) nouveau_bios(nvkm_device(a))
+#define nvkm_fb(a) nouveau_fb(nvkm_device(a))
+#define nvkm_vmmgr(a) nouveau_vmmgr(nvkm_device(a))
+#define nvkm_bar(a) nouveau_bar(nvkm_device(a))
+#define nvkm_gpio(a) nouveau_gpio(nvkm_device(a))
+#define nvkm_clock(a) nouveau_clock(nvkm_device(a))
+#define nvkm_i2c(a) nouveau_i2c(nvkm_device(a))
+#define nvkm_timer(a) nouveau_timer(nvkm_device(a))
+#define nvkm_wait(a,b,c,d) nv_wait(nvkm_timer(a), (b), (c), (d))
+#define nvkm_wait_cb(a,b,c) nv_wait_cb(nvkm_timer(a), (b), (c))
+#define nvkm_therm(a) nouveau_therm(nvkm_device(a))
+
+#include <engine/device.h>
+#include <engine/fifo.h>
+#include <engine/graph.h>
+#include <engine/software.h>
+
+#define nvkm_fifo(a) nouveau_fifo(nvkm_device(a))
+#define nvkm_fifo_chan(a) ((struct nouveau_fifo_chan *)nvkm_object(a))
+#define nvkm_gr(a) ((struct nouveau_graph *)nouveau_engine(nvkm_object(a), NVDEV_ENGINE_GR))
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvif/driver.h b/drivers/gpu/drm/nouveau/nvif/driver.h
new file mode 100644
index 000000000000..b72a8f0c2758
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/driver.h
@@ -0,0 +1,21 @@
+#ifndef __NVIF_DRIVER_H__
+#define __NVIF_DRIVER_H__
+
+struct nvif_driver {
+        const char *name;
+        int (*init)(const char *name, u64 device, const char *cfg,
+                    const char *dbg, void **priv);
+        void (*fini)(void *priv);
+        int (*suspend)(void *priv);
+        int (*resume)(void *priv);
+        int (*ioctl)(void *priv, bool super, void *data, u32 size, void **hack);
+        void *(*map)(void *priv, u64 handle, u32 size);
+        void (*unmap)(void *priv, void *ptr, u32 size);
+        bool keep;
+};
+
+extern const struct nvif_driver nvif_driver_nvkm;
+extern const struct nvif_driver nvif_driver_drm;
+extern const struct nvif_driver nvif_driver_lib;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvif/event.h b/drivers/gpu/drm/nouveau/nvif/event.h
new file mode 100644
index 000000000000..21764499b4be
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/event.h
@@ -0,0 +1,62 @@
+#ifndef __NVIF_EVENT_H__
+#define __NVIF_EVENT_H__
+
+struct nvif_notify_req_v0 {
+        __u8  version;
+        __u8  reply;
+        __u8  pad02[5];
+#define NVIF_NOTIFY_V0_ROUTE_NVIF                                          0x00
+        __u8  route;
+        __u64 token;    /* must be unique */
+        __u8  data[];   /* request data (below) */
+};
+
+struct nvif_notify_rep_v0 {
+        __u8  version;
+        __u8  pad01[6];
+        __u8  route;
+        __u64 token;
+        __u8  data[];   /* reply data (below) */
+};
+
+struct nvif_notify_head_req_v0 {
+        /* nvif_notify_req ... */
+        __u8  version;
+        __u8  head;
+        __u8  pad02[6];
+};
+
+struct nvif_notify_head_rep_v0 {
+        /* nvif_notify_rep ... */
+        __u8  version;
+        __u8  pad01[7];
+};
+
+struct nvif_notify_conn_req_v0 {
+        /* nvif_notify_req ... */
+        __u8  version;
+#define NVIF_NOTIFY_CONN_V0_PLUG                                           0x01
+#define NVIF_NOTIFY_CONN_V0_UNPLUG                                         0x02
+#define NVIF_NOTIFY_CONN_V0_IRQ                                            0x04
+#define NVIF_NOTIFY_CONN_V0_ANY                                            0x07
+        __u8  mask;
+        __u8  conn;
+        __u8  pad03[5];
+};
+
+struct nvif_notify_conn_rep_v0 {
+        /* nvif_notify_rep ... */
+        __u8  version;
+        __u8  mask;
+        __u8  pad02[6];
+};
+
+struct nvif_notify_uevent_req {
+        /* nvif_notify_req ... */
+};
+
+struct nvif_notify_uevent_rep {
+        /* nvif_notify_rep ... */
+};
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvif/ioctl.h b/drivers/gpu/drm/nouveau/nvif/ioctl.h
new file mode 100644
index 000000000000..4cd8e323b23d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/ioctl.h
@@ -0,0 +1,128 @@
+#ifndef __NVIF_IOCTL_H__
+#define __NVIF_IOCTL_H__
+
+struct nvif_ioctl_v0 {
+        __u8  version;
+#define NVIF_IOCTL_V0_OWNER_NVIF                                           0x00
+#define NVIF_IOCTL_V0_OWNER_ANY                                            0xff
+        __u8  owner;
+#define NVIF_IOCTL_V0_NOP                                                  0x00
+#define NVIF_IOCTL_V0_SCLASS                                               0x01
+#define NVIF_IOCTL_V0_NEW                                                  0x02
+#define NVIF_IOCTL_V0_DEL                                                  0x03
+#define NVIF_IOCTL_V0_MTHD                                                 0x04
+#define NVIF_IOCTL_V0_RD                                                   0x05
+#define NVIF_IOCTL_V0_WR                                                   0x06
+#define NVIF_IOCTL_V0_MAP                                                  0x07
+#define NVIF_IOCTL_V0_UNMAP                                                0x08
+#define NVIF_IOCTL_V0_NTFY_NEW                                             0x09
+#define NVIF_IOCTL_V0_NTFY_DEL                                             0x0a
+#define NVIF_IOCTL_V0_NTFY_GET                                             0x0b
+#define NVIF_IOCTL_V0_NTFY_PUT                                             0x0c
+        __u8  type;
+        __u8  path_nr;
+#define NVIF_IOCTL_V0_ROUTE_NVIF                                           0x00
+#define NVIF_IOCTL_V0_ROUTE_HIDDEN                                         0xff
+        __u8  pad04[3];
+        __u8  route;
+        __u64 token;
+        __u32 path[8];          /* in reverse */
+        __u8  data[];           /* ioctl data (below) */
+};
+
+struct nvif_ioctl_nop {
+};
+
+struct nvif_ioctl_sclass_v0 {
+        /* nvif_ioctl ... */
+        __u8  version;
+        __u8  count;
+        __u8  pad02[6];
+        __u32 oclass[];
+};
+
+struct nvif_ioctl_new_v0 {
+        /* nvif_ioctl ... */
+        __u8  version;
+        __u8  pad01[6];
+        __u8  route;
+        __u64 token;
+        __u32 handle;
+/* these class numbers are made up by us, and not nvidia-assigned */
+#define NVIF_IOCTL_NEW_V0_PERFCTR                                    0x0000ffff
+#define NVIF_IOCTL_NEW_V0_CONTROL                                    0x0000fffe
+        __u32 oclass;
+        __u8  data[];           /* class data (class.h) */
+};
+
+struct nvif_ioctl_del {
+};
+
+struct nvif_ioctl_rd_v0 {
+        /* nvif_ioctl ... */
+        __u8  version;
+        __u8  size;
+        __u8  pad02[2];
+        __u32 data;
+        __u64 addr;
+};
+
+struct nvif_ioctl_wr_v0 {
+        /* nvif_ioctl ... */
+        __u8  version;
+        __u8  size;
+        __u8  pad02[2];
+        __u32 data;
+        __u64 addr;
+};
+
+struct nvif_ioctl_map_v0 {
+        /* nvif_ioctl ... */
+        __u8  version;
+        __u8  pad01[3];
+        __u32 length;
+        __u64 handle;
+};
+
+struct nvif_ioctl_unmap {
+};
+
+struct nvif_ioctl_ntfy_new_v0 {
+        /* nvif_ioctl ... */
+        __u8  version;
+        __u8  event;
+        __u8  index;
+        __u8  pad03[5];
+        __u8  data[];           /* event request data (event.h) */
+};
+
+struct nvif_ioctl_ntfy_del_v0 {
+        /* nvif_ioctl ... */
+        __u8  version;
+        __u8  index;
+        __u8  pad02[6];
+};
+
+struct nvif_ioctl_ntfy_get_v0 {
+        /* nvif_ioctl ... */
+        __u8  version;
+        __u8  index;
+        __u8  pad02[6];
+};
+
+struct nvif_ioctl_ntfy_put_v0 {
+        /* nvif_ioctl ... */
+        __u8  version;
+        __u8  index;
+        __u8  pad02[6];
+};
+
+struct nvif_ioctl_mthd_v0 {
+        /* nvif_ioctl ... */
+        __u8  version;
+        __u8  method;
+        __u8  pad02[6];
+        __u8  data[];           /* method data (class.h) */
+};
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvif/list.h b/drivers/gpu/drm/nouveau/nvif/list.h
new file mode 100644
index 000000000000..8af5d144ecb0
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/list.h
@@ -0,0 +1,353 @@
+/*
+ * Copyright © 2010 Intel Corporation
+ * Copyright © 2010 Francisco Jerez <currojerez@riseup.net>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+/* Modified by Ben Skeggs <bskeggs@redhat.com> to match kernel list APIs */
+
+#ifndef _XORG_LIST_H_
+#define _XORG_LIST_H_
+
+/**
+ * @file Classic doubly-link circular list implementation.
+ * For real usage examples of the linked list, see the file test/list.c
+ *
+ * Example:
+ * We need to keep a list of struct foo in the parent struct bar, i.e. what
+ * we want is something like this.
+ *
+ *     struct bar {
+ *          ...
+ *          struct foo *list_of_foos; -----> struct foo {}, struct foo {}, struct foo{}
+ *          ...
+ *     }
+ *
+ * We need one list head in bar and a list element in all list_of_foos (both are of
+ * data type 'struct list_head').
+ *
+ *     struct bar {
+ *          ...
+ *          struct list_head list_of_foos;
+ *          ...
+ *     }
+ *
+ *     struct foo {
+ *          ...
+ *          struct list_head entry;
+ *          ...
+ *     }
+ *
+ * Now we initialize the list head:
+ *
+ *     struct bar bar;
+ *     ...
+ *     INIT_LIST_HEAD(&bar.list_of_foos);
+ *
+ * Then we create the first element and add it to this list:
+ *
+ *     struct foo *foo = malloc(...);
+ *     ....
+ *     list_add(&foo->entry, &bar.list_of_foos);
+ *
+ * Repeat the above for each element you want to add to the list. Deleting
+ * works with the element itself.
+ *      list_del(&foo->entry);
+ *      free(foo);
+ *
+ * Note: calling list_del(&bar.list_of_foos) will set bar.list_of_foos to an empty
+ * list again.
+ *
+ * Looping through the list requires a 'struct foo' as iterator and the
+ * name of the field the subnodes use.
+ *
+ * struct foo *iterator;
+ * list_for_each_entry(iterator, &bar.list_of_foos, entry) {
+ *      if (iterator->something == ...)
+ *             ...
+ * }
+ *
+ * Note: You must not call list_del() on the iterator if you continue the
+ * loop. You need to run the safe for-each loop instead:
+ *
+ * struct foo *iterator, *next;
+ * list_for_each_entry_safe(iterator, next, &bar.list_of_foos, entry) {
+ *      if (...)
+ *              list_del(&iterator->entry);
+ * }
+ *
+ */
+
+/**
+ * The linkage struct for list nodes. This struct must be part of your
+ * to-be-linked struct. struct list_head is required for both the head of the
+ * list and for each list node.
+ *
+ * Position and name of the struct list_head field is irrelevant.
+ * There are no requirements that elements of a list are of the same type.
+ * There are no requirements for a list head, any struct list_head can be a list
+ * head.
+ */
+struct list_head {
+    struct list_head *next, *prev;
+};
+
+/**
+ * Initialize the list as an empty list.
+ *
+ * Example:
+ * INIT_LIST_HEAD(&bar->list_of_foos);
+ *
+ * @param The list to initialized.
+ */
+#define LIST_HEAD_INIT(name) { &(name), &(name) }
+
+#define LIST_HEAD(name) \
+        struct list_head name = LIST_HEAD_INIT(name)
+
+static inline void
+INIT_LIST_HEAD(struct list_head *list)
+{
+    list->next = list->prev = list;
+}
+
+static inline void
+__list_add(struct list_head *entry,
+                struct list_head *prev, struct list_head *next)
+{
+    next->prev = entry;
+    entry->next = next;
+    entry->prev = prev;
+    prev->next = entry;
+}
+
+/**
+ * Insert a new element after the given list head. The new element does not
+ * need to be initialised as empty list.
+ * The list changes from:
+ *      head → some element → ...
+ * to
+ *      head → new element → older element → ...
+ *
+ * Example:
+ * struct foo *newfoo = malloc(...);
+ * list_add(&newfoo->entry, &bar->list_of_foos);
+ *
+ * @param entry The new element to prepend to the list.
+ * @param head The existing list.
+ */
+static inline void
+list_add(struct list_head *entry, struct list_head *head)
+{
+    __list_add(entry, head, head->next);
+}
+
+/**
+ * Append a new element to the end of the list given with this list head.
+ *
+ * The list changes from:
+ *      head → some element → ... → lastelement
+ * to
+ *      head → some element → ... → lastelement → new element
+ *
+ * Example:
+ * struct foo *newfoo = malloc(...);
+ * list_add_tail(&newfoo->entry, &bar->list_of_foos);
+ *
+ * @param entry The new element to prepend to the list.
+ * @param head The existing list.
+ */
+static inline void
+list_add_tail(struct list_head *entry, struct list_head *head)
+{
+    __list_add(entry, head->prev, head);
+}
+
+static inline void
+__list_del(struct list_head *prev, struct list_head *next)
+{
+    next->prev = prev;
+    prev->next = next;
+}
+
+/**
+ * Remove the element from the list it is in. Using this function will reset
+ * the pointers to/from this element so it is removed from the list. It does
+ * NOT free the element itself or manipulate it otherwise.
+ *
+ * Using list_del on a pure list head (like in the example at the top of
+ * this file) will NOT remove the first element from
+ * the list but rather reset the list as empty list.
+ *
+ * Example:
+ * list_del(&foo->entry);
+ *
+ * @param entry The element to remove.
+ */
+static inline void
+list_del(struct list_head *entry)
+{
+    __list_del(entry->prev, entry->next);
+}
+
+static inline void
+list_del_init(struct list_head *entry)
+{
+    __list_del(entry->prev, entry->next);
+    INIT_LIST_HEAD(entry);
+}
+
+static inline void list_move_tail(struct list_head *list,
+                                  struct list_head *head)
+{
+        __list_del(list->prev, list->next);
+        list_add_tail(list, head);
+}
+
+/**
+ * Check if the list is empty.
+ *
+ * Example:
+ * list_empty(&bar->list_of_foos);
+ *
+ * @return True if the list contains one or more elements or False otherwise.
+ */
+static inline bool
+list_empty(struct list_head *head)
+{
+    return head->next == head;
+}
+
+/**
+ * Returns a pointer to the container of this list element.
+ *
+ * Example:
+ * struct foo* f;
+ * f = container_of(&foo->entry, struct foo, entry);
+ * assert(f == foo);
+ *
+ * @param ptr Pointer to the struct list_head.
+ * @param type Data type of the list element.
+ * @param member Member name of the struct list_head field in the list element.
+ * @return A pointer to the data struct containing the list head.
+ */
+#ifndef container_of
+#define container_of(ptr, type, member) \
+    (type *)((char *)(ptr) - (char *) &((type *)0)->member)
+#endif
+
+/**
+ * Alias of container_of
+ */
+#define list_entry(ptr, type, member) \
+    container_of(ptr, type, member)
+
+/**
+ * Retrieve the first list entry for the given list pointer.
+ *
+ * Example:
+ * struct foo *first;
+ * first = list_first_entry(&bar->list_of_foos, struct foo, list_of_foos);
+ *
+ * @param ptr The list head
+ * @param type Data type of the list element to retrieve
+ * @param member Member name of the struct list_head field in the list element.
+ * @return A pointer to the first list element.
+ */
+#define list_first_entry(ptr, type, member) \
+    list_entry((ptr)->next, type, member)
+
+/**
+ * Retrieve the last list entry for the given listpointer.
+ *
+ * Example:
+ * struct foo *first;
+ * first = list_last_entry(&bar->list_of_foos, struct foo, list_of_foos);
+ *
+ * @param ptr The list head
+ * @param type Data type of the list element to retrieve
+ * @param member Member name of the struct list_head field in the list element.
+ * @return A pointer to the last list element.
+ */
+#define list_last_entry(ptr, type, member) \
+    list_entry((ptr)->prev, type, member)
+
+#define __container_of(ptr, sample, member)                             \
+    (void *)container_of((ptr), typeof(*(sample)), member)
+
+/**
+ * Loop through the list given by head and set pos to struct in the list.
+ *
+ * Example:
+ * struct foo *iterator;
+ * list_for_each_entry(iterator, &bar->list_of_foos, entry) {
+ *      [modify iterator]
+ * }
+ *
+ * This macro is not safe for node deletion. Use list_for_each_entry_safe
+ * instead.
+ *
+ * @param pos Iterator variable of the type of the list elements.
+ * @param head List head
+ * @param member Member name of the struct list_head in the list elements.
+ *
+ */
+#define list_for_each_entry(pos, head, member)                          \
+    for (pos = __container_of((head)->next, pos, member);               \
+         &pos->member != (head);                                        \
+         pos = __container_of(pos->member.next, pos, member))
+
+/**
+ * Loop through the list, keeping a backup pointer to the element. This
+ * macro allows for the deletion of a list element while looping through the
+ * list.
+ *
+ * See list_for_each_entry for more details.
+ */
+#define list_for_each_entry_safe(pos, tmp, head, member)                \
+    for (pos = __container_of((head)->next, pos, member),               \
+         tmp = __container_of(pos->member.next, pos, member);           \
+         &pos->member != (head);                                        \
+         pos = tmp, tmp = __container_of(pos->member.next, tmp, member))
+
+
+#define list_for_each_entry_reverse(pos, head, member)                  \
+        for (pos = __container_of((head)->prev, pos, member);           \
+             &pos->member != (head);                                    \
+             pos = __container_of(pos->member.prev, pos, member))
+
+#define list_for_each_entry_continue(pos, head, member)                 \
+        for (pos = __container_of(pos->member.next, pos, member);       \
+             &pos->member != (head);                                    \
+             pos = __container_of(pos->member.next, pos, member))
+
+#define list_for_each_entry_continue_reverse(pos, head, member)         \
+        for (pos = __container_of(pos->member.prev, pos, member);       \
+             &pos->member != (head);                                    \
+             pos = __container_of(pos->member.prev, pos, member))
+
+#define list_for_each_entry_from(pos, head, member)                     \
+        for (;                                                          \
+             &pos->member != (head);                                    \
+             pos = __container_of(pos->member.next, pos, member))
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvif/notify.c b/drivers/gpu/drm/nouveau/nvif/notify.c
new file mode 100644
index 000000000000..7c06123a559c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/notify.c
@@ -0,0 +1,237 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <nvif/client.h>
+#include <nvif/driver.h>
+#include <nvif/notify.h>
+#include <nvif/object.h>
+#include <nvif/ioctl.h>
+#include <nvif/event.h>
+
+static inline int
+nvif_notify_put_(struct nvif_notify *notify)
+{
+        struct nvif_object *object = notify->object;
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_ntfy_put_v0 ntfy;
+        } args = {
+                .ioctl.type = NVIF_IOCTL_V0_NTFY_PUT,
+                .ntfy.index = notify->index,
+        };
+
+        if (atomic_inc_return(&notify->putcnt) != 1)
+                return 0;
+
+        return nvif_object_ioctl(object, &args, sizeof(args), NULL);
+}
+
+int
+nvif_notify_put(struct nvif_notify *notify)
+{
+        if (likely(notify->object) &&
+            test_and_clear_bit(NVIF_NOTIFY_USER, &notify->flags)) {
+                int ret = nvif_notify_put_(notify);
+                if (test_bit(NVIF_NOTIFY_WORK, &notify->flags))
+                        flush_work(&notify->work);
+                return ret;
+        }
+        return 0;
+}
+
+static inline int
+nvif_notify_get_(struct nvif_notify *notify)
+{
+        struct nvif_object *object = notify->object;
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_ntfy_get_v0 ntfy;
+        } args = {
+                .ioctl.type = NVIF_IOCTL_V0_NTFY_GET,
+                .ntfy.index = notify->index,
+        };
+
+        if (atomic_dec_return(&notify->putcnt) != 0)
+                return 0;
+
+        return nvif_object_ioctl(object, &args, sizeof(args), NULL);
+}
+
+int
+nvif_notify_get(struct nvif_notify *notify)
+{
+        if (likely(notify->object) &&
+            !test_and_set_bit(NVIF_NOTIFY_USER, &notify->flags))
+                return nvif_notify_get_(notify);
+        return 0;
+}
+
+static void
+nvif_notify_work(struct work_struct *work)
+{
+        struct nvif_notify *notify = container_of(work, typeof(*notify), work);
+        if (notify->func(notify) == NVIF_NOTIFY_KEEP)
+                nvif_notify_get_(notify);
+}
+
+int
+nvif_notify(const void *header, u32 length, const void *data, u32 size)
+{
+        struct nvif_notify *notify = NULL;
+        const union {
+                struct nvif_notify_rep_v0 v0;
+        } *args = header;
+        int ret = NVIF_NOTIFY_DROP;
+
+        if (length == sizeof(args->v0) && args->v0.version == 0) {
+                if (WARN_ON(args->v0.route))
+                        return NVIF_NOTIFY_DROP;
+                notify = (void *)(unsigned long)args->v0.token;
+        }
+
+        if (!WARN_ON(notify == NULL)) {
+                struct nvif_client *client = nvif_client(notify->object);
+                if (!WARN_ON(notify->size != size)) {
+                        if (test_bit(NVIF_NOTIFY_WORK, &notify->flags)) {
+                                atomic_inc(&notify->putcnt);
+                                memcpy((void *)notify->data, data, size);
+                                schedule_work(&notify->work);
+                                return NVIF_NOTIFY_DROP;
+                        }
+                        notify->data = data;
+                        ret = notify->func(notify);
+                        notify->data = NULL;
+                        if (ret != NVIF_NOTIFY_DROP && client->driver->keep) {
+                                atomic_inc(&notify->putcnt);
+                                nvif_notify_get_(notify);
+                        }
+                }
+        }
+
+        return ret;
+}
+
+int
+nvif_notify_fini(struct nvif_notify *notify)
+{
+        struct nvif_object *object = notify->object;
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_ntfy_del_v0 ntfy;
+        } args = {
+                .ioctl.type = NVIF_IOCTL_V0_NTFY_DEL,
+                .ntfy.index = notify->index,
+        };
+        int ret = nvif_notify_put(notify);
+        if (ret >= 0 && object) {
+                ret = nvif_object_ioctl(object, &args, sizeof(args), NULL);
+                if (ret == 0) {
+                        nvif_object_ref(NULL, &notify->object);
+                        kfree((void *)notify->data);
+                }
+        }
+        return ret;
+}
+
+int
+nvif_notify_init(struct nvif_object *object, void (*dtor)(struct nvif_notify *),
+                 int (*func)(struct nvif_notify *), bool work, u8 event,
+                 void *data, u32 size, u32 reply, struct nvif_notify *notify)
+{
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_ntfy_new_v0 ntfy;
+                struct nvif_notify_req_v0 req;
+        } *args;
+        int ret = -ENOMEM;
+
+        notify->object = NULL;
+        nvif_object_ref(object, &notify->object);
+        notify->flags = 0;
+        atomic_set(&notify->putcnt, 1);
+        notify->dtor = dtor;
+        notify->func = func;
+        notify->data = NULL;
+        notify->size = reply;
+        if (work) {
+                INIT_WORK(&notify->work, nvif_notify_work);
+                set_bit(NVIF_NOTIFY_WORK, &notify->flags);
+                notify->data = kmalloc(notify->size, GFP_KERNEL);
+                if (!notify->data)
+                        goto done;
+        }
+
+        if (!(args = kmalloc(sizeof(*args) + size, GFP_KERNEL)))
+                goto done;
+        args->ioctl.version = 0;
+        args->ioctl.type = NVIF_IOCTL_V0_NTFY_NEW;
+        args->ntfy.version = 0;
+        args->ntfy.event = event;
+        args->req.version = 0;
+        args->req.reply = notify->size;
+        args->req.route = 0;
+        args->req.token = (unsigned long)(void *)notify;
+
+        memcpy(args->req.data, data, size);
+        ret = nvif_object_ioctl(object, args, sizeof(*args) + size, NULL);
+        notify->index = args->ntfy.index;
+        kfree(args);
+done:
+        if (ret)
+                nvif_notify_fini(notify);
+        return ret;
+}
+
+static void
+nvif_notify_del(struct nvif_notify *notify)
+{
+        nvif_notify_fini(notify);
+        kfree(notify);
+}
+
+void
+nvif_notify_ref(struct nvif_notify *notify, struct nvif_notify **pnotify)
+{
+        BUG_ON(notify != NULL);
+        if (*pnotify)
+                (*pnotify)->dtor(*pnotify);
+        *pnotify = notify;
+}
+
+int
+nvif_notify_new(struct nvif_object *object, int (*func)(struct nvif_notify *),
+                bool work, u8 type, void *data, u32 size, u32 reply,
+                struct nvif_notify **pnotify)
+{
+        struct nvif_notify *notify = kzalloc(sizeof(*notify), GFP_KERNEL);
+        if (notify) {
+                int ret = nvif_notify_init(object, nvif_notify_del, func, work,
+                                           type, data, size, reply, notify);
+                if (ret)
+                        kfree(notify);
+                *pnotify = notify;
+                return ret;
+        }
+        return -ENOMEM;
+}
diff --git a/drivers/gpu/drm/nouveau/nvif/notify.h b/drivers/gpu/drm/nouveau/nvif/notify.h
new file mode 100644
index 000000000000..9ebfa3b45e76
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/notify.h
@@ -0,0 +1,39 @@
+#ifndef __NVIF_NOTIFY_H__
+#define __NVIF_NOTIFY_H__
+
+struct nvif_notify {
+        struct nvif_object *object;
+        int index;
+
+#define NVIF_NOTIFY_USER 0
+#define NVIF_NOTIFY_WORK 1
+        unsigned long flags;
+        atomic_t putcnt;
+        void (*dtor)(struct nvif_notify *);
+#define NVIF_NOTIFY_DROP 0
+#define NVIF_NOTIFY_KEEP 1
+        int  (*func)(struct nvif_notify *);
+
+        /* this is const for a *very* good reason - the data might be on the
+         * stack from an irq handler.  if you're not nvif/notify.c then you
+         * should probably think twice before casting it away...
+         */
+        const void *data;
+        u32 size;
+        struct work_struct work;
+};
+
+int  nvif_notify_init(struct nvif_object *, void (*dtor)(struct nvif_notify *),
+                      int (*func)(struct nvif_notify *), bool work, u8 type,
+                      void *data, u32 size, u32 reply, struct nvif_notify *);
+int  nvif_notify_fini(struct nvif_notify *);
+int  nvif_notify_get(struct nvif_notify *);
+int  nvif_notify_put(struct nvif_notify *);
+int  nvif_notify(const void *, u32, const void *, u32);
+
+int  nvif_notify_new(struct nvif_object *, int (*func)(struct nvif_notify *),
+                     bool work, u8 type, void *data, u32 size, u32 reply,
+                     struct nvif_notify **);
+void nvif_notify_ref(struct nvif_notify *, struct nvif_notify **);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvif/object.c b/drivers/gpu/drm/nouveau/nvif/object.c
new file mode 100644
index 000000000000..b0c82206ece2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/object.c
@@ -0,0 +1,302 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include "object.h"
+#include "client.h"
+#include "driver.h"
+#include "ioctl.h"
+
+int
+nvif_object_ioctl(struct nvif_object *object, void *data, u32 size, void **hack)
+{
+        struct nvif_client *client = nvif_client(object);
+        union {
+                struct nvif_ioctl_v0 v0;
+        } *args = data;
+
+        if (size >= sizeof(*args) && args->v0.version == 0) {
+                args->v0.owner = NVIF_IOCTL_V0_OWNER_ANY;
+                args->v0.path_nr = 0;
+                while (args->v0.path_nr < ARRAY_SIZE(args->v0.path)) {
+                        args->v0.path[args->v0.path_nr++] = object->handle;
+                        if (object->parent == object)
+                                break;
+                        object = object->parent;
+                }
+        } else
+                return -ENOSYS;
+
+        return client->driver->ioctl(client->base.priv, client->super, data, size, hack);
+}
+
+int
+nvif_object_sclass(struct nvif_object *object, u32 *oclass, int count)
+{
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_sclass_v0 sclass;
+        } *args;
+        u32 size = count * sizeof(args->sclass.oclass[0]);
+        int ret;
+
+        if (!(args = kmalloc(sizeof(*args) + size, GFP_KERNEL)))
+                return -ENOMEM;
+        args->ioctl.version = 0;
+        args->ioctl.type = NVIF_IOCTL_V0_SCLASS;
+        args->sclass.version = 0;
+        args->sclass.count = count;
+
+        memcpy(args->sclass.oclass, oclass, size);
+        ret = nvif_object_ioctl(object, args, sizeof(*args) + size, NULL);
+        ret = ret ? ret : args->sclass.count;
+        memcpy(oclass, args->sclass.oclass, size);
+        kfree(args);
+        return ret;
+}
+
+u32
+nvif_object_rd(struct nvif_object *object, int size, u64 addr)
+{
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_rd_v0 rd;
+        } args = {
+                .ioctl.type = NVIF_IOCTL_V0_RD,
+                .rd.size = size,
+                .rd.addr = addr,
+        };
+        int ret = nvif_object_ioctl(object, &args, sizeof(args), NULL);
+        if (ret) {
+                /*XXX: warn? */
+                return 0;
+        }
+        return args.rd.data;
+}
+
+void
+nvif_object_wr(struct nvif_object *object, int size, u64 addr, u32 data)
+{
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_wr_v0 wr;
+        } args = {
+                .ioctl.type = NVIF_IOCTL_V0_WR,
+                .wr.size = size,
+                .wr.addr = addr,
+                .wr.data = data,
+        };
+        int ret = nvif_object_ioctl(object, &args, sizeof(args), NULL);
+        if (ret) {
+                /*XXX: warn? */
+        }
+}
+
+int
+nvif_object_mthd(struct nvif_object *object, u32 mthd, void *data, u32 size)
+{
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_mthd_v0 mthd;
+        } *args;
+        u8 stack[128];
+        int ret;
+
+        if (sizeof(*args) + size > sizeof(stack)) {
+                if (!(args = kmalloc(sizeof(*args) + size, GFP_KERNEL)))
+                        return -ENOMEM;
+        } else {
+                args = (void *)stack;
+        }
+        args->ioctl.version = 0;
+        args->ioctl.type = NVIF_IOCTL_V0_MTHD;
+        args->mthd.version = 0;
+        args->mthd.method = mthd;
+
+        memcpy(args->mthd.data, data, size);
+        ret = nvif_object_ioctl(object, args, sizeof(*args) + size, NULL);
+        memcpy(data, args->mthd.data, size);
+        if (args != (void *)stack)
+                kfree(args);
+        return ret;
+}
+
+void
+nvif_object_unmap(struct nvif_object *object)
+{
+        if (object->map.size) {
+                struct nvif_client *client = nvif_client(object);
+                struct {
+                        struct nvif_ioctl_v0 ioctl;
+                        struct nvif_ioctl_unmap unmap;
+                } args = {
+                        .ioctl.type = NVIF_IOCTL_V0_UNMAP,
+                };
+
+                if (object->map.ptr) {
+                        client->driver->unmap(client, object->map.ptr,
+                                                      object->map.size);
+                        object->map.ptr = NULL;
+                }
+
+                nvif_object_ioctl(object, &args, sizeof(args), NULL);
+                object->map.size = 0;
+        }
+}
+
+int
+nvif_object_map(struct nvif_object *object)
+{
+        struct nvif_client *client = nvif_client(object);
+        struct {
+                struct nvif_ioctl_v0 ioctl;
+                struct nvif_ioctl_map_v0 map;
+        } args = {
+                .ioctl.type = NVIF_IOCTL_V0_MAP,
+        };
+        int ret = nvif_object_ioctl(object, &args, sizeof(args), NULL);
+        if (ret == 0) {
+                object->map.size = args.map.length;
+                object->map.ptr = client->driver->map(client, args.map.handle,
+                                                      object->map.size);
+                if (ret = -ENOMEM, object->map.ptr)
+                        return 0;
+                nvif_object_unmap(object);
+        }
+        return ret;
+}
+
+struct ctor {
+        struct nvif_ioctl_v0 ioctl;
+        struct nvif_ioctl_new_v0 new;
+};
+
+void
+nvif_object_fini(struct nvif_object *object)
+{
+        struct ctor *ctor = container_of(object->data, typeof(*ctor), new.data);
+        if (object->parent) {
+                struct {
+                        struct nvif_ioctl_v0 ioctl;
+                        struct nvif_ioctl_del del;
+                } args = {
+                        .ioctl.type = NVIF_IOCTL_V0_DEL,
+                };
+
+                nvif_object_unmap(object);
+                nvif_object_ioctl(object, &args, sizeof(args), NULL);
+                if (object->data) {
+                        object->size = 0;
+                        object->data = NULL;
+                        kfree(ctor);
+                }
+                nvif_object_ref(NULL, &object->parent);
+        }
+}
+
+int
+nvif_object_init(struct nvif_object *parent, void (*dtor)(struct nvif_object *),
+                 u32 handle, u32 oclass, void *data, u32 size,
+                 struct nvif_object *object)
+{
+        struct ctor *ctor;
+        int ret = 0;
+
+        object->parent = NULL;
+        object->object = object;
+        nvif_object_ref(parent, &object->parent);
+        kref_init(&object->refcount);
+        object->handle = handle;
+        object->oclass = oclass;
+        object->data = NULL;
+        object->size = 0;
+        object->dtor = dtor;
+        object->map.ptr = NULL;
+        object->map.size = 0;
+
+        if (object->parent) {
+                if (!(ctor = kmalloc(sizeof(*ctor) + size, GFP_KERNEL))) {
+                        nvif_object_fini(object);
+                        return -ENOMEM;
+                }
+                object->data = ctor->new.data;
+                object->size = size;
+                memcpy(object->data, data, size);
+
+                ctor->ioctl.version = 0;
+                ctor->ioctl.type = NVIF_IOCTL_V0_NEW;
+                ctor->new.version = 0;
+                ctor->new.route = NVIF_IOCTL_V0_ROUTE_NVIF;
+                ctor->new.token = (unsigned long)(void *)object;
+                ctor->new.handle = handle;
+                ctor->new.oclass = oclass;
+
+                ret = nvif_object_ioctl(parent, ctor, sizeof(*ctor) +
+                                        object->size, &object->priv);
+        }
+
+        if (ret)
+                nvif_object_fini(object);
+        return ret;
+}
+
+static void
+nvif_object_del(struct nvif_object *object)
+{
+        nvif_object_fini(object);
+        kfree(object);
+}
+
+int
+nvif_object_new(struct nvif_object *parent, u32 handle, u32 oclass,
+                void *data, u32 size, struct nvif_object **pobject)
+{
+        struct nvif_object *object = kzalloc(sizeof(*object), GFP_KERNEL);
+        if (object) {
+                int ret = nvif_object_init(parent, nvif_object_del, handle,
+                                           oclass, data, size, object);
+                if (ret)
+                        kfree(object);
+                *pobject = object;
+                return ret;
+        }
+        return -ENOMEM;
+}
+
+static void
+nvif_object_put(struct kref *kref)
+{
+        struct nvif_object *object =
+                container_of(kref, typeof(*object), refcount);
+        object->dtor(object);
+}
+
+void
+nvif_object_ref(struct nvif_object *object, struct nvif_object **pobject)
+{
+        if (object)
+                kref_get(&object->refcount);
+        if (*pobject)
+                kref_put(&(*pobject)->refcount, nvif_object_put);
+        *pobject = object;
+}
diff --git a/drivers/gpu/drm/nouveau/nvif/object.h b/drivers/gpu/drm/nouveau/nvif/object.h
new file mode 100644
index 000000000000..fac3a3bbec44
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/object.h
@@ -0,0 +1,75 @@
+#ifndef __NVIF_OBJECT_H__
+#define __NVIF_OBJECT_H__
+
+#include <nvif/os.h>
+
+struct nvif_object {
+        struct nvif_object *parent;
+        struct nvif_object *object; /*XXX: hack for nvif_object() */
+        struct kref refcount;
+        u32 handle;
+        u32 oclass;
+        void *data;
+        u32   size;
+        void *priv; /*XXX: hack */
+        void (*dtor)(struct nvif_object *);
+        struct {
+                void *ptr;
+                u32 size;
+        } map;
+};
+
+int  nvif_object_init(struct nvif_object *, void (*dtor)(struct nvif_object *),
+                      u32 handle, u32 oclass, void *, u32,
+                      struct nvif_object *);
+void nvif_object_fini(struct nvif_object *);
+int  nvif_object_new(struct nvif_object *, u32 handle, u32 oclass,
+                     void *, u32, struct nvif_object **);
+void nvif_object_ref(struct nvif_object *, struct nvif_object **);
+int  nvif_object_ioctl(struct nvif_object *, void *, u32, void **);
+int  nvif_object_sclass(struct nvif_object *, u32 *, int);
+u32  nvif_object_rd(struct nvif_object *, int, u64);
+void nvif_object_wr(struct nvif_object *, int, u64, u32);
+int  nvif_object_mthd(struct nvif_object *, u32, void *, u32);
+int  nvif_object_map(struct nvif_object *);
+void nvif_object_unmap(struct nvif_object *);
+
+#define nvif_object(a) (a)->object
+
+#define ioread8_native ioread8
+#define iowrite8_native iowrite8
+#define nvif_rd(a,b,c) ({                                                      \
+        struct nvif_object *_object = nvif_object(a);                          \
+        u32 _data;                                                             \
+        if (likely(_object->map.ptr))                                          \
+                _data = ioread##b##_native((u8 *)_object->map.ptr + (c));      \
+        else                                                                   \
+                _data = nvif_object_rd(_object, (b) / 8, (c));                 \
+        _data;                                                                 \
+})
+#define nvif_wr(a,b,c,d) ({                                                    \
+        struct nvif_object *_object = nvif_object(a);                          \
+        if (likely(_object->map.ptr))                                          \
+                iowrite##b##_native((d), (u8 *)_object->map.ptr + (c));        \
+        else                                                                   \
+                nvif_object_wr(_object, (b) / 8, (c), (d));                    \
+})
+#define nvif_rd08(a,b) ({ u8  _v = nvif_rd((a), 8, (b)); _v; })
+#define nvif_rd16(a,b) ({ u16 _v = nvif_rd((a), 16, (b)); _v; })
+#define nvif_rd32(a,b) ({ u32 _v = nvif_rd((a), 32, (b)); _v; })
+#define nvif_wr08(a,b,c) nvif_wr((a), 8, (b), (u8)(c))
+#define nvif_wr16(a,b,c) nvif_wr((a), 16, (b), (u16)(c))
+#define nvif_wr32(a,b,c) nvif_wr((a), 32, (b), (u32)(c))
+#define nvif_mask(a,b,c,d) ({                                                  \
+        u32 _v = nvif_rd32(nvif_object(a), (b));                               \
+        nvif_wr32(nvif_object(a), (b), (_v & ~(c)) | (d));                     \
+        _v;                                                                    \
+})
+
+#define nvif_mthd(a,b,c,d) nvif_object_mthd(nvif_object(a), (b), (c), (d))
+
+/*XXX*/
+#include <core/object.h>
+#define nvkm_object(a) ((struct nouveau_object *)nvif_object(a)->priv)
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvif/os.h b/drivers/gpu/drm/nouveau/nvif/os.h
new file mode 120000
index 000000000000..bd744b2cf5cf
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/os.h
@@ -0,0 +1 @@
+../core/os.h
\ No newline at end of file
diff --git a/drivers/gpu/drm/nouveau/nvif/unpack.h b/drivers/gpu/drm/nouveau/nvif/unpack.h
new file mode 100644
index 000000000000..5933188b4a77
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/unpack.h
@@ -0,0 +1,24 @@
+#ifndef __NVIF_UNPACK_H__
+#define __NVIF_UNPACK_H__
+
+#define nvif_unvers(d) ({                                                      \
+        ret = (size == sizeof(d)) ? 0 : -ENOSYS;                               \
+        (ret == 0);                                                            \
+})
+
+#define nvif_unpack(d,vl,vh,m) ({                                              \
+        if ((vl) == 0 || ret == -ENOSYS) {                                     \
+                int _size = sizeof(d);                                         \
+                if (_size <= size && (d).version >= (vl) &&                    \
+                                     (d).version <= (vh)) {                    \
+                        data = (u8 *)data + _size;                             \
+                        size = size - _size;                                   \
+                        ret = ((m) || !size) ? 0 : -E2BIG;                     \
+                } else {                                                       \
+                        ret = -ENOSYS;                                         \
+                }                                                              \
+        }                                                                      \
+        (ret == 0);                                                            \
+})
+
+#endif
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
index ca65df144765..c96db433f8af 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
@@ -848,6 +848,7 @@ static int ttm_dma_pool_get_pages(struct dma_pool *pool,
         if (count) {
                 d_page = list_first_entry(&pool->free_list, struct dma_page, page_list);
                 ttm->pages[index] = d_page->p;
+                ttm_dma->cpu_address[index] = d_page->vaddr;
                 ttm_dma->dma_address[index] = d_page->dma;
                 list_move_tail(&d_page->page_list, &ttm_dma->pages_list);
                 r = 0;
@@ -979,6 +980,7 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
         INIT_LIST_HEAD(&ttm_dma->pages_list);
         for (i = 0; i < ttm->num_pages; i++) {
                 ttm->pages[i] = NULL;
+                ttm_dma->cpu_address[i] = 0;
                 ttm_dma->dma_address[i] = 0;
         }
 
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index 75f319090043..bf080abc86d1 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -55,9 +55,12 @@ static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
 
 static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm)
 {
-        ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, sizeof(void*));
-        ttm->dma_address = drm_calloc_large(ttm->ttm.num_pages,
-                                            sizeof(*ttm->dma_address));
+        ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages,
+                                          sizeof(*ttm->ttm.pages) +
+                                          sizeof(*ttm->dma_address) +
+                                          sizeof(*ttm->cpu_address));
+        ttm->cpu_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages);
+        ttm->dma_address = (void *) (ttm->cpu_address + ttm->ttm.num_pages);
 }
 
 #ifdef CONFIG_X86
@@ -228,7 +231,7 @@ int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev,
 
         INIT_LIST_HEAD(&ttm_dma->pages_list);
         ttm_dma_tt_alloc_page_directory(ttm_dma);
-        if (!ttm->pages || !ttm_dma->dma_address) {
+        if (!ttm->pages) {
                 ttm_tt_destroy(ttm);
                 pr_err("Failed allocating page table\n");
                 return -ENOMEM;
@@ -243,7 +246,7 @@ void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma)
 
         drm_free_large(ttm->pages);
         ttm->pages = NULL;
-        drm_free_large(ttm_dma->dma_address);
+        ttm_dma->cpu_address = NULL;
         ttm_dma->dma_address = NULL;
 }
 EXPORT_SYMBOL(ttm_dma_tt_fini);
diff --git a/drivers/i2c/Kconfig b/drivers/i2c/Kconfig
index 7b7ea320a258..3e3b680dc007 100644
--- a/drivers/i2c/Kconfig
+++ b/drivers/i2c/Kconfig
@@ -2,7 +2,9 @@
 # I2C subsystem configuration
 #
 
-menuconfig I2C
+menu "I2C support"
+
+config I2C
         tristate "I2C support"
         select RT_MUTEXES
         ---help---
@@ -21,6 +23,18 @@ menuconfig I2C
           This I2C support can also be built as a module.  If so, the module
           will be called i2c-core.
 
+config I2C_ACPI
+        bool "I2C ACPI support"
+        select I2C
+        depends on ACPI
+        default y
+        help
+          Say Y here if you want to enable ACPI I2C support. This includes support
+          for automatic enumeration of I2C slave devices and support for ACPI I2C
+          Operation Regions. Operation Regions allow firmware (BIOS) code to
+          access I2C slave devices, such as smart batteries through an I2C host
+          controller driver.
+
 if I2C
 
 config I2C_BOARDINFO
@@ -124,3 +138,5 @@ config I2C_DEBUG_BUS
           on.
 
 endif # I2C
+
+endmenu
diff --git a/drivers/i2c/Makefile b/drivers/i2c/Makefile
index 1722f50f2473..a1f590cbb435 100644
--- a/drivers/i2c/Makefile
+++ b/drivers/i2c/Makefile
@@ -2,8 +2,11 @@
 # Makefile for the i2c core.
 #
 
+i2ccore-y := i2c-core.o
+i2ccore-$(CONFIG_I2C_ACPI)      += i2c-acpi.o
+
 obj-$(CONFIG_I2C_BOARDINFO)     += i2c-boardinfo.o
-obj-$(CONFIG_I2C)               += i2c-core.o
+obj-$(CONFIG_I2C)               += i2ccore.o
 obj-$(CONFIG_I2C_SMBUS)         += i2c-smbus.o
 obj-$(CONFIG_I2C_CHARDEV)       += i2c-dev.o
 obj-$(CONFIG_I2C_MUX)           += i2c-mux.o
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index 9f7d5859cf65..2ac87fa3058d 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -109,6 +109,7 @@ config I2C_I801
             Avoton (SOC)
             Wellsburg (PCH)
             Coleto Creek (PCH)
+            Wildcat Point (PCH)
             Wildcat Point-LP (PCH)
             BayTrail (SOC)
 
@@ -465,9 +466,9 @@ config I2C_EG20T
 config I2C_EXYNOS5
         tristate "Exynos5 high-speed I2C driver"
         depends on ARCH_EXYNOS5 && OF
+        default y
         help
-          Say Y here to include support for high-speed I2C controller in the
-          Exynos5 based Samsung SoCs.
+          High-speed I2C controller on Exynos5 based Samsung SoCs.
 
 config I2C_GPIO
         tristate "GPIO-based bitbanging I2C"
@@ -700,16 +701,6 @@ config I2C_S3C2410
           Say Y here to include support for I2C controller in the
           Samsung SoCs.
 
-config I2C_S6000
-        tristate "S6000 I2C support"
-        depends on XTENSA_VARIANT_S6000
-        help
-          This driver supports the on chip I2C device on the
-          S6000 xtensa processor family.
-
-          To compile this driver as a module, choose M here. The module
-          will be called i2c-s6000.
-
 config I2C_SH7760
         tristate "Renesas SH7760 I2C Controller"
         depends on CPU_SUBTYPE_SH7760
@@ -1018,37 +1009,6 @@ config I2C_CROS_EC_TUNNEL
           connected there. This will work whatever the interface used to
           talk to the EC (SPI, I2C or LPC).
 
-config SCx200_I2C
-        tristate "NatSemi SCx200 I2C using GPIO pins (DEPRECATED)"
-        depends on SCx200_GPIO
-        select I2C_ALGOBIT
-        help
-          Enable the use of two GPIO pins of a SCx200 processor as an I2C bus.
-
-          If you don't know what to do here, say N.
-
-          This support is also available as a module.  If so, the module
-          will be called scx200_i2c.
-
-          This driver is deprecated and will be dropped soon. Use i2c-gpio
-          (or scx200_acb) instead.
-
-config SCx200_I2C_SCL
-        int "GPIO pin used for SCL"
-        depends on SCx200_I2C
-        default "12"
-        help
-          Enter the GPIO pin number used for the SCL signal.  This value can
-          also be specified with a module parameter.
-
-config SCx200_I2C_SDA
-        int "GPIO pin used for SDA"
-        depends on SCx200_I2C
-        default "13"
-        help
-          Enter the GPIO pin number used for the SSA signal.  This value can
-          also be specified with a module parameter.
-
 config SCx200_ACB
         tristate "Geode ACCESS.bus support"
         depends on X86_32 && PCI
diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile
index dd9a7f8e873f..49bf07e5ef4d 100644
--- a/drivers/i2c/busses/Makefile
+++ b/drivers/i2c/busses/Makefile
@@ -68,7 +68,6 @@ obj-$(CONFIG_I2C_QUP)		+= i2c-qup.o
 obj-$(CONFIG_I2C_RIIC)          += i2c-riic.o
 obj-$(CONFIG_I2C_RK3X)          += i2c-rk3x.o
 obj-$(CONFIG_I2C_S3C2410)       += i2c-s3c2410.o
-obj-$(CONFIG_I2C_S6000)         += i2c-s6000.o
 obj-$(CONFIG_I2C_SH7760)        += i2c-sh7760.o
 obj-$(CONFIG_I2C_SH_MOBILE)     += i2c-sh_mobile.o
 obj-$(CONFIG_I2C_SIMTEC)        += i2c-simtec.o
@@ -101,6 +100,5 @@ obj-$(CONFIG_I2C_ELEKTOR)	+= i2c-elektor.o
 obj-$(CONFIG_I2C_PCA_ISA)       += i2c-pca-isa.o
 obj-$(CONFIG_I2C_SIBYTE)        += i2c-sibyte.o
 obj-$(CONFIG_SCx200_ACB)        += scx200_acb.o
-obj-$(CONFIG_SCx200_I2C)        += scx200_i2c.o
 
 ccflags-$(CONFIG_I2C_DEBUG_BUS) := -DDEBUG
diff --git a/drivers/i2c/busses/i2c-at91.c b/drivers/i2c/busses/i2c-at91.c
index e95f9ba96790..79a68999a696 100644
--- a/drivers/i2c/busses/i2c-at91.c
+++ b/drivers/i2c/busses/i2c-at91.c
@@ -210,7 +210,7 @@ static void at91_twi_write_data_dma_callback(void *data)
         struct at91_twi_dev *dev = (struct at91_twi_dev *)data;
 
         dma_unmap_single(dev->dev, sg_dma_address(&dev->dma.sg),
-                         dev->buf_len, DMA_MEM_TO_DEV);
+                         dev->buf_len, DMA_TO_DEVICE);
 
         at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_STOP);
 }
@@ -289,7 +289,7 @@ static void at91_twi_read_data_dma_callback(void *data)
         struct at91_twi_dev *dev = (struct at91_twi_dev *)data;
 
         dma_unmap_single(dev->dev, sg_dma_address(&dev->dma.sg),
-                         dev->buf_len, DMA_DEV_TO_MEM);
+                         dev->buf_len, DMA_FROM_DEVICE);
 
         /* The last two bytes have to be read without using dma */
         dev->buf += dev->buf_len - 2;
@@ -768,7 +768,7 @@ static int at91_twi_probe(struct platform_device *pdev)
         snprintf(dev->adapter.name, sizeof(dev->adapter.name), "AT91");
         i2c_set_adapdata(&dev->adapter, dev);
         dev->adapter.owner = THIS_MODULE;
-        dev->adapter.class = I2C_CLASS_HWMON | I2C_CLASS_DEPRECATED;
+        dev->adapter.class = I2C_CLASS_DEPRECATED;
         dev->adapter.algo = &at91_twi_algorithm;
         dev->adapter.dev.parent = dev->dev;
         dev->adapter.nr = pdev->id;
diff --git a/drivers/i2c/busses/i2c-bcm2835.c b/drivers/i2c/busses/i2c-bcm2835.c
index 214ff9700efe..4b8ecd0b3661 100644
--- a/drivers/i2c/busses/i2c-bcm2835.c
+++ b/drivers/i2c/busses/i2c-bcm2835.c
@@ -277,7 +277,7 @@ static int bcm2835_i2c_probe(struct platform_device *pdev)
         adap = &i2c_dev->adapter;
         i2c_set_adapdata(adap, i2c_dev);
         adap->owner = THIS_MODULE;
-        adap->class = I2C_CLASS_HWMON | I2C_CLASS_DEPRECATED;
+        adap->class = I2C_CLASS_DEPRECATED;
         strlcpy(adap->name, "bcm2835 I2C adapter", sizeof(adap->name));
         adap->algo = &bcm2835_i2c_algo;
         adap->dev.parent = &pdev->dev;
diff --git a/drivers/i2c/busses/i2c-bfin-twi.c b/drivers/i2c/busses/i2c-bfin-twi.c
index 3e271e7558d3..067c1615e968 100644
--- a/drivers/i2c/busses/i2c-bfin-twi.c
+++ b/drivers/i2c/busses/i2c-bfin-twi.c
@@ -648,7 +648,7 @@ static int i2c_bfin_twi_probe(struct platform_device *pdev)
         strlcpy(p_adap->name, pdev->name, sizeof(p_adap->name));
         p_adap->algo = &bfin_twi_algorithm;
         p_adap->algo_data = iface;
-        p_adap->class = I2C_CLASS_HWMON | I2C_CLASS_SPD | I2C_CLASS_DEPRECATED;
+        p_adap->class = I2C_CLASS_DEPRECATED;
         p_adap->dev.parent = &pdev->dev;
         p_adap->timeout = 5 * HZ;
         p_adap->retries = 3;
diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c
index 389bc68c55ad..4d9614719128 100644
--- a/drivers/i2c/busses/i2c-davinci.c
+++ b/drivers/i2c/busses/i2c-davinci.c
@@ -712,7 +712,7 @@ static int davinci_i2c_probe(struct platform_device *pdev)
         adap = &dev->adapter;
         i2c_set_adapdata(adap, dev);
         adap->owner = THIS_MODULE;
-        adap->class = I2C_CLASS_HWMON | I2C_CLASS_DEPRECATED;
+        adap->class = I2C_CLASS_DEPRECATED;
         strlcpy(adap->name, "DaVinci I2C adapter", sizeof(adap->name));
         adap->algo = &i2c_davinci_algo;
         adap->dev.parent = &pdev->dev;
diff --git a/drivers/i2c/busses/i2c-designware-pcidrv.c b/drivers/i2c/busses/i2c-designware-pcidrv.c
index 3356f7ab9f79..d31d313ab4f7 100644
--- a/drivers/i2c/busses/i2c-designware-pcidrv.c
+++ b/drivers/i2c/busses/i2c-designware-pcidrv.c
@@ -188,6 +188,7 @@ static struct  dw_pci_controller  dw_pci_controllers[] = {
                 .scl_sda_cfg = &hsw_config,
         },
 };
+
 static struct i2c_algorithm i2c_dw_algo = {
         .master_xfer    = i2c_dw_xfer,
         .functionality  = i2c_dw_func,
@@ -350,6 +351,14 @@ static const struct pci_device_id i2_designware_pci_ids[] = {
         /* Haswell */
         { PCI_VDEVICE(INTEL, 0x9c61), haswell },
         { PCI_VDEVICE(INTEL, 0x9c62), haswell },
+        /* Braswell / Cherrytrail */
+        { PCI_VDEVICE(INTEL, 0x22C1), baytrail,},
+        { PCI_VDEVICE(INTEL, 0x22C2), baytrail },
+        { PCI_VDEVICE(INTEL, 0x22C3), baytrail },
+        { PCI_VDEVICE(INTEL, 0x22C4), baytrail },
+        { PCI_VDEVICE(INTEL, 0x22C5), baytrail },
+        { PCI_VDEVICE(INTEL, 0x22C6), baytrail },
+        { PCI_VDEVICE(INTEL, 0x22C7), baytrail },
         { 0,}
 };
 MODULE_DEVICE_TABLE(pci, i2_designware_pci_ids);
diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c
index 402ec3970fed..bc8773333155 100644
--- a/drivers/i2c/busses/i2c-designware-platdrv.c
+++ b/drivers/i2c/busses/i2c-designware-platdrv.c
@@ -106,6 +106,7 @@ static const struct acpi_device_id dw_i2c_acpi_match[] = {
         { "INT3432", 0 },
         { "INT3433", 0 },
         { "80860F41", 0 },
+        { "808622C1", 0 },
         { }
 };
 MODULE_DEVICE_TABLE(acpi, dw_i2c_acpi_match);
@@ -202,7 +203,7 @@ static int dw_i2c_probe(struct platform_device *pdev)
         adap = &dev->adapter;
         i2c_set_adapdata(adap, dev);
         adap->owner = THIS_MODULE;
-        adap->class = I2C_CLASS_HWMON | I2C_CLASS_DEPRECATED;
+        adap->class = I2C_CLASS_DEPRECATED;
         strlcpy(adap->name, "Synopsys DesignWare I2C adapter",
                         sizeof(adap->name));
         adap->algo = &i2c_dw_algo;
diff --git a/drivers/i2c/busses/i2c-efm32.c b/drivers/i2c/busses/i2c-efm32.c
index f7eccd682de9..10b8323b08d4 100644
--- a/drivers/i2c/busses/i2c-efm32.c
+++ b/drivers/i2c/busses/i2c-efm32.c
@@ -370,7 +370,13 @@ static int efm32_i2c_probe(struct platform_device *pdev)
                 return ret;
         }
 
-        ret = of_property_read_u32(np, "efm32,location", &location);
+
+        ret = of_property_read_u32(np, "energymicro,location", &location);
+
+        if (ret)
+                /* fall back to wrongly namespaced property */
+                ret = of_property_read_u32(np, "efm32,location", &location);
+
         if (!ret) {
                 dev_dbg(&pdev->dev, "using location %u\n", location);
         } else {
diff --git a/drivers/i2c/busses/i2c-exynos5.c b/drivers/i2c/busses/i2c-exynos5.c
index 63d229202854..28073f1d6d47 100644
--- a/drivers/i2c/busses/i2c-exynos5.c
+++ b/drivers/i2c/busses/i2c-exynos5.c
@@ -405,7 +405,6 @@ static irqreturn_t exynos5_i2c_irq(int irqno, void *dev_id)
 
         int_status = readl(i2c->regs + HSI2C_INT_STATUS);
         writel(int_status, i2c->regs + HSI2C_INT_STATUS);
-        fifo_status = readl(i2c->regs + HSI2C_FIFO_STATUS);
 
         /* handle interrupt related to the transfer status */
         if (int_status & HSI2C_INT_I2C) {
@@ -526,7 +525,7 @@ static void exynos5_i2c_message_start(struct exynos5_i2c *i2c, int stop)
         if (i2c->msg->flags & I2C_M_RD) {
                 i2c_ctl |= HSI2C_RXCHON;
 
-                i2c_auto_conf = HSI2C_READ_WRITE;
+                i2c_auto_conf |= HSI2C_READ_WRITE;
 
                 trig_lvl = (i2c->msg->len > i2c->variant->fifo_depth) ?
                         (i2c->variant->fifo_depth * 3 / 4) : i2c->msg->len;
@@ -549,7 +548,6 @@ static void exynos5_i2c_message_start(struct exynos5_i2c *i2c, int stop)
         writel(fifo_ctl, i2c->regs + HSI2C_FIFO_CTL);
         writel(i2c_ctl, i2c->regs + HSI2C_CTL);
 
-
         /*
          * Enable interrupts before starting the transfer so that we don't
          * miss any INT_I2C interrupts.
@@ -789,8 +787,16 @@ static int exynos5_i2c_resume_noirq(struct device *dev)
 }
 #endif
 
-static SIMPLE_DEV_PM_OPS(exynos5_i2c_dev_pm_ops, exynos5_i2c_suspend_noirq,
-                         exynos5_i2c_resume_noirq);
+static const struct dev_pm_ops exynos5_i2c_dev_pm_ops = {
+#ifdef CONFIG_PM_SLEEP
+        .suspend_noirq = exynos5_i2c_suspend_noirq,
+        .resume_noirq = exynos5_i2c_resume_noirq,
+        .freeze_noirq = exynos5_i2c_suspend_noirq,
+        .thaw_noirq = exynos5_i2c_resume_noirq,
+        .poweroff_noirq = exynos5_i2c_suspend_noirq,
+        .restore_noirq = exynos5_i2c_resume_noirq,
+#endif
+};
 
 static struct platform_driver exynos5_i2c_driver = {
         .probe          = exynos5_i2c_probe,
diff --git a/drivers/i2c/busses/i2c-gpio.c b/drivers/i2c/busses/i2c-gpio.c
index 71a45b210a24..933f1e453e41 100644
--- a/drivers/i2c/busses/i2c-gpio.c
+++ b/drivers/i2c/busses/i2c-gpio.c
@@ -238,12 +238,10 @@ static int i2c_gpio_probe(struct platform_device *pdev)
 static int i2c_gpio_remove(struct platform_device *pdev)
 {
         struct i2c_gpio_private_data *priv;
-        struct i2c_gpio_platform_data *pdata;
         struct i2c_adapter *adap;
 
         priv = platform_get_drvdata(pdev);
         adap = &priv->adap;
-        pdata = &priv->pdata;
 
         i2c_del_adapter(adap);
 
diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c
index 6777cd6f8776..2994690b26e9 100644
--- a/drivers/i2c/busses/i2c-i801.c
+++ b/drivers/i2c/busses/i2c-i801.c
@@ -22,57 +22,58 @@
 */
 
 /*
-  Supports the following Intel I/O Controller Hubs (ICH):
-
-                                  I/O                     Block   I2C
-                                  region  SMBus   Block   proc.   block
-  Chip name             PCI ID    size    PEC     buffer  call    read
-  ----------------------------------------------------------------------
-  82801AA  (ICH)        0x2413     16      no      no      no      no
-  82801AB  (ICH0)       0x2423     16      no      no      no      no
-  82801BA  (ICH2)       0x2443     16      no      no      no      no
-  82801CA  (ICH3)       0x2483     32     soft     no      no      no
-  82801DB  (ICH4)       0x24c3     32     hard     yes     no      no
-  82801E   (ICH5)       0x24d3     32     hard     yes     yes     yes
-  6300ESB               0x25a4     32     hard     yes     yes     yes
-  82801F   (ICH6)       0x266a     32     hard     yes     yes     yes
-  6310ESB/6320ESB       0x269b     32     hard     yes     yes     yes
-  82801G   (ICH7)       0x27da     32     hard     yes     yes     yes
-  82801H   (ICH8)       0x283e     32     hard     yes     yes     yes
-  82801I   (ICH9)       0x2930     32     hard     yes     yes     yes
-  EP80579 (Tolapai)     0x5032     32     hard     yes     yes     yes
-  ICH10                 0x3a30     32     hard     yes     yes     yes
-  ICH10                 0x3a60     32     hard     yes     yes     yes
-  5/3400 Series (PCH)   0x3b30     32     hard     yes     yes     yes
-  6 Series (PCH)        0x1c22     32     hard     yes     yes     yes
-  Patsburg (PCH)        0x1d22     32     hard     yes     yes     yes
-  Patsburg (PCH) IDF    0x1d70     32     hard     yes     yes     yes
-  Patsburg (PCH) IDF    0x1d71     32     hard     yes     yes     yes
-  Patsburg (PCH) IDF    0x1d72     32     hard     yes     yes     yes
-  DH89xxCC (PCH)        0x2330     32     hard     yes     yes     yes
-  Panther Point (PCH)   0x1e22     32     hard     yes     yes     yes
-  Lynx Point (PCH)      0x8c22     32     hard     yes     yes     yes
-  Lynx Point-LP (PCH)   0x9c22     32     hard     yes     yes     yes
-  Avoton (SOC)          0x1f3c     32     hard     yes     yes     yes
-  Wellsburg (PCH)       0x8d22     32     hard     yes     yes     yes
-  Wellsburg (PCH) MS    0x8d7d     32     hard     yes     yes     yes
-  Wellsburg (PCH) MS    0x8d7e     32     hard     yes     yes     yes
-  Wellsburg (PCH) MS    0x8d7f     32     hard     yes     yes     yes
-  Coleto Creek (PCH)    0x23b0     32     hard     yes     yes     yes
-  Wildcat Point-LP (PCH)   0x9ca2     32     hard     yes     yes     yes
-  BayTrail (SOC)        0x0f12     32     hard     yes     yes     yes
-
-  Features supported by this driver:
-  Software PEC                     no
-  Hardware PEC                     yes
-  Block buffer                     yes
-  Block process call transaction   no
-  I2C block read transaction       yes  (doesn't use the block buffer)
-  Slave mode                       no
-  Interrupt processing             yes
-
-  See the file Documentation/i2c/busses/i2c-i801 for details.
-*/
+ * Supports the following Intel I/O Controller Hubs (ICH):
+ *
+ *                                      I/O                     Block   I2C
+ *                                      region  SMBus   Block   proc.   block
+ * Chip name                    PCI ID  size    PEC     buffer  call    read
+ * ---------------------------------------------------------------------------
+ * 82801AA (ICH)                0x2413  16      no      no      no      no
+ * 82801AB (ICH0)               0x2423  16      no      no      no      no
+ * 82801BA (ICH2)               0x2443  16      no      no      no      no
+ * 82801CA (ICH3)               0x2483  32      soft    no      no      no
+ * 82801DB (ICH4)               0x24c3  32      hard    yes     no      no
+ * 82801E (ICH5)                0x24d3  32      hard    yes     yes     yes
+ * 6300ESB                      0x25a4  32      hard    yes     yes     yes
+ * 82801F (ICH6)                0x266a  32      hard    yes     yes     yes
+ * 6310ESB/6320ESB              0x269b  32      hard    yes     yes     yes
+ * 82801G (ICH7)                0x27da  32      hard    yes     yes     yes
+ * 82801H (ICH8)                0x283e  32      hard    yes     yes     yes
+ * 82801I (ICH9)                0x2930  32      hard    yes     yes     yes
+ * EP80579 (Tolapai)            0x5032  32      hard    yes     yes     yes
+ * ICH10                        0x3a30  32      hard    yes     yes     yes
+ * ICH10                        0x3a60  32      hard    yes     yes     yes
+ * 5/3400 Series (PCH)          0x3b30  32      hard    yes     yes     yes
+ * 6 Series (PCH)               0x1c22  32      hard    yes     yes     yes
+ * Patsburg (PCH)               0x1d22  32      hard    yes     yes     yes
+ * Patsburg (PCH) IDF           0x1d70  32      hard    yes     yes     yes
+ * Patsburg (PCH) IDF           0x1d71  32      hard    yes     yes     yes
+ * Patsburg (PCH) IDF           0x1d72  32      hard    yes     yes     yes
+ * DH89xxCC (PCH)               0x2330  32      hard    yes     yes     yes
+ * Panther Point (PCH)          0x1e22  32      hard    yes     yes     yes
+ * Lynx Point (PCH)             0x8c22  32      hard    yes     yes     yes
+ * Lynx Point-LP (PCH)          0x9c22  32      hard    yes     yes     yes
+ * Avoton (SOC)                 0x1f3c  32      hard    yes     yes     yes
+ * Wellsburg (PCH)              0x8d22  32      hard    yes     yes     yes
+ * Wellsburg (PCH) MS           0x8d7d  32      hard    yes     yes     yes
+ * Wellsburg (PCH) MS           0x8d7e  32      hard    yes     yes     yes
+ * Wellsburg (PCH) MS           0x8d7f  32      hard    yes     yes     yes
+ * Coleto Creek (PCH)           0x23b0  32      hard    yes     yes     yes
+ * Wildcat Point (PCH)          0x8ca2  32      hard    yes     yes     yes
+ * Wildcat Point-LP (PCH)       0x9ca2  32      hard    yes     yes     yes
+ * BayTrail (SOC)               0x0f12  32      hard    yes     yes     yes
+ *
+ * Features supported by this driver:
+ * Software PEC                         no
+ * Hardware PEC                         yes
+ * Block buffer                         yes
+ * Block process call transaction       no
+ * I2C block read transaction           yes (doesn't use the block buffer)
+ * Slave mode                           no
+ * Interrupt processing                 yes
+ *
+ * See the file Documentation/i2c/busses/i2c-i801 for details.
+ */
 
 #include <linux/interrupt.h>
 #include <linux/module.h>
@@ -162,24 +163,25 @@
                                  STATUS_ERROR_FLAGS)
 
 /* Older devices have their ID defined in <linux/pci_ids.h> */
-#define PCI_DEVICE_ID_INTEL_BAYTRAIL_SMBUS      0x0f12
-#define PCI_DEVICE_ID_INTEL_COUGARPOINT_SMBUS   0x1c22
-#define PCI_DEVICE_ID_INTEL_PATSBURG_SMBUS      0x1d22
+#define PCI_DEVICE_ID_INTEL_BAYTRAIL_SMBUS              0x0f12
+#define PCI_DEVICE_ID_INTEL_COUGARPOINT_SMBUS           0x1c22
+#define PCI_DEVICE_ID_INTEL_PATSBURG_SMBUS              0x1d22
 /* Patsburg also has three 'Integrated Device Function' SMBus controllers */
-#define PCI_DEVICE_ID_INTEL_PATSBURG_SMBUS_IDF0 0x1d70
-#define PCI_DEVICE_ID_INTEL_PATSBURG_SMBUS_IDF1 0x1d71
-#define PCI_DEVICE_ID_INTEL_PATSBURG_SMBUS_IDF2 0x1d72
-#define PCI_DEVICE_ID_INTEL_PANTHERPOINT_SMBUS  0x1e22
-#define PCI_DEVICE_ID_INTEL_AVOTON_SMBUS        0x1f3c
-#define PCI_DEVICE_ID_INTEL_DH89XXCC_SMBUS      0x2330
-#define PCI_DEVICE_ID_INTEL_COLETOCREEK_SMBUS   0x23b0
-#define PCI_DEVICE_ID_INTEL_5_3400_SERIES_SMBUS 0x3b30
-#define PCI_DEVICE_ID_INTEL_LYNXPOINT_SMBUS     0x8c22
-#define PCI_DEVICE_ID_INTEL_WELLSBURG_SMBUS     0x8d22
-#define PCI_DEVICE_ID_INTEL_WELLSBURG_SMBUS_MS0 0x8d7d
-#define PCI_DEVICE_ID_INTEL_WELLSBURG_SMBUS_MS1 0x8d7e
-#define PCI_DEVICE_ID_INTEL_WELLSBURG_SMBUS_MS2 0x8d7f
-#define PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_SMBUS  0x9c22
+#define PCI_DEVICE_ID_INTEL_PATSBURG_SMBUS_IDF0         0x1d70
+#define PCI_DEVICE_ID_INTEL_PATSBURG_SMBUS_IDF1         0x1d71
+#define PCI_DEVICE_ID_INTEL_PATSBURG_SMBUS_IDF2         0x1d72
+#define PCI_DEVICE_ID_INTEL_PANTHERPOINT_SMBUS          0x1e22
+#define PCI_DEVICE_ID_INTEL_AVOTON_SMBUS                0x1f3c
+#define PCI_DEVICE_ID_INTEL_DH89XXCC_SMBUS              0x2330
+#define PCI_DEVICE_ID_INTEL_COLETOCREEK_SMBUS           0x23b0
+#define PCI_DEVICE_ID_INTEL_5_3400_SERIES_SMBUS         0x3b30
+#define PCI_DEVICE_ID_INTEL_LYNXPOINT_SMBUS             0x8c22
+#define PCI_DEVICE_ID_INTEL_WILDCATPOINT_SMBUS          0x8ca2
+#define PCI_DEVICE_ID_INTEL_WELLSBURG_SMBUS             0x8d22
+#define PCI_DEVICE_ID_INTEL_WELLSBURG_SMBUS_MS0         0x8d7d
+#define PCI_DEVICE_ID_INTEL_WELLSBURG_SMBUS_MS1         0x8d7e
+#define PCI_DEVICE_ID_INTEL_WELLSBURG_SMBUS_MS2         0x8d7f
+#define PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_SMBUS          0x9c22
 #define PCI_DEVICE_ID_INTEL_WILDCATPOINT_LP_SMBUS       0x9ca2
 
 struct i801_mux_config {
@@ -823,6 +825,7 @@ static const struct pci_device_id i801_ids[] = {
         { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WELLSBURG_SMBUS_MS1) },
         { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WELLSBURG_SMBUS_MS2) },
         { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_COLETOCREEK_SMBUS) },
+        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WILDCATPOINT_SMBUS) },
         { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WILDCATPOINT_LP_SMBUS) },
         { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BAYTRAIL_SMBUS) },
         { 0, }
diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c
index aa8bc146718b..613069bc561a 100644
--- a/drivers/i2c/busses/i2c-imx.c
+++ b/drivers/i2c/busses/i2c-imx.c
@@ -735,10 +735,7 @@ static int i2c_imx_probe(struct platform_device *pdev)
         clk_disable_unprepare(i2c_imx->clk);
 
         dev_dbg(&i2c_imx->adapter.dev, "claimed irq %d\n", irq);
-        dev_dbg(&i2c_imx->adapter.dev, "device resources from 0x%x to 0x%x\n",
-                res->start, res->end);
-        dev_dbg(&i2c_imx->adapter.dev, "allocated %d bytes at 0x%x\n",
-                resource_size(res), res->start);
+        dev_dbg(&i2c_imx->adapter.dev, "device resources: %pR\n", res);
         dev_dbg(&i2c_imx->adapter.dev, "adapter name: \"%s\"\n",
                 i2c_imx->adapter.name);
         dev_info(&i2c_imx->adapter.dev, "IMX I2C adapter registered\n");
diff --git a/drivers/i2c/busses/i2c-mpc.c b/drivers/i2c/busses/i2c-mpc.c
index 6a32aa095f83..0edf630b099a 100644
--- a/drivers/i2c/busses/i2c-mpc.c
+++ b/drivers/i2c/busses/i2c-mpc.c
@@ -341,8 +341,7 @@ static u32 mpc_i2c_get_sec_cfg_8xxx(void)
                         iounmap(reg);
                 }
         }
-        if (node)
-                of_node_put(node);
+        of_node_put(node);
 
         return val;
 }
diff --git a/drivers/i2c/busses/i2c-mv64xxx.c b/drivers/i2c/busses/i2c-mv64xxx.c
index 9f4b775e2e39..6dc5ded86f62 100644
--- a/drivers/i2c/busses/i2c-mv64xxx.c
+++ b/drivers/i2c/busses/i2c-mv64xxx.c
@@ -863,7 +863,7 @@ mv64xxx_i2c_probe(struct platform_device *pd)
         drv_data->adapter.dev.parent = &pd->dev;
         drv_data->adapter.algo = &mv64xxx_i2c_algo;
         drv_data->adapter.owner = THIS_MODULE;
-        drv_data->adapter.class = I2C_CLASS_HWMON | I2C_CLASS_SPD | I2C_CLASS_DEPRECATED;
+        drv_data->adapter.class = I2C_CLASS_DEPRECATED;
         drv_data->adapter.nr = pd->id;
         drv_data->adapter.dev.of_node = pd->dev.of_node;
         platform_set_drvdata(pd, drv_data);
diff --git a/drivers/i2c/busses/i2c-nomadik.c b/drivers/i2c/busses/i2c-nomadik.c
index 0e55d85fd4ed..9ad038d223c4 100644
--- a/drivers/i2c/busses/i2c-nomadik.c
+++ b/drivers/i2c/busses/i2c-nomadik.c
@@ -1032,10 +1032,10 @@ static int nmk_i2c_probe(struct amba_device *adev, const struct amba_id *id)
         adap = &dev->adap;
         adap->dev.of_node = np;
         adap->dev.parent = &adev->dev;
-        adap->owner     = THIS_MODULE;
-        adap->class     = I2C_CLASS_HWMON | I2C_CLASS_SPD | I2C_CLASS_DEPRECATED;
-        adap->algo      = &nmk_i2c_algo;
-        adap->timeout   = msecs_to_jiffies(dev->timeout);
+        adap->owner = THIS_MODULE;
+        adap->class = I2C_CLASS_DEPRECATED;
+        adap->algo = &nmk_i2c_algo;
+        adap->timeout = msecs_to_jiffies(dev->timeout);
         snprintf(adap->name, sizeof(adap->name),
                  "Nomadik I2C at %pR", &adev->res);
 
diff --git a/drivers/i2c/busses/i2c-ocores.c b/drivers/i2c/busses/i2c-ocores.c
index 0e10cc6182f0..2a4fe0b7cfb7 100644
--- a/drivers/i2c/busses/i2c-ocores.c
+++ b/drivers/i2c/busses/i2c-ocores.c
@@ -239,15 +239,15 @@ static u32 ocores_func(struct i2c_adapter *adap)
 }
 
 static const struct i2c_algorithm ocores_algorithm = {
-        .master_xfer    = ocores_xfer,
-        .functionality  = ocores_func,
+        .master_xfer = ocores_xfer,
+        .functionality = ocores_func,
 };
 
 static struct i2c_adapter ocores_adapter = {
-        .owner          = THIS_MODULE,
-        .name           = "i2c-ocores",
-        .class          = I2C_CLASS_HWMON | I2C_CLASS_SPD | I2C_CLASS_DEPRECATED,
-        .algo           = &ocores_algorithm,
+        .owner = THIS_MODULE,
+        .name = "i2c-ocores",
+        .class = I2C_CLASS_DEPRECATED,
+        .algo = &ocores_algorithm,
 };
 
 static const struct of_device_id ocores_i2c_match[] = {
diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c
index b182793a4051..0dffb0e62c3b 100644
--- a/drivers/i2c/busses/i2c-omap.c
+++ b/drivers/i2c/busses/i2c-omap.c
@@ -1236,7 +1236,7 @@ omap_i2c_probe(struct platform_device *pdev)
         adap = &dev->adapter;
         i2c_set_adapdata(adap, dev);
         adap->owner = THIS_MODULE;
-        adap->class = I2C_CLASS_HWMON | I2C_CLASS_DEPRECATED;
+        adap->class = I2C_CLASS_DEPRECATED;
         strlcpy(adap->name, "OMAP I2C adapter", sizeof(adap->name));
         adap->algo = &omap_i2c_algo;
         adap->dev.parent = &pdev->dev;
diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c
index 2a5efb5b487c..3a4d64e1dfb1 100644
--- a/drivers/i2c/busses/i2c-qup.c
+++ b/drivers/i2c/busses/i2c-qup.c
@@ -633,13 +633,17 @@ static int qup_i2c_probe(struct platform_device *pdev)
          * associated with each byte written/received
          */
         size = QUP_OUTPUT_BLOCK_SIZE(io_mode);
-        if (size >= ARRAY_SIZE(blk_sizes))
-                return -EIO;
+        if (size >= ARRAY_SIZE(blk_sizes)) {
+                ret = -EIO;
+                goto fail;
+        }
         qup->out_blk_sz = blk_sizes[size] / 2;
 
         size = QUP_INPUT_BLOCK_SIZE(io_mode);
-        if (size >= ARRAY_SIZE(blk_sizes))
-                return -EIO;
+        if (size >= ARRAY_SIZE(blk_sizes)) {
+                ret = -EIO;
+                goto fail;
+        }
         qup->in_blk_sz = blk_sizes[size] / 2;
 
         size = QUP_OUTPUT_FIFO_SIZE(io_mode);
diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c
index 899405923678..f3c7139dfa25 100644
--- a/drivers/i2c/busses/i2c-rcar.c
+++ b/drivers/i2c/busses/i2c-rcar.c
@@ -541,13 +541,13 @@ static int rcar_i2c_probe(struct platform_device *pdev)
         irq = platform_get_irq(pdev, 0);
         init_waitqueue_head(&priv->wait);
 
-        adap                    = &priv->adap;
-        adap->nr                = pdev->id;
-        adap->algo              = &rcar_i2c_algo;
-        adap->class             = I2C_CLASS_HWMON | I2C_CLASS_SPD | I2C_CLASS_DEPRECATED;
-        adap->retries           = 3;
-        adap->dev.parent        = dev;
-        adap->dev.of_node       = dev->of_node;
+        adap = &priv->adap;
+        adap->nr = pdev->id;
+        adap->algo = &rcar_i2c_algo;
+        adap->class = I2C_CLASS_DEPRECATED;
+        adap->retries = 3;
+        adap->dev.parent = dev;
+        adap->dev.of_node = dev->of_node;
         i2c_set_adapdata(adap, priv);
         strlcpy(adap->name, pdev->name, sizeof(adap->name));
 
diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c
index a9791509966a..69e11853e8bf 100644
--- a/drivers/i2c/busses/i2c-rk3x.c
+++ b/drivers/i2c/busses/i2c-rk3x.c
@@ -399,7 +399,7 @@ static irqreturn_t rk3x_i2c_irq(int irqno, void *dev_id)
         }
 
         /* is there anything left to handle? */
-        if (unlikely(ipd == 0))
+        if (unlikely((ipd & REG_INT_ALL) == 0))
                 goto out;
 
         switch (i2c->state) {
diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c
index e828a1dba0e5..e086fb075f2b 100644
--- a/drivers/i2c/busses/i2c-s3c2410.c
+++ b/drivers/i2c/busses/i2c-s3c2410.c
@@ -1128,11 +1128,11 @@ static int s3c24xx_i2c_probe(struct platform_device *pdev)
                 s3c24xx_i2c_parse_dt(pdev->dev.of_node, i2c);
 
         strlcpy(i2c->adap.name, "s3c2410-i2c", sizeof(i2c->adap.name));
-        i2c->adap.owner   = THIS_MODULE;
-        i2c->adap.algo    = &s3c24xx_i2c_algorithm;
+        i2c->adap.owner = THIS_MODULE;
+        i2c->adap.algo = &s3c24xx_i2c_algorithm;
         i2c->adap.retries = 2;
-        i2c->adap.class   = I2C_CLASS_HWMON | I2C_CLASS_SPD | I2C_CLASS_DEPRECATED;
-        i2c->tx_setup     = 50;
+        i2c->adap.class = I2C_CLASS_DEPRECATED;
+        i2c->tx_setup = 50;
 
         init_waitqueue_head(&i2c->wait);
 
@@ -1267,7 +1267,7 @@ static int s3c24xx_i2c_suspend_noirq(struct device *dev)
         return 0;
 }
 
-static int s3c24xx_i2c_resume(struct device *dev)
+static int s3c24xx_i2c_resume_noirq(struct device *dev)
 {
         struct platform_device *pdev = to_platform_device(dev);
         struct s3c24xx_i2c *i2c = platform_get_drvdata(pdev);
@@ -1285,7 +1285,11 @@ static int s3c24xx_i2c_resume(struct device *dev)
 static const struct dev_pm_ops s3c24xx_i2c_dev_pm_ops = {
 #ifdef CONFIG_PM_SLEEP
         .suspend_noirq = s3c24xx_i2c_suspend_noirq,
-        .resume = s3c24xx_i2c_resume,
+        .resume_noirq = s3c24xx_i2c_resume_noirq,
+        .freeze_noirq = s3c24xx_i2c_suspend_noirq,
+        .thaw_noirq = s3c24xx_i2c_resume_noirq,
+        .poweroff_noirq = s3c24xx_i2c_suspend_noirq,
+        .restore_noirq = s3c24xx_i2c_resume_noirq,
 #endif
 };
 
diff --git a/drivers/i2c/busses/i2c-s6000.c b/drivers/i2c/busses/i2c-s6000.c
deleted file mode 100644
index dd186a037684..000000000000
--- a/drivers/i2c/busses/i2c-s6000.c
+++ /dev/null
@@ -1,404 +0,0 @@
-/*
- * drivers/i2c/busses/i2c-s6000.c
- *
- * Description: Driver for S6000 Family I2C Interface
- * Copyright (c) 2008 emlix GmbH
- * Author:      Oskar Schirmer <oskar@scara.com>
- *
- * Partially based on i2c-bfin-twi.c driver by <sonic.zhang@analog.com>
- * Copyright (c) 2005-2007 Analog Devices, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <linux/clk.h>
-#include <linux/err.h>
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/delay.h>
-#include <linux/i2c.h>
-#include <linux/i2c/s6000.h>
-#include <linux/timer.h>
-#include <linux/spinlock.h>
-#include <linux/completion.h>
-#include <linux/interrupt.h>
-#include <linux/platform_device.h>
-#include <linux/io.h>
-
-#include "i2c-s6000.h"
-
-#define DRV_NAME "i2c-s6000"
-
-#define POLL_TIMEOUT    (2 * HZ)
-
-struct s6i2c_if {
-        u8 __iomem              *reg; /* memory mapped registers */
-        int                     irq;
-        spinlock_t              lock;
-        struct i2c_msg          *msgs; /* messages currently handled */
-        int                     msgs_num; /* nb of msgs to do */
-        int                     msgs_push; /* nb of msgs read/written */
-        int                     msgs_done; /* nb of msgs finally handled */
-        unsigned                push; /* nb of bytes read/written in msg */
-        unsigned                done; /* nb of bytes finally handled */
-        int                     timeout_count; /* timeout retries left */
-        struct timer_list       timeout_timer;
-        struct i2c_adapter      adap;
-        struct completion       complete;
-        struct clk              *clk;
-        struct resource         *res;
-};
-
-static inline u16 i2c_rd16(struct s6i2c_if *iface, unsigned n)
-{
-        return readw(iface->reg + (n));
-}
-
-static inline void i2c_wr16(struct s6i2c_if *iface, unsigned n, u16 v)
-{
-        writew(v, iface->reg + (n));
-}
-
-static inline u32 i2c_rd32(struct s6i2c_if *iface, unsigned n)
-{
-        return readl(iface->reg + (n));
-}
-
-static inline void i2c_wr32(struct s6i2c_if *iface, unsigned n, u32 v)
-{
-        writel(v, iface->reg + (n));
-}
-
-static struct s6i2c_if s6i2c_if;
-
-static void s6i2c_handle_interrupt(struct s6i2c_if *iface)
-{
-        if (i2c_rd16(iface, S6_I2C_INTRSTAT) & (1 << S6_I2C_INTR_TXABRT)) {
-                i2c_rd16(iface, S6_I2C_CLRTXABRT);
-                i2c_wr16(iface, S6_I2C_INTRMASK, 0);
-                complete(&iface->complete);
-                return;
-        }
-        if (iface->msgs_done >= iface->msgs_num) {
-                dev_err(&iface->adap.dev, "s6i2c: spurious I2C irq: %04x\n",
-                        i2c_rd16(iface, S6_I2C_INTRSTAT));
-                i2c_wr16(iface, S6_I2C_INTRMASK, 0);
-                return;
-        }
-        while ((iface->msgs_push < iface->msgs_num)
-            && (i2c_rd16(iface, S6_I2C_STATUS) & (1 << S6_I2C_STATUS_TFNF))) {
-                struct i2c_msg *m = &iface->msgs[iface->msgs_push];
-                if (!(m->flags & I2C_M_RD))
-                        i2c_wr16(iface, S6_I2C_DATACMD, m->buf[iface->push]);
-                else
-                        i2c_wr16(iface, S6_I2C_DATACMD,
-                                 1 << S6_I2C_DATACMD_READ);
-                if (++iface->push >= m->len) {
-                        iface->push = 0;
-                        iface->msgs_push += 1;
-                }
-        }
-        do {
-                struct i2c_msg *m = &iface->msgs[iface->msgs_done];
-                if (!(m->flags & I2C_M_RD)) {
-                        if (iface->msgs_done < iface->msgs_push)
-                                iface->msgs_done += 1;
-                        else
-                                break;
-                } else if (i2c_rd16(iface, S6_I2C_STATUS)
-                                & (1 << S6_I2C_STATUS_RFNE)) {
-                        m->buf[iface->done] = i2c_rd16(iface, S6_I2C_DATACMD);
-                        if (++iface->done >= m->len) {
-                                iface->done = 0;
-                                iface->msgs_done += 1;
-                        }
-                } else{
-                        break;
-                }
-        } while (iface->msgs_done < iface->msgs_num);
-        if (iface->msgs_done >= iface->msgs_num) {
-                i2c_wr16(iface, S6_I2C_INTRMASK, 1 << S6_I2C_INTR_TXABRT);
-                complete(&iface->complete);
-        } else if (iface->msgs_push >= iface->msgs_num) {
-                i2c_wr16(iface, S6_I2C_INTRMASK, (1 << S6_I2C_INTR_TXABRT) |
-                                                 (1 << S6_I2C_INTR_RXFULL));
-        } else {
-                i2c_wr16(iface, S6_I2C_INTRMASK, (1 << S6_I2C_INTR_TXABRT) |
-                                                 (1 << S6_I2C_INTR_TXEMPTY) |
-                                                 (1 << S6_I2C_INTR_RXFULL));
-        }
-}
-
-static irqreturn_t s6i2c_interrupt_entry(int irq, void *dev_id)
-{
-        struct s6i2c_if *iface = dev_id;
-        if (!(i2c_rd16(iface, S6_I2C_STATUS) & ((1 << S6_I2C_INTR_RXUNDER)
-                                              | (1 << S6_I2C_INTR_RXOVER)
-                                              | (1 << S6_I2C_INTR_RXFULL)
-                                              | (1 << S6_I2C_INTR_TXOVER)
-                                              | (1 << S6_I2C_INTR_TXEMPTY)
-                                              | (1 << S6_I2C_INTR_RDREQ)
-                                              | (1 << S6_I2C_INTR_TXABRT)
-                                              | (1 << S6_I2C_INTR_RXDONE)
-                                              | (1 << S6_I2C_INTR_ACTIVITY)
-                                              | (1 << S6_I2C_INTR_STOPDET)
-                                              | (1 << S6_I2C_INTR_STARTDET)
-                                              | (1 << S6_I2C_INTR_GENCALL))))
-                return IRQ_NONE;
-
-        spin_lock(&iface->lock);
-        del_timer(&iface->timeout_timer);
-        s6i2c_handle_interrupt(iface);
-        spin_unlock(&iface->lock);
-        return IRQ_HANDLED;
-}
-
-static void s6i2c_timeout(unsigned long data)
-{
-        struct s6i2c_if *iface = (struct s6i2c_if *)data;
-        unsigned long flags;
-
-        spin_lock_irqsave(&iface->lock, flags);
-        s6i2c_handle_interrupt(iface);
-        if (--iface->timeout_count > 0) {
-                iface->timeout_timer.expires = jiffies + POLL_TIMEOUT;
-                add_timer(&iface->timeout_timer);
-        } else {
-                complete(&iface->complete);
-                i2c_wr16(iface, S6_I2C_INTRMASK, 0);
-        }
-        spin_unlock_irqrestore(&iface->lock, flags);
-}
-
-static int s6i2c_master_xfer(struct i2c_adapter *adap,
-                                struct i2c_msg *msgs, int num)
-{
-        struct s6i2c_if *iface = adap->algo_data;
-        int i;
-        if (num == 0)
-                return 0;
-        if (i2c_rd16(iface, S6_I2C_STATUS) & (1 << S6_I2C_STATUS_ACTIVITY))
-                yield();
-        i2c_wr16(iface, S6_I2C_INTRMASK, 0);
-        i2c_rd16(iface, S6_I2C_CLRINTR);
-        for (i = 0; i < num; i++) {
-                if (msgs[i].flags & I2C_M_TEN) {
-                        dev_err(&adap->dev,
-                                "s6i2c: 10 bits addr not supported\n");
-                        return -EINVAL;
-                }
-                if (msgs[i].len == 0) {
-                        dev_err(&adap->dev,
-                                "s6i2c: zero length message not supported\n");
-                        return -EINVAL;
-                }
-                if (msgs[i].addr != msgs[0].addr) {
-                        dev_err(&adap->dev,
-                                "s6i2c: multiple xfer cannot change target\n");
-                        return -EINVAL;
-                }
-        }
-
-        iface->msgs = msgs;
-        iface->msgs_num = num;
-        iface->msgs_push = 0;
-        iface->msgs_done = 0;
-        iface->push = 0;
-        iface->done = 0;
-        iface->timeout_count = 10;
-        i2c_wr16(iface, S6_I2C_TAR, msgs[0].addr);
-        i2c_wr16(iface, S6_I2C_ENABLE, 1);
-        i2c_wr16(iface, S6_I2C_INTRMASK, (1 << S6_I2C_INTR_TXEMPTY) |
-                                         (1 << S6_I2C_INTR_TXABRT));
-
-        iface->timeout_timer.expires = jiffies + POLL_TIMEOUT;
-        add_timer(&iface->timeout_timer);
-        wait_for_completion(&iface->complete);
-        del_timer_sync(&iface->timeout_timer);
-        while (i2c_rd32(iface, S6_I2C_TXFLR) > 0)
-                schedule();
-        while (i2c_rd16(iface, S6_I2C_STATUS) & (1 << S6_I2C_STATUS_ACTIVITY))
-                schedule();
-
-        i2c_wr16(iface, S6_I2C_INTRMASK, 0);
-        i2c_wr16(iface, S6_I2C_ENABLE, 0);
-        return iface->msgs_done;
-}
-
-static u32 s6i2c_functionality(struct i2c_adapter *adap)
-{
-        return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
-}
-
-static struct i2c_algorithm s6i2c_algorithm = {
-        .master_xfer   = s6i2c_master_xfer,
-        .functionality = s6i2c_functionality,
-};
-
-static u16 nanoseconds_on_clk(struct s6i2c_if *iface, u32 ns)
-{
-        u32 dividend = ((clk_get_rate(iface->clk) / 1000) * ns) / 1000000;
-        if (dividend > 0xffff)
-                return 0xffff;
-        return dividend;
-}
-
-static int s6i2c_probe(struct platform_device *dev)
-{
-        struct s6i2c_if *iface = &s6i2c_if;
-        struct i2c_adapter *p_adap;
-        const char *clock;
-        int bus_num, rc;
-        spin_lock_init(&iface->lock);
-        init_completion(&iface->complete);
-        iface->irq = platform_get_irq(dev, 0);
-        if (iface->irq < 0) {
-                rc = iface->irq;
-                goto err_out;
-        }
-        iface->res = platform_get_resource(dev, IORESOURCE_MEM, 0);
-        if (!iface->res) {
-                rc = -ENXIO;
-                goto err_out;
-        }
-        iface->res = request_mem_region(iface->res->start,
-                                        resource_size(iface->res),
-                                        dev->dev.bus_id);
-        if (!iface->res) {
-                rc = -EBUSY;
-                goto err_out;
-        }
-        iface->reg = ioremap_nocache(iface->res->start,
-                                     resource_size(iface->res));
-        if (!iface->reg) {
-                rc = -ENOMEM;
-                goto err_reg;
-        }
-
-        clock = 0;
-        bus_num = -1;
-        if (dev_get_platdata(&dev->dev)) {
-                struct s6_i2c_platform_data *pdata =
-                        dev_get_platdata(&dev->dev);
-                bus_num = pdata->bus_num;
-                clock = pdata->clock;
-        }
-        iface->clk = clk_get(&dev->dev, clock);
-        if (IS_ERR(iface->clk)) {
-                rc = PTR_ERR(iface->clk);
-                goto err_map;
-        }
-        rc = clk_enable(iface->clk);
-        if (rc < 0)
-                goto err_clk_put;
-        init_timer(&iface->timeout_timer);
-        iface->timeout_timer.function = s6i2c_timeout;
-        iface->timeout_timer.data = (unsigned long)iface;
-
-        p_adap = &iface->adap;
-        strlcpy(p_adap->name, dev->name, sizeof(p_adap->name));
-        p_adap->algo = &s6i2c_algorithm;
-        p_adap->algo_data = iface;
-        p_adap->nr = bus_num;
-        p_adap->class = 0;
-        p_adap->dev.parent = &dev->dev;
-        i2c_wr16(iface, S6_I2C_INTRMASK, 0);
-        rc = request_irq(iface->irq, s6i2c_interrupt_entry,
-                         IRQF_SHARED, dev->name, iface);
-        if (rc) {
-                dev_err(&p_adap->dev, "s6i2c: can't get IRQ %d\n", iface->irq);
-                goto err_clk_dis;
-        }
-
-        i2c_wr16(iface, S6_I2C_ENABLE, 0);
-        udelay(1);
-        i2c_wr32(iface, S6_I2C_SRESET, 1 << S6_I2C_SRESET_IC_SRST);
-        i2c_wr16(iface, S6_I2C_CLRTXABRT, 1);
-        i2c_wr16(iface, S6_I2C_CON,
-                        (1 << S6_I2C_CON_MASTER) |
-                        (S6_I2C_CON_SPEED_NORMAL << S6_I2C_CON_SPEED) |
-                        (0 << S6_I2C_CON_10BITSLAVE) |
-                        (0 << S6_I2C_CON_10BITMASTER) |
-                        (1 << S6_I2C_CON_RESTARTENA) |
-                        (1 << S6_I2C_CON_SLAVEDISABLE));
-        i2c_wr16(iface, S6_I2C_SSHCNT, nanoseconds_on_clk(iface, 4000));
-        i2c_wr16(iface, S6_I2C_SSLCNT, nanoseconds_on_clk(iface, 4700));
-        i2c_wr16(iface, S6_I2C_FSHCNT, nanoseconds_on_clk(iface, 600));
-        i2c_wr16(iface, S6_I2C_FSLCNT, nanoseconds_on_clk(iface, 1300));
-        i2c_wr16(iface, S6_I2C_RXTL, 0);
-        i2c_wr16(iface, S6_I2C_TXTL, 0);
-
-        platform_set_drvdata(dev, iface);
-        rc = i2c_add_numbered_adapter(p_adap);
-        if (rc)
-                goto err_irq_free;
-        return 0;
-
-err_irq_free:
-        free_irq(iface->irq, iface);
-err_clk_dis:
-        clk_disable(iface->clk);
-err_clk_put:
-        clk_put(iface->clk);
-err_map:
-        iounmap(iface->reg);
-err_reg:
-        release_mem_region(iface->res->start,
-                           resource_size(iface->res));
-err_out:
-        return rc;
-}
-
-static int s6i2c_remove(struct platform_device *pdev)
-{
-        struct s6i2c_if *iface = platform_get_drvdata(pdev);
-        i2c_wr16(iface, S6_I2C_ENABLE, 0);
-        i2c_del_adapter(&iface->adap);
-        free_irq(iface->irq, iface);
-        clk_disable(iface->clk);
-        clk_put(iface->clk);
-        iounmap(iface->reg);
-        release_mem_region(iface->res->start,
-                           resource_size(iface->res));
-        return 0;
-}
-
-static struct platform_driver s6i2c_driver = {
-        .probe          = s6i2c_probe,
-        .remove         = s6i2c_remove,
-        .driver         = {
-                .name   = DRV_NAME,
-                .owner  = THIS_MODULE,
-        },
-};
-
-static int __init s6i2c_init(void)
-{
-        pr_info("I2C: S6000 I2C driver\n");
-        return platform_driver_register(&s6i2c_driver);
-}
-
-static void __exit s6i2c_exit(void)
-{
-        platform_driver_unregister(&s6i2c_driver);
-}
-
-MODULE_DESCRIPTION("I2C-Bus adapter routines for S6000 I2C");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:" DRV_NAME);
-
-subsys_initcall(s6i2c_init);
-module_exit(s6i2c_exit);
diff --git a/drivers/i2c/busses/i2c-s6000.h b/drivers/i2c/busses/i2c-s6000.h
deleted file mode 100644
index 4936f9f2256f..000000000000
--- a/drivers/i2c/busses/i2c-s6000.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * drivers/i2c/busses/i2c-s6000.h
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (C) 2008 Emlix GmbH <info@emlix.com>
- * Author:      Oskar Schirmer <oskar@scara.com>
- */
-
-#ifndef __DRIVERS_I2C_BUSSES_I2C_S6000_H
-#define __DRIVERS_I2C_BUSSES_I2C_S6000_H
-
-#define S6_I2C_CON              0x000
-#define S6_I2C_CON_MASTER               0
-#define S6_I2C_CON_SPEED                1
-#define S6_I2C_CON_SPEED_NORMAL                 1
-#define S6_I2C_CON_SPEED_FAST                   2
-#define S6_I2C_CON_SPEED_MASK                   3
-#define S6_I2C_CON_10BITSLAVE           3
-#define S6_I2C_CON_10BITMASTER          4
-#define S6_I2C_CON_RESTARTENA           5
-#define S6_I2C_CON_SLAVEDISABLE         6
-#define S6_I2C_TAR              0x004
-#define S6_I2C_TAR_GCORSTART            10
-#define S6_I2C_TAR_SPECIAL              11
-#define S6_I2C_SAR              0x008
-#define S6_I2C_HSMADDR          0x00C
-#define S6_I2C_DATACMD          0x010
-#define S6_I2C_DATACMD_READ             8
-#define S6_I2C_SSHCNT           0x014
-#define S6_I2C_SSLCNT           0x018
-#define S6_I2C_FSHCNT           0x01C
-#define S6_I2C_FSLCNT           0x020
-#define S6_I2C_INTRSTAT         0x02C
-#define S6_I2C_INTRMASK         0x030
-#define S6_I2C_RAWINTR          0x034
-#define S6_I2C_INTR_RXUNDER             0
-#define S6_I2C_INTR_RXOVER              1
-#define S6_I2C_INTR_RXFULL              2
-#define S6_I2C_INTR_TXOVER              3
-#define S6_I2C_INTR_TXEMPTY             4
-#define S6_I2C_INTR_RDREQ               5
-#define S6_I2C_INTR_TXABRT              6
-#define S6_I2C_INTR_RXDONE              7
-#define S6_I2C_INTR_ACTIVITY            8
-#define S6_I2C_INTR_STOPDET             9
-#define S6_I2C_INTR_STARTDET            10
-#define S6_I2C_INTR_GENCALL             11
-#define S6_I2C_RXTL             0x038
-#define S6_I2C_TXTL             0x03C
-#define S6_I2C_CLRINTR          0x040
-#define S6_I2C_CLRRXUNDER       0x044
-#define S6_I2C_CLRRXOVER        0x048
-#define S6_I2C_CLRTXOVER        0x04C
-#define S6_I2C_CLRRDREQ         0x050
-#define S6_I2C_CLRTXABRT        0x054
-#define S6_I2C_CLRRXDONE        0x058
-#define S6_I2C_CLRACTIVITY      0x05C
-#define S6_I2C_CLRSTOPDET       0x060
-#define S6_I2C_CLRSTARTDET      0x064
-#define S6_I2C_CLRGENCALL       0x068
-#define S6_I2C_ENABLE           0x06C
-#define S6_I2C_STATUS           0x070
-#define S6_I2C_STATUS_ACTIVITY          0
-#define S6_I2C_STATUS_TFNF              1
-#define S6_I2C_STATUS_TFE               2
-#define S6_I2C_STATUS_RFNE              3
-#define S6_I2C_STATUS_RFF               4
-#define S6_I2C_TXFLR            0x074
-#define S6_I2C_RXFLR            0x078
-#define S6_I2C_SRESET           0x07C
-#define S6_I2C_SRESET_IC_SRST           0
-#define S6_I2C_SRESET_IC_MASTER_SRST    1
-#define S6_I2C_SRESET_IC_SLAVE_SRST     2
-#define S6_I2C_TXABRTSOURCE     0x080
-
-#endif
diff --git a/drivers/i2c/busses/i2c-sirf.c b/drivers/i2c/busses/i2c-sirf.c
index a3216defc1d3..b1336d5f0531 100644
--- a/drivers/i2c/busses/i2c-sirf.c
+++ b/drivers/i2c/busses/i2c-sirf.c
@@ -311,7 +311,7 @@ static int i2c_sirfsoc_probe(struct platform_device *pdev)
                 goto out;
         }
         adap = &siic->adapter;
-        adap->class = I2C_CLASS_HWMON | I2C_CLASS_DEPRECATED;
+        adap->class = I2C_CLASS_DEPRECATED;
 
         mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
         siic->base = devm_ioremap_resource(&pdev->dev, mem_res);
diff --git a/drivers/i2c/busses/i2c-st.c b/drivers/i2c/busses/i2c-st.c
index 95b947670386..2e4eccd6599a 100644
--- a/drivers/i2c/busses/i2c-st.c
+++ b/drivers/i2c/busses/i2c-st.c
@@ -206,25 +206,31 @@ static inline void st_i2c_clr_bits(void __iomem *reg, u32 mask)
         writel_relaxed(readl_relaxed(reg) & ~mask, reg);
 }
 
-/* From I2C Specifications v0.5 */
+/*
+ * From I2C Specifications v0.5.
+ *
+ * All the values below have +10% margin added to be
+ * compatible with some out-of-spec devices,
+ * like HDMI link of the Toshiba 19AV600 TV.
+ */
 static struct st_i2c_timings i2c_timings[] = {
         [I2C_MODE_STANDARD] = {
                 .rate                   = 100000,
-                .rep_start_hold         = 4000,
-                .rep_start_setup        = 4700,
-                .start_hold             = 4000,
-                .data_setup_time        = 250,
-                .stop_setup_time        = 4000,
-                .bus_free_time          = 4700,
+                .rep_start_hold         = 4400,
+                .rep_start_setup        = 5170,
+                .start_hold             = 4400,
+                .data_setup_time        = 275,
+                .stop_setup_time        = 4400,
+                .bus_free_time          = 5170,
         },
         [I2C_MODE_FAST] = {
                 .rate                   = 400000,
-                .rep_start_hold         = 600,
-                .rep_start_setup        = 600,
-                .start_hold             = 600,
-                .data_setup_time        = 100,
-                .stop_setup_time        = 600,
-                .bus_free_time          = 1300,
+                .rep_start_hold         = 660,
+                .rep_start_setup        = 660,
+                .start_hold             = 660,
+                .data_setup_time        = 110,
+                .stop_setup_time        = 660,
+                .bus_free_time          = 1430,
         },
 };
 
@@ -815,7 +821,7 @@ static int st_i2c_probe(struct platform_device *pdev)
 
         adap = &i2c_dev->adap;
         i2c_set_adapdata(adap, i2c_dev);
-        snprintf(adap->name, sizeof(adap->name), "ST I2C(0x%x)", res->start);
+        snprintf(adap->name, sizeof(adap->name), "ST I2C(0x%pa)", &res->start);
         adap->owner = THIS_MODULE;
         adap->timeout = 2 * HZ;
         adap->retries = 0;
diff --git a/drivers/i2c/busses/i2c-stu300.c b/drivers/i2c/busses/i2c-stu300.c
index fefb1c19ec1d..6a44f37798c8 100644
--- a/drivers/i2c/busses/i2c-stu300.c
+++ b/drivers/i2c/busses/i2c-stu300.c
@@ -909,7 +909,7 @@ static int stu300_probe(struct platform_device *pdev)
         adap = &dev->adapter;
         adap->owner = THIS_MODULE;
         /* DDC class but actually often used for more generic I2C */
-        adap->class = I2C_CLASS_DDC | I2C_CLASS_DEPRECATED;
+        adap->class = I2C_CLASS_DEPRECATED;
         strlcpy(adap->name, "ST Microelectronics DDC I2C adapter",
                 sizeof(adap->name));
         adap->nr = bus_nr;
diff --git a/drivers/i2c/busses/i2c-taos-evm.c b/drivers/i2c/busses/i2c-taos-evm.c
index 057602683553..10855a0b7e7f 100644
--- a/drivers/i2c/busses/i2c-taos-evm.c
+++ b/drivers/i2c/busses/i2c-taos-evm.c
@@ -311,19 +311,8 @@ static struct serio_driver taos_drv = {
         .interrupt      = taos_interrupt,
 };
 
-static int __init taos_init(void)
-{
-        return serio_register_driver(&taos_drv);
-}
-
-static void __exit taos_exit(void)
-{
-        serio_unregister_driver(&taos_drv);
-}
+module_serio_driver(taos_drv);
 
 MODULE_AUTHOR("Jean Delvare <jdelvare@suse.de>");
 MODULE_DESCRIPTION("TAOS evaluation module driver");
 MODULE_LICENSE("GPL");
-
-module_init(taos_init);
-module_exit(taos_exit);
diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c
index f1bb2fc06791..87d0371cebb7 100644
--- a/drivers/i2c/busses/i2c-tegra.c
+++ b/drivers/i2c/busses/i2c-tegra.c
@@ -792,7 +792,7 @@ static int tegra_i2c_probe(struct platform_device *pdev)
 
         i2c_set_adapdata(&i2c_dev->adapter, i2c_dev);
         i2c_dev->adapter.owner = THIS_MODULE;
-        i2c_dev->adapter.class = I2C_CLASS_HWMON | I2C_CLASS_DEPRECATED;
+        i2c_dev->adapter.class = I2C_CLASS_DEPRECATED;
         strlcpy(i2c_dev->adapter.name, "Tegra I2C adapter",
                 sizeof(i2c_dev->adapter.name));
         i2c_dev->adapter.algo = &tegra_i2c_algo;
diff --git a/drivers/i2c/busses/i2c-xiic.c b/drivers/i2c/busses/i2c-xiic.c
index 7731f1795869..ade9223912d3 100644
--- a/drivers/i2c/busses/i2c-xiic.c
+++ b/drivers/i2c/busses/i2c-xiic.c
@@ -677,15 +677,15 @@ static u32 xiic_func(struct i2c_adapter *adap)
 }
 
 static const struct i2c_algorithm xiic_algorithm = {
-        .master_xfer    = xiic_xfer,
-        .functionality  = xiic_func,
+        .master_xfer = xiic_xfer,
+        .functionality = xiic_func,
 };
 
 static struct i2c_adapter xiic_adapter = {
-        .owner          = THIS_MODULE,
-        .name           = DRIVER_NAME,
-        .class          = I2C_CLASS_HWMON | I2C_CLASS_SPD | I2C_CLASS_DEPRECATED,
-        .algo           = &xiic_algorithm,
+        .owner = THIS_MODULE,
+        .name = DRIVER_NAME,
+        .class = I2C_CLASS_DEPRECATED,
+        .algo = &xiic_algorithm,
 };
 
 
diff --git a/drivers/i2c/busses/scx200_i2c.c b/drivers/i2c/busses/scx200_i2c.c
deleted file mode 100644
index 8eadf0f47ad7..000000000000
--- a/drivers/i2c/busses/scx200_i2c.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/* linux/drivers/i2c/busses/scx200_i2c.c
-
-   Copyright (c) 2001,2002 Christer Weinigel <wingel@nano-system.com>
-
-   National Semiconductor SCx200 I2C bus on GPIO pins
-
-   Based on i2c-velleman.c Copyright (C) 1995-96, 2000 Simon G. Vogl
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 2 of the License, or
-   (at your option) any later version.
-   
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-   
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.                 
-*/
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/module.h>
-#include <linux/errno.h>
-#include <linux/kernel.h>
-#include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
-#include <linux/io.h>
-
-#include <linux/scx200_gpio.h>
-
-MODULE_AUTHOR("Christer Weinigel <wingel@nano-system.com>");
-MODULE_DESCRIPTION("NatSemi SCx200 I2C Driver");
-MODULE_LICENSE("GPL");
-
-static int scl = CONFIG_SCx200_I2C_SCL;
-static int sda = CONFIG_SCx200_I2C_SDA;
-
-module_param(scl, int, 0);
-MODULE_PARM_DESC(scl, "GPIO line for SCL");
-module_param(sda, int, 0);
-MODULE_PARM_DESC(sda, "GPIO line for SDA");
-
-static void scx200_i2c_setscl(void *data, int state)
-{
-        scx200_gpio_set(scl, state);
-}
-
-static void scx200_i2c_setsda(void *data, int state)
-{
-        scx200_gpio_set(sda, state);
-} 
-
-static int scx200_i2c_getscl(void *data)
-{
-        return scx200_gpio_get(scl);
-}
-
-static int scx200_i2c_getsda(void *data)
-{
-        return scx200_gpio_get(sda);
-}
-
-/* ------------------------------------------------------------------------
- * Encapsulate the above functions in the correct operations structure.
- * This is only done when more than one hardware adapter is supported.
- */
-
-static struct i2c_algo_bit_data scx200_i2c_data = {
-        .setsda         = scx200_i2c_setsda,
-        .setscl         = scx200_i2c_setscl,
-        .getsda         = scx200_i2c_getsda,
-        .getscl         = scx200_i2c_getscl,
-        .udelay         = 10,
-        .timeout        = HZ,
-};
-
-static struct i2c_adapter scx200_i2c_ops = {
-        .owner             = THIS_MODULE,
-        .class             = I2C_CLASS_HWMON | I2C_CLASS_SPD,
-        .algo_data         = &scx200_i2c_data,
-        .name   = "NatSemi SCx200 I2C",
-};
-
-static int scx200_i2c_init(void)
-{
-        pr_debug("NatSemi SCx200 I2C Driver\n");
-
-        if (!scx200_gpio_present()) {
-                pr_err("no SCx200 gpio pins available\n");
-                return -ENODEV;
-        }
-
-        pr_debug("SCL=GPIO%02u, SDA=GPIO%02u\n", scl, sda);
-
-        if (scl == -1 || sda == -1 || scl == sda) {
-                pr_err("scl and sda must be specified\n");
-                return -EINVAL;
-        }
-
-        /* Configure GPIOs as open collector outputs */
-        scx200_gpio_configure(scl, ~2, 5);
-        scx200_gpio_configure(sda, ~2, 5);
-
-        if (i2c_bit_add_bus(&scx200_i2c_ops) < 0) {
-                pr_err("adapter %s registration failed\n", scx200_i2c_ops.name);
-                return -ENODEV;
-        }
-        
-        return 0;
-}
-
-static void scx200_i2c_cleanup(void)
-{
-        i2c_del_adapter(&scx200_i2c_ops);
-}
-
-module_init(scx200_i2c_init);
-module_exit(scx200_i2c_cleanup);
-
-/*
-    Local variables:
-        compile-command: "make -k -C ../.. SUBDIRS=drivers/i2c modules"
-        c-basic-offset: 8
-    End:
-*/
diff --git a/drivers/i2c/i2c-acpi.c b/drivers/i2c/i2c-acpi.c
new file mode 100644
index 000000000000..e8b61967334b
--- /dev/null
+++ b/drivers/i2c/i2c-acpi.c
@@ -0,0 +1,362 @@
+/*
+ * I2C ACPI code
+ *
+ * Copyright (C) 2014 Intel Corp
+ *
+ * Author: Lan Tianyu <tianyu.lan@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ */
+#define pr_fmt(fmt) "I2C/ACPI : " fmt
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/acpi.h>
+
+struct acpi_i2c_handler_data {
+        struct acpi_connection_info info;
+        struct i2c_adapter *adapter;
+};
+
+struct gsb_buffer {
+        u8      status;
+        u8      len;
+        union {
+                u16     wdata;
+                u8      bdata;
+                u8      data[0];
+        };
+} __packed;
+
+static int acpi_i2c_add_resource(struct acpi_resource *ares, void *data)
+{
+        struct i2c_board_info *info = data;
+
+        if (ares->type == ACPI_RESOURCE_TYPE_SERIAL_BUS) {
+                struct acpi_resource_i2c_serialbus *sb;
+
+                sb = &ares->data.i2c_serial_bus;
+                if (sb->type == ACPI_RESOURCE_SERIAL_TYPE_I2C) {
+                        info->addr = sb->slave_address;
+                        if (sb->access_mode == ACPI_I2C_10BIT_MODE)
+                                info->flags |= I2C_CLIENT_TEN;
+                }
+        } else if (info->irq < 0) {
+                struct resource r;
+
+                if (acpi_dev_resource_interrupt(ares, 0, &r))
+                        info->irq = r.start;
+        }
+
+        /* Tell the ACPI core to skip this resource */
+        return 1;
+}
+
+static acpi_status acpi_i2c_add_device(acpi_handle handle, u32 level,
+                                       void *data, void **return_value)
+{
+        struct i2c_adapter *adapter = data;
+        struct list_head resource_list;
+        struct i2c_board_info info;
+        struct acpi_device *adev;
+        int ret;
+
+        if (acpi_bus_get_device(handle, &adev))
+                return AE_OK;
+        if (acpi_bus_get_status(adev) || !adev->status.present)
+                return AE_OK;
+
+        memset(&info, 0, sizeof(info));
+        info.acpi_node.companion = adev;
+        info.irq = -1;
+
+        INIT_LIST_HEAD(&resource_list);
+        ret = acpi_dev_get_resources(adev, &resource_list,
+                                     acpi_i2c_add_resource, &info);
+        acpi_dev_free_resource_list(&resource_list);
+
+        if (ret < 0 || !info.addr)
+                return AE_OK;
+
+        adev->power.flags.ignore_parent = true;
+        strlcpy(info.type, dev_name(&adev->dev), sizeof(info.type));
+        if (!i2c_new_device(adapter, &info)) {
+                adev->power.flags.ignore_parent = false;
+                dev_err(&adapter->dev,
+                        "failed to add I2C device %s from ACPI\n",
+                        dev_name(&adev->dev));
+        }
+
+        return AE_OK;
+}
+
+/**
+ * acpi_i2c_register_devices - enumerate I2C slave devices behind adapter
+ * @adap: pointer to adapter
+ *
+ * Enumerate all I2C slave devices behind this adapter by walking the ACPI
+ * namespace. When a device is found it will be added to the Linux device
+ * model and bound to the corresponding ACPI handle.
+ */
+void acpi_i2c_register_devices(struct i2c_adapter *adap)
+{
+        acpi_handle handle;
+        acpi_status status;
+
+        if (!adap->dev.parent)
+                return;
+
+        handle = ACPI_HANDLE(adap->dev.parent);
+        if (!handle)
+                return;
+
+        status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle, 1,
+                                     acpi_i2c_add_device, NULL,
+                                     adap, NULL);
+        if (ACPI_FAILURE(status))
+                dev_warn(&adap->dev, "failed to enumerate I2C slaves\n");
+}
+
+static int acpi_gsb_i2c_read_bytes(struct i2c_client *client,
+                u8 cmd, u8 *data, u8 data_len)
+{
+
+        struct i2c_msg msgs[2];
+        int ret;
+        u8 *buffer;
+
+        buffer = kzalloc(data_len, GFP_KERNEL);
+        if (!buffer)
+                return AE_NO_MEMORY;
+
+        msgs[0].addr = client->addr;
+        msgs[0].flags = client->flags;
+        msgs[0].len = 1;
+        msgs[0].buf = &cmd;
+
+        msgs[1].addr = client->addr;
+        msgs[1].flags = client->flags | I2C_M_RD;
+        msgs[1].len = data_len;
+        msgs[1].buf = buffer;
+
+        ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
+        if (ret < 0)
+                dev_err(&client->adapter->dev, "i2c read failed\n");
+        else
+                memcpy(data, buffer, data_len);
+
+        kfree(buffer);
+        return ret;
+}
+
+static int acpi_gsb_i2c_write_bytes(struct i2c_client *client,
+                u8 cmd, u8 *data, u8 data_len)
+{
+
+        struct i2c_msg msgs[1];
+        u8 *buffer;
+        int ret = AE_OK;
+
+        buffer = kzalloc(data_len + 1, GFP_KERNEL);
+        if (!buffer)
+                return AE_NO_MEMORY;
+
+        buffer[0] = cmd;
+        memcpy(buffer + 1, data, data_len);
+
+        msgs[0].addr = client->addr;
+        msgs[0].flags = client->flags;
+        msgs[0].len = data_len + 1;
+        msgs[0].buf = buffer;
+
+        ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
+        if (ret < 0)
+                dev_err(&client->adapter->dev, "i2c write failed\n");
+
+        kfree(buffer);
+        return ret;
+}
+
+static acpi_status
+acpi_i2c_space_handler(u32 function, acpi_physical_address command,
+                        u32 bits, u64 *value64,
+                        void *handler_context, void *region_context)
+{
+        struct gsb_buffer *gsb = (struct gsb_buffer *)value64;
+        struct acpi_i2c_handler_data *data = handler_context;
+        struct acpi_connection_info *info = &data->info;
+        struct acpi_resource_i2c_serialbus *sb;
+        struct i2c_adapter *adapter = data->adapter;
+        struct i2c_client client;
+        struct acpi_resource *ares;
+        u32 accessor_type = function >> 16;
+        u8 action = function & ACPI_IO_MASK;
+        acpi_status ret = AE_OK;
+        int status;
+
+        ret = acpi_buffer_to_resource(info->connection, info->length, &ares);
+        if (ACPI_FAILURE(ret))
+                return ret;
+
+        if (!value64 || ares->type != ACPI_RESOURCE_TYPE_SERIAL_BUS) {
+                ret = AE_BAD_PARAMETER;
+                goto err;
+        }
+
+        sb = &ares->data.i2c_serial_bus;
+        if (sb->type != ACPI_RESOURCE_SERIAL_TYPE_I2C) {
+                ret = AE_BAD_PARAMETER;
+                goto err;
+        }
+
+        memset(&client, 0, sizeof(client));
+        client.adapter = adapter;
+        client.addr = sb->slave_address;
+        client.flags = 0;
+
+        if (sb->access_mode == ACPI_I2C_10BIT_MODE)
+                client.flags |= I2C_CLIENT_TEN;
+
+        switch (accessor_type) {
+        case ACPI_GSB_ACCESS_ATTRIB_SEND_RCV:
+                if (action == ACPI_READ) {
+                        status = i2c_smbus_read_byte(&client);
+                        if (status >= 0) {
+                                gsb->bdata = status;
+                                status = 0;
+                        }
+                } else {
+                        status = i2c_smbus_write_byte(&client, gsb->bdata);
+                }
+                break;
+
+        case ACPI_GSB_ACCESS_ATTRIB_BYTE:
+                if (action == ACPI_READ) {
+                        status = i2c_smbus_read_byte_data(&client, command);
+                        if (status >= 0) {
+                                gsb->bdata = status;
+                                status = 0;
+                        }
+                } else {
+                        status = i2c_smbus_write_byte_data(&client, command,
+                                        gsb->bdata);
+                }
+                break;
+
+        case ACPI_GSB_ACCESS_ATTRIB_WORD:
+                if (action == ACPI_READ) {
+                        status = i2c_smbus_read_word_data(&client, command);
+                        if (status >= 0) {
+                                gsb->wdata = status;
+                                status = 0;
+                        }
+                } else {
+                        status = i2c_smbus_write_word_data(&client, command,
+                                        gsb->wdata);
+                }
+                break;
+
+        case ACPI_GSB_ACCESS_ATTRIB_BLOCK:
+                if (action == ACPI_READ) {
+                        status = i2c_smbus_read_block_data(&client, command,
+                                        gsb->data);
+                        if (status >= 0) {
+                                gsb->len = status;
+                                status = 0;
+                        }
+                } else {
+                        status = i2c_smbus_write_block_data(&client, command,
+                                        gsb->len, gsb->data);
+                }
+                break;
+
+        case ACPI_GSB_ACCESS_ATTRIB_MULTIBYTE:
+                if (action == ACPI_READ) {
+                        status = acpi_gsb_i2c_read_bytes(&client, command,
+                                        gsb->data, info->access_length);
+                        if (status > 0)
+                                status = 0;
+                } else {
+                        status = acpi_gsb_i2c_write_bytes(&client, command,
+                                        gsb->data, info->access_length);
+                }
+                break;
+
+        default:
+                pr_info("protocol(0x%02x) is not supported.\n", accessor_type);
+                ret = AE_BAD_PARAMETER;
+                goto err;
+        }
+
+        gsb->status = status;
+
+ err:
+        ACPI_FREE(ares);
+        return ret;
+}
+
+
+int acpi_i2c_install_space_handler(struct i2c_adapter *adapter)
+{
+        acpi_handle handle = ACPI_HANDLE(adapter->dev.parent);
+        struct acpi_i2c_handler_data *data;
+        acpi_status status;
+
+        if (!handle)
+                return -ENODEV;
+
+        data = kzalloc(sizeof(struct acpi_i2c_handler_data),
+                            GFP_KERNEL);
+        if (!data)
+                return -ENOMEM;
+
+        data->adapter = adapter;
+        status = acpi_bus_attach_private_data(handle, (void *)data);
+        if (ACPI_FAILURE(status)) {
+                kfree(data);
+                return -ENOMEM;
+        }
+
+        status = acpi_install_address_space_handler(handle,
+                                ACPI_ADR_SPACE_GSBUS,
+                                &acpi_i2c_space_handler,
+                                NULL,
+                                data);
+        if (ACPI_FAILURE(status)) {
+                dev_err(&adapter->dev, "Error installing i2c space handler\n");
+                acpi_bus_detach_private_data(handle);
+                kfree(data);
+                return -ENOMEM;
+        }
+
+        return 0;
+}
+
+void acpi_i2c_remove_space_handler(struct i2c_adapter *adapter)
+{
+        acpi_handle handle = ACPI_HANDLE(adapter->dev.parent);
+        struct acpi_i2c_handler_data *data;
+        acpi_status status;
+
+        if (!handle)
+                return;
+
+        acpi_remove_address_space_handler(handle,
+                                ACPI_ADR_SPACE_GSBUS,
+                                &acpi_i2c_space_handler);
+
+        status = acpi_bus_get_private_data(handle, (void **)&data);
+        if (ACPI_SUCCESS(status))
+                kfree(data);
+
+        acpi_bus_detach_private_data(handle);
+}
diff --git a/drivers/i2c/i2c-core.c b/drivers/i2c/i2c-core.c
index 66aa83b99383..632057a44615 100644
--- a/drivers/i2c/i2c-core.c
+++ b/drivers/i2c/i2c-core.c
@@ -1097,101 +1097,6 @@ EXPORT_SYMBOL(of_find_i2c_adapter_by_node);
 static void of_i2c_register_devices(struct i2c_adapter *adap) { }
 #endif /* CONFIG_OF */
 
-/* ACPI support code */
-
-#if IS_ENABLED(CONFIG_ACPI)
-static int acpi_i2c_add_resource(struct acpi_resource *ares, void *data)
-{
-        struct i2c_board_info *info = data;
-
-        if (ares->type == ACPI_RESOURCE_TYPE_SERIAL_BUS) {
-                struct acpi_resource_i2c_serialbus *sb;
-
-                sb = &ares->data.i2c_serial_bus;
-                if (sb->type == ACPI_RESOURCE_SERIAL_TYPE_I2C) {
-                        info->addr = sb->slave_address;
-                        if (sb->access_mode == ACPI_I2C_10BIT_MODE)
-                                info->flags |= I2C_CLIENT_TEN;
-                }
-        } else if (info->irq < 0) {
-                struct resource r;
-
-                if (acpi_dev_resource_interrupt(ares, 0, &r))
-                        info->irq = r.start;
-        }
-
-        /* Tell the ACPI core to skip this resource */
-        return 1;
-}
-
-static acpi_status acpi_i2c_add_device(acpi_handle handle, u32 level,
-                                       void *data, void **return_value)
-{
-        struct i2c_adapter *adapter = data;
-        struct list_head resource_list;
-        struct i2c_board_info info;
-        struct acpi_device *adev;
-        int ret;
-
-        if (acpi_bus_get_device(handle, &adev))
-                return AE_OK;
-        if (acpi_bus_get_status(adev) || !adev->status.present)
-                return AE_OK;
-
-        memset(&info, 0, sizeof(info));
-        info.acpi_node.companion = adev;
-        info.irq = -1;
-
-        INIT_LIST_HEAD(&resource_list);
-        ret = acpi_dev_get_resources(adev, &resource_list,
-                                     acpi_i2c_add_resource, &info);
-        acpi_dev_free_resource_list(&resource_list);
-
-        if (ret < 0 || !info.addr)
-                return AE_OK;
-
-        adev->power.flags.ignore_parent = true;
-        strlcpy(info.type, dev_name(&adev->dev), sizeof(info.type));
-        if (!i2c_new_device(adapter, &info)) {
-                adev->power.flags.ignore_parent = false;
-                dev_err(&adapter->dev,
-                        "failed to add I2C device %s from ACPI\n",
-                        dev_name(&adev->dev));
-        }
-
-        return AE_OK;
-}
-
-/**
- * acpi_i2c_register_devices - enumerate I2C slave devices behind adapter
- * @adap: pointer to adapter
- *
- * Enumerate all I2C slave devices behind this adapter by walking the ACPI
- * namespace. When a device is found it will be added to the Linux device
- * model and bound to the corresponding ACPI handle.
- */
-static void acpi_i2c_register_devices(struct i2c_adapter *adap)
-{
-        acpi_handle handle;
-        acpi_status status;
-
-        if (!adap->dev.parent)
-                return;
-
-        handle = ACPI_HANDLE(adap->dev.parent);
-        if (!handle)
-                return;
-
-        status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle, 1,
-                                     acpi_i2c_add_device, NULL,
-                                     adap, NULL);
-        if (ACPI_FAILURE(status))
-                dev_warn(&adap->dev, "failed to enumerate I2C slaves\n");
-}
-#else
-static inline void acpi_i2c_register_devices(struct i2c_adapter *adap) {}
-#endif /* CONFIG_ACPI */
-
 static int i2c_do_add_adapter(struct i2c_driver *driver,
                               struct i2c_adapter *adap)
 {
@@ -1298,6 +1203,7 @@ exit_recovery:
         /* create pre-declared device nodes */
         of_i2c_register_devices(adap);
         acpi_i2c_register_devices(adap);
+        acpi_i2c_install_space_handler(adap);
 
         if (adap->nr < __i2c_first_dynamic_bus_num)
                 i2c_scan_static_board_info(adap);
@@ -1471,6 +1377,7 @@ void i2c_del_adapter(struct i2c_adapter *adap)
                 return;
         }
 
+        acpi_i2c_remove_space_handler(adap);
         /* Tell drivers about this removal */
         mutex_lock(&core_lock);
         bus_for_each_drv(&i2c_bus_type, NULL, adap,
@@ -2013,6 +1920,16 @@ static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver)
         if (!driver->detect || !address_list)
                 return 0;
 
+        /* Warn that the adapter lost class based instantiation */
+        if (adapter->class == I2C_CLASS_DEPRECATED) {
+                dev_dbg(&adapter->dev,
+                        "This adapter dropped support for I2C classes and "
+                        "won't auto-detect %s devices anymore. If you need it, check "
+                        "'Documentation/i2c/instantiating-devices' for alternatives.\n",
+                        driver->driver.name);
+                return 0;
+        }
+
         /* Stop here if the classes do not match */
         if (!(adapter->class & driver->class))
                 return 0;
diff --git a/drivers/i2c/i2c-stub.c b/drivers/i2c/i2c-stub.c
index 77e4849d2f2a..d241aa295d96 100644
--- a/drivers/i2c/i2c-stub.c
+++ b/drivers/i2c/i2c-stub.c
@@ -2,7 +2,7 @@
     i2c-stub.c - I2C/SMBus chip emulator
 
     Copyright (c) 2004 Mark M. Hoffman <mhoffman@lightlink.com>
-    Copyright (C) 2007, 2012 Jean Delvare <jdelvare@suse.de>
+    Copyright (C) 2007-2014 Jean Delvare <jdelvare@suse.de>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -27,28 +27,109 @@
 #include <linux/slab.h>
 #include <linux/errno.h>
 #include <linux/i2c.h>
+#include <linux/list.h>
 
 #define MAX_CHIPS 10
-#define STUB_FUNC (I2C_FUNC_SMBUS_QUICK | I2C_FUNC_SMBUS_BYTE | \
-                   I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA | \
-                   I2C_FUNC_SMBUS_I2C_BLOCK)
+
+/*
+ * Support for I2C_FUNC_SMBUS_BLOCK_DATA is disabled by default and must
+ * be enabled explicitly by setting the I2C_FUNC_SMBUS_BLOCK_DATA bits
+ * in the 'functionality' module parameter.
+ */
+#define STUB_FUNC_DEFAULT \
+                (I2C_FUNC_SMBUS_QUICK | I2C_FUNC_SMBUS_BYTE | \
+                 I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA | \
+                 I2C_FUNC_SMBUS_I2C_BLOCK)
+
+#define STUB_FUNC_ALL \
+                (STUB_FUNC_DEFAULT | I2C_FUNC_SMBUS_BLOCK_DATA)
 
 static unsigned short chip_addr[MAX_CHIPS];
 module_param_array(chip_addr, ushort, NULL, S_IRUGO);
 MODULE_PARM_DESC(chip_addr,
                  "Chip addresses (up to 10, between 0x03 and 0x77)");
 
-static unsigned long functionality = STUB_FUNC;
+static unsigned long functionality = STUB_FUNC_DEFAULT;
 module_param(functionality, ulong, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(functionality, "Override functionality bitfield");
 
+/* Some chips have banked register ranges */
+
+static u8 bank_reg[MAX_CHIPS];
+module_param_array(bank_reg, byte, NULL, S_IRUGO);
+MODULE_PARM_DESC(bank_reg, "Bank register");
+
+static u8 bank_mask[MAX_CHIPS];
+module_param_array(bank_mask, byte, NULL, S_IRUGO);
+MODULE_PARM_DESC(bank_mask, "Bank value mask");
+
+static u8 bank_start[MAX_CHIPS];
+module_param_array(bank_start, byte, NULL, S_IRUGO);
+MODULE_PARM_DESC(bank_start, "First banked register");
+
+static u8 bank_end[MAX_CHIPS];
+module_param_array(bank_end, byte, NULL, S_IRUGO);
+MODULE_PARM_DESC(bank_end, "Last banked register");
+
+struct smbus_block_data {
+        struct list_head node;
+        u8 command;
+        u8 len;
+        u8 block[I2C_SMBUS_BLOCK_MAX];
+};
+
 struct stub_chip {
         u8 pointer;
         u16 words[256];         /* Byte operations use the LSB as per SMBus
                                    specification */
+        struct list_head smbus_blocks;
+
+        /* For chips with banks, extra registers are allocated dynamically */
+        u8 bank_reg;
+        u8 bank_shift;
+        u8 bank_mask;
+        u8 bank_sel;            /* Currently selected bank */
+        u8 bank_start;
+        u8 bank_end;
+        u16 bank_size;
+        u16 *bank_words;        /* Room for bank_mask * bank_size registers */
 };
 
 static struct stub_chip *stub_chips;
+static int stub_chips_nr;
+
+static struct smbus_block_data *stub_find_block(struct device *dev,
+                                                struct stub_chip *chip,
+                                                u8 command, bool create)
+{
+        struct smbus_block_data *b, *rb = NULL;
+
+        list_for_each_entry(b, &chip->smbus_blocks, node) {
+                if (b->command == command) {
+                        rb = b;
+                        break;
+                }
+        }
+        if (rb == NULL && create) {
+                rb = devm_kzalloc(dev, sizeof(*rb), GFP_KERNEL);
+                if (rb == NULL)
+                        return rb;
+                rb->command = command;
+                list_add(&rb->node, &chip->smbus_blocks);
+        }
+        return rb;
+}
+
+static u16 *stub_get_wordp(struct stub_chip *chip, u8 offset)
+{
+        if (chip->bank_sel &&
+            offset >= chip->bank_start && offset <= chip->bank_end)
+                return chip->bank_words +
+                       (chip->bank_sel - 1) * chip->bank_size +
+                       offset - chip->bank_start;
+        else
+                return chip->words + offset;
+}
 
 /* Return negative errno on error. */
 static s32 stub_xfer(struct i2c_adapter *adap, u16 addr, unsigned short flags,
@@ -57,9 +138,11 @@ static s32 stub_xfer(struct i2c_adapter *adap, u16 addr, unsigned short flags,
         s32 ret;
         int i, len;
         struct stub_chip *chip = NULL;
+        struct smbus_block_data *b;
+        u16 *wordp;
 
         /* Search for the right chip */
-        for (i = 0; i < MAX_CHIPS && chip_addr[i]; i++) {
+        for (i = 0; i < stub_chips_nr; i++) {
                 if (addr == chip_addr[i]) {
                         chip = stub_chips + i;
                         break;
@@ -82,7 +165,8 @@ static s32 stub_xfer(struct i2c_adapter *adap, u16 addr, unsigned short flags,
                                 "smbus byte - addr 0x%02x, wrote 0x%02x.\n",
                                 addr, command);
                 } else {
-                        data->byte = chip->words[chip->pointer++] & 0xff;
+                        wordp = stub_get_wordp(chip, chip->pointer++);
+                        data->byte = *wordp & 0xff;
                         dev_dbg(&adap->dev,
                                 "smbus byte - addr 0x%02x, read  0x%02x.\n",
                                 addr, data->byte);
@@ -92,14 +176,25 @@ static s32 stub_xfer(struct i2c_adapter *adap, u16 addr, unsigned short flags,
                 break;
 
         case I2C_SMBUS_BYTE_DATA:
+                wordp = stub_get_wordp(chip, command);
                 if (read_write == I2C_SMBUS_WRITE) {
-                        chip->words[command] &= 0xff00;
-                        chip->words[command] |= data->byte;
+                        *wordp &= 0xff00;
+                        *wordp |= data->byte;
                         dev_dbg(&adap->dev,
                                 "smbus byte data - addr 0x%02x, wrote 0x%02x at 0x%02x.\n",
                                 addr, data->byte, command);
+
+                        /* Set the bank as needed */
+                        if (chip->bank_words && command == chip->bank_reg) {
+                                chip->bank_sel =
+                                        (data->byte >> chip->bank_shift)
+                                        & chip->bank_mask;
+                                dev_dbg(&adap->dev,
+                                        "switching to bank %u.\n",
+                                        chip->bank_sel);
+                        }
                 } else {
-                        data->byte = chip->words[command] & 0xff;
+                        data->byte = *wordp & 0xff;
                         dev_dbg(&adap->dev,
                                 "smbus byte data - addr 0x%02x, read  0x%02x at 0x%02x.\n",
                                 addr, data->byte, command);
@@ -110,13 +205,14 @@ static s32 stub_xfer(struct i2c_adapter *adap, u16 addr, unsigned short flags,
                 break;
 
         case I2C_SMBUS_WORD_DATA:
+                wordp = stub_get_wordp(chip, command);
                 if (read_write == I2C_SMBUS_WRITE) {
-                        chip->words[command] = data->word;
+                        *wordp = data->word;
                         dev_dbg(&adap->dev,
                                 "smbus word data - addr 0x%02x, wrote 0x%04x at 0x%02x.\n",
                                 addr, data->word, command);
                 } else {
-                        data->word = chip->words[command];
+                        data->word = *wordp;
                         dev_dbg(&adap->dev,
                                 "smbus word data - addr 0x%02x, read  0x%04x at 0x%02x.\n",
                                 addr, data->word, command);
@@ -126,6 +222,12 @@ static s32 stub_xfer(struct i2c_adapter *adap, u16 addr, unsigned short flags,
                 break;
 
         case I2C_SMBUS_I2C_BLOCK_DATA:
+                /*
+                 * We ignore banks here, because banked chips don't use I2C
+                 * block transfers
+                 */
+                if (data->block[0] > 256 - command)     /* Avoid overrun */
+                        data->block[0] = 256 - command;
                 len = data->block[0];
                 if (read_write == I2C_SMBUS_WRITE) {
                         for (i = 0; i < len; i++) {
@@ -148,6 +250,55 @@ static s32 stub_xfer(struct i2c_adapter *adap, u16 addr, unsigned short flags,
                 ret = 0;
                 break;
 
+        case I2C_SMBUS_BLOCK_DATA:
+                /*
+                 * We ignore banks here, because chips typically don't use both
+                 * banks and SMBus block transfers
+                 */
+                b = stub_find_block(&adap->dev, chip, command, false);
+                if (read_write == I2C_SMBUS_WRITE) {
+                        len = data->block[0];
+                        if (len == 0 || len > I2C_SMBUS_BLOCK_MAX) {
+                                ret = -EINVAL;
+                                break;
+                        }
+                        if (b == NULL) {
+                                b = stub_find_block(&adap->dev, chip, command,
+                                                    true);
+                                if (b == NULL) {
+                                        ret = -ENOMEM;
+                                        break;
+                                }
+                        }
+                        /* Largest write sets read block length */
+                        if (len > b->len)
+                                b->len = len;
+                        for (i = 0; i < len; i++)
+                                b->block[i] = data->block[i + 1];
+                        /* update for byte and word commands */
+                        chip->words[command] = (b->block[0] << 8) | b->len;
+                        dev_dbg(&adap->dev,
+                                "smbus block data - addr 0x%02x, wrote %d bytes at 0x%02x.\n",
+                                addr, len, command);
+                } else {
+                        if (b == NULL) {
+                                dev_dbg(&adap->dev,
+                                        "SMBus block read command without prior block write not supported\n");
+                                ret = -EOPNOTSUPP;
+                                break;
+                        }
+                        len = b->len;
+                        data->block[0] = len;
+                        for (i = 0; i < len; i++)
+                                data->block[i + 1] = b->block[i];
+                        dev_dbg(&adap->dev,
+                                "smbus block data - addr 0x%02x, read  %d bytes at 0x%02x.\n",
+                                addr, len, command);
+                }
+
+                ret = 0;
+                break;
+
         default:
                 dev_dbg(&adap->dev, "Unsupported I2C/SMBus command\n");
                 ret = -EOPNOTSUPP;
@@ -159,7 +310,7 @@ static s32 stub_xfer(struct i2c_adapter *adap, u16 addr, unsigned short flags,
 
 static u32 stub_func(struct i2c_adapter *adapter)
 {
-        return STUB_FUNC & functionality;
+        return STUB_FUNC_ALL & functionality;
 }
 
 static const struct i2c_algorithm smbus_algorithm = {
@@ -174,6 +325,43 @@ static struct i2c_adapter stub_adapter = {
         .name           = "SMBus stub driver",
 };
 
+static int __init i2c_stub_allocate_banks(int i)
+{
+        struct stub_chip *chip = stub_chips + i;
+
+        chip->bank_reg = bank_reg[i];
+        chip->bank_start = bank_start[i];
+        chip->bank_end = bank_end[i];
+        chip->bank_size = bank_end[i] - bank_start[i] + 1;
+
+        /* We assume that all bits in the mask are contiguous */
+        chip->bank_mask = bank_mask[i];
+        while (!(chip->bank_mask & 1)) {
+                chip->bank_shift++;
+                chip->bank_mask >>= 1;
+        }
+
+        chip->bank_words = kzalloc(chip->bank_mask * chip->bank_size *
+                                   sizeof(u16), GFP_KERNEL);
+        if (!chip->bank_words)
+                return -ENOMEM;
+
+        pr_debug("i2c-stub: Allocated %u banks of %u words each (registers 0x%02x to 0x%02x)\n",
+                 chip->bank_mask, chip->bank_size, chip->bank_start,
+                 chip->bank_end);
+
+        return 0;
+}
+
+static void i2c_stub_free(void)
+{
+        int i;
+
+        for (i = 0; i < stub_chips_nr; i++)
+                kfree(stub_chips[i].bank_words);
+        kfree(stub_chips);
+}
+
 static int __init i2c_stub_init(void)
 {
         int i, ret;
@@ -194,22 +382,39 @@ static int __init i2c_stub_init(void)
         }
 
         /* Allocate memory for all chips at once */
-        stub_chips = kzalloc(i * sizeof(struct stub_chip), GFP_KERNEL);
+        stub_chips_nr = i;
+        stub_chips = kcalloc(stub_chips_nr, sizeof(struct stub_chip),
+                             GFP_KERNEL);
         if (!stub_chips) {
                 pr_err("i2c-stub: Out of memory\n");
                 return -ENOMEM;
         }
+        for (i = 0; i < stub_chips_nr; i++) {
+                INIT_LIST_HEAD(&stub_chips[i].smbus_blocks);
+
+                /* Allocate extra memory for banked register ranges */
+                if (bank_mask[i]) {
+                        ret = i2c_stub_allocate_banks(i);
+                        if (ret)
+                                goto fail_free;
+                }
+        }
 
         ret = i2c_add_adapter(&stub_adapter);
         if (ret)
-                kfree(stub_chips);
+                goto fail_free;
+
+        return 0;
+
+ fail_free:
+        i2c_stub_free();
         return ret;
 }
 
 static void __exit i2c_stub_exit(void)
 {
         i2c_del_adapter(&stub_adapter);
-        kfree(stub_chips);
+        i2c_stub_free();
 }
 
 MODULE_AUTHOR("Mark M. Hoffman <mhoffman@lightlink.com>");
diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c
index 9bd4212782ab..ec11b404b433 100644
--- a/drivers/i2c/muxes/i2c-mux-pca954x.c
+++ b/drivers/i2c/muxes/i2c-mux-pca954x.c
@@ -41,6 +41,7 @@
 #include <linux/i2c-mux.h>
 #include <linux/i2c/pca954x.h>
 #include <linux/module.h>
+#include <linux/pm.h>
 #include <linux/slab.h>
 
 #define PCA954X_MAX_NCHANS 8
@@ -273,9 +274,23 @@ static int pca954x_remove(struct i2c_client *client)
         return 0;
 }
 
+#ifdef CONFIG_PM_SLEEP
+static int pca954x_resume(struct device *dev)
+{
+        struct i2c_client *client = to_i2c_client(dev);
+        struct pca954x *data = i2c_get_clientdata(client);
+
+        data->last_chan = 0;
+        return i2c_smbus_write_byte(client, 0);
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(pca954x_pm, NULL, pca954x_resume);
+
 static struct i2c_driver pca954x_driver = {
         .driver         = {
                 .name   = "pca954x",
+                .pm     = &pca954x_pm,
                 .owner  = THIS_MODULE,
         },
         .probe          = pca954x_probe,
diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c
index f3ac4154cbb6..44ec72684df5 100644
--- a/fs/cifs/cifs_debug.c
+++ b/fs/cifs/cifs_debug.c
@@ -213,7 +213,7 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v)
                                                    tcon->nativeFileSystem);
                                 }
                                 seq_printf(m, "DevInfo: 0x%x Attributes: 0x%x"
-                                        "\n\tPathComponentMax: %d Status: 0x%d",
+                                        "\n\tPathComponentMax: %d Status: %d",
                                         le32_to_cpu(tcon->fsDevInfo.DeviceCharacteristics),
                                         le32_to_cpu(tcon->fsAttrInfo.Attributes),
                                         le32_to_cpu(tcon->fsAttrInfo.MaxPathNameComponentLength),
diff --git a/fs/cifs/cifsfs.h b/fs/cifs/cifsfs.h
index 70f178a7c759..560480263336 100644
--- a/fs/cifs/cifsfs.h
+++ b/fs/cifs/cifsfs.h
@@ -136,5 +136,5 @@ extern long cifs_ioctl(struct file *filep, unsigned int cmd, unsigned long arg);
 extern const struct export_operations cifs_export_ops;
 #endif /* CONFIG_CIFS_NFSD_EXPORT */
 
-#define CIFS_VERSION   "2.03"
+#define CIFS_VERSION   "2.04"
 #endif                          /* _CIFSFS_H */
diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h
index de6aed8c78e5..0012e1e291d4 100644
--- a/fs/cifs/cifsglob.h
+++ b/fs/cifs/cifsglob.h
@@ -404,6 +404,11 @@ struct smb_version_operations {
                         const struct cifs_fid *, u32 *);
         int (*set_acl)(struct cifs_ntsd *, __u32, struct inode *, const char *,
                         int);
+        /* writepages retry size */
+        unsigned int (*wp_retry_size)(struct inode *);
+        /* get mtu credits */
+        int (*wait_mtu_credits)(struct TCP_Server_Info *, unsigned int,
+                                unsigned int *, unsigned int *);
 };
 
 struct smb_version_values {
@@ -640,6 +645,16 @@ add_credits(struct TCP_Server_Info *server, const unsigned int add,
 }
 
 static inline void
+add_credits_and_wake_if(struct TCP_Server_Info *server, const unsigned int add,
+                        const int optype)
+{
+        if (add) {
+                server->ops->add_credits(server, add, optype);
+                wake_up(&server->request_q);
+        }
+}
+
+static inline void
 set_credits(struct TCP_Server_Info *server, const int val)
 {
         server->ops->set_credits(server, val);
@@ -1044,6 +1059,7 @@ struct cifs_readdata {
         struct address_space            *mapping;
         __u64                           offset;
         unsigned int                    bytes;
+        unsigned int                    got_bytes;
         pid_t                           pid;
         int                             result;
         struct work_struct              work;
@@ -1053,6 +1069,7 @@ struct cifs_readdata {
         struct kvec                     iov;
         unsigned int                    pagesz;
         unsigned int                    tailsz;
+        unsigned int                    credits;
         unsigned int                    nr_pages;
         struct page                     *pages[];
 };
@@ -1073,6 +1090,7 @@ struct cifs_writedata {
         int                             result;
         unsigned int                    pagesz;
         unsigned int                    tailsz;
+        unsigned int                    credits;
         unsigned int                    nr_pages;
         struct page                     *pages[];
 };
@@ -1398,6 +1416,7 @@ static inline void free_dfs_info_array(struct dfs_info3_param *param,
 #define   CIFS_OBREAK_OP   0x0100    /* oplock break request */
 #define   CIFS_NEG_OP      0x0200    /* negotiate request */
 #define   CIFS_OP_MASK     0x0380    /* mask request type */
+#define   CIFS_HAS_CREDITS 0x0400    /* already has credits */
 
 /* Security Flags: indicate type of session setup needed */
 #define   CIFSSEC_MAY_SIGN      0x00001
diff --git a/fs/cifs/cifsproto.h b/fs/cifs/cifsproto.h
index ca7980a1e303..c31ce98c1704 100644
--- a/fs/cifs/cifsproto.h
+++ b/fs/cifs/cifsproto.h
@@ -36,6 +36,7 @@ extern struct smb_hdr *cifs_buf_get(void);
 extern void cifs_buf_release(void *);
 extern struct smb_hdr *cifs_small_buf_get(void);
 extern void cifs_small_buf_release(void *);
+extern void free_rsp_buf(int, void *);
 extern void cifs_rqst_page_to_kvec(struct smb_rqst *rqst, unsigned int idx,
                                         struct kvec *iov);
 extern int smb_send(struct TCP_Server_Info *, struct smb_hdr *,
@@ -89,6 +90,9 @@ extern struct mid_q_entry *cifs_setup_async_request(struct TCP_Server_Info *,
                                                 struct smb_rqst *);
 extern int cifs_check_receive(struct mid_q_entry *mid,
                         struct TCP_Server_Info *server, bool log_error);
+extern int cifs_wait_mtu_credits(struct TCP_Server_Info *server,
+                                 unsigned int size, unsigned int *num,
+                                 unsigned int *credits);
 extern int SendReceive2(const unsigned int /* xid */ , struct cifs_ses *,
                         struct kvec *, int /* nvec to send */,
                         int * /* type of buf returned */ , const int flags);
diff --git a/fs/cifs/cifssmb.c b/fs/cifs/cifssmb.c
index 6ce4e0954b98..66f65001a6d8 100644
--- a/fs/cifs/cifssmb.c
+++ b/fs/cifs/cifssmb.c
@@ -196,10 +196,6 @@ cifs_reconnect_tcon(struct cifs_tcon *tcon, int smb_command)
         if (rc)
                 goto out;
 
-        /*
-         * FIXME: check if wsize needs updated due to negotiated smb buffer
-         *        size shrinking
-         */
         atomic_inc(&tconInfoReconnectCount);
 
         /* tell server Unix caps we support */
@@ -1517,7 +1513,6 @@ cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid)
                 return length;
 
         server->total_read += length;
-        rdata->bytes = length;
 
         cifs_dbg(FYI, "total_read=%u buflen=%u remaining=%u\n",
                  server->total_read, buflen, data_len);
@@ -1560,12 +1555,18 @@ cifs_readv_callback(struct mid_q_entry *mid)
                                          rc);
                 }
                 /* FIXME: should this be counted toward the initiating task? */
-                task_io_account_read(rdata->bytes);
-                cifs_stats_bytes_read(tcon, rdata->bytes);
+                task_io_account_read(rdata->got_bytes);
+                cifs_stats_bytes_read(tcon, rdata->got_bytes);
                 break;
         case MID_REQUEST_SUBMITTED:
         case MID_RETRY_NEEDED:
                 rdata->result = -EAGAIN;
+                if (server->sign && rdata->got_bytes)
+                        /* reset bytes number since we can not check a sign */
+                        rdata->got_bytes = 0;
+                /* FIXME: should this be counted toward the initiating task? */
+                task_io_account_read(rdata->got_bytes);
+                cifs_stats_bytes_read(tcon, rdata->got_bytes);
                 break;
         default:
                 rdata->result = -EIO;
@@ -1734,10 +1735,7 @@ CIFSSMBRead(const unsigned int xid, struct cifs_io_parms *io_parms,
 
 /*      cifs_small_buf_release(pSMB); */ /* Freed earlier now in SendReceive2 */
         if (*buf) {
-                if (resp_buf_type == CIFS_SMALL_BUFFER)
-                        cifs_small_buf_release(iov[0].iov_base);
-                else if (resp_buf_type == CIFS_LARGE_BUFFER)
-                        cifs_buf_release(iov[0].iov_base);
+                free_rsp_buf(resp_buf_type, iov[0].iov_base);
         } else if (resp_buf_type != CIFS_NO_BUFFER) {
                 /* return buffer to caller to free */
                 *buf = iov[0].iov_base;
@@ -1899,28 +1897,80 @@ cifs_writedata_release(struct kref *refcount)
 static void
 cifs_writev_requeue(struct cifs_writedata *wdata)
 {
-        int i, rc;
+        int i, rc = 0;
         struct inode *inode = wdata->cfile->dentry->d_inode;
         struct TCP_Server_Info *server;
+        unsigned int rest_len;
 
-        for (i = 0; i < wdata->nr_pages; i++) {
-                lock_page(wdata->pages[i]);
-                clear_page_dirty_for_io(wdata->pages[i]);
-        }
-
+        server = tlink_tcon(wdata->cfile->tlink)->ses->server;
+        i = 0;
+        rest_len = wdata->bytes;
         do {
-                server = tlink_tcon(wdata->cfile->tlink)->ses->server;
-                rc = server->ops->async_writev(wdata, cifs_writedata_release);
-        } while (rc == -EAGAIN);
+                struct cifs_writedata *wdata2;
+                unsigned int j, nr_pages, wsize, tailsz, cur_len;
+
+                wsize = server->ops->wp_retry_size(inode);
+                if (wsize < rest_len) {
+                        nr_pages = wsize / PAGE_CACHE_SIZE;
+                        if (!nr_pages) {
+                                rc = -ENOTSUPP;
+                                break;
+                        }
+                        cur_len = nr_pages * PAGE_CACHE_SIZE;
+                        tailsz = PAGE_CACHE_SIZE;
+                } else {
+                        nr_pages = DIV_ROUND_UP(rest_len, PAGE_CACHE_SIZE);
+                        cur_len = rest_len;
+                        tailsz = rest_len - (nr_pages - 1) * PAGE_CACHE_SIZE;
+                }
 
-        for (i = 0; i < wdata->nr_pages; i++) {
-                unlock_page(wdata->pages[i]);
-                if (rc != 0) {
-                        SetPageError(wdata->pages[i]);
-                        end_page_writeback(wdata->pages[i]);
-                        page_cache_release(wdata->pages[i]);
+                wdata2 = cifs_writedata_alloc(nr_pages, cifs_writev_complete);
+                if (!wdata2) {
+                        rc = -ENOMEM;
+                        break;
                 }
-        }
+
+                for (j = 0; j < nr_pages; j++) {
+                        wdata2->pages[j] = wdata->pages[i + j];
+                        lock_page(wdata2->pages[j]);
+                        clear_page_dirty_for_io(wdata2->pages[j]);
+                }
+
+                wdata2->sync_mode = wdata->sync_mode;
+                wdata2->nr_pages = nr_pages;
+                wdata2->offset = page_offset(wdata2->pages[0]);
+                wdata2->pagesz = PAGE_CACHE_SIZE;
+                wdata2->tailsz = tailsz;
+                wdata2->bytes = cur_len;
+
+                wdata2->cfile = find_writable_file(CIFS_I(inode), false);
+                if (!wdata2->cfile) {
+                        cifs_dbg(VFS, "No writable handles for inode\n");
+                        rc = -EBADF;
+                        break;
+                }
+                wdata2->pid = wdata2->cfile->pid;
+                rc = server->ops->async_writev(wdata2, cifs_writedata_release);
+
+                for (j = 0; j < nr_pages; j++) {
+                        unlock_page(wdata2->pages[j]);
+                        if (rc != 0 && rc != -EAGAIN) {
+                                SetPageError(wdata2->pages[j]);
+                                end_page_writeback(wdata2->pages[j]);
+                                page_cache_release(wdata2->pages[j]);
+                        }
+                }
+
+                if (rc) {
+                        kref_put(&wdata2->refcount, cifs_writedata_release);
+                        if (rc == -EAGAIN)
+                                continue;
+                        break;
+                }
+
+                rest_len -= cur_len;
+                i += nr_pages;
+        } while (i < wdata->nr_pages);
 
         mapping_set_error(inode->i_mapping, rc);
         kref_put(&wdata->refcount, cifs_writedata_release);
@@ -2203,10 +2253,7 @@ CIFSSMBWrite2(const unsigned int xid, struct cifs_io_parms *io_parms,
         }
 
 /*      cifs_small_buf_release(pSMB); */ /* Freed earlier now in SendReceive2 */
-        if (resp_buf_type == CIFS_SMALL_BUFFER)
-                cifs_small_buf_release(iov[0].iov_base);
-        else if (resp_buf_type == CIFS_LARGE_BUFFER)
-                cifs_buf_release(iov[0].iov_base);
+        free_rsp_buf(resp_buf_type, iov[0].iov_base);
 
         /* Note: On -EAGAIN error only caller can retry on handle based calls
                 since file handle passed in no longer valid */
@@ -2451,10 +2498,7 @@ plk_err_exit:
         if (pSMB)
                 cifs_small_buf_release(pSMB);
 
-        if (resp_buf_type == CIFS_SMALL_BUFFER)
-                cifs_small_buf_release(iov[0].iov_base);
-        else if (resp_buf_type == CIFS_LARGE_BUFFER)
-                cifs_buf_release(iov[0].iov_base);
+        free_rsp_buf(resp_buf_type, iov[0].iov_base);
 
         /* Note: On -EAGAIN error only caller can retry on handle based calls
            since file handle passed in no longer valid */
@@ -3838,10 +3882,7 @@ CIFSSMBGetCIFSACL(const unsigned int xid, struct cifs_tcon *tcon, __u16 fid,
                 }
         }
 qsec_out:
-        if (buf_type == CIFS_SMALL_BUFFER)
-                cifs_small_buf_release(iov[0].iov_base);
-        else if (buf_type == CIFS_LARGE_BUFFER)
-                cifs_buf_release(iov[0].iov_base);
+        free_rsp_buf(buf_type, iov[0].iov_base);
 /*      cifs_small_buf_release(pSMB); */ /* Freed earlier now in SendReceive2 */
         return rc;
 }
diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c
index b98366f21f9e..03ed8a09581c 100644
--- a/fs/cifs/connect.c
+++ b/fs/cifs/connect.c
@@ -557,7 +557,7 @@ cifs_readv_from_socket(struct TCP_Server_Info *server, struct kvec *iov_orig,
                 try_to_freeze();
 
                 if (server_unresponsive(server)) {
-                        total_read = -EAGAIN;
+                        total_read = -ECONNABORTED;
                         break;
                 }
 
@@ -571,7 +571,7 @@ cifs_readv_from_socket(struct TCP_Server_Info *server, struct kvec *iov_orig,
                         break;
                 } else if (server->tcpStatus == CifsNeedReconnect) {
                         cifs_reconnect(server);
-                        total_read = -EAGAIN;
+                        total_read = -ECONNABORTED;
                         break;
                 } else if (length == -ERESTARTSYS ||
                            length == -EAGAIN ||
@@ -588,7 +588,7 @@ cifs_readv_from_socket(struct TCP_Server_Info *server, struct kvec *iov_orig,
                         cifs_dbg(FYI, "Received no data or error: expecting %d\n"
                                  "got %d", to_read, length);
                         cifs_reconnect(server);
-                        total_read = -EAGAIN;
+                        total_read = -ECONNABORTED;
                         break;
                 }
         }
@@ -786,7 +786,7 @@ standard_receive3(struct TCP_Server_Info *server, struct mid_q_entry *mid)
                 cifs_dbg(VFS, "SMB response too long (%u bytes)\n", pdu_length);
                 cifs_reconnect(server);
                 wake_up(&server->response_q);
-                return -EAGAIN;
+                return -ECONNABORTED;
         }
 
         /* switch to large buffer if too big for a small one */
diff --git a/fs/cifs/file.c b/fs/cifs/file.c
index b88b1ade4d3d..4ab2f79ffa7a 100644
--- a/fs/cifs/file.c
+++ b/fs/cifs/file.c
@@ -1670,8 +1670,8 @@ cifs_write(struct cifsFileInfo *open_file, __u32 pid, const char *write_data,
                                         break;
                         }
 
-                        len = min((size_t)cifs_sb->wsize,
-                                  write_size - total_written);
+                        len = min(server->ops->wp_retry_size(dentry->d_inode),
+                                  (unsigned int)write_size - total_written);
                         /* iov[0] is reserved for smb header */
                         iov[1].iov_base = (char *)write_data + total_written;
                         iov[1].iov_len = len;
@@ -1878,15 +1878,163 @@ static int cifs_partialpagewrite(struct page *page, unsigned from, unsigned to)
         return rc;
 }
 
+static struct cifs_writedata *
+wdata_alloc_and_fillpages(pgoff_t tofind, struct address_space *mapping,
+                          pgoff_t end, pgoff_t *index,
+                          unsigned int *found_pages)
+{
+        unsigned int nr_pages;
+        struct page **pages;
+        struct cifs_writedata *wdata;
+
+        wdata = cifs_writedata_alloc((unsigned int)tofind,
+                                     cifs_writev_complete);
+        if (!wdata)
+                return NULL;
+
+        /*
+         * find_get_pages_tag seems to return a max of 256 on each
+         * iteration, so we must call it several times in order to
+         * fill the array or the wsize is effectively limited to
+         * 256 * PAGE_CACHE_SIZE.
+         */
+        *found_pages = 0;
+        pages = wdata->pages;
+        do {
+                nr_pages = find_get_pages_tag(mapping, index,
+                                              PAGECACHE_TAG_DIRTY, tofind,
+                                              pages);
+                *found_pages += nr_pages;
+                tofind -= nr_pages;
+                pages += nr_pages;
+        } while (nr_pages && tofind && *index <= end);
+
+        return wdata;
+}
+
+static unsigned int
+wdata_prepare_pages(struct cifs_writedata *wdata, unsigned int found_pages,
+                    struct address_space *mapping,
+                    struct writeback_control *wbc,
+                    pgoff_t end, pgoff_t *index, pgoff_t *next, bool *done)
+{
+        unsigned int nr_pages = 0, i;
+        struct page *page;
+
+        for (i = 0; i < found_pages; i++) {
+                page = wdata->pages[i];
+                /*
+                 * At this point we hold neither mapping->tree_lock nor
+                 * lock on the page itself: the page may be truncated or
+                 * invalidated (changing page->mapping to NULL), or even
+                 * swizzled back from swapper_space to tmpfs file
+                 * mapping
+                 */
+
+                if (nr_pages == 0)
+                        lock_page(page);
+                else if (!trylock_page(page))
+                        break;
+
+                if (unlikely(page->mapping != mapping)) {
+                        unlock_page(page);
+                        break;
+                }
+
+                if (!wbc->range_cyclic && page->index > end) {
+                        *done = true;
+                        unlock_page(page);
+                        break;
+                }
+
+                if (*next && (page->index != *next)) {
+                        /* Not next consecutive page */
+                        unlock_page(page);
+                        break;
+                }
+
+                if (wbc->sync_mode != WB_SYNC_NONE)
+                        wait_on_page_writeback(page);
+
+                if (PageWriteback(page) ||
+                                !clear_page_dirty_for_io(page)) {
+                        unlock_page(page);
+                        break;
+                }
+
+                /*
+                 * This actually clears the dirty bit in the radix tree.
+                 * See cifs_writepage() for more commentary.
+                 */
+                set_page_writeback(page);
+                if (page_offset(page) >= i_size_read(mapping->host)) {
+                        *done = true;
+                        unlock_page(page);
+                        end_page_writeback(page);
+                        break;
+                }
+
+                wdata->pages[i] = page;
+                *next = page->index + 1;
+                ++nr_pages;
+        }
+
+        /* reset index to refind any pages skipped */
+        if (nr_pages == 0)
+                *index = wdata->pages[0]->index + 1;
+
+        /* put any pages we aren't going to use */
+        for (i = nr_pages; i < found_pages; i++) {
+                page_cache_release(wdata->pages[i]);
+                wdata->pages[i] = NULL;
+        }
+
+        return nr_pages;
+}
+
+static int
+wdata_send_pages(struct cifs_writedata *wdata, unsigned int nr_pages,
+                 struct address_space *mapping, struct writeback_control *wbc)
+{
+        int rc = 0;
+        struct TCP_Server_Info *server;
+        unsigned int i;
+
+        wdata->sync_mode = wbc->sync_mode;
+        wdata->nr_pages = nr_pages;
+        wdata->offset = page_offset(wdata->pages[0]);
+        wdata->pagesz = PAGE_CACHE_SIZE;
+        wdata->tailsz = min(i_size_read(mapping->host) -
+                        page_offset(wdata->pages[nr_pages - 1]),
+                        (loff_t)PAGE_CACHE_SIZE);
+        wdata->bytes = ((nr_pages - 1) * PAGE_CACHE_SIZE) + wdata->tailsz;
+
+        if (wdata->cfile != NULL)
+                cifsFileInfo_put(wdata->cfile);
+        wdata->cfile = find_writable_file(CIFS_I(mapping->host), false);
+        if (!wdata->cfile) {
+                cifs_dbg(VFS, "No writable handles for inode\n");
+                rc = -EBADF;
+        } else {
+                wdata->pid = wdata->cfile->pid;
+                server = tlink_tcon(wdata->cfile->tlink)->ses->server;
+                rc = server->ops->async_writev(wdata, cifs_writedata_release);
+        }
+
+        for (i = 0; i < nr_pages; ++i)
+                unlock_page(wdata->pages[i]);
+
+        return rc;
+}
+
 static int cifs_writepages(struct address_space *mapping,
                            struct writeback_control *wbc)
 {
         struct cifs_sb_info *cifs_sb = CIFS_SB(mapping->host->i_sb);
+        struct TCP_Server_Info *server;
         bool done = false, scanned = false, range_whole = false;
         pgoff_t end, index;
         struct cifs_writedata *wdata;
-        struct TCP_Server_Info *server;
-        struct page *page;
         int rc = 0;
 
         /*
@@ -1906,152 +2054,50 @@ static int cifs_writepages(struct address_space *mapping,
                         range_whole = true;
                 scanned = true;
         }
+        server = cifs_sb_master_tcon(cifs_sb)->ses->server;
 retry:
         while (!done && index <= end) {
-                unsigned int i, nr_pages, found_pages;
-                pgoff_t next = 0, tofind;
-                struct page **pages;
+                unsigned int i, nr_pages, found_pages, wsize, credits;
+                pgoff_t next = 0, tofind, saved_index = index;
+
+                rc = server->ops->wait_mtu_credits(server, cifs_sb->wsize,
+                                                   &wsize, &credits);
+                if (rc)
+                        break;
 
-                tofind = min((cifs_sb->wsize / PAGE_CACHE_SIZE) - 1,
-                                end - index) + 1;
+                tofind = min((wsize / PAGE_CACHE_SIZE) - 1, end - index) + 1;
 
-                wdata = cifs_writedata_alloc((unsigned int)tofind,
-                                             cifs_writev_complete);
+                wdata = wdata_alloc_and_fillpages(tofind, mapping, end, &index,
+                                                  &found_pages);
                 if (!wdata) {
                         rc = -ENOMEM;
+                        add_credits_and_wake_if(server, credits, 0);
                         break;
                 }
 
-                /*
-                 * find_get_pages_tag seems to return a max of 256 on each
-                 * iteration, so we must call it several times in order to
-                 * fill the array or the wsize is effectively limited to
-                 * 256 * PAGE_CACHE_SIZE.
-                 */
-                found_pages = 0;
-                pages = wdata->pages;
-                do {
-                        nr_pages = find_get_pages_tag(mapping, &index,
-                                                        PAGECACHE_TAG_DIRTY,
-                                                        tofind, pages);
-                        found_pages += nr_pages;
-                        tofind -= nr_pages;
-                        pages += nr_pages;
-                } while (nr_pages && tofind && index <= end);
-
                 if (found_pages == 0) {
                         kref_put(&wdata->refcount, cifs_writedata_release);
+                        add_credits_and_wake_if(server, credits, 0);
                         break;
                 }
 
-                nr_pages = 0;
-                for (i = 0; i < found_pages; i++) {
-                        page = wdata->pages[i];
-                        /*
-                         * At this point we hold neither mapping->tree_lock nor
-                         * lock on the page itself: the page may be truncated or
-                         * invalidated (changing page->mapping to NULL), or even
-                         * swizzled back from swapper_space to tmpfs file
-                         * mapping
-                         */
-
-                        if (nr_pages == 0)
-                                lock_page(page);
-                        else if (!trylock_page(page))
-                                break;
-
-                        if (unlikely(page->mapping != mapping)) {
-                                unlock_page(page);
-                                break;
-                        }
-
-                        if (!wbc->range_cyclic && page->index > end) {
-                                done = true;
-                                unlock_page(page);
-                                break;
-                        }
-
-                        if (next && (page->index != next)) {
-                                /* Not next consecutive page */
-                                unlock_page(page);
-                                break;
-                        }
-
-                        if (wbc->sync_mode != WB_SYNC_NONE)
-                                wait_on_page_writeback(page);
-
-                        if (PageWriteback(page) ||
-                                        !clear_page_dirty_for_io(page)) {
-                                unlock_page(page);
-                                break;
-                        }
-
-                        /*
-                         * This actually clears the dirty bit in the radix tree.
-                         * See cifs_writepage() for more commentary.
-                         */
-                        set_page_writeback(page);
-
-                        if (page_offset(page) >= i_size_read(mapping->host)) {
-                                done = true;
-                                unlock_page(page);
-                                end_page_writeback(page);
-                                break;
-                        }
-
-                        wdata->pages[i] = page;
-                        next = page->index + 1;
-                        ++nr_pages;
-                }
-
-                /* reset index to refind any pages skipped */
-                if (nr_pages == 0)
-                        index = wdata->pages[0]->index + 1;
-
-                /* put any pages we aren't going to use */
-                for (i = nr_pages; i < found_pages; i++) {
-                        page_cache_release(wdata->pages[i]);
-                        wdata->pages[i] = NULL;
-                }
+                nr_pages = wdata_prepare_pages(wdata, found_pages, mapping, wbc,
+                                               end, &index, &next, &done);
 
                 /* nothing to write? */
                 if (nr_pages == 0) {
                         kref_put(&wdata->refcount, cifs_writedata_release);
+                        add_credits_and_wake_if(server, credits, 0);
                         continue;
                 }
 
-                wdata->sync_mode = wbc->sync_mode;
-                wdata->nr_pages = nr_pages;
-                wdata->offset = page_offset(wdata->pages[0]);
-                wdata->pagesz = PAGE_CACHE_SIZE;
-                wdata->tailsz =
-                        min(i_size_read(mapping->host) -
-                            page_offset(wdata->pages[nr_pages - 1]),
-                            (loff_t)PAGE_CACHE_SIZE);
-                wdata->bytes = ((nr_pages - 1) * PAGE_CACHE_SIZE) +
-                                        wdata->tailsz;
-
-                do {
-                        if (wdata->cfile != NULL)
-                                cifsFileInfo_put(wdata->cfile);
-                        wdata->cfile = find_writable_file(CIFS_I(mapping->host),
-                                                          false);
-                        if (!wdata->cfile) {
-                                cifs_dbg(VFS, "No writable handles for inode\n");
-                                rc = -EBADF;
-                                break;
-                        }
-                        wdata->pid = wdata->cfile->pid;
-                        server = tlink_tcon(wdata->cfile->tlink)->ses->server;
-                        rc = server->ops->async_writev(wdata,
-                                                        cifs_writedata_release);
-                } while (wbc->sync_mode == WB_SYNC_ALL && rc == -EAGAIN);
+                wdata->credits = credits;
 
-                for (i = 0; i < nr_pages; ++i)
-                        unlock_page(wdata->pages[i]);
+                rc = wdata_send_pages(wdata, nr_pages, mapping, wbc);
 
                 /* send failure -- clean up the mess */
                 if (rc != 0) {
+                        add_credits_and_wake_if(server, wdata->credits, 0);
                         for (i = 0; i < nr_pages; ++i) {
                                 if (rc == -EAGAIN)
                                         redirty_page_for_writepage(wbc,
@@ -2066,6 +2112,11 @@ retry:
                 }
                 kref_put(&wdata->refcount, cifs_writedata_release);
 
+                if (wbc->sync_mode == WB_SYNC_ALL && rc == -EAGAIN) {
+                        index = saved_index;
+                        continue;
+                }
+
                 wbc->nr_to_write -= nr_pages;
                 if (wbc->nr_to_write <= 0)
                         done = true;
@@ -2362,123 +2413,109 @@ cifs_uncached_writev_complete(struct work_struct *work)
         kref_put(&wdata->refcount, cifs_uncached_writedata_release);
 }
 
-/* attempt to send write to server, retry on any -EAGAIN errors */
 static int
-cifs_uncached_retry_writev(struct cifs_writedata *wdata)
+wdata_fill_from_iovec(struct cifs_writedata *wdata, struct iov_iter *from,
+                      size_t *len, unsigned long *num_pages)
 {
-        int rc;
-        struct TCP_Server_Info *server;
+        size_t save_len, copied, bytes, cur_len = *len;
+        unsigned long i, nr_pages = *num_pages;
 
-        server = tlink_tcon(wdata->cfile->tlink)->ses->server;
+        save_len = cur_len;
+        for (i = 0; i < nr_pages; i++) {
+                bytes = min_t(const size_t, cur_len, PAGE_SIZE);
+                copied = copy_page_from_iter(wdata->pages[i], 0, bytes, from);
+                cur_len -= copied;
+                /*
+                 * If we didn't copy as much as we expected, then that
+                 * may mean we trod into an unmapped area. Stop copying
+                 * at that point. On the next pass through the big
+                 * loop, we'll likely end up getting a zero-length
+                 * write and bailing out of it.
+                 */
+                if (copied < bytes)
+                        break;
+        }
+        cur_len = save_len - cur_len;
+        *len = cur_len;
 
-        do {
-                if (wdata->cfile->invalidHandle) {
-                        rc = cifs_reopen_file(wdata->cfile, false);
-                        if (rc != 0)
-                                continue;
-                }
-                rc = server->ops->async_writev(wdata,
-                                               cifs_uncached_writedata_release);
-        } while (rc == -EAGAIN);
+        /*
+         * If we have no data to send, then that probably means that
+         * the copy above failed altogether. That's most likely because
+         * the address in the iovec was bogus. Return -EFAULT and let
+         * the caller free anything we allocated and bail out.
+         */
+        if (!cur_len)
+                return -EFAULT;
 
-        return rc;
+        /*
+         * i + 1 now represents the number of pages we actually used in
+         * the copy phase above.
+         */
+        *num_pages = i + 1;
+        return 0;
 }
 
-static ssize_t
-cifs_iovec_write(struct file *file, struct iov_iter *from, loff_t *poffset)
+static int
+cifs_write_from_iter(loff_t offset, size_t len, struct iov_iter *from,
+                     struct cifsFileInfo *open_file,
+                     struct cifs_sb_info *cifs_sb, struct list_head *wdata_list)
 {
-        unsigned long nr_pages, i;
-        size_t bytes, copied, len, cur_len;
-        ssize_t total_written = 0;
-        loff_t offset;
-        struct cifsFileInfo *open_file;
-        struct cifs_tcon *tcon;
-        struct cifs_sb_info *cifs_sb;
-        struct cifs_writedata *wdata, *tmp;
-        struct list_head wdata_list;
-        int rc;
+        int rc = 0;
+        size_t cur_len;
+        unsigned long nr_pages, num_pages, i;
+        struct cifs_writedata *wdata;
+        struct iov_iter saved_from;
+        loff_t saved_offset = offset;
         pid_t pid;
-
-        len = iov_iter_count(from);
-        rc = generic_write_checks(file, poffset, &len, 0);
-        if (rc)
-                return rc;
-
-        if (!len)
-                return 0;
-
-        iov_iter_truncate(from, len);
-
-        INIT_LIST_HEAD(&wdata_list);
-        cifs_sb = CIFS_SB(file->f_path.dentry->d_sb);
-        open_file = file->private_data;
-        tcon = tlink_tcon(open_file->tlink);
-
-        if (!tcon->ses->server->ops->async_writev)
-                return -ENOSYS;
-
-        offset = *poffset;
+        struct TCP_Server_Info *server;
 
         if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_RWPIDFORWARD)
                 pid = open_file->pid;
         else
                 pid = current->tgid;
 
+        server = tlink_tcon(open_file->tlink)->ses->server;
+        memcpy(&saved_from, from, sizeof(struct iov_iter));
+
         do {
-                size_t save_len;
+                unsigned int wsize, credits;
+
+                rc = server->ops->wait_mtu_credits(server, cifs_sb->wsize,
+                                                   &wsize, &credits);
+                if (rc)
+                        break;
 
-                nr_pages = get_numpages(cifs_sb->wsize, len, &cur_len);
+                nr_pages = get_numpages(wsize, len, &cur_len);
                 wdata = cifs_writedata_alloc(nr_pages,
                                              cifs_uncached_writev_complete);
                 if (!wdata) {
                         rc = -ENOMEM;
+                        add_credits_and_wake_if(server, credits, 0);
                         break;
                 }
 
                 rc = cifs_write_allocate_pages(wdata->pages, nr_pages);
                 if (rc) {
                         kfree(wdata);
+                        add_credits_and_wake_if(server, credits, 0);
                         break;
                 }
 
-                save_len = cur_len;
-                for (i = 0; i < nr_pages; i++) {
-                        bytes = min_t(size_t, cur_len, PAGE_SIZE);
-                        copied = copy_page_from_iter(wdata->pages[i], 0, bytes,
-                                                     from);
-                        cur_len -= copied;
-                        /*
-                         * If we didn't copy as much as we expected, then that
-                         * may mean we trod into an unmapped area. Stop copying
-                         * at that point. On the next pass through the big
-                         * loop, we'll likely end up getting a zero-length
-                         * write and bailing out of it.
-                         */
-                        if (copied < bytes)
-                                break;
-                }
-                cur_len = save_len - cur_len;
-
-                /*
-                 * If we have no data to send, then that probably means that
-                 * the copy above failed altogether. That's most likely because
-                 * the address in the iovec was bogus. Set the rc to -EFAULT,
-                 * free anything we allocated and bail out.
-                 */
-                if (!cur_len) {
+                num_pages = nr_pages;
+                rc = wdata_fill_from_iovec(wdata, from, &cur_len, &num_pages);
+                if (rc) {
                         for (i = 0; i < nr_pages; i++)
                                 put_page(wdata->pages[i]);
                         kfree(wdata);
-                        rc = -EFAULT;
+                        add_credits_and_wake_if(server, credits, 0);
                         break;
                 }
 
                 /*
-                 * i + 1 now represents the number of pages we actually used in
-                 * the copy phase above. Bring nr_pages down to that, and free
-                 * any pages that we didn't use.
+                 * Bring nr_pages down to the number of pages we actually used,
+                 * and free any pages that we didn't use.
                  */
-                for ( ; nr_pages > i + 1; nr_pages--)
+                for ( ; nr_pages > num_pages; nr_pages--)
                         put_page(wdata->pages[nr_pages - 1]);
 
                 wdata->sync_mode = WB_SYNC_ALL;
@@ -2489,18 +2526,69 @@ cifs_iovec_write(struct file *file, struct iov_iter *from, loff_t *poffset)
                 wdata->bytes = cur_len;
                 wdata->pagesz = PAGE_SIZE;
                 wdata->tailsz = cur_len - ((nr_pages - 1) * PAGE_SIZE);
-                rc = cifs_uncached_retry_writev(wdata);
+                wdata->credits = credits;
+
+                if (!wdata->cfile->invalidHandle ||
+                    !cifs_reopen_file(wdata->cfile, false))
+                        rc = server->ops->async_writev(wdata,
+                                        cifs_uncached_writedata_release);
                 if (rc) {
+                        add_credits_and_wake_if(server, wdata->credits, 0);
                         kref_put(&wdata->refcount,
                                  cifs_uncached_writedata_release);
+                        if (rc == -EAGAIN) {
+                                memcpy(from, &saved_from,
+                                       sizeof(struct iov_iter));
+                                iov_iter_advance(from, offset - saved_offset);
+                                continue;
+                        }
                         break;
                 }
 
-                list_add_tail(&wdata->list, &wdata_list);
+                list_add_tail(&wdata->list, wdata_list);
                 offset += cur_len;
                 len -= cur_len;
         } while (len > 0);
 
+        return rc;
+}
+
+static ssize_t
+cifs_iovec_write(struct file *file, struct iov_iter *from, loff_t *poffset)
+{
+        size_t len;
+        ssize_t total_written = 0;
+        struct cifsFileInfo *open_file;
+        struct cifs_tcon *tcon;
+        struct cifs_sb_info *cifs_sb;
+        struct cifs_writedata *wdata, *tmp;
+        struct list_head wdata_list;
+        struct iov_iter saved_from;
+        int rc;
+
+        len = iov_iter_count(from);
+        rc = generic_write_checks(file, poffset, &len, 0);
+        if (rc)
+                return rc;
+
+        if (!len)
+                return 0;
+
+        iov_iter_truncate(from, len);
+
+        INIT_LIST_HEAD(&wdata_list);
+        cifs_sb = CIFS_SB(file->f_path.dentry->d_sb);
+        open_file = file->private_data;
+        tcon = tlink_tcon(open_file->tlink);
+
+        if (!tcon->ses->server->ops->async_writev)
+                return -ENOSYS;
+
+        memcpy(&saved_from, from, sizeof(struct iov_iter));
+
+        rc = cifs_write_from_iter(*poffset, len, from, open_file, cifs_sb,
+                                  &wdata_list);
+
         /*
          * If at least one write was successfully sent, then discard any rc
          * value from the later writes. If the other write succeeds, then
@@ -2529,7 +2617,25 @@ restart_loop:
 
                         /* resend call if it's a retryable error */
                         if (rc == -EAGAIN) {
-                                rc = cifs_uncached_retry_writev(wdata);
+                                struct list_head tmp_list;
+                                struct iov_iter tmp_from;
+
+                                INIT_LIST_HEAD(&tmp_list);
+                                list_del_init(&wdata->list);
+
+                                memcpy(&tmp_from, &saved_from,
+                                       sizeof(struct iov_iter));
+                                iov_iter_advance(&tmp_from,
+                                                 wdata->offset - *poffset);
+
+                                rc = cifs_write_from_iter(wdata->offset,
+                                                wdata->bytes, &tmp_from,
+                                                open_file, cifs_sb, &tmp_list);
+
+                                list_splice(&tmp_list, &wdata_list);
+
+                                kref_put(&wdata->refcount,
+                                         cifs_uncached_writedata_release);
                                 goto restart_loop;
                         }
                 }
@@ -2722,26 +2828,6 @@ cifs_uncached_readdata_release(struct kref *refcount)
         cifs_readdata_release(refcount);
 }
 
-static int
-cifs_retry_async_readv(struct cifs_readdata *rdata)
-{
-        int rc;
-        struct TCP_Server_Info *server;
-
-        server = tlink_tcon(rdata->cfile->tlink)->ses->server;
-
-        do {
-                if (rdata->cfile->invalidHandle) {
-                        rc = cifs_reopen_file(rdata->cfile, true);
-                        if (rc != 0)
-                                continue;
-                }
-                rc = server->ops->async_readv(rdata);
-        } while (rc == -EAGAIN);
-
-        return rc;
-}
-
 /**
  * cifs_readdata_to_iov - copy data from pages in response to an iovec
  * @rdata:      the readdata response with list of pages holding data
@@ -2754,7 +2840,7 @@ cifs_retry_async_readv(struct cifs_readdata *rdata)
 static int
 cifs_readdata_to_iov(struct cifs_readdata *rdata, struct iov_iter *iter)
 {
-        size_t remaining = rdata->bytes;
+        size_t remaining = rdata->got_bytes;
         unsigned int i;
 
         for (i = 0; i < rdata->nr_pages; i++) {
@@ -2782,11 +2868,12 @@ static int
 cifs_uncached_read_into_pages(struct TCP_Server_Info *server,
                         struct cifs_readdata *rdata, unsigned int len)
 {
-        int total_read = 0, result = 0;
+        int result = 0;
         unsigned int i;
         unsigned int nr_pages = rdata->nr_pages;
         struct kvec iov;
 
+        rdata->got_bytes = 0;
         rdata->tailsz = PAGE_SIZE;
         for (i = 0; i < nr_pages; i++) {
                 struct page *page = rdata->pages[i];
@@ -2820,55 +2907,45 @@ cifs_uncached_read_into_pages(struct TCP_Server_Info *server,
                 if (result < 0)
                         break;
 
-                total_read += result;
+                rdata->got_bytes += result;
         }
 
-        return total_read > 0 ? total_read : result;
+        return rdata->got_bytes > 0 && result != -ECONNABORTED ?
+                                                rdata->got_bytes : result;
 }
 
-ssize_t cifs_user_readv(struct kiocb *iocb, struct iov_iter *to)
+static int
+cifs_send_async_read(loff_t offset, size_t len, struct cifsFileInfo *open_file,
+                     struct cifs_sb_info *cifs_sb, struct list_head *rdata_list)
 {
-        struct file *file = iocb->ki_filp;
-        ssize_t rc;
-        size_t len, cur_len;
-        ssize_t total_read = 0;
-        loff_t offset = iocb->ki_pos;
-        unsigned int npages;
-        struct cifs_sb_info *cifs_sb;
-        struct cifs_tcon *tcon;
-        struct cifsFileInfo *open_file;
-        struct cifs_readdata *rdata, *tmp;
-        struct list_head rdata_list;
+        struct cifs_readdata *rdata;
+        unsigned int npages, rsize, credits;
+        size_t cur_len;
+        int rc;
         pid_t pid;
+        struct TCP_Server_Info *server;
 
-        len = iov_iter_count(to);
-        if (!len)
-                return 0;
-
-        INIT_LIST_HEAD(&rdata_list);
-        cifs_sb = CIFS_SB(file->f_path.dentry->d_sb);
-        open_file = file->private_data;
-        tcon = tlink_tcon(open_file->tlink);
-
-        if (!tcon->ses->server->ops->async_readv)
-                return -ENOSYS;
+        server = tlink_tcon(open_file->tlink)->ses->server;
 
         if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_RWPIDFORWARD)
                 pid = open_file->pid;
         else
                 pid = current->tgid;
 
-        if ((file->f_flags & O_ACCMODE) == O_WRONLY)
-                cifs_dbg(FYI, "attempting read on write only file instance\n");
-
         do {
-                cur_len = min_t(const size_t, len - total_read, cifs_sb->rsize);
+                rc = server->ops->wait_mtu_credits(server, cifs_sb->rsize,
+                                                   &rsize, &credits);
+                if (rc)
+                        break;
+
+                cur_len = min_t(const size_t, len, rsize);
                 npages = DIV_ROUND_UP(cur_len, PAGE_SIZE);
 
                 /* allocate a readdata struct */
                 rdata = cifs_readdata_alloc(npages,
                                             cifs_uncached_readv_complete);
                 if (!rdata) {
+                        add_credits_and_wake_if(server, credits, 0);
                         rc = -ENOMEM;
                         break;
                 }
@@ -2884,44 +2961,113 @@ ssize_t cifs_user_readv(struct kiocb *iocb, struct iov_iter *to)
                 rdata->pid = pid;
                 rdata->pagesz = PAGE_SIZE;
                 rdata->read_into_pages = cifs_uncached_read_into_pages;
+                rdata->credits = credits;
 
-                rc = cifs_retry_async_readv(rdata);
+                if (!rdata->cfile->invalidHandle ||
+                    !cifs_reopen_file(rdata->cfile, true))
+                        rc = server->ops->async_readv(rdata);
 error:
                 if (rc) {
+                        add_credits_and_wake_if(server, rdata->credits, 0);
                         kref_put(&rdata->refcount,
                                  cifs_uncached_readdata_release);
+                        if (rc == -EAGAIN)
+                                continue;
                         break;
                 }
 
-                list_add_tail(&rdata->list, &rdata_list);
+                list_add_tail(&rdata->list, rdata_list);
                 offset += cur_len;
                 len -= cur_len;
         } while (len > 0);
 
+        return rc;
+}
+
+ssize_t cifs_user_readv(struct kiocb *iocb, struct iov_iter *to)
+{
+        struct file *file = iocb->ki_filp;
+        ssize_t rc;
+        size_t len;
+        ssize_t total_read = 0;
+        loff_t offset = iocb->ki_pos;
+        struct cifs_sb_info *cifs_sb;
+        struct cifs_tcon *tcon;
+        struct cifsFileInfo *open_file;
+        struct cifs_readdata *rdata, *tmp;
+        struct list_head rdata_list;
+
+        len = iov_iter_count(to);
+        if (!len)
+                return 0;
+
+        INIT_LIST_HEAD(&rdata_list);
+        cifs_sb = CIFS_SB(file->f_path.dentry->d_sb);
+        open_file = file->private_data;
+        tcon = tlink_tcon(open_file->tlink);
+
+        if (!tcon->ses->server->ops->async_readv)
+                return -ENOSYS;
+
+        if ((file->f_flags & O_ACCMODE) == O_WRONLY)
+                cifs_dbg(FYI, "attempting read on write only file instance\n");
+
+        rc = cifs_send_async_read(offset, len, open_file, cifs_sb, &rdata_list);
+
         /* if at least one read request send succeeded, then reset rc */
         if (!list_empty(&rdata_list))
                 rc = 0;
 
         len = iov_iter_count(to);
         /* the loop below should proceed in the order of increasing offsets */
+again:
         list_for_each_entry_safe(rdata, tmp, &rdata_list, list) {
-        again:
                 if (!rc) {
                         /* FIXME: freezable sleep too? */
                         rc = wait_for_completion_killable(&rdata->done);
                         if (rc)
                                 rc = -EINTR;
-                        else if (rdata->result) {
-                                rc = rdata->result;
+                        else if (rdata->result == -EAGAIN) {
                                 /* resend call if it's a retryable error */
-                                if (rc == -EAGAIN) {
-                                        rc = cifs_retry_async_readv(rdata);
-                                        goto again;
+                                struct list_head tmp_list;
+                                unsigned int got_bytes = rdata->got_bytes;
+
+                                list_del_init(&rdata->list);
+                                INIT_LIST_HEAD(&tmp_list);
+
+                                /*
+                                 * Got a part of data and then reconnect has
+                                 * happened -- fill the buffer and continue
+                                 * reading.
+                                 */
+                                if (got_bytes && got_bytes < rdata->bytes) {
+                                        rc = cifs_readdata_to_iov(rdata, to);
+                                        if (rc) {
+                                                kref_put(&rdata->refcount,
+                                                cifs_uncached_readdata_release);
+                                                continue;
+                                        }
                                 }
-                        } else {
+
+                                rc = cifs_send_async_read(
+                                                rdata->offset + got_bytes,
+                                                rdata->bytes - got_bytes,
+                                                rdata->cfile, cifs_sb,
+                                                &tmp_list);
+
+                                list_splice(&tmp_list, &rdata_list);
+
+                                kref_put(&rdata->refcount,
+                                         cifs_uncached_readdata_release);
+                                goto again;
+                        } else if (rdata->result)
+                                rc = rdata->result;
+                        else
                                 rc = cifs_readdata_to_iov(rdata, to);
-                        }
 
+                        /* if there was a short read -- discard anything left */
+                        if (rdata->got_bytes && rdata->got_bytes < rdata->bytes)
+                                rc = -ENODATA;
                 }
                 list_del_init(&rdata->list);
                 kref_put(&rdata->refcount, cifs_uncached_readdata_release);
@@ -3030,18 +3176,19 @@ cifs_read(struct file *file, char *read_data, size_t read_size, loff_t *offset)
 
         for (total_read = 0, cur_offset = read_data; read_size > total_read;
              total_read += bytes_read, cur_offset += bytes_read) {
-                current_read_size = min_t(uint, read_size - total_read, rsize);
-                /*
-                 * For windows me and 9x we do not want to request more than it
-                 * negotiated since it will refuse the read then.
-                 */
-                if ((tcon->ses) && !(tcon->ses->capabilities &
+                do {
+                        current_read_size = min_t(uint, read_size - total_read,
+                                                  rsize);
+                        /*
+                         * For windows me and 9x we do not want to request more
+                         * than it negotiated since it will refuse the read
+                         * then.
+                         */
+                        if ((tcon->ses) && !(tcon->ses->capabilities &
                                 tcon->ses->server->vals->cap_large_files)) {
-                        current_read_size = min_t(uint, current_read_size,
-                                        CIFSMaxBufSize);
-                }
-                rc = -EAGAIN;
-                while (rc == -EAGAIN) {
+                                current_read_size = min_t(uint,
+                                        current_read_size, CIFSMaxBufSize);
+                        }
                         if (open_file->invalidHandle) {
                                 rc = cifs_reopen_file(open_file, true);
                                 if (rc != 0)
@@ -3054,7 +3201,8 @@ cifs_read(struct file *file, char *read_data, size_t read_size, loff_t *offset)
                         rc = server->ops->sync_read(xid, open_file, &io_parms,
                                                     &bytes_read, &cur_offset,
                                                     &buf_type);
-                }
+                } while (rc == -EAGAIN);
+
                 if (rc || (bytes_read == 0)) {
                         if (total_read) {
                                 break;
@@ -3133,25 +3281,30 @@ int cifs_file_mmap(struct file *file, struct vm_area_struct *vma)
 static void
 cifs_readv_complete(struct work_struct *work)
 {
-        unsigned int i;
+        unsigned int i, got_bytes;
         struct cifs_readdata *rdata = container_of(work,
                                                 struct cifs_readdata, work);
 
+        got_bytes = rdata->got_bytes;
         for (i = 0; i < rdata->nr_pages; i++) {
                 struct page *page = rdata->pages[i];
 
                 lru_cache_add_file(page);
 
-                if (rdata->result == 0) {
+                if (rdata->result == 0 ||
+                    (rdata->result == -EAGAIN && got_bytes)) {
                         flush_dcache_page(page);
                         SetPageUptodate(page);
                 }
 
                 unlock_page(page);
 
-                if (rdata->result == 0)
+                if (rdata->result == 0 ||
+                    (rdata->result == -EAGAIN && got_bytes))
                         cifs_readpage_to_fscache(rdata->mapping->host, page);
 
+                got_bytes -= min_t(unsigned int, PAGE_CACHE_SIZE, got_bytes);
+
                 page_cache_release(page);
                 rdata->pages[i] = NULL;
         }
@@ -3162,7 +3315,7 @@ static int
 cifs_readpages_read_into_pages(struct TCP_Server_Info *server,
                         struct cifs_readdata *rdata, unsigned int len)
 {
-        int total_read = 0, result = 0;
+        int result = 0;
         unsigned int i;
         u64 eof;
         pgoff_t eof_index;
@@ -3174,6 +3327,7 @@ cifs_readpages_read_into_pages(struct TCP_Server_Info *server,
         eof_index = eof ? (eof - 1) >> PAGE_CACHE_SHIFT : 0;
         cifs_dbg(FYI, "eof=%llu eof_index=%lu\n", eof, eof_index);
 
+        rdata->got_bytes = 0;
         rdata->tailsz = PAGE_CACHE_SIZE;
         for (i = 0; i < nr_pages; i++) {
                 struct page *page = rdata->pages[i];
@@ -3228,10 +3382,70 @@ cifs_readpages_read_into_pages(struct TCP_Server_Info *server,
                 if (result < 0)
                         break;
 
-                total_read += result;
+                rdata->got_bytes += result;
         }
 
-        return total_read > 0 ? total_read : result;
+        return rdata->got_bytes > 0 && result != -ECONNABORTED ?
+                                                rdata->got_bytes : result;
+}
+
+static int
+readpages_get_pages(struct address_space *mapping, struct list_head *page_list,
+                    unsigned int rsize, struct list_head *tmplist,
+                    unsigned int *nr_pages, loff_t *offset, unsigned int *bytes)
+{
+        struct page *page, *tpage;
+        unsigned int expected_index;
+        int rc;
+
+        INIT_LIST_HEAD(tmplist);
+
+        page = list_entry(page_list->prev, struct page, lru);
+
+        /*
+         * Lock the page and put it in the cache. Since no one else
+         * should have access to this page, we're safe to simply set
+         * PG_locked without checking it first.
+         */
+        __set_page_locked(page);
+        rc = add_to_page_cache_locked(page, mapping,
+                                      page->index, GFP_KERNEL);
+
+        /* give up if we can't stick it in the cache */
+        if (rc) {
+                __clear_page_locked(page);
+                return rc;
+        }
+
+        /* move first page to the tmplist */
+        *offset = (loff_t)page->index << PAGE_CACHE_SHIFT;
+        *bytes = PAGE_CACHE_SIZE;
+        *nr_pages = 1;
+        list_move_tail(&page->lru, tmplist);
+
+        /* now try and add more pages onto the request */
+        expected_index = page->index + 1;
+        list_for_each_entry_safe_reverse(page, tpage, page_list, lru) {
+                /* discontinuity ? */
+                if (page->index != expected_index)
+                        break;
+
+                /* would this page push the read over the rsize? */
+                if (*bytes + PAGE_CACHE_SIZE > rsize)
+                        break;
+
+                __set_page_locked(page);
+                if (add_to_page_cache_locked(page, mapping, page->index,
+                                                                GFP_KERNEL)) {
+                        __clear_page_locked(page);
+                        break;
+                }
+                list_move_tail(&page->lru, tmplist);
+                (*bytes) += PAGE_CACHE_SIZE;
+                expected_index++;
+                (*nr_pages)++;
+        }
+        return rc;
 }
 
 static int cifs_readpages(struct file *file, struct address_space *mapping,
@@ -3241,19 +3455,10 @@ static int cifs_readpages(struct file *file, struct address_space *mapping,
         struct list_head tmplist;
         struct cifsFileInfo *open_file = file->private_data;
         struct cifs_sb_info *cifs_sb = CIFS_SB(file->f_path.dentry->d_sb);
-        unsigned int rsize = cifs_sb->rsize;
+        struct TCP_Server_Info *server;
         pid_t pid;
 
         /*
-         * Give up immediately if rsize is too small to read an entire page.
-         * The VFS will fall back to readpage. We should never reach this
-         * point however since we set ra_pages to 0 when the rsize is smaller
-         * than a cache page.
-         */
-        if (unlikely(rsize < PAGE_CACHE_SIZE))
-                return 0;
-
-        /*
          * Reads as many pages as possible from fscache. Returns -ENOBUFS
          * immediately if the cookie is negative
          *
@@ -3271,7 +3476,7 @@ static int cifs_readpages(struct file *file, struct address_space *mapping,
                 pid = current->tgid;
 
         rc = 0;
-        INIT_LIST_HEAD(&tmplist);
+        server = tlink_tcon(open_file->tlink)->ses->server;
 
         cifs_dbg(FYI, "%s: file=%p mapping=%p num_pages=%u\n",
                  __func__, file, mapping, num_pages);
@@ -3288,58 +3493,35 @@ static int cifs_readpages(struct file *file, struct address_space *mapping,
          * the rdata->pages, then we want them in increasing order.
          */
         while (!list_empty(page_list)) {
-                unsigned int i;
-                unsigned int bytes = PAGE_CACHE_SIZE;
-                unsigned int expected_index;
-                unsigned int nr_pages = 1;
+                unsigned int i, nr_pages, bytes, rsize;
                 loff_t offset;
                 struct page *page, *tpage;
                 struct cifs_readdata *rdata;
+                unsigned credits;
 
-                page = list_entry(page_list->prev, struct page, lru);
+                rc = server->ops->wait_mtu_credits(server, cifs_sb->rsize,
+                                                   &rsize, &credits);
+                if (rc)
+                        break;
 
                 /*
-                 * Lock the page and put it in the cache. Since no one else
-                 * should have access to this page, we're safe to simply set
-                 * PG_locked without checking it first.
+                 * Give up immediately if rsize is too small to read an entire
+                 * page. The VFS will fall back to readpage. We should never
+                 * reach this point however since we set ra_pages to 0 when the
+                 * rsize is smaller than a cache page.
                  */
-                __set_page_locked(page);
-                rc = add_to_page_cache_locked(page, mapping,
-                                              page->index, GFP_KERNEL);
+                if (unlikely(rsize < PAGE_CACHE_SIZE)) {
+                        add_credits_and_wake_if(server, credits, 0);
+                        return 0;
+                }
 
-                /* give up if we can't stick it in the cache */
+                rc = readpages_get_pages(mapping, page_list, rsize, &tmplist,
+                                         &nr_pages, &offset, &bytes);
                 if (rc) {
-                        __clear_page_locked(page);
+                        add_credits_and_wake_if(server, credits, 0);
                         break;
                 }
 
-                /* move first page to the tmplist */
-                offset = (loff_t)page->index << PAGE_CACHE_SHIFT;
-                list_move_tail(&page->lru, &tmplist);
-
-                /* now try and add more pages onto the request */
-                expected_index = page->index + 1;
-                list_for_each_entry_safe_reverse(page, tpage, page_list, lru) {
-                        /* discontinuity ? */
-                        if (page->index != expected_index)
-                                break;
-
-                        /* would this page push the read over the rsize? */
-                        if (bytes + PAGE_CACHE_SIZE > rsize)
-                                break;
-
-                        __set_page_locked(page);
-                        if (add_to_page_cache_locked(page, mapping,
-                                                page->index, GFP_KERNEL)) {
-                                __clear_page_locked(page);
-                                break;
-                        }
-                        list_move_tail(&page->lru, &tmplist);
-                        bytes += PAGE_CACHE_SIZE;
-                        expected_index++;
-                        nr_pages++;
-                }
-
                 rdata = cifs_readdata_alloc(nr_pages, cifs_readv_complete);
                 if (!rdata) {
                         /* best to give up if we're out of mem */
@@ -3350,6 +3532,7 @@ static int cifs_readpages(struct file *file, struct address_space *mapping,
                                 page_cache_release(page);
                         }
                         rc = -ENOMEM;
+                        add_credits_and_wake_if(server, credits, 0);
                         break;
                 }
 
@@ -3360,21 +3543,32 @@ static int cifs_readpages(struct file *file, struct address_space *mapping,
                 rdata->pid = pid;
                 rdata->pagesz = PAGE_CACHE_SIZE;
                 rdata->read_into_pages = cifs_readpages_read_into_pages;
+                rdata->credits = credits;
 
                 list_for_each_entry_safe(page, tpage, &tmplist, lru) {
                         list_del(&page->lru);
                         rdata->pages[rdata->nr_pages++] = page;
                 }
 
-                rc = cifs_retry_async_readv(rdata);
-                if (rc != 0) {
+                if (!rdata->cfile->invalidHandle ||
+                    !cifs_reopen_file(rdata->cfile, true))
+                        rc = server->ops->async_readv(rdata);
+                if (rc) {
+                        add_credits_and_wake_if(server, rdata->credits, 0);
                         for (i = 0; i < rdata->nr_pages; i++) {
                                 page = rdata->pages[i];
                                 lru_cache_add_file(page);
                                 unlock_page(page);
                                 page_cache_release(page);
+                                if (rc == -EAGAIN)
+                                        list_add_tail(&page->lru, &tmplist);
                         }
                         kref_put(&rdata->refcount, cifs_readdata_release);
+                        if (rc == -EAGAIN) {
+                                /* Re-add pages to the page_list and retry */
+                                list_splice(&tmplist, page_list);
+                                continue;
+                        }
                         break;
                 }
 
diff --git a/fs/cifs/misc.c b/fs/cifs/misc.c
index 6bf55d0ed494..81340c6253eb 100644
--- a/fs/cifs/misc.c
+++ b/fs/cifs/misc.c
@@ -226,6 +226,15 @@ cifs_small_buf_release(void *buf_to_free)
         return;
 }
 
+void
+free_rsp_buf(int resp_buftype, void *rsp)
+{
+        if (resp_buftype == CIFS_SMALL_BUFFER)
+                cifs_small_buf_release(rsp);
+        else if (resp_buftype == CIFS_LARGE_BUFFER)
+                cifs_buf_release(rsp);
+}
+
 /* NB: MID can not be set if treeCon not passed in, in that
    case it is responsbility of caller to set the mid */
 void
@@ -414,7 +423,7 @@ is_valid_oplock_break(char *buffer, struct TCP_Server_Info *srv)
                         return true;
                 }
                 if (pSMBr->hdr.Status.CifsError) {
-                        cifs_dbg(FYI, "notify err 0x%d\n",
+                        cifs_dbg(FYI, "notify err 0x%x\n",
                                  pSMBr->hdr.Status.CifsError);
                         return true;
                 }
@@ -441,7 +450,7 @@ is_valid_oplock_break(char *buffer, struct TCP_Server_Info *srv)
         if (pSMB->hdr.WordCount != 8)
                 return false;
 
-        cifs_dbg(FYI, "oplock type 0x%d level 0x%d\n",
+        cifs_dbg(FYI, "oplock type 0x%x level 0x%x\n",
                  pSMB->LockType, pSMB->OplockLevel);
         if (!(pSMB->LockType & LOCKING_ANDX_OPLOCK_RELEASE))
                 return false;
diff --git a/fs/cifs/sess.c b/fs/cifs/sess.c
index e87387dbf39f..39ee32688eac 100644
--- a/fs/cifs/sess.c
+++ b/fs/cifs/sess.c
@@ -520,382 +520,559 @@ select_sectype(struct TCP_Server_Info *server, enum securityEnum requested)
         }
 }
 
-int
-CIFS_SessSetup(const unsigned int xid, struct cifs_ses *ses,
-               const struct nls_table *nls_cp)
+struct sess_data {
+        unsigned int xid;
+        struct cifs_ses *ses;
+        struct nls_table *nls_cp;
+        void (*func)(struct sess_data *);
+        int result;
+
+        /* we will send the SMB in three pieces:
+         * a fixed length beginning part, an optional
+         * SPNEGO blob (which can be zero length), and a
+         * last part which will include the strings
+         * and rest of bcc area. This allows us to avoid
+         * a large buffer 17K allocation
+         */
+        int buf0_type;
+        struct kvec iov[3];
+};
+
+static int
+sess_alloc_buffer(struct sess_data *sess_data, int wct)
 {
-        int rc = 0;
-        int wct;
+        int rc;
+        struct cifs_ses *ses = sess_data->ses;
         struct smb_hdr *smb_buf;
-        char *bcc_ptr;
-        char *str_area;
-        SESSION_SETUP_ANDX *pSMB;
-        __u32 capabilities;
-        __u16 count;
-        int resp_buf_type;
-        struct kvec iov[3];
-        enum securityEnum type;
-        __u16 action, bytes_remaining;
-        struct key *spnego_key = NULL;
-        __le32 phase = NtLmNegotiate; /* NTLMSSP, if needed, is multistage */
-        u16 blob_len;
-        char *ntlmsspblob = NULL;
 
-        if (ses == NULL) {
-                WARN(1, "%s: ses == NULL!", __func__);
-                return -EINVAL;
-        }
+        rc = small_smb_init_no_tc(SMB_COM_SESSION_SETUP_ANDX, wct, ses,
+                                  (void **)&smb_buf);
 
-        type = select_sectype(ses->server, ses->sectype);
-        cifs_dbg(FYI, "sess setup type %d\n", type);
-        if (type == Unspecified) {
-                cifs_dbg(VFS,
-                        "Unable to select appropriate authentication method!");
-                return -EINVAL;
+        if (rc)
+                return rc;
+
+        sess_data->iov[0].iov_base = (char *)smb_buf;
+        sess_data->iov[0].iov_len = be32_to_cpu(smb_buf->smb_buf_length) + 4;
+        /*
+         * This variable will be used to clear the buffer
+         * allocated above in case of any error in the calling function.
+         */
+        sess_data->buf0_type = CIFS_SMALL_BUFFER;
+
+        /* 2000 big enough to fit max user, domain, NOS name etc. */
+        sess_data->iov[2].iov_base = kmalloc(2000, GFP_KERNEL);
+        if (!sess_data->iov[2].iov_base) {
+                rc = -ENOMEM;
+                goto out_free_smb_buf;
         }
 
-        if (type == RawNTLMSSP) {
-                /* if memory allocation is successful, caller of this function
-                 * frees it.
-                 */
-                ses->ntlmssp = kmalloc(sizeof(struct ntlmssp_auth), GFP_KERNEL);
-                if (!ses->ntlmssp)
-                        return -ENOMEM;
-                ses->ntlmssp->sesskey_per_smbsess = false;
+        return 0;
+
+out_free_smb_buf:
+        kfree(smb_buf);
+        sess_data->iov[0].iov_base = NULL;
+        sess_data->iov[0].iov_len = 0;
+        sess_data->buf0_type = CIFS_NO_BUFFER;
+        return rc;
+}
+
+static void
+sess_free_buffer(struct sess_data *sess_data)
+{
 
+        free_rsp_buf(sess_data->buf0_type, sess_data->iov[0].iov_base);
+        sess_data->buf0_type = CIFS_NO_BUFFER;
+        kfree(sess_data->iov[2].iov_base);
+}
+
+static int
+sess_establish_session(struct sess_data *sess_data)
+{
+        struct cifs_ses *ses = sess_data->ses;
+
+        mutex_lock(&ses->server->srv_mutex);
+        if (!ses->server->session_estab) {
+                if (ses->server->sign) {
+                        ses->server->session_key.response =
+                                kmemdup(ses->auth_key.response,
+                                ses->auth_key.len, GFP_KERNEL);
+                        if (!ses->server->session_key.response) {
+                                mutex_unlock(&ses->server->srv_mutex);
+                                return -ENOMEM;
+                        }
+                        ses->server->session_key.len =
+                                                ses->auth_key.len;
+                }
+                ses->server->sequence_number = 0x2;
+                ses->server->session_estab = true;
         }
+        mutex_unlock(&ses->server->srv_mutex);
 
-ssetup_ntlmssp_authenticate:
-        if (phase == NtLmChallenge)
-                phase = NtLmAuthenticate; /* if ntlmssp, now final phase */
+        cifs_dbg(FYI, "CIFS session established successfully\n");
+        spin_lock(&GlobalMid_Lock);
+        ses->status = CifsGood;
+        ses->need_reconnect = false;
+        spin_unlock(&GlobalMid_Lock);
 
-        if (type == LANMAN) {
-#ifndef CONFIG_CIFS_WEAK_PW_HASH
-                /* LANMAN and plaintext are less secure and off by default.
-                So we make this explicitly be turned on in kconfig (in the
-                build) and turned on at runtime (changed from the default)
-                in proc/fs/cifs or via mount parm.  Unfortunately this is
-                needed for old Win (e.g. Win95), some obscure NAS and OS/2 */
-                return -EOPNOTSUPP;
-#endif
-                wct = 10; /* lanman 2 style sessionsetup */
-        } else if ((type == NTLM) || (type == NTLMv2)) {
-                /* For NTLMv2 failures eventually may need to retry NTLM */
-                wct = 13; /* old style NTLM sessionsetup */
-        } else /* same size: negotiate or auth, NTLMSSP or extended security */
-                wct = 12;
+        return 0;
+}
 
-        rc = small_smb_init_no_tc(SMB_COM_SESSION_SETUP_ANDX, wct, ses,
-                            (void **)&smb_buf);
-        if (rc)
-                return rc;
+static int
+sess_sendreceive(struct sess_data *sess_data)
+{
+        int rc;
+        struct smb_hdr *smb_buf = (struct smb_hdr *) sess_data->iov[0].iov_base;
+        __u16 count;
 
-        pSMB = (SESSION_SETUP_ANDX *)smb_buf;
+        count = sess_data->iov[1].iov_len + sess_data->iov[2].iov_len;
+        smb_buf->smb_buf_length =
+                cpu_to_be32(be32_to_cpu(smb_buf->smb_buf_length) + count);
+        put_bcc(count, smb_buf);
+
+        rc = SendReceive2(sess_data->xid, sess_data->ses,
+                          sess_data->iov, 3 /* num_iovecs */,
+                          &sess_data->buf0_type,
+                          CIFS_LOG_ERROR);
+
+        return rc;
+}
 
+/*
+ * LANMAN and plaintext are less secure and off by default.
+ * So we make this explicitly be turned on in kconfig (in the
+ * build) and turned on at runtime (changed from the default)
+ * in proc/fs/cifs or via mount parm.  Unfortunately this is
+ * needed for old Win (e.g. Win95), some obscure NAS and OS/2
+ */
+#ifdef CONFIG_CIFS_WEAK_PW_HASH
+static void
+sess_auth_lanman(struct sess_data *sess_data)
+{
+        int rc = 0;
+        struct smb_hdr *smb_buf;
+        SESSION_SETUP_ANDX *pSMB;
+        char *bcc_ptr;
+        struct cifs_ses *ses = sess_data->ses;
+        char lnm_session_key[CIFS_AUTH_RESP_SIZE];
+        __u32 capabilities;
+        __u16 bytes_remaining;
+
+        /* lanman 2 style sessionsetup */
+        /* wct = 10 */
+        rc = sess_alloc_buffer(sess_data, 10);
+        if (rc)
+                goto out;
+
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        bcc_ptr = sess_data->iov[2].iov_base;
         capabilities = cifs_ssetup_hdr(ses, pSMB);
 
-        /* we will send the SMB in three pieces:
-        a fixed length beginning part, an optional
-        SPNEGO blob (which can be zero length), and a
-        last part which will include the strings
-        and rest of bcc area. This allows us to avoid
-        a large buffer 17K allocation */
-        iov[0].iov_base = (char *)pSMB;
-        iov[0].iov_len = be32_to_cpu(smb_buf->smb_buf_length) + 4;
-
-        /* setting this here allows the code at the end of the function
-           to free the request buffer if there's an error */
-        resp_buf_type = CIFS_SMALL_BUFFER;
+        pSMB->req.hdr.Flags2 &= ~SMBFLG2_UNICODE;
 
-        /* 2000 big enough to fit max user, domain, NOS name etc. */
-        str_area = kmalloc(2000, GFP_KERNEL);
-        if (str_area == NULL) {
-                rc = -ENOMEM;
-                goto ssetup_exit;
-        }
-        bcc_ptr = str_area;
+        /* no capabilities flags in old lanman negotiation */
+        pSMB->old_req.PasswordLength = cpu_to_le16(CIFS_AUTH_RESP_SIZE);
 
-        iov[1].iov_base = NULL;
-        iov[1].iov_len = 0;
+        /* Calculate hash with password and copy into bcc_ptr.
+         * Encryption Key (stored as in cryptkey) gets used if the
+         * security mode bit in Negottiate Protocol response states
+         * to use challenge/response method (i.e. Password bit is 1).
+         */
+        rc = calc_lanman_hash(ses->password, ses->server->cryptkey,
+                              ses->server->sec_mode & SECMODE_PW_ENCRYPT ?
+                              true : false, lnm_session_key);
 
-        if (type == LANMAN) {
-#ifdef CONFIG_CIFS_WEAK_PW_HASH
-                char lnm_session_key[CIFS_AUTH_RESP_SIZE];
+        memcpy(bcc_ptr, (char *)lnm_session_key, CIFS_AUTH_RESP_SIZE);
+        bcc_ptr += CIFS_AUTH_RESP_SIZE;
+
+        /*
+         * can not sign if LANMAN negotiated so no need
+         * to calculate signing key? but what if server
+         * changed to do higher than lanman dialect and
+         * we reconnected would we ever calc signing_key?
+         */
 
-                pSMB->req.hdr.Flags2 &= ~SMBFLG2_UNICODE;
+        cifs_dbg(FYI, "Negotiating LANMAN setting up strings\n");
+        /* Unicode not allowed for LANMAN dialects */
+        ascii_ssetup_strings(&bcc_ptr, ses, sess_data->nls_cp);
 
-                /* no capabilities flags in old lanman negotiation */
+        sess_data->iov[2].iov_len = (long) bcc_ptr -
+                        (long) sess_data->iov[2].iov_base;
 
-                pSMB->old_req.PasswordLength = cpu_to_le16(CIFS_AUTH_RESP_SIZE);
+        rc = sess_sendreceive(sess_data);
+        if (rc)
+                goto out;
 
-                /* Calculate hash with password and copy into bcc_ptr.
-                 * Encryption Key (stored as in cryptkey) gets used if the
-                 * security mode bit in Negottiate Protocol response states
-                 * to use challenge/response method (i.e. Password bit is 1).
-                 */
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        smb_buf = (struct smb_hdr *)sess_data->iov[0].iov_base;
 
-                rc = calc_lanman_hash(ses->password, ses->server->cryptkey,
-                                 ses->server->sec_mode & SECMODE_PW_ENCRYPT ?
-                                        true : false, lnm_session_key);
+        /* lanman response has a word count of 3 */
+        if (smb_buf->WordCount != 3) {
+                rc = -EIO;
+                cifs_dbg(VFS, "bad word count %d\n", smb_buf->WordCount);
+                goto out;
+        }
 
-                memcpy(bcc_ptr, (char *)lnm_session_key, CIFS_AUTH_RESP_SIZE);
-                bcc_ptr += CIFS_AUTH_RESP_SIZE;
+        if (le16_to_cpu(pSMB->resp.Action) & GUEST_LOGIN)
+                cifs_dbg(FYI, "Guest login\n"); /* BB mark SesInfo struct? */
+
+        ses->Suid = smb_buf->Uid;   /* UID left in wire format (le) */
+        cifs_dbg(FYI, "UID = %llu\n", ses->Suid);
 
-                /* can not sign if LANMAN negotiated so no need
-                to calculate signing key? but what if server
-                changed to do higher than lanman dialect and
-                we reconnected would we ever calc signing_key? */
+        bytes_remaining = get_bcc(smb_buf);
+        bcc_ptr = pByteArea(smb_buf);
 
-                cifs_dbg(FYI, "Negotiating LANMAN setting up strings\n");
-                /* Unicode not allowed for LANMAN dialects */
-                ascii_ssetup_strings(&bcc_ptr, ses, nls_cp);
+        /* BB check if Unicode and decode strings */
+        if (bytes_remaining == 0) {
+                /* no string area to decode, do nothing */
+        } else if (smb_buf->Flags2 & SMBFLG2_UNICODE) {
+                /* unicode string area must be word-aligned */
+                if (((unsigned long) bcc_ptr - (unsigned long) smb_buf) % 2) {
+                        ++bcc_ptr;
+                        --bytes_remaining;
+                }
+                decode_unicode_ssetup(&bcc_ptr, bytes_remaining, ses,
+                                      sess_data->nls_cp);
+        } else {
+                decode_ascii_ssetup(&bcc_ptr, bytes_remaining, ses,
+                                    sess_data->nls_cp);
+        }
+
+        rc = sess_establish_session(sess_data);
+out:
+        sess_data->result = rc;
+        sess_data->func = NULL;
+        sess_free_buffer(sess_data);
+}
+
+#else
+
+static void
+sess_auth_lanman(struct sess_data *sess_data)
+{
+        sess_data->result = -EOPNOTSUPP;
+        sess_data->func = NULL;
+}
 #endif
-        } else if (type == NTLM) {
-                pSMB->req_no_secext.Capabilities = cpu_to_le32(capabilities);
-                pSMB->req_no_secext.CaseInsensitivePasswordLength =
+
+static void
+sess_auth_ntlm(struct sess_data *sess_data)
+{
+        int rc = 0;
+        struct smb_hdr *smb_buf;
+        SESSION_SETUP_ANDX *pSMB;
+        char *bcc_ptr;
+        struct cifs_ses *ses = sess_data->ses;
+        __u32 capabilities;
+        __u16 bytes_remaining;
+
+        /* old style NTLM sessionsetup */
+        /* wct = 13 */
+        rc = sess_alloc_buffer(sess_data, 13);
+        if (rc)
+                goto out;
+
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        bcc_ptr = sess_data->iov[2].iov_base;
+        capabilities = cifs_ssetup_hdr(ses, pSMB);
+
+        pSMB->req_no_secext.Capabilities = cpu_to_le32(capabilities);
+        pSMB->req_no_secext.CaseInsensitivePasswordLength =
                         cpu_to_le16(CIFS_AUTH_RESP_SIZE);
-                pSMB->req_no_secext.CaseSensitivePasswordLength =
+        pSMB->req_no_secext.CaseSensitivePasswordLength =
                         cpu_to_le16(CIFS_AUTH_RESP_SIZE);
 
-                /* calculate ntlm response and session key */
-                rc = setup_ntlm_response(ses, nls_cp);
-                if (rc) {
-                        cifs_dbg(VFS, "Error %d during NTLM authentication\n",
+        /* calculate ntlm response and session key */
+        rc = setup_ntlm_response(ses, sess_data->nls_cp);
+        if (rc) {
+                cifs_dbg(VFS, "Error %d during NTLM authentication\n",
                                  rc);
-                        goto ssetup_exit;
-                }
+                goto out;
+        }
 
-                /* copy ntlm response */
-                memcpy(bcc_ptr, ses->auth_key.response + CIFS_SESS_KEY_SIZE,
-                                CIFS_AUTH_RESP_SIZE);
-                bcc_ptr += CIFS_AUTH_RESP_SIZE;
-                memcpy(bcc_ptr, ses->auth_key.response + CIFS_SESS_KEY_SIZE,
-                                CIFS_AUTH_RESP_SIZE);
-                bcc_ptr += CIFS_AUTH_RESP_SIZE;
-
-                if (ses->capabilities & CAP_UNICODE) {
-                        /* unicode strings must be word aligned */
-                        if (iov[0].iov_len % 2) {
-                                *bcc_ptr = 0;
-                                bcc_ptr++;
-                        }
-                        unicode_ssetup_strings(&bcc_ptr, ses, nls_cp);
-                } else
-                        ascii_ssetup_strings(&bcc_ptr, ses, nls_cp);
-        } else if (type == NTLMv2) {
-                pSMB->req_no_secext.Capabilities = cpu_to_le32(capabilities);
-
-                /* LM2 password would be here if we supported it */
-                pSMB->req_no_secext.CaseInsensitivePasswordLength = 0;
-
-                /* calculate nlmv2 response and session key */
-                rc = setup_ntlmv2_rsp(ses, nls_cp);
-                if (rc) {
-                        cifs_dbg(VFS, "Error %d during NTLMv2 authentication\n",
-                                 rc);
-                        goto ssetup_exit;
+        /* copy ntlm response */
+        memcpy(bcc_ptr, ses->auth_key.response + CIFS_SESS_KEY_SIZE,
+                        CIFS_AUTH_RESP_SIZE);
+        bcc_ptr += CIFS_AUTH_RESP_SIZE;
+        memcpy(bcc_ptr, ses->auth_key.response + CIFS_SESS_KEY_SIZE,
+                        CIFS_AUTH_RESP_SIZE);
+        bcc_ptr += CIFS_AUTH_RESP_SIZE;
+
+        if (ses->capabilities & CAP_UNICODE) {
+                /* unicode strings must be word aligned */
+                if (sess_data->iov[0].iov_len % 2) {
+                        *bcc_ptr = 0;
+                        bcc_ptr++;
                 }
-                memcpy(bcc_ptr, ses->auth_key.response + CIFS_SESS_KEY_SIZE,
-                                ses->auth_key.len - CIFS_SESS_KEY_SIZE);
-                bcc_ptr += ses->auth_key.len - CIFS_SESS_KEY_SIZE;
-
-                /* set case sensitive password length after tilen may get
-                 * assigned, tilen is 0 otherwise.
-                 */
-                pSMB->req_no_secext.CaseSensitivePasswordLength =
-                        cpu_to_le16(ses->auth_key.len - CIFS_SESS_KEY_SIZE);
+                unicode_ssetup_strings(&bcc_ptr, ses, sess_data->nls_cp);
+        } else {
+                ascii_ssetup_strings(&bcc_ptr, ses, sess_data->nls_cp);
+        }
 
-                if (ses->capabilities & CAP_UNICODE) {
-                        if (iov[0].iov_len % 2) {
-                                *bcc_ptr = 0;
-                                bcc_ptr++;
-                        }
-                        unicode_ssetup_strings(&bcc_ptr, ses, nls_cp);
-                } else
-                        ascii_ssetup_strings(&bcc_ptr, ses, nls_cp);
-        } else if (type == Kerberos) {
-#ifdef CONFIG_CIFS_UPCALL
-                struct cifs_spnego_msg *msg;
 
-                spnego_key = cifs_get_spnego_key(ses);
-                if (IS_ERR(spnego_key)) {
-                        rc = PTR_ERR(spnego_key);
-                        spnego_key = NULL;
-                        goto ssetup_exit;
-                }
+        sess_data->iov[2].iov_len = (long) bcc_ptr -
+                        (long) sess_data->iov[2].iov_base;
 
-                msg = spnego_key->payload.data;
-                /* check version field to make sure that cifs.upcall is
-                   sending us a response in an expected form */
-                if (msg->version != CIFS_SPNEGO_UPCALL_VERSION) {
-                        cifs_dbg(VFS, "incorrect version of cifs.upcall "
-                                   "expected %d but got %d)",
-                                   CIFS_SPNEGO_UPCALL_VERSION, msg->version);
-                        rc = -EKEYREJECTED;
-                        goto ssetup_exit;
-                }
+        rc = sess_sendreceive(sess_data);
+        if (rc)
+                goto out;
 
-                ses->auth_key.response = kmemdup(msg->data, msg->sesskey_len,
-                                                 GFP_KERNEL);
-                if (!ses->auth_key.response) {
-                        cifs_dbg(VFS,
-                                "Kerberos can't allocate (%u bytes) memory",
-                                msg->sesskey_len);
-                        rc = -ENOMEM;
-                        goto ssetup_exit;
-                }
-                ses->auth_key.len = msg->sesskey_len;
-
-                pSMB->req.hdr.Flags2 |= SMBFLG2_EXT_SEC;
-                capabilities |= CAP_EXTENDED_SECURITY;
-                pSMB->req.Capabilities = cpu_to_le32(capabilities);
-                iov[1].iov_base = msg->data + msg->sesskey_len;
-                iov[1].iov_len = msg->secblob_len;
-                pSMB->req.SecurityBlobLength = cpu_to_le16(iov[1].iov_len);
-
-                if (ses->capabilities & CAP_UNICODE) {
-                        /* unicode strings must be word aligned */
-                        if ((iov[0].iov_len + iov[1].iov_len) % 2) {
-                                *bcc_ptr = 0;
-                                bcc_ptr++;
-                        }
-                        unicode_oslm_strings(&bcc_ptr, nls_cp);
-                        unicode_domain_string(&bcc_ptr, ses, nls_cp);
-                } else
-                /* BB: is this right? */
-                        ascii_ssetup_strings(&bcc_ptr, ses, nls_cp);
-#else /* ! CONFIG_CIFS_UPCALL */
-                cifs_dbg(VFS, "Kerberos negotiated but upcall support disabled!\n");
-                rc = -ENOSYS;
-                goto ssetup_exit;
-#endif /* CONFIG_CIFS_UPCALL */
-        } else if (type == RawNTLMSSP) {
-                if ((pSMB->req.hdr.Flags2 & SMBFLG2_UNICODE) == 0) {
-                        cifs_dbg(VFS, "NTLMSSP requires Unicode support\n");
-                        rc = -ENOSYS;
-                        goto ssetup_exit;
-                }
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        smb_buf = (struct smb_hdr *)sess_data->iov[0].iov_base;
 
-                cifs_dbg(FYI, "ntlmssp session setup phase %d\n", phase);
-                pSMB->req.hdr.Flags2 |= SMBFLG2_EXT_SEC;
-                capabilities |= CAP_EXTENDED_SECURITY;
-                pSMB->req.Capabilities |= cpu_to_le32(capabilities);
-                switch(phase) {
-                case NtLmNegotiate:
-                        build_ntlmssp_negotiate_blob(
-                                pSMB->req.SecurityBlob, ses);
-                        iov[1].iov_len = sizeof(NEGOTIATE_MESSAGE);
-                        iov[1].iov_base = pSMB->req.SecurityBlob;
-                        pSMB->req.SecurityBlobLength =
-                                cpu_to_le16(sizeof(NEGOTIATE_MESSAGE));
-                        break;
-                case NtLmAuthenticate:
-                        /*
-                         * 5 is an empirical value, large enough to hold
-                         * authenticate message plus max 10 of av paris,
-                         * domain, user, workstation names, flags, etc.
-                         */
-                        ntlmsspblob = kzalloc(
-                                5*sizeof(struct _AUTHENTICATE_MESSAGE),
-                                GFP_KERNEL);
-                        if (!ntlmsspblob) {
-                                rc = -ENOMEM;
-                                goto ssetup_exit;
-                        }
+        if (smb_buf->WordCount != 3) {
+                rc = -EIO;
+                cifs_dbg(VFS, "bad word count %d\n", smb_buf->WordCount);
+                goto out;
+        }
 
-                        rc = build_ntlmssp_auth_blob(ntlmsspblob,
-                                                &blob_len, ses, nls_cp);
-                        if (rc)
-                                goto ssetup_exit;
-                        iov[1].iov_len = blob_len;
-                        iov[1].iov_base = ntlmsspblob;
-                        pSMB->req.SecurityBlobLength = cpu_to_le16(blob_len);
-                        /*
-                         * Make sure that we tell the server that we are using
-                         * the uid that it just gave us back on the response
-                         * (challenge)
-                         */
-                        smb_buf->Uid = ses->Suid;
-                        break;
-                default:
-                        cifs_dbg(VFS, "invalid phase %d\n", phase);
-                        rc = -ENOSYS;
-                        goto ssetup_exit;
+        if (le16_to_cpu(pSMB->resp.Action) & GUEST_LOGIN)
+                cifs_dbg(FYI, "Guest login\n"); /* BB mark SesInfo struct? */
+
+        ses->Suid = smb_buf->Uid;   /* UID left in wire format (le) */
+        cifs_dbg(FYI, "UID = %llu\n", ses->Suid);
+
+        bytes_remaining = get_bcc(smb_buf);
+        bcc_ptr = pByteArea(smb_buf);
+
+        /* BB check if Unicode and decode strings */
+        if (bytes_remaining == 0) {
+                /* no string area to decode, do nothing */
+        } else if (smb_buf->Flags2 & SMBFLG2_UNICODE) {
+                /* unicode string area must be word-aligned */
+                if (((unsigned long) bcc_ptr - (unsigned long) smb_buf) % 2) {
+                        ++bcc_ptr;
+                        --bytes_remaining;
                 }
-                /* unicode strings must be word aligned */
-                if ((iov[0].iov_len + iov[1].iov_len) % 2) {
+                decode_unicode_ssetup(&bcc_ptr, bytes_remaining, ses,
+                                      sess_data->nls_cp);
+        } else {
+                decode_ascii_ssetup(&bcc_ptr, bytes_remaining, ses,
+                                    sess_data->nls_cp);
+        }
+
+        rc = sess_establish_session(sess_data);
+out:
+        sess_data->result = rc;
+        sess_data->func = NULL;
+        sess_free_buffer(sess_data);
+        kfree(ses->auth_key.response);
+        ses->auth_key.response = NULL;
+}
+
+static void
+sess_auth_ntlmv2(struct sess_data *sess_data)
+{
+        int rc = 0;
+        struct smb_hdr *smb_buf;
+        SESSION_SETUP_ANDX *pSMB;
+        char *bcc_ptr;
+        struct cifs_ses *ses = sess_data->ses;
+        __u32 capabilities;
+        __u16 bytes_remaining;
+
+        /* old style NTLM sessionsetup */
+        /* wct = 13 */
+        rc = sess_alloc_buffer(sess_data, 13);
+        if (rc)
+                goto out;
+
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        bcc_ptr = sess_data->iov[2].iov_base;
+        capabilities = cifs_ssetup_hdr(ses, pSMB);
+
+        pSMB->req_no_secext.Capabilities = cpu_to_le32(capabilities);
+
+        /* LM2 password would be here if we supported it */
+        pSMB->req_no_secext.CaseInsensitivePasswordLength = 0;
+
+        /* calculate nlmv2 response and session key */
+        rc = setup_ntlmv2_rsp(ses, sess_data->nls_cp);
+        if (rc) {
+                cifs_dbg(VFS, "Error %d during NTLMv2 authentication\n", rc);
+                goto out;
+        }
+
+        memcpy(bcc_ptr, ses->auth_key.response + CIFS_SESS_KEY_SIZE,
+                        ses->auth_key.len - CIFS_SESS_KEY_SIZE);
+        bcc_ptr += ses->auth_key.len - CIFS_SESS_KEY_SIZE;
+
+        /* set case sensitive password length after tilen may get
+         * assigned, tilen is 0 otherwise.
+         */
+        pSMB->req_no_secext.CaseSensitivePasswordLength =
+                cpu_to_le16(ses->auth_key.len - CIFS_SESS_KEY_SIZE);
+
+        if (ses->capabilities & CAP_UNICODE) {
+                if (sess_data->iov[0].iov_len % 2) {
                         *bcc_ptr = 0;
                         bcc_ptr++;
                 }
-                unicode_oslm_strings(&bcc_ptr, nls_cp);
+                unicode_ssetup_strings(&bcc_ptr, ses, sess_data->nls_cp);
         } else {
-                cifs_dbg(VFS, "secType %d not supported!\n", type);
-                rc = -ENOSYS;
-                goto ssetup_exit;
+                ascii_ssetup_strings(&bcc_ptr, ses, sess_data->nls_cp);
         }
 
-        iov[2].iov_base = str_area;
-        iov[2].iov_len = (long) bcc_ptr - (long) str_area;
 
-        count = iov[1].iov_len + iov[2].iov_len;
-        smb_buf->smb_buf_length =
-                cpu_to_be32(be32_to_cpu(smb_buf->smb_buf_length) + count);
+        sess_data->iov[2].iov_len = (long) bcc_ptr -
+                        (long) sess_data->iov[2].iov_base;
 
-        put_bcc(count, smb_buf);
+        rc = sess_sendreceive(sess_data);
+        if (rc)
+                goto out;
 
-        rc = SendReceive2(xid, ses, iov, 3 /* num_iovecs */, &resp_buf_type,
-                          CIFS_LOG_ERROR);
-        /* SMB request buf freed in SendReceive2 */
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        smb_buf = (struct smb_hdr *)sess_data->iov[0].iov_base;
+
+        if (smb_buf->WordCount != 3) {
+                rc = -EIO;
+                cifs_dbg(VFS, "bad word count %d\n", smb_buf->WordCount);
+                goto out;
+        }
+
+        if (le16_to_cpu(pSMB->resp.Action) & GUEST_LOGIN)
+                cifs_dbg(FYI, "Guest login\n"); /* BB mark SesInfo struct? */
+
+        ses->Suid = smb_buf->Uid;   /* UID left in wire format (le) */
+        cifs_dbg(FYI, "UID = %llu\n", ses->Suid);
 
-        pSMB = (SESSION_SETUP_ANDX *)iov[0].iov_base;
-        smb_buf = (struct smb_hdr *)iov[0].iov_base;
+        bytes_remaining = get_bcc(smb_buf);
+        bcc_ptr = pByteArea(smb_buf);
 
-        if ((type == RawNTLMSSP) && (resp_buf_type != CIFS_NO_BUFFER) &&
-            (smb_buf->Status.CifsError ==
-                        cpu_to_le32(NT_STATUS_MORE_PROCESSING_REQUIRED))) {
-                if (phase != NtLmNegotiate) {
-                        cifs_dbg(VFS, "Unexpected more processing error\n");
-                        goto ssetup_exit;
+        /* BB check if Unicode and decode strings */
+        if (bytes_remaining == 0) {
+                /* no string area to decode, do nothing */
+        } else if (smb_buf->Flags2 & SMBFLG2_UNICODE) {
+                /* unicode string area must be word-aligned */
+                if (((unsigned long) bcc_ptr - (unsigned long) smb_buf) % 2) {
+                        ++bcc_ptr;
+                        --bytes_remaining;
                 }
-                /* NTLMSSP Negotiate sent now processing challenge (response) */
-                phase = NtLmChallenge; /* process ntlmssp challenge */
-                rc = 0; /* MORE_PROC rc is not an error here, but expected */
+                decode_unicode_ssetup(&bcc_ptr, bytes_remaining, ses,
+                                      sess_data->nls_cp);
+        } else {
+                decode_ascii_ssetup(&bcc_ptr, bytes_remaining, ses,
+                                    sess_data->nls_cp);
         }
+
+        rc = sess_establish_session(sess_data);
+out:
+        sess_data->result = rc;
+        sess_data->func = NULL;
+        sess_free_buffer(sess_data);
+        kfree(ses->auth_key.response);
+        ses->auth_key.response = NULL;
+}
+
+#ifdef CONFIG_CIFS_UPCALL
+static void
+sess_auth_kerberos(struct sess_data *sess_data)
+{
+        int rc = 0;
+        struct smb_hdr *smb_buf;
+        SESSION_SETUP_ANDX *pSMB;
+        char *bcc_ptr;
+        struct cifs_ses *ses = sess_data->ses;
+        __u32 capabilities;
+        __u16 bytes_remaining;
+        struct key *spnego_key = NULL;
+        struct cifs_spnego_msg *msg;
+        u16 blob_len;
+
+        /* extended security */
+        /* wct = 12 */
+        rc = sess_alloc_buffer(sess_data, 12);
         if (rc)
-                goto ssetup_exit;
+                goto out;
 
-        if ((smb_buf->WordCount != 3) && (smb_buf->WordCount != 4)) {
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        bcc_ptr = sess_data->iov[2].iov_base;
+        capabilities = cifs_ssetup_hdr(ses, pSMB);
+
+        spnego_key = cifs_get_spnego_key(ses);
+        if (IS_ERR(spnego_key)) {
+                rc = PTR_ERR(spnego_key);
+                spnego_key = NULL;
+                goto out;
+        }
+
+        msg = spnego_key->payload.data;
+        /*
+         * check version field to make sure that cifs.upcall is
+         * sending us a response in an expected form
+         */
+        if (msg->version != CIFS_SPNEGO_UPCALL_VERSION) {
+                cifs_dbg(VFS,
+                  "incorrect version of cifs.upcall (expected %d but got %d)",
+                              CIFS_SPNEGO_UPCALL_VERSION, msg->version);
+                rc = -EKEYREJECTED;
+                goto out_put_spnego_key;
+        }
+
+        ses->auth_key.response = kmemdup(msg->data, msg->sesskey_len,
+                                         GFP_KERNEL);
+        if (!ses->auth_key.response) {
+                cifs_dbg(VFS, "Kerberos can't allocate (%u bytes) memory",
+                                msg->sesskey_len);
+                rc = -ENOMEM;
+                goto out_put_spnego_key;
+        }
+        ses->auth_key.len = msg->sesskey_len;
+
+        pSMB->req.hdr.Flags2 |= SMBFLG2_EXT_SEC;
+        capabilities |= CAP_EXTENDED_SECURITY;
+        pSMB->req.Capabilities = cpu_to_le32(capabilities);
+        sess_data->iov[1].iov_base = msg->data + msg->sesskey_len;
+        sess_data->iov[1].iov_len = msg->secblob_len;
+        pSMB->req.SecurityBlobLength = cpu_to_le16(sess_data->iov[1].iov_len);
+
+        if (ses->capabilities & CAP_UNICODE) {
+                /* unicode strings must be word aligned */
+                if ((sess_data->iov[0].iov_len
+                        + sess_data->iov[1].iov_len) % 2) {
+                        *bcc_ptr = 0;
+                        bcc_ptr++;
+                }
+                unicode_oslm_strings(&bcc_ptr, sess_data->nls_cp);
+                unicode_domain_string(&bcc_ptr, ses, sess_data->nls_cp);
+        } else {
+                /* BB: is this right? */
+                ascii_ssetup_strings(&bcc_ptr, ses, sess_data->nls_cp);
+        }
+
+        sess_data->iov[2].iov_len = (long) bcc_ptr -
+                        (long) sess_data->iov[2].iov_base;
+
+        rc = sess_sendreceive(sess_data);
+        if (rc)
+                goto out_put_spnego_key;
+
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        smb_buf = (struct smb_hdr *)sess_data->iov[0].iov_base;
+
+        if (smb_buf->WordCount != 4) {
                 rc = -EIO;
                 cifs_dbg(VFS, "bad word count %d\n", smb_buf->WordCount);
-                goto ssetup_exit;
+                goto out_put_spnego_key;
         }
-        action = le16_to_cpu(pSMB->resp.Action);
-        if (action & GUEST_LOGIN)
+
+        if (le16_to_cpu(pSMB->resp.Action) & GUEST_LOGIN)
                 cifs_dbg(FYI, "Guest login\n"); /* BB mark SesInfo struct? */
+
         ses->Suid = smb_buf->Uid;   /* UID left in wire format (le) */
         cifs_dbg(FYI, "UID = %llu\n", ses->Suid);
-        /* response can have either 3 or 4 word count - Samba sends 3 */
-        /* and lanman response is 3 */
+
         bytes_remaining = get_bcc(smb_buf);
         bcc_ptr = pByteArea(smb_buf);
 
-        if (smb_buf->WordCount == 4) {
-                blob_len = le16_to_cpu(pSMB->resp.SecurityBlobLength);
-                if (blob_len > bytes_remaining) {
-                        cifs_dbg(VFS, "bad security blob length %d\n",
-                                 blob_len);
-                        rc = -EINVAL;
-                        goto ssetup_exit;
-                }
-                if (phase == NtLmChallenge) {
-                        rc = decode_ntlmssp_challenge(bcc_ptr, blob_len, ses);
-                        /* now goto beginning for ntlmssp authenticate phase */
-                        if (rc)
-                                goto ssetup_exit;
-                }
-                bcc_ptr += blob_len;
-                bytes_remaining -= blob_len;
+        blob_len = le16_to_cpu(pSMB->resp.SecurityBlobLength);
+        if (blob_len > bytes_remaining) {
+                cifs_dbg(VFS, "bad security blob length %d\n",
+                                blob_len);
+                rc = -EINVAL;
+                goto out_put_spnego_key;
         }
+        bcc_ptr += blob_len;
+        bytes_remaining -= blob_len;
 
         /* BB check if Unicode and decode strings */
         if (bytes_remaining == 0) {
@@ -906,60 +1083,371 @@ ssetup_ntlmssp_authenticate:
                         ++bcc_ptr;
                         --bytes_remaining;
                 }
-                decode_unicode_ssetup(&bcc_ptr, bytes_remaining, ses, nls_cp);
+                decode_unicode_ssetup(&bcc_ptr, bytes_remaining, ses,
+                                      sess_data->nls_cp);
         } else {
-                decode_ascii_ssetup(&bcc_ptr, bytes_remaining, ses, nls_cp);
+                decode_ascii_ssetup(&bcc_ptr, bytes_remaining, ses,
+                                    sess_data->nls_cp);
         }
 
-ssetup_exit:
-        if (spnego_key) {
-                key_invalidate(spnego_key);
-                key_put(spnego_key);
+        rc = sess_establish_session(sess_data);
+out_put_spnego_key:
+        key_invalidate(spnego_key);
+        key_put(spnego_key);
+out:
+        sess_data->result = rc;
+        sess_data->func = NULL;
+        sess_free_buffer(sess_data);
+        kfree(ses->auth_key.response);
+        ses->auth_key.response = NULL;
+}
+
+#else
+
+static void
+sess_auth_kerberos(struct sess_data *sess_data)
+{
+        cifs_dbg(VFS, "Kerberos negotiated but upcall support disabled!\n");
+        sess_data->result = -ENOSYS;
+        sess_data->func = NULL;
+}
+#endif /* ! CONFIG_CIFS_UPCALL */
+
+/*
+ * The required kvec buffers have to be allocated before calling this
+ * function.
+ */
+static int
+_sess_auth_rawntlmssp_assemble_req(struct sess_data *sess_data)
+{
+        struct smb_hdr *smb_buf;
+        SESSION_SETUP_ANDX *pSMB;
+        struct cifs_ses *ses = sess_data->ses;
+        __u32 capabilities;
+        char *bcc_ptr;
+
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        smb_buf = (struct smb_hdr *)pSMB;
+
+        capabilities = cifs_ssetup_hdr(ses, pSMB);
+        if ((pSMB->req.hdr.Flags2 & SMBFLG2_UNICODE) == 0) {
+                cifs_dbg(VFS, "NTLMSSP requires Unicode support\n");
+                return -ENOSYS;
         }
-        kfree(str_area);
-        kfree(ntlmsspblob);
-        ntlmsspblob = NULL;
-        if (resp_buf_type == CIFS_SMALL_BUFFER) {
-                cifs_dbg(FYI, "ssetup freeing small buf %p\n", iov[0].iov_base);
-                cifs_small_buf_release(iov[0].iov_base);
-        } else if (resp_buf_type == CIFS_LARGE_BUFFER)
-                cifs_buf_release(iov[0].iov_base);
 
-        /* if ntlmssp, and negotiate succeeded, proceed to authenticate phase */
-        if ((phase == NtLmChallenge) && (rc == 0))
-                goto ssetup_ntlmssp_authenticate;
+        pSMB->req.hdr.Flags2 |= SMBFLG2_EXT_SEC;
+        capabilities |= CAP_EXTENDED_SECURITY;
+        pSMB->req.Capabilities |= cpu_to_le32(capabilities);
+
+        bcc_ptr = sess_data->iov[2].iov_base;
+        /* unicode strings must be word aligned */
+        if ((sess_data->iov[0].iov_len + sess_data->iov[1].iov_len) % 2) {
+                *bcc_ptr = 0;
+                bcc_ptr++;
+        }
+        unicode_oslm_strings(&bcc_ptr, sess_data->nls_cp);
+
+        sess_data->iov[2].iov_len = (long) bcc_ptr -
+                                        (long) sess_data->iov[2].iov_base;
+
+        return 0;
+}
+
+static void
+sess_auth_rawntlmssp_authenticate(struct sess_data *sess_data);
+
+static void
+sess_auth_rawntlmssp_negotiate(struct sess_data *sess_data)
+{
+        int rc;
+        struct smb_hdr *smb_buf;
+        SESSION_SETUP_ANDX *pSMB;
+        struct cifs_ses *ses = sess_data->ses;
+        __u16 bytes_remaining;
+        char *bcc_ptr;
+        u16 blob_len;
+
+        cifs_dbg(FYI, "rawntlmssp session setup negotiate phase\n");
+
+        /*
+         * if memory allocation is successful, caller of this function
+         * frees it.
+         */
+        ses->ntlmssp = kmalloc(sizeof(struct ntlmssp_auth), GFP_KERNEL);
+        if (!ses->ntlmssp) {
+                rc = -ENOMEM;
+                goto out;
+        }
+        ses->ntlmssp->sesskey_per_smbsess = false;
+
+        /* wct = 12 */
+        rc = sess_alloc_buffer(sess_data, 12);
+        if (rc)
+                goto out;
+
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+
+        /* Build security blob before we assemble the request */
+        build_ntlmssp_negotiate_blob(pSMB->req.SecurityBlob, ses);
+        sess_data->iov[1].iov_len = sizeof(NEGOTIATE_MESSAGE);
+        sess_data->iov[1].iov_base = pSMB->req.SecurityBlob;
+        pSMB->req.SecurityBlobLength = cpu_to_le16(sizeof(NEGOTIATE_MESSAGE));
+
+        rc = _sess_auth_rawntlmssp_assemble_req(sess_data);
+        if (rc)
+                goto out;
+
+        rc = sess_sendreceive(sess_data);
+
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        smb_buf = (struct smb_hdr *)sess_data->iov[0].iov_base;
+
+        /* If true, rc here is expected and not an error */
+        if (sess_data->buf0_type != CIFS_NO_BUFFER &&
+            smb_buf->Status.CifsError ==
+                        cpu_to_le32(NT_STATUS_MORE_PROCESSING_REQUIRED))
+                rc = 0;
+
+        if (rc)
+                goto out;
+
+        cifs_dbg(FYI, "rawntlmssp session setup challenge phase\n");
+
+        if (smb_buf->WordCount != 4) {
+                rc = -EIO;
+                cifs_dbg(VFS, "bad word count %d\n", smb_buf->WordCount);
+                goto out;
+        }
+
+        ses->Suid = smb_buf->Uid;   /* UID left in wire format (le) */
+        cifs_dbg(FYI, "UID = %llu\n", ses->Suid);
+
+        bytes_remaining = get_bcc(smb_buf);
+        bcc_ptr = pByteArea(smb_buf);
+
+        blob_len = le16_to_cpu(pSMB->resp.SecurityBlobLength);
+        if (blob_len > bytes_remaining) {
+                cifs_dbg(VFS, "bad security blob length %d\n",
+                                blob_len);
+                rc = -EINVAL;
+                goto out;
+        }
+
+        rc = decode_ntlmssp_challenge(bcc_ptr, blob_len, ses);
+out:
+        sess_free_buffer(sess_data);
 
         if (!rc) {
-                mutex_lock(&ses->server->srv_mutex);
-                if (!ses->server->session_estab) {
-                        if (ses->server->sign) {
-                                ses->server->session_key.response =
-                                        kmemdup(ses->auth_key.response,
-                                        ses->auth_key.len, GFP_KERNEL);
-                                if (!ses->server->session_key.response) {
-                                        rc = -ENOMEM;
-                                        mutex_unlock(&ses->server->srv_mutex);
-                                        goto keycp_exit;
-                                }
-                                ses->server->session_key.len =
-                                                        ses->auth_key.len;
-                        }
-                        ses->server->sequence_number = 0x2;
-                        ses->server->session_estab = true;
-                }
-                mutex_unlock(&ses->server->srv_mutex);
+                sess_data->func = sess_auth_rawntlmssp_authenticate;
+                return;
+        }
+
+        /* Else error. Cleanup */
+        kfree(ses->auth_key.response);
+        ses->auth_key.response = NULL;
+        kfree(ses->ntlmssp);
+        ses->ntlmssp = NULL;
+
+        sess_data->func = NULL;
+        sess_data->result = rc;
+}
 
-                cifs_dbg(FYI, "CIFS session established successfully\n");
-                spin_lock(&GlobalMid_Lock);
-                ses->status = CifsGood;
-                ses->need_reconnect = false;
-                spin_unlock(&GlobalMid_Lock);
+static void
+sess_auth_rawntlmssp_authenticate(struct sess_data *sess_data)
+{
+        int rc;
+        struct smb_hdr *smb_buf;
+        SESSION_SETUP_ANDX *pSMB;
+        struct cifs_ses *ses = sess_data->ses;
+        __u16 bytes_remaining;
+        char *bcc_ptr;
+        char *ntlmsspblob = NULL;
+        u16 blob_len;
+
+        cifs_dbg(FYI, "rawntlmssp session setup authenticate phase\n");
+
+        /* wct = 12 */
+        rc = sess_alloc_buffer(sess_data, 12);
+        if (rc)
+                goto out;
+
+        /* Build security blob before we assemble the request */
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        smb_buf = (struct smb_hdr *)pSMB;
+        /*
+         * 5 is an empirical value, large enough to hold
+         * authenticate message plus max 10 of av paris,
+         * domain, user, workstation names, flags, etc.
+         */
+        ntlmsspblob = kzalloc(5*sizeof(struct _AUTHENTICATE_MESSAGE),
+                                GFP_KERNEL);
+        if (!ntlmsspblob) {
+                rc = -ENOMEM;
+                goto out;
         }
 
-keycp_exit:
+        rc = build_ntlmssp_auth_blob(ntlmsspblob,
+                                        &blob_len, ses, sess_data->nls_cp);
+        if (rc)
+                goto out_free_ntlmsspblob;
+        sess_data->iov[1].iov_len = blob_len;
+        sess_data->iov[1].iov_base = ntlmsspblob;
+        pSMB->req.SecurityBlobLength = cpu_to_le16(blob_len);
+        /*
+         * Make sure that we tell the server that we are using
+         * the uid that it just gave us back on the response
+         * (challenge)
+         */
+        smb_buf->Uid = ses->Suid;
+
+        rc = _sess_auth_rawntlmssp_assemble_req(sess_data);
+        if (rc)
+                goto out_free_ntlmsspblob;
+
+        rc = sess_sendreceive(sess_data);
+        if (rc)
+                goto out_free_ntlmsspblob;
+
+        pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base;
+        smb_buf = (struct smb_hdr *)sess_data->iov[0].iov_base;
+        if (smb_buf->WordCount != 4) {
+                rc = -EIO;
+                cifs_dbg(VFS, "bad word count %d\n", smb_buf->WordCount);
+                goto out_free_ntlmsspblob;
+        }
+
+        if (le16_to_cpu(pSMB->resp.Action) & GUEST_LOGIN)
+                cifs_dbg(FYI, "Guest login\n"); /* BB mark SesInfo struct? */
+
+        bytes_remaining = get_bcc(smb_buf);
+        bcc_ptr = pByteArea(smb_buf);
+        blob_len = le16_to_cpu(pSMB->resp.SecurityBlobLength);
+        if (blob_len > bytes_remaining) {
+                cifs_dbg(VFS, "bad security blob length %d\n",
+                                blob_len);
+                rc = -EINVAL;
+                goto out_free_ntlmsspblob;
+        }
+        bcc_ptr += blob_len;
+        bytes_remaining -= blob_len;
+
+
+        /* BB check if Unicode and decode strings */
+        if (bytes_remaining == 0) {
+                /* no string area to decode, do nothing */
+        } else if (smb_buf->Flags2 & SMBFLG2_UNICODE) {
+                /* unicode string area must be word-aligned */
+                if (((unsigned long) bcc_ptr - (unsigned long) smb_buf) % 2) {
+                        ++bcc_ptr;
+                        --bytes_remaining;
+                }
+                decode_unicode_ssetup(&bcc_ptr, bytes_remaining, ses,
+                                      sess_data->nls_cp);
+        } else {
+                decode_ascii_ssetup(&bcc_ptr, bytes_remaining, ses,
+                                    sess_data->nls_cp);
+        }
+
+out_free_ntlmsspblob:
+        kfree(ntlmsspblob);
+out:
+        sess_free_buffer(sess_data);
+
+         if (!rc)
+                rc = sess_establish_session(sess_data);
+
+        /* Cleanup */
         kfree(ses->auth_key.response);
         ses->auth_key.response = NULL;
         kfree(ses->ntlmssp);
+        ses->ntlmssp = NULL;
+
+        sess_data->func = NULL;
+        sess_data->result = rc;
+}
+
+static int select_sec(struct cifs_ses *ses, struct sess_data *sess_data)
+{
+        int type;
+
+        type = select_sectype(ses->server, ses->sectype);
+        cifs_dbg(FYI, "sess setup type %d\n", type);
+        if (type == Unspecified) {
+                cifs_dbg(VFS,
+                        "Unable to select appropriate authentication method!");
+                return -EINVAL;
+        }
+
+        switch (type) {
+        case LANMAN:
+                /* LANMAN and plaintext are less secure and off by default.
+                 * So we make this explicitly be turned on in kconfig (in the
+                 * build) and turned on at runtime (changed from the default)
+                 * in proc/fs/cifs or via mount parm.  Unfortunately this is
+                 * needed for old Win (e.g. Win95), some obscure NAS and OS/2 */
+#ifdef CONFIG_CIFS_WEAK_PW_HASH
+                sess_data->func = sess_auth_lanman;
+                break;
+#else
+                return -EOPNOTSUPP;
+#endif
+        case NTLM:
+                sess_data->func = sess_auth_ntlm;
+                break;
+        case NTLMv2:
+                sess_data->func = sess_auth_ntlmv2;
+                break;
+        case Kerberos:
+#ifdef CONFIG_CIFS_UPCALL
+                sess_data->func = sess_auth_kerberos;
+                break;
+#else
+                cifs_dbg(VFS, "Kerberos negotiated but upcall support disabled!\n");
+                return -ENOSYS;
+                break;
+#endif /* CONFIG_CIFS_UPCALL */
+        case RawNTLMSSP:
+                sess_data->func = sess_auth_rawntlmssp_negotiate;
+                break;
+        default:
+                cifs_dbg(VFS, "secType %d not supported!\n", type);
+                return -ENOSYS;
+        }
+
+        return 0;
+}
+
+int CIFS_SessSetup(const unsigned int xid, struct cifs_ses *ses,
+                    const struct nls_table *nls_cp)
+{
+        int rc = 0;
+        struct sess_data *sess_data;
+
+        if (ses == NULL) {
+                WARN(1, "%s: ses == NULL!", __func__);
+                return -EINVAL;
+        }
+
+        sess_data = kzalloc(sizeof(struct sess_data), GFP_KERNEL);
+        if (!sess_data)
+                return -ENOMEM;
+
+        rc = select_sec(ses, sess_data);
+        if (rc)
+                goto out;
+
+        sess_data->xid = xid;
+        sess_data->ses = ses;
+        sess_data->buf0_type = CIFS_NO_BUFFER;
+        sess_data->nls_cp = (struct nls_table *) nls_cp;
+
+        while (sess_data->func)
+                sess_data->func(sess_data);
+
+        /* Store result before we free sess_data */
+        rc = sess_data->result;
 
+out:
+        kfree(sess_data);
         return rc;
 }
diff --git a/fs/cifs/smb1ops.c b/fs/cifs/smb1ops.c
index d1fdfa848703..5e8c22d6c7b9 100644
--- a/fs/cifs/smb1ops.c
+++ b/fs/cifs/smb1ops.c
@@ -1009,6 +1009,12 @@ cifs_is_read_op(__u32 oplock)
         return oplock == OPLOCK_READ;
 }
 
+static unsigned int
+cifs_wp_retry_size(struct inode *inode)
+{
+        return CIFS_SB(inode->i_sb)->wsize;
+}
+
 struct smb_version_operations smb1_operations = {
         .send_cancel = send_nt_cancel,
         .compare_fids = cifs_compare_fids,
@@ -1019,6 +1025,7 @@ struct smb_version_operations smb1_operations = {
         .set_credits = cifs_set_credits,
         .get_credits_field = cifs_get_credits_field,
         .get_credits = cifs_get_credits,
+        .wait_mtu_credits = cifs_wait_mtu_credits,
         .get_next_mid = cifs_get_next_mid,
         .read_data_offset = cifs_read_data_offset,
         .read_data_length = cifs_read_data_length,
@@ -1078,6 +1085,7 @@ struct smb_version_operations smb1_operations = {
         .query_mf_symlink = cifs_query_mf_symlink,
         .create_mf_symlink = cifs_create_mf_symlink,
         .is_read_op = cifs_is_read_op,
+        .wp_retry_size = cifs_wp_retry_size,
 #ifdef CONFIG_CIFS_XATTR
         .query_all_EAs = CIFSSMBQAllEAs,
         .set_EA = CIFSSMBSetEA,
diff --git a/fs/cifs/smb2inode.c b/fs/cifs/smb2inode.c
index 84c012a6aba0..0150182a4494 100644
--- a/fs/cifs/smb2inode.c
+++ b/fs/cifs/smb2inode.c
@@ -91,7 +91,7 @@ smb2_open_op_close(const unsigned int xid, struct cifs_tcon *tcon,
         case SMB2_OP_SET_EOF:
                 tmprc = SMB2_set_eof(xid, tcon, fid.persistent_fid,
                                      fid.volatile_fid, current->tgid,
-                                     (__le64 *)data);
+                                     (__le64 *)data, false);
                 break;
         case SMB2_OP_SET_INFO:
                 tmprc = SMB2_set_info(xid, tcon, fid.persistent_fid,
diff --git a/fs/cifs/smb2maperror.c b/fs/cifs/smb2maperror.c
index 94bd4fbb13d3..e31a9dfdcd39 100644
--- a/fs/cifs/smb2maperror.c
+++ b/fs/cifs/smb2maperror.c
@@ -605,7 +605,7 @@ static const struct status_to_posix_error smb2_error_map_table[] = {
         {STATUS_MAPPED_FILE_SIZE_ZERO, -EIO, "STATUS_MAPPED_FILE_SIZE_ZERO"},
         {STATUS_TOO_MANY_OPENED_FILES, -EMFILE, "STATUS_TOO_MANY_OPENED_FILES"},
         {STATUS_CANCELLED, -EIO, "STATUS_CANCELLED"},
-        {STATUS_CANNOT_DELETE, -EIO, "STATUS_CANNOT_DELETE"},
+        {STATUS_CANNOT_DELETE, -EACCES, "STATUS_CANNOT_DELETE"},
         {STATUS_INVALID_COMPUTER_NAME, -EIO, "STATUS_INVALID_COMPUTER_NAME"},
         {STATUS_FILE_DELETED, -EIO, "STATUS_FILE_DELETED"},
         {STATUS_SPECIAL_ACCOUNT, -EIO, "STATUS_SPECIAL_ACCOUNT"},
diff --git a/fs/cifs/smb2misc.c b/fs/cifs/smb2misc.c
index b8021fde987d..f2e6ac29a8d6 100644
--- a/fs/cifs/smb2misc.c
+++ b/fs/cifs/smb2misc.c
@@ -437,7 +437,7 @@ smb2_tcon_has_lease(struct cifs_tcon *tcon, struct smb2_lease_break *rsp,
                         continue;
 
                 cifs_dbg(FYI, "found in the open list\n");
-                cifs_dbg(FYI, "lease key match, lease break 0x%d\n",
+                cifs_dbg(FYI, "lease key match, lease break 0x%x\n",
                          le32_to_cpu(rsp->NewLeaseState));
 
                 server->ops->set_oplock_level(cinode, lease_state, 0, NULL);
@@ -467,7 +467,7 @@ smb2_tcon_has_lease(struct cifs_tcon *tcon, struct smb2_lease_break *rsp,
                 }
 
                 cifs_dbg(FYI, "found in the pending open list\n");
-                cifs_dbg(FYI, "lease key match, lease break 0x%d\n",
+                cifs_dbg(FYI, "lease key match, lease break 0x%x\n",
                          le32_to_cpu(rsp->NewLeaseState));
 
                 open->oplock = lease_state;
@@ -546,7 +546,7 @@ smb2_is_valid_oplock_break(char *buffer, struct TCP_Server_Info *server)
                         return false;
         }
 
-        cifs_dbg(FYI, "oplock level 0x%d\n", rsp->OplockLevel);
+        cifs_dbg(FYI, "oplock level 0x%x\n", rsp->OplockLevel);
 
         /* look up tcon based on tid & uid */
         spin_lock(&cifs_tcp_ses_lock);
diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c
index 787844bde384..77f8aeb9c2fc 100644
--- a/fs/cifs/smb2ops.c
+++ b/fs/cifs/smb2ops.c
@@ -19,6 +19,7 @@
 
 #include <linux/pagemap.h>
 #include <linux/vfs.h>
+#include <linux/falloc.h>
 #include "cifsglob.h"
 #include "smb2pdu.h"
 #include "smb2proto.h"
@@ -112,6 +113,53 @@ smb2_get_credits(struct mid_q_entry *mid)
         return le16_to_cpu(((struct smb2_hdr *)mid->resp_buf)->CreditRequest);
 }
 
+static int
+smb2_wait_mtu_credits(struct TCP_Server_Info *server, unsigned int size,
+                      unsigned int *num, unsigned int *credits)
+{
+        int rc = 0;
+        unsigned int scredits;
+
+        spin_lock(&server->req_lock);
+        while (1) {
+                if (server->credits <= 0) {
+                        spin_unlock(&server->req_lock);
+                        cifs_num_waiters_inc(server);
+                        rc = wait_event_killable(server->request_q,
+                                        has_credits(server, &server->credits));
+                        cifs_num_waiters_dec(server);
+                        if (rc)
+                                return rc;
+                        spin_lock(&server->req_lock);
+                } else {
+                        if (server->tcpStatus == CifsExiting) {
+                                spin_unlock(&server->req_lock);
+                                return -ENOENT;
+                        }
+
+                        scredits = server->credits;
+                        /* can deadlock with reopen */
+                        if (scredits == 1) {
+                                *num = SMB2_MAX_BUFFER_SIZE;
+                                *credits = 0;
+                                break;
+                        }
+
+                        /* leave one credit for a possible reopen */
+                        scredits--;
+                        *num = min_t(unsigned int, size,
+                                     scredits * SMB2_MAX_BUFFER_SIZE);
+
+                        *credits = DIV_ROUND_UP(*num, SMB2_MAX_BUFFER_SIZE);
+                        server->credits -= *credits;
+                        server->in_flight++;
+                        break;
+                }
+        }
+        spin_unlock(&server->req_lock);
+        return rc;
+}
+
 static __u64
 smb2_get_next_mid(struct TCP_Server_Info *server)
 {
@@ -182,8 +230,9 @@ smb2_negotiate_wsize(struct cifs_tcon *tcon, struct smb_vol *volume_info)
         /* start with specified wsize, or default */
         wsize = volume_info->wsize ? volume_info->wsize : CIFS_DEFAULT_IOSIZE;
         wsize = min_t(unsigned int, wsize, server->max_write);
-        /* set it to the maximum buffer size value we can send with 1 credit */
-        wsize = min_t(unsigned int, wsize, SMB2_MAX_BUFFER_SIZE);
+
+        if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU))
+                wsize = min_t(unsigned int, wsize, SMB2_MAX_BUFFER_SIZE);
 
         return wsize;
 }
@@ -197,8 +246,9 @@ smb2_negotiate_rsize(struct cifs_tcon *tcon, struct smb_vol *volume_info)
         /* start with specified rsize, or default */
         rsize = volume_info->rsize ? volume_info->rsize : CIFS_DEFAULT_IOSIZE;
         rsize = min_t(unsigned int, rsize, server->max_read);
-        /* set it to the maximum buffer size value we can send with 1 credit */
-        rsize = min_t(unsigned int, rsize, SMB2_MAX_BUFFER_SIZE);
+
+        if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU))
+                rsize = min_t(unsigned int, rsize, SMB2_MAX_BUFFER_SIZE);
 
         return rsize;
 }
@@ -687,7 +737,7 @@ smb2_set_file_size(const unsigned int xid, struct cifs_tcon *tcon,
 {
         __le64 eof = cpu_to_le64(size);
         return SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid,
-                            cfile->fid.volatile_fid, cfile->pid, &eof);
+                            cfile->fid.volatile_fid, cfile->pid, &eof, false);
 }
 
 static int
@@ -1104,6 +1154,13 @@ smb3_parse_lease_buf(void *buf, unsigned int *epoch)
         return le32_to_cpu(lc->lcontext.LeaseState);
 }
 
+static unsigned int
+smb2_wp_retry_size(struct inode *inode)
+{
+        return min_t(unsigned int, CIFS_SB(inode->i_sb)->wsize,
+                     SMB2_MAX_BUFFER_SIZE);
+}
+
 struct smb_version_operations smb20_operations = {
         .compare_fids = smb2_compare_fids,
         .setup_request = smb2_setup_request,
@@ -1113,6 +1170,7 @@ struct smb_version_operations smb20_operations = {
         .set_credits = smb2_set_credits,
         .get_credits_field = smb2_get_credits_field,
         .get_credits = smb2_get_credits,
+        .wait_mtu_credits = cifs_wait_mtu_credits,
         .get_next_mid = smb2_get_next_mid,
         .read_data_offset = smb2_read_data_offset,
         .read_data_length = smb2_read_data_length,
@@ -1177,6 +1235,7 @@ struct smb_version_operations smb20_operations = {
         .create_lease_buf = smb2_create_lease_buf,
         .parse_lease_buf = smb2_parse_lease_buf,
         .clone_range = smb2_clone_range,
+        .wp_retry_size = smb2_wp_retry_size,
 };
 
 struct smb_version_operations smb21_operations = {
@@ -1188,6 +1247,7 @@ struct smb_version_operations smb21_operations = {
         .set_credits = smb2_set_credits,
         .get_credits_field = smb2_get_credits_field,
         .get_credits = smb2_get_credits,
+        .wait_mtu_credits = smb2_wait_mtu_credits,
         .get_next_mid = smb2_get_next_mid,
         .read_data_offset = smb2_read_data_offset,
         .read_data_length = smb2_read_data_length,
@@ -1252,6 +1312,7 @@ struct smb_version_operations smb21_operations = {
         .create_lease_buf = smb2_create_lease_buf,
         .parse_lease_buf = smb2_parse_lease_buf,
         .clone_range = smb2_clone_range,
+        .wp_retry_size = smb2_wp_retry_size,
 };
 
 struct smb_version_operations smb30_operations = {
@@ -1263,6 +1324,7 @@ struct smb_version_operations smb30_operations = {
         .set_credits = smb2_set_credits,
         .get_credits_field = smb2_get_credits_field,
         .get_credits = smb2_get_credits,
+        .wait_mtu_credits = smb2_wait_mtu_credits,
         .get_next_mid = smb2_get_next_mid,
         .read_data_offset = smb2_read_data_offset,
         .read_data_length = smb2_read_data_length,
@@ -1330,6 +1392,7 @@ struct smb_version_operations smb30_operations = {
         .parse_lease_buf = smb3_parse_lease_buf,
         .clone_range = smb2_clone_range,
         .validate_negotiate = smb3_validate_negotiate,
+        .wp_retry_size = smb2_wp_retry_size,
 };
 
 struct smb_version_values smb20_values = {
diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c
index b0b260dbb19d..42ebc1a8be6c 100644
--- a/fs/cifs/smb2pdu.c
+++ b/fs/cifs/smb2pdu.c
@@ -108,7 +108,6 @@ smb2_hdr_assemble(struct smb2_hdr *hdr, __le16 smb2_cmd /* command */ ,
         if (!tcon)
                 goto out;
 
-        /* BB FIXME when we do write > 64K add +1 for every 64K in req or rsp */
         /* GLOBAL_CAP_LARGE_MTU will only be set if dialect > SMB2.02 */
         /* See sections 2.2.4 and 3.2.4.1.5 of MS-SMB2 */
         if ((tcon->ses) &&
@@ -245,10 +244,6 @@ smb2_reconnect(__le16 smb2_command, struct cifs_tcon *tcon)
         if (rc)
                 goto out;
         atomic_inc(&tconInfoReconnectCount);
-        /*
-         * BB FIXME add code to check if wsize needs update due to negotiated
-         * smb buffer size shrinking.
-         */
 out:
         /*
          * Check if handle based operation so we know whether we can continue
@@ -309,16 +304,6 @@ small_smb2_init(__le16 smb2_command, struct cifs_tcon *tcon,
         return rc;
 }
 
-static void
-free_rsp_buf(int resp_buftype, void *rsp)
-{
-        if (resp_buftype == CIFS_SMALL_BUFFER)
-                cifs_small_buf_release(rsp);
-        else if (resp_buftype == CIFS_LARGE_BUFFER)
-                cifs_buf_release(rsp);
-}
-
-
 /*
  *
  *      SMB2 Worker functions follow:
@@ -1738,12 +1723,18 @@ smb2_readv_callback(struct mid_q_entry *mid)
                                          rc);
                 }
                 /* FIXME: should this be counted toward the initiating task? */
-                task_io_account_read(rdata->bytes);
-                cifs_stats_bytes_read(tcon, rdata->bytes);
+                task_io_account_read(rdata->got_bytes);
+                cifs_stats_bytes_read(tcon, rdata->got_bytes);
                 break;
         case MID_REQUEST_SUBMITTED:
         case MID_RETRY_NEEDED:
                 rdata->result = -EAGAIN;
+                if (server->sign && rdata->got_bytes)
+                        /* reset bytes number since we can not check a sign */
+                        rdata->got_bytes = 0;
+                /* FIXME: should this be counted toward the initiating task? */
+                task_io_account_read(rdata->got_bytes);
+                cifs_stats_bytes_read(tcon, rdata->got_bytes);
                 break;
         default:
                 if (rdata->result != -ENODATA)
@@ -1762,11 +1753,12 @@ smb2_readv_callback(struct mid_q_entry *mid)
 int
 smb2_async_readv(struct cifs_readdata *rdata)
 {
-        int rc;
+        int rc, flags = 0;
         struct smb2_hdr *buf;
         struct cifs_io_parms io_parms;
         struct smb_rqst rqst = { .rq_iov = &rdata->iov,
                                  .rq_nvec = 1 };
+        struct TCP_Server_Info *server;
 
         cifs_dbg(FYI, "%s: offset=%llu bytes=%u\n",
                  __func__, rdata->offset, rdata->bytes);
@@ -1777,18 +1769,41 @@ smb2_async_readv(struct cifs_readdata *rdata)
         io_parms.persistent_fid = rdata->cfile->fid.persistent_fid;
         io_parms.volatile_fid = rdata->cfile->fid.volatile_fid;
         io_parms.pid = rdata->pid;
+
+        server = io_parms.tcon->ses->server;
+
         rc = smb2_new_read_req(&rdata->iov, &io_parms, 0, 0);
-        if (rc)
+        if (rc) {
+                if (rc == -EAGAIN && rdata->credits) {
+                        /* credits was reset by reconnect */
+                        rdata->credits = 0;
+                        /* reduce in_flight value since we won't send the req */
+                        spin_lock(&server->req_lock);
+                        server->in_flight--;
+                        spin_unlock(&server->req_lock);
+                }
                 return rc;
+        }
 
         buf = (struct smb2_hdr *)rdata->iov.iov_base;
         /* 4 for rfc1002 length field */
         rdata->iov.iov_len = get_rfc1002_length(rdata->iov.iov_base) + 4;
 
+        if (rdata->credits) {
+                buf->CreditCharge = cpu_to_le16(DIV_ROUND_UP(rdata->bytes,
+                                                SMB2_MAX_BUFFER_SIZE));
+                spin_lock(&server->req_lock);
+                server->credits += rdata->credits -
+                                                le16_to_cpu(buf->CreditCharge);
+                spin_unlock(&server->req_lock);
+                wake_up(&server->request_q);
+                flags = CIFS_HAS_CREDITS;
+        }
+
         kref_get(&rdata->refcount);
         rc = cifs_call_async(io_parms.tcon->ses->server, &rqst,
                              cifs_readv_receive, smb2_readv_callback,
-                             rdata, 0);
+                             rdata, flags);
         if (rc) {
                 kref_put(&rdata->refcount, cifs_readdata_release);
                 cifs_stats_fail_inc(io_parms.tcon, SMB2_READ_HE);
@@ -1906,15 +1921,25 @@ int
 smb2_async_writev(struct cifs_writedata *wdata,
                   void (*release)(struct kref *kref))
 {
-        int rc = -EACCES;
+        int rc = -EACCES, flags = 0;
         struct smb2_write_req *req = NULL;
         struct cifs_tcon *tcon = tlink_tcon(wdata->cfile->tlink);
+        struct TCP_Server_Info *server = tcon->ses->server;
         struct kvec iov;
         struct smb_rqst rqst;
 
         rc = small_smb2_init(SMB2_WRITE, tcon, (void **) &req);
-        if (rc)
+        if (rc) {
+                if (rc == -EAGAIN && wdata->credits) {
+                        /* credits was reset by reconnect */
+                        wdata->credits = 0;
+                        /* reduce in_flight value since we won't send the req */
+                        spin_lock(&server->req_lock);
+                        server->in_flight--;
+                        spin_unlock(&server->req_lock);
+                }
                 goto async_writev_out;
+        }
 
         req->hdr.ProcessId = cpu_to_le32(wdata->cfile->pid);
 
@@ -1947,9 +1972,20 @@ smb2_async_writev(struct cifs_writedata *wdata,
 
         inc_rfc1001_len(&req->hdr, wdata->bytes - 1 /* Buffer */);
 
+        if (wdata->credits) {
+                req->hdr.CreditCharge = cpu_to_le16(DIV_ROUND_UP(wdata->bytes,
+                                                    SMB2_MAX_BUFFER_SIZE));
+                spin_lock(&server->req_lock);
+                server->credits += wdata->credits -
+                                        le16_to_cpu(req->hdr.CreditCharge);
+                spin_unlock(&server->req_lock);
+                wake_up(&server->request_q);
+                flags = CIFS_HAS_CREDITS;
+        }
+
         kref_get(&wdata->refcount);
-        rc = cifs_call_async(tcon->ses->server, &rqst, NULL,
-                                smb2_writev_callback, wdata, 0);
+        rc = cifs_call_async(server, &rqst, NULL, smb2_writev_callback, wdata,
+                             flags);
 
         if (rc) {
                 kref_put(&wdata->refcount, release);
@@ -2325,7 +2361,7 @@ SMB2_set_hardlink(const unsigned int xid, struct cifs_tcon *tcon,
 
 int
 SMB2_set_eof(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid,
-             u64 volatile_fid, u32 pid, __le64 *eof)
+             u64 volatile_fid, u32 pid, __le64 *eof, bool is_falloc)
 {
         struct smb2_file_eof_info info;
         void *data;
@@ -2336,8 +2372,12 @@ SMB2_set_eof(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid,
         data = &info;
         size = sizeof(struct smb2_file_eof_info);
 
-        return send_set_info(xid, tcon, persistent_fid, volatile_fid, pid,
-                             FILE_END_OF_FILE_INFORMATION, 1, &data, &size);
+        if (is_falloc)
+                return send_set_info(xid, tcon, persistent_fid, volatile_fid,
+                        pid, FILE_ALLOCATION_INFORMATION, 1, &data, &size);
+        else
+                return send_set_info(xid, tcon, persistent_fid, volatile_fid,
+                        pid, FILE_END_OF_FILE_INFORMATION, 1, &data, &size);
 }
 
 int
diff --git a/fs/cifs/smb2proto.h b/fs/cifs/smb2proto.h
index 0ce48db20a65..67e8ce8055de 100644
--- a/fs/cifs/smb2proto.h
+++ b/fs/cifs/smb2proto.h
@@ -139,7 +139,7 @@ extern int SMB2_set_hardlink(const unsigned int xid, struct cifs_tcon *tcon,
                              __le16 *target_file);
 extern int SMB2_set_eof(const unsigned int xid, struct cifs_tcon *tcon,
                         u64 persistent_fid, u64 volatile_fid, u32 pid,
-                        __le64 *eof);
+                        __le64 *eof, bool is_fallocate);
 extern int SMB2_set_info(const unsigned int xid, struct cifs_tcon *tcon,
                          u64 persistent_fid, u64 volatile_fid,
                          FILE_BASIC_INFO *buf);
diff --git a/fs/cifs/smb2transport.c b/fs/cifs/smb2transport.c
index 59c748ce872f..5111e7272db6 100644
--- a/fs/cifs/smb2transport.c
+++ b/fs/cifs/smb2transport.c
@@ -466,7 +466,12 @@ smb2_verify_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server)
 static inline void
 smb2_seq_num_into_buf(struct TCP_Server_Info *server, struct smb2_hdr *hdr)
 {
+        unsigned int i, num = le16_to_cpu(hdr->CreditCharge);
+
         hdr->MessageId = get_next_mid64(server);
+        /* skip message numbers according to CreditCharge field */
+        for (i = 1; i < num; i++)
+                get_next_mid(server);
 }
 
 static struct mid_q_entry *
diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c
index 18cd5650a5fc..9d087f4e7d4e 100644
--- a/fs/cifs/transport.c
+++ b/fs/cifs/transport.c
@@ -448,6 +448,15 @@ wait_for_free_request(struct TCP_Server_Info *server, const int timeout,
         return wait_for_free_credits(server, timeout, val);
 }
 
+int
+cifs_wait_mtu_credits(struct TCP_Server_Info *server, unsigned int size,
+                      unsigned int *num, unsigned int *credits)
+{
+        *num = size;
+        *credits = 0;
+        return 0;
+}
+
 static int allocate_mid(struct cifs_ses *ses, struct smb_hdr *in_buf,
                         struct mid_q_entry **ppmidQ)
 {
@@ -531,20 +540,23 @@ cifs_call_async(struct TCP_Server_Info *server, struct smb_rqst *rqst,
 {
         int rc, timeout, optype;
         struct mid_q_entry *mid;
+        unsigned int credits = 0;
 
         timeout = flags & CIFS_TIMEOUT_MASK;
         optype = flags & CIFS_OP_MASK;
 
-        rc = wait_for_free_request(server, timeout, optype);
-        if (rc)
-                return rc;
+        if ((flags & CIFS_HAS_CREDITS) == 0) {
+                rc = wait_for_free_request(server, timeout, optype);
+                if (rc)
+                        return rc;
+                credits = 1;
+        }
 
         mutex_lock(&server->srv_mutex);
         mid = server->ops->setup_async_request(server, rqst);
         if (IS_ERR(mid)) {
                 mutex_unlock(&server->srv_mutex);
-                add_credits(server, 1, optype);
-                wake_up(&server->request_q);
+                add_credits_and_wake_if(server, credits, optype);
                 return PTR_ERR(mid);
         }
 
@@ -572,8 +584,7 @@ cifs_call_async(struct TCP_Server_Info *server, struct smb_rqst *rqst,
                 return 0;
 
         cifs_delete_mid(mid);
-        add_credits(server, 1, optype);
-        wake_up(&server->request_q);
+        add_credits_and_wake_if(server, credits, optype);
         return rc;
 }
 
diff --git a/fs/namespace.c b/fs/namespace.c
index 2a1447c946e7..0acabea58319 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -890,8 +890,21 @@ static struct mount *clone_mnt(struct mount *old, struct dentry *root,
 
         mnt->mnt.mnt_flags = old->mnt.mnt_flags & ~(MNT_WRITE_HOLD|MNT_MARKED);
         /* Don't allow unprivileged users to change mount flags */
-        if ((flag & CL_UNPRIVILEGED) && (mnt->mnt.mnt_flags & MNT_READONLY))
-                mnt->mnt.mnt_flags |= MNT_LOCK_READONLY;
+        if (flag & CL_UNPRIVILEGED) {
+                mnt->mnt.mnt_flags |= MNT_LOCK_ATIME;
+
+                if (mnt->mnt.mnt_flags & MNT_READONLY)
+                        mnt->mnt.mnt_flags |= MNT_LOCK_READONLY;
+
+                if (mnt->mnt.mnt_flags & MNT_NODEV)
+                        mnt->mnt.mnt_flags |= MNT_LOCK_NODEV;
+
+                if (mnt->mnt.mnt_flags & MNT_NOSUID)
+                        mnt->mnt.mnt_flags |= MNT_LOCK_NOSUID;
+
+                if (mnt->mnt.mnt_flags & MNT_NOEXEC)
+                        mnt->mnt.mnt_flags |= MNT_LOCK_NOEXEC;
+        }
 
         /* Don't allow unprivileged users to reveal what is under a mount */
         if ((flag & CL_UNPRIVILEGED) && list_empty(&old->mnt_expire))
@@ -1896,9 +1909,6 @@ static int change_mount_flags(struct vfsmount *mnt, int ms_flags)
         if (readonly_request == __mnt_is_readonly(mnt))
                 return 0;
 
-        if (mnt->mnt_flags & MNT_LOCK_READONLY)
-                return -EPERM;
-
         if (readonly_request)
                 error = mnt_make_readonly(real_mount(mnt));
         else
@@ -1924,6 +1934,33 @@ static int do_remount(struct path *path, int flags, int mnt_flags,
         if (path->dentry != path->mnt->mnt_root)
                 return -EINVAL;
 
+        /* Don't allow changing of locked mnt flags.
+         *
+         * No locks need to be held here while testing the various
+         * MNT_LOCK flags because those flags can never be cleared
+         * once they are set.
+         */
+        if ((mnt->mnt.mnt_flags & MNT_LOCK_READONLY) &&
+            !(mnt_flags & MNT_READONLY)) {
+                return -EPERM;
+        }
+        if ((mnt->mnt.mnt_flags & MNT_LOCK_NODEV) &&
+            !(mnt_flags & MNT_NODEV)) {
+                return -EPERM;
+        }
+        if ((mnt->mnt.mnt_flags & MNT_LOCK_NOSUID) &&
+            !(mnt_flags & MNT_NOSUID)) {
+                return -EPERM;
+        }
+        if ((mnt->mnt.mnt_flags & MNT_LOCK_NOEXEC) &&
+            !(mnt_flags & MNT_NOEXEC)) {
+                return -EPERM;
+        }
+        if ((mnt->mnt.mnt_flags & MNT_LOCK_ATIME) &&
+            ((mnt->mnt.mnt_flags & MNT_ATIME_MASK) != (mnt_flags & MNT_ATIME_MASK))) {
+                return -EPERM;
+        }
+
         err = security_sb_remount(sb, data);
         if (err)
                 return err;
@@ -1937,7 +1974,7 @@ static int do_remount(struct path *path, int flags, int mnt_flags,
                 err = do_remount_sb(sb, flags, data, 0);
         if (!err) {
                 lock_mount_hash();
-                mnt_flags |= mnt->mnt.mnt_flags & MNT_PROPAGATION_MASK;
+                mnt_flags |= mnt->mnt.mnt_flags & ~MNT_USER_SETTABLE_MASK;
                 mnt->mnt.mnt_flags = mnt_flags;
                 touch_mnt_namespace(mnt->mnt_ns);
                 unlock_mount_hash();
@@ -2122,7 +2159,7 @@ static int do_new_mount(struct path *path, const char *fstype, int flags,
                  */
                 if (!(type->fs_flags & FS_USERNS_DEV_MOUNT)) {
                         flags |= MS_NODEV;
-                        mnt_flags |= MNT_NODEV;
+                        mnt_flags |= MNT_NODEV | MNT_LOCK_NODEV;
                 }
         }
 
@@ -2436,6 +2473,14 @@ long do_mount(const char *dev_name, const char *dir_name,
         if (flags & MS_RDONLY)
                 mnt_flags |= MNT_READONLY;
 
+        /* The default atime for remount is preservation */
+        if ((flags & MS_REMOUNT) &&
+            ((flags & (MS_NOATIME | MS_NODIRATIME | MS_RELATIME |
+                       MS_STRICTATIME)) == 0)) {
+                mnt_flags &= ~MNT_ATIME_MASK;
+                mnt_flags |= path.mnt->mnt_flags & MNT_ATIME_MASK;
+        }
+
         flags &= ~(MS_NOSUID | MS_NOEXEC | MS_NODEV | MS_ACTIVE | MS_BORN |
                    MS_NOATIME | MS_NODIRATIME | MS_RELATIME| MS_KERNMOUNT |
                    MS_STRICTATIME);
@@ -2972,13 +3017,13 @@ static void *mntns_get(struct task_struct *task)
         struct mnt_namespace *ns = NULL;
         struct nsproxy *nsproxy;
 
-        rcu_read_lock();
-        nsproxy = task_nsproxy(task);
+        task_lock(task);
+        nsproxy = task->nsproxy;
         if (nsproxy) {
                 ns = nsproxy->mnt_ns;
                 get_mnt_ns(ns);
         }
-        rcu_read_unlock();
+        task_unlock(task);
 
         return ns;
 }
diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index 1d09289c8f0e..180d1ec9c32e 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -1205,7 +1205,7 @@ static const struct file_operations nfs_server_list_fops = {
         .open           = nfs_server_list_open,
         .read           = seq_read,
         .llseek         = seq_lseek,
-        .release        = seq_release,
+        .release        = seq_release_net,
         .owner          = THIS_MODULE,
 };
 
@@ -1226,7 +1226,7 @@ static const struct file_operations nfs_volume_list_fops = {
         .open           = nfs_volume_list_open,
         .read           = seq_read,
         .llseek         = seq_lseek,
-        .release        = seq_release,
+        .release        = seq_release_net,
         .owner          = THIS_MODULE,
 };
 
@@ -1236,19 +1236,8 @@ static const struct file_operations nfs_volume_list_fops = {
  */
 static int nfs_server_list_open(struct inode *inode, struct file *file)
 {
-        struct seq_file *m;
-        int ret;
-        struct pid_namespace *pid_ns = file->f_dentry->d_sb->s_fs_info;
-        struct net *net = pid_ns->child_reaper->nsproxy->net_ns;
-
-        ret = seq_open(file, &nfs_server_list_ops);
-        if (ret < 0)
-                return ret;
-
-        m = file->private_data;
-        m->private = net;
-
-        return 0;
+        return seq_open_net(inode, file, &nfs_server_list_ops,
+                           sizeof(struct seq_net_private));
 }
 
 /*
@@ -1256,7 +1245,7 @@ static int nfs_server_list_open(struct inode *inode, struct file *file)
  */
 static void *nfs_server_list_start(struct seq_file *m, loff_t *_pos)
 {
-        struct nfs_net *nn = net_generic(m->private, nfs_net_id);
+        struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id);
 
         /* lock the list against modification */
         spin_lock(&nn->nfs_client_lock);
@@ -1268,7 +1257,7 @@ static void *nfs_server_list_start(struct seq_file *m, loff_t *_pos)
  */
 static void *nfs_server_list_next(struct seq_file *p, void *v, loff_t *pos)
 {
-        struct nfs_net *nn = net_generic(p->private, nfs_net_id);
+        struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id);
 
         return seq_list_next(v, &nn->nfs_client_list, pos);
 }
@@ -1278,7 +1267,7 @@ static void *nfs_server_list_next(struct seq_file *p, void *v, loff_t *pos)
  */
 static void nfs_server_list_stop(struct seq_file *p, void *v)
 {
-        struct nfs_net *nn = net_generic(p->private, nfs_net_id);
+        struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id);
 
         spin_unlock(&nn->nfs_client_lock);
 }
@@ -1289,7 +1278,7 @@ static void nfs_server_list_stop(struct seq_file *p, void *v)
 static int nfs_server_list_show(struct seq_file *m, void *v)
 {
         struct nfs_client *clp;
-        struct nfs_net *nn = net_generic(m->private, nfs_net_id);
+        struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id);
 
         /* display header on line 1 */
         if (v == &nn->nfs_client_list) {
@@ -1321,19 +1310,8 @@ static int nfs_server_list_show(struct seq_file *m, void *v)
  */
 static int nfs_volume_list_open(struct inode *inode, struct file *file)
 {
-        struct seq_file *m;
-        int ret;
-        struct pid_namespace *pid_ns = file->f_dentry->d_sb->s_fs_info;
-        struct net *net = pid_ns->child_reaper->nsproxy->net_ns;
-
-        ret = seq_open(file, &nfs_volume_list_ops);
-        if (ret < 0)
-                return ret;
-
-        m = file->private_data;
-        m->private = net;
-
-        return 0;
+        return seq_open_net(inode, file, &nfs_server_list_ops,
+                           sizeof(struct seq_net_private));
 }
 
 /*
@@ -1341,7 +1319,7 @@ static int nfs_volume_list_open(struct inode *inode, struct file *file)
  */
 static void *nfs_volume_list_start(struct seq_file *m, loff_t *_pos)
 {
-        struct nfs_net *nn = net_generic(m->private, nfs_net_id);
+        struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id);
 
         /* lock the list against modification */
         spin_lock(&nn->nfs_client_lock);
@@ -1353,7 +1331,7 @@ static void *nfs_volume_list_start(struct seq_file *m, loff_t *_pos)
  */
 static void *nfs_volume_list_next(struct seq_file *p, void *v, loff_t *pos)
 {
-        struct nfs_net *nn = net_generic(p->private, nfs_net_id);
+        struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id);
 
         return seq_list_next(v, &nn->nfs_volume_list, pos);
 }
@@ -1363,7 +1341,7 @@ static void *nfs_volume_list_next(struct seq_file *p, void *v, loff_t *pos)
  */
 static void nfs_volume_list_stop(struct seq_file *p, void *v)
 {
-        struct nfs_net *nn = net_generic(p->private, nfs_net_id);
+        struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id);
 
         spin_unlock(&nn->nfs_client_lock);
 }
@@ -1376,7 +1354,7 @@ static int nfs_volume_list_show(struct seq_file *m, void *v)
         struct nfs_server *server;
         struct nfs_client *clp;
         char dev[8], fsid[17];
-        struct nfs_net *nn = net_generic(m->private, nfs_net_id);
+        struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id);
 
         /* display header on line 1 */
         if (v == &nn->nfs_volume_list) {
@@ -1407,6 +1385,45 @@ static int nfs_volume_list_show(struct seq_file *m, void *v)
         return 0;
 }
 
+int nfs_fs_proc_net_init(struct net *net)
+{
+        struct nfs_net *nn = net_generic(net, nfs_net_id);
+        struct proc_dir_entry *p;
+
+        nn->proc_nfsfs = proc_net_mkdir(net, "nfsfs", net->proc_net);
+        if (!nn->proc_nfsfs)
+                goto error_0;
+
+        /* a file of servers with which we're dealing */
+        p = proc_create("servers", S_IFREG|S_IRUGO,
+                        nn->proc_nfsfs, &nfs_server_list_fops);
+        if (!p)
+                goto error_1;
+
+        /* a file of volumes that we have mounted */
+        p = proc_create("volumes", S_IFREG|S_IRUGO,
+                        nn->proc_nfsfs, &nfs_volume_list_fops);
+        if (!p)
+                goto error_2;
+        return 0;
+
+error_2:
+        remove_proc_entry("servers", nn->proc_nfsfs);
+error_1:
+        remove_proc_entry("fs/nfsfs", NULL);
+error_0:
+        return -ENOMEM;
+}
+
+void nfs_fs_proc_net_exit(struct net *net)
+{
+        struct nfs_net *nn = net_generic(net, nfs_net_id);
+
+        remove_proc_entry("volumes", nn->proc_nfsfs);
+        remove_proc_entry("servers", nn->proc_nfsfs);
+        remove_proc_entry("fs/nfsfs", NULL);
+}
+
 /*
  * initialise the /proc/fs/nfsfs/ directory
  */
@@ -1419,14 +1436,12 @@ int __init nfs_fs_proc_init(void)
                 goto error_0;
 
         /* a file of servers with which we're dealing */
-        p = proc_create("servers", S_IFREG|S_IRUGO,
-                        proc_fs_nfs, &nfs_server_list_fops);
+        p = proc_symlink("servers", proc_fs_nfs, "../../net/nfsfs/servers");
         if (!p)
                 goto error_1;
 
         /* a file of volumes that we have mounted */
-        p = proc_create("volumes", S_IFREG|S_IRUGO,
-                        proc_fs_nfs, &nfs_volume_list_fops);
+        p = proc_symlink("volumes", proc_fs_nfs, "../../net/nfsfs/volumes");
         if (!p)
                 goto error_2;
         return 0;
diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index abd37a380535..68921b01b792 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -1840,11 +1840,12 @@ EXPORT_SYMBOL_GPL(nfs_net_id);
 static int nfs_net_init(struct net *net)
 {
         nfs_clients_init(net);
-        return 0;
+        return nfs_fs_proc_net_init(net);
 }
 
 static void nfs_net_exit(struct net *net)
 {
+        nfs_fs_proc_net_exit(net);
         nfs_cleanup_cb_ident_idr(net);
 }
 
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 617f36611d4a..e2a45ae5014e 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -195,7 +195,16 @@ extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *,
 #ifdef CONFIG_PROC_FS
 extern int __init nfs_fs_proc_init(void);
 extern void nfs_fs_proc_exit(void);
+extern int nfs_fs_proc_net_init(struct net *net);
+extern void nfs_fs_proc_net_exit(struct net *net);
 #else
+static inline int nfs_fs_proc_net_init(struct net *net)
+{
+        return 0;
+}
+static inline void nfs_fs_proc_net_exit(struct net *net)
+{
+}
 static inline int nfs_fs_proc_init(void)
 {
         return 0;
diff --git a/fs/nfs/netns.h b/fs/nfs/netns.h
index 8ee1fab83268..ef221fb8a183 100644
--- a/fs/nfs/netns.h
+++ b/fs/nfs/netns.h
@@ -29,6 +29,9 @@ struct nfs_net {
 #endif
         spinlock_t nfs_client_lock;
         struct timespec boot_time;
+#ifdef CONFIG_PROC_FS
+        struct proc_dir_entry *proc_nfsfs;
+#endif
 };
 
 extern int nfs_net_id;
diff --git a/fs/nfsd/acl.h b/fs/nfsd/acl.h
index a986ceb6fd0d..4cd7c69a6cb9 100644
--- a/fs/nfsd/acl.h
+++ b/fs/nfsd/acl.h
@@ -47,7 +47,7 @@ struct svc_rqst;
 #define NFS4_ACL_MAX ((PAGE_SIZE - sizeof(struct nfs4_acl)) \
                         / sizeof(struct nfs4_ace))
 
-struct nfs4_acl *nfs4_acl_new(int);
+int nfs4_acl_bytes(int entries);
 int nfs4_acl_get_whotype(char *, u32);
 __be32 nfs4_acl_write_who(struct xdr_stream *xdr, int who);
 
diff --git a/fs/nfsd/auth.c b/fs/nfsd/auth.c
index 72f44823adbb..9d46a0bdd9f9 100644
--- a/fs/nfsd/auth.c
+++ b/fs/nfsd/auth.c
@@ -28,7 +28,7 @@ int nfsd_setuser(struct svc_rqst *rqstp, struct svc_export *exp)
         validate_process_creds();
 
         /* discard any old override before preparing the new set */
-        revert_creds(get_cred(current->real_cred));
+        revert_creds(get_cred(current_real_cred()));
         new = prepare_creds();
         if (!new)
                 return -ENOMEM;
diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c
index 13b85f94d9e2..72ffd7cce3c3 100644
--- a/fs/nfsd/export.c
+++ b/fs/nfsd/export.c
@@ -698,8 +698,8 @@ static void svc_export_init(struct cache_head *cnew, struct cache_head *citem)
 
         kref_get(&item->ex_client->ref);
         new->ex_client = item->ex_client;
-        new->ex_path.dentry = dget(item->ex_path.dentry);
-        new->ex_path.mnt = mntget(item->ex_path.mnt);
+        new->ex_path = item->ex_path;
+        path_get(&item->ex_path);
         new->ex_fslocs.locations = NULL;
         new->ex_fslocs.locations_count = 0;
         new->ex_fslocs.migrated = 0;
@@ -1253,7 +1253,7 @@ static int e_show(struct seq_file *m, void *p)
                 return 0;
         }
 
-        cache_get(&exp->h);
+        exp_get(exp);
         if (cache_check(cd, &exp->h, NULL))
                 return 0;
         exp_put(exp);
diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h
index cfeea85c5bed..04dc8c167b0c 100644
--- a/fs/nfsd/export.h
+++ b/fs/nfsd/export.h
@@ -101,9 +101,10 @@ static inline void exp_put(struct svc_export *exp)
         cache_put(&exp->h, exp->cd);
 }
 
-static inline void exp_get(struct svc_export *exp)
+static inline struct svc_export *exp_get(struct svc_export *exp)
 {
         cache_get(&exp->h);
+        return exp;
 }
 struct svc_export * rqst_exp_find(struct svc_rqst *, int, u32 *);
 
diff --git a/fs/nfsd/fault_inject.c b/fs/nfsd/fault_inject.c
index 2ed05c3cd43d..c16bf5af6831 100644
--- a/fs/nfsd/fault_inject.c
+++ b/fs/nfsd/fault_inject.c
@@ -17,81 +17,13 @@
 
 struct nfsd_fault_inject_op {
         char *file;
-        u64 (*forget)(struct nfs4_client *, u64);
-        u64 (*print)(struct nfs4_client *, u64);
+        u64 (*get)(void);
+        u64 (*set_val)(u64);
+        u64 (*set_clnt)(struct sockaddr_storage *, size_t);
 };
 
-static struct nfsd_fault_inject_op inject_ops[] = {
-        {
-                .file   = "forget_clients",
-                .forget = nfsd_forget_client,
-                .print  = nfsd_print_client,
-        },
-        {
-                .file   = "forget_locks",
-                .forget = nfsd_forget_client_locks,
-                .print  = nfsd_print_client_locks,
-        },
-        {
-                .file   = "forget_openowners",
-                .forget = nfsd_forget_client_openowners,
-                .print  = nfsd_print_client_openowners,
-        },
-        {
-                .file   = "forget_delegations",
-                .forget = nfsd_forget_client_delegations,
-                .print  = nfsd_print_client_delegations,
-        },
-        {
-                .file   = "recall_delegations",
-                .forget = nfsd_recall_client_delegations,
-                .print  = nfsd_print_client_delegations,
-        },
-};
-
-static long int NUM_INJECT_OPS = sizeof(inject_ops) / sizeof(struct nfsd_fault_inject_op);
 static struct dentry *debug_dir;
 
-static void nfsd_inject_set(struct nfsd_fault_inject_op *op, u64 val)
-{
-        u64 count = 0;
-
-        if (val == 0)
-                printk(KERN_INFO "NFSD Fault Injection: %s (all)", op->file);
-        else
-                printk(KERN_INFO "NFSD Fault Injection: %s (n = %llu)", op->file, val);
-
-        nfs4_lock_state();
-        count = nfsd_for_n_state(val, op->forget);
-        nfs4_unlock_state();
-        printk(KERN_INFO "NFSD: %s: found %llu", op->file, count);
-}
-
-static void nfsd_inject_set_client(struct nfsd_fault_inject_op *op,
-                                   struct sockaddr_storage *addr,
-                                   size_t addr_size)
-{
-        char buf[INET6_ADDRSTRLEN];
-        struct nfs4_client *clp;
-        u64 count;
-
-        nfs4_lock_state();
-        clp = nfsd_find_client(addr, addr_size);
-        if (clp) {
-                count = op->forget(clp, 0);
-                rpc_ntop((struct sockaddr *)&clp->cl_addr, buf, sizeof(buf));
-                printk(KERN_INFO "NFSD [%s]: Client %s had %llu state object(s)\n", op->file, buf, count);
-        }
-        nfs4_unlock_state();
-}
-
-static void nfsd_inject_get(struct nfsd_fault_inject_op *op, u64 *val)
-{
-        nfs4_lock_state();
-        *val = nfsd_for_n_state(0, op->print);
-        nfs4_unlock_state();
-}
-
 static ssize_t fault_inject_read(struct file *file, char __user *buf,
                                  size_t len, loff_t *ppos)
 {
@@ -99,9 +31,10 @@ static ssize_t fault_inject_read(struct file *file, char __user *buf,
         char read_buf[25];
         size_t size;
         loff_t pos = *ppos;
+        struct nfsd_fault_inject_op *op = file_inode(file)->i_private;
 
         if (!pos)
-                nfsd_inject_get(file_inode(file)->i_private, &val);
+                val = op->get();
         size = scnprintf(read_buf, sizeof(read_buf), "%llu\n", val);
 
         return simple_read_from_buffer(buf, len, ppos, read_buf, size);
@@ -114,18 +47,36 @@ static ssize_t fault_inject_write(struct file *file, const char __user *buf,
         size_t size = min(sizeof(write_buf) - 1, len);
         struct net *net = current->nsproxy->net_ns;
         struct sockaddr_storage sa;
+        struct nfsd_fault_inject_op *op = file_inode(file)->i_private;
         u64 val;
+        char *nl;
 
         if (copy_from_user(write_buf, buf, size))
                 return -EFAULT;
         write_buf[size] = '\0';
 
+        /* Deal with any embedded newlines in the string */
+        nl = strchr(write_buf, '\n');
+        if (nl) {
+                size = nl - write_buf;
+                *nl = '\0';
+        }
+
         size = rpc_pton(net, write_buf, size, (struct sockaddr *)&sa, sizeof(sa));
-        if (size > 0)
-                nfsd_inject_set_client(file_inode(file)->i_private, &sa, size);
-        else {
+        if (size > 0) {
+                val = op->set_clnt(&sa, size);
+                if (val)
+                        pr_info("NFSD [%s]: Client %s had %llu state object(s)\n",
+                                op->file, write_buf, val);
+        } else {
                 val = simple_strtoll(write_buf, NULL, 0);
-                nfsd_inject_set(file_inode(file)->i_private, val);
+                if (val == 0)
+                        pr_info("NFSD Fault Injection: %s (all)", op->file);
+                else
+                        pr_info("NFSD Fault Injection: %s (n = %llu)",
+                                op->file, val);
+                val = op->set_val(val);
+                pr_info("NFSD: %s: found %llu", op->file, val);
         }
         return len; /* on success, claim we got the whole input */
 }
@@ -141,6 +92,41 @@ void nfsd_fault_inject_cleanup(void)
         debugfs_remove_recursive(debug_dir);
 }
 
+static struct nfsd_fault_inject_op inject_ops[] = {
+        {
+                .file     = "forget_clients",
+                .get      = nfsd_inject_print_clients,
+                .set_val  = nfsd_inject_forget_clients,
+                .set_clnt = nfsd_inject_forget_client,
+        },
+        {
+                .file     = "forget_locks",
+                .get      = nfsd_inject_print_locks,
+                .set_val  = nfsd_inject_forget_locks,
+                .set_clnt = nfsd_inject_forget_client_locks,
+        },
+        {
+                .file     = "forget_openowners",
+                .get      = nfsd_inject_print_openowners,
+                .set_val  = nfsd_inject_forget_openowners,
+                .set_clnt = nfsd_inject_forget_client_openowners,
+        },
+        {
+                .file     = "forget_delegations",
+                .get      = nfsd_inject_print_delegations,
+                .set_val  = nfsd_inject_forget_delegations,
+                .set_clnt = nfsd_inject_forget_client_delegations,
+        },
+        {
+                .file     = "recall_delegations",
+                .get      = nfsd_inject_print_delegations,
+                .set_val  = nfsd_inject_recall_delegations,
+                .set_clnt = nfsd_inject_recall_client_delegations,
+        },
+};
+
+#define NUM_INJECT_OPS (sizeof(inject_ops)/sizeof(struct nfsd_fault_inject_op))
+
 int nfsd_fault_inject_init(void)
 {
         unsigned int i;
diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
index d32b3aa6600d..ea6749a32760 100644
--- a/fs/nfsd/netns.h
+++ b/fs/nfsd/netns.h
@@ -29,14 +29,19 @@
 #define CLIENT_HASH_SIZE                (1 << CLIENT_HASH_BITS)
 #define CLIENT_HASH_MASK                (CLIENT_HASH_SIZE - 1)
 
-#define LOCKOWNER_INO_HASH_BITS         8
-#define LOCKOWNER_INO_HASH_SIZE         (1 << LOCKOWNER_INO_HASH_BITS)
-
 #define SESSION_HASH_SIZE       512
 
 struct cld_net;
 struct nfsd4_client_tracking_ops;
 
+/*
+ * Represents a nfsd "container". With respect to nfsv4 state tracking, the
+ * fields of interest are the *_id_hashtbls and the *_name_tree. These track
+ * the nfs4_client objects by either short or long form clientid.
+ *
+ * Each nfsd_net runs a nfs4_laundromat workqueue job when necessary to clean
+ * up expired clients and delegations within the container.
+ */
 struct nfsd_net {
         struct cld_net *cld_net;
 
@@ -66,8 +71,6 @@ struct nfsd_net {
         struct rb_root conf_name_tree;
         struct list_head *unconf_id_hashtbl;
         struct rb_root unconf_name_tree;
-        struct list_head *ownerstr_hashtbl;
-        struct list_head *lockowner_ino_hashtbl;
         struct list_head *sessionid_hashtbl;
         /*
          * client_lru holds client queue ordered by nfs4_client.cl_time
@@ -97,10 +100,16 @@ struct nfsd_net {
         bool nfsd_net_up;
         bool lockd_up;
 
+        /* Time of server startup */
+        struct timeval nfssvc_boot;
+
         /*
-         * Time of server startup
+         * Max number of connections this nfsd container will allow. Defaults
+         * to '0' which is means that it bases this on the number of threads.
          */
-        struct timeval nfssvc_boot;
+        unsigned int max_connections;
+
+        u32 clientid_counter;
 
         struct svc_serv *nfsd_serv;
 };
diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c
index 12b023a7ab7d..ac54ea60b3f6 100644
--- a/fs/nfsd/nfs2acl.c
+++ b/fs/nfsd/nfs2acl.c
@@ -54,14 +54,14 @@ static __be32 nfsacld_proc_getacl(struct svc_rqst * rqstp,
 
         if (resp->mask & (NFS_ACL|NFS_ACLCNT)) {
                 acl = get_acl(inode, ACL_TYPE_ACCESS);
-                if (IS_ERR(acl)) {
-                        nfserr = nfserrno(PTR_ERR(acl));
-                        goto fail;
-                }
                 if (acl == NULL) {
                         /* Solaris returns the inode's minimum ACL. */
                         acl = posix_acl_from_mode(inode->i_mode, GFP_KERNEL);
                 }
+                if (IS_ERR(acl)) {
+                        nfserr = nfserrno(PTR_ERR(acl));
+                        goto fail;
+                }
                 resp->acl_access = acl;
         }
         if (resp->mask & (NFS_DFACL|NFS_DFACLCNT)) {
diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c
index 2a514e21dc74..34cbbab6abd7 100644
--- a/fs/nfsd/nfs3acl.c
+++ b/fs/nfsd/nfs3acl.c
@@ -47,14 +47,14 @@ static __be32 nfsd3_proc_getacl(struct svc_rqst * rqstp,
 
         if (resp->mask & (NFS_ACL|NFS_ACLCNT)) {
                 acl = get_acl(inode, ACL_TYPE_ACCESS);
-                if (IS_ERR(acl)) {
-                        nfserr = nfserrno(PTR_ERR(acl));
-                        goto fail;
-                }
                 if (acl == NULL) {
                         /* Solaris returns the inode's minimum ACL. */
                         acl = posix_acl_from_mode(inode->i_mode, GFP_KERNEL);
                 }
+                if (IS_ERR(acl)) {
+                        nfserr = nfserrno(PTR_ERR(acl));
+                        goto fail;
+                }
                 resp->acl_access = acl;
         }
         if (resp->mask & (NFS_DFACL|NFS_DFACLCNT)) {
diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c
index 401289913130..fa2525b2e9d7 100644
--- a/fs/nfsd/nfs3proc.c
+++ b/fs/nfsd/nfs3proc.c
@@ -157,11 +157,7 @@ nfsd3_proc_read(struct svc_rqst *rqstp, struct nfsd3_readargs *argp,
          * 1 (status) + 22 (post_op_attr) + 1 (count) + 1 (eof)
          * + 1 (xdr opaque byte count) = 26
          */
-
-        resp->count = argp->count;
-        if (max_blocksize < resp->count)
-                resp->count = max_blocksize;
-
+        resp->count = min(argp->count, max_blocksize);
         svc_reserve_auth(rqstp, ((1 + NFS3_POST_OP_ATTR_WORDS + 3)<<2) + resp->count +4);
 
         fh_copy(&resp->fh, &argp->fh);
@@ -286,8 +282,7 @@ nfsd3_proc_symlink(struct svc_rqst *rqstp, struct nfsd3_symlinkargs *argp,
         fh_copy(&resp->dirfh, &argp->ffh);
         fh_init(&resp->fh, NFS3_FHSIZE);
         nfserr = nfsd_symlink(rqstp, &resp->dirfh, argp->fname, argp->flen,
-                                                   argp->tname, argp->tlen,
-                                                   &resp->fh, &argp->attrs);
+                                                   argp->tname, &resp->fh);
         RETURN_STATUS(nfserr);
 }
 
diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c
index e6c01e80325e..39c5eb3ad33a 100644
--- a/fs/nfsd/nfs3xdr.c
+++ b/fs/nfsd/nfs3xdr.c
@@ -120,10 +120,7 @@ decode_sattr3(__be32 *p, struct iattr *iap)
 
                 iap->ia_valid |= ATTR_SIZE;
                 p = xdr_decode_hyper(p, &newsize);
-                if (newsize <= NFS_OFFSET_MAX)
-                        iap->ia_size = newsize;
-                else
-                        iap->ia_size = NFS_OFFSET_MAX;
+                iap->ia_size = min_t(u64, newsize, NFS_OFFSET_MAX);
         }
         if ((tmp = ntohl(*p++)) == 1) { /* set to server time */
                 iap->ia_valid |= ATTR_ATIME;
@@ -338,10 +335,8 @@ nfs3svc_decode_readargs(struct svc_rqst *rqstp, __be32 *p,
                 return 0;
         p = xdr_decode_hyper(p, &args->offset);
 
-        len = args->count = ntohl(*p++);
-
-        if (len > max_blocksize)
-                len = max_blocksize;
+        args->count = ntohl(*p++);
+        len = min(args->count, max_blocksize);
 
         /* set up the kvec */
         v=0;
@@ -349,7 +344,7 @@ nfs3svc_decode_readargs(struct svc_rqst *rqstp, __be32 *p,
                 struct page *p = *(rqstp->rq_next_page++);
 
                 rqstp->rq_vec[v].iov_base = page_address(p);
-                rqstp->rq_vec[v].iov_len = len < PAGE_SIZE? len : PAGE_SIZE;
+                rqstp->rq_vec[v].iov_len = min_t(unsigned int, len, PAGE_SIZE);
                 len -= rqstp->rq_vec[v].iov_len;
                 v++;
         }
@@ -484,9 +479,7 @@ nfs3svc_decode_symlinkargs(struct svc_rqst *rqstp, __be32 *p,
         }
         /* now copy next page if there is one */
         if (len && !avail && rqstp->rq_arg.page_len) {
-                avail = rqstp->rq_arg.page_len;
-                if (avail > PAGE_SIZE)
-                        avail = PAGE_SIZE;
+                avail = min_t(unsigned int, rqstp->rq_arg.page_len, PAGE_SIZE);
                 old = page_address(rqstp->rq_arg.pages[0]);
         }
         while (len && avail && *old) {
@@ -571,10 +564,7 @@ nfs3svc_decode_readdirargs(struct svc_rqst *rqstp, __be32 *p,
         args->verf   = p; p += 2;
         args->dircount = ~0;
         args->count  = ntohl(*p++);
-
-        if (args->count > PAGE_SIZE)
-                args->count = PAGE_SIZE;
-
+        args->count  = min_t(u32, args->count, PAGE_SIZE);
         args->buffer = page_address(*(rqstp->rq_next_page++));
 
         return xdr_argsize_check(rqstp, p);
@@ -595,10 +585,7 @@ nfs3svc_decode_readdirplusargs(struct svc_rqst *rqstp, __be32 *p,
         args->dircount = ntohl(*p++);
         args->count    = ntohl(*p++);
 
-        len = (args->count > max_blocksize) ? max_blocksize :
-                                                  args->count;
-        args->count = len;
-
+        len = args->count = min(args->count, max_blocksize);
         while (len > 0) {
                 struct page *p = *(rqstp->rq_next_page++);
                 if (!args->buffer)
@@ -913,8 +900,7 @@ encode_entry(struct readdir_cd *ccd, const char *name, int namlen,
          */
 
         /* truncate filename if too long */
-        if (namlen > NFS3_MAXNAMLEN)
-                namlen = NFS3_MAXNAMLEN;
+        namlen = min(namlen, NFS3_MAXNAMLEN);
 
         slen = XDR_QUADLEN(namlen);
         elen = slen + NFS3_ENTRY_BAGGAGE
diff --git a/fs/nfsd/nfs4acl.c b/fs/nfsd/nfs4acl.c
index d714156a19fd..59fd76651781 100644
--- a/fs/nfsd/nfs4acl.c
+++ b/fs/nfsd/nfs4acl.c
@@ -146,35 +146,43 @@ nfsd4_get_nfs4_acl(struct svc_rqst *rqstp, struct dentry *dentry,
         int size = 0;
 
         pacl = get_acl(inode, ACL_TYPE_ACCESS);
-        if (!pacl) {
+        if (!pacl)
                 pacl = posix_acl_from_mode(inode->i_mode, GFP_KERNEL);
-                if (IS_ERR(pacl))
-                        return PTR_ERR(pacl);
-        }
+
+        if (IS_ERR(pacl))
+                return PTR_ERR(pacl);
+
         /* allocate for worst case: one (deny, allow) pair each: */
         size += 2 * pacl->a_count;
 
         if (S_ISDIR(inode->i_mode)) {
                 flags = NFS4_ACL_DIR;
                 dpacl = get_acl(inode, ACL_TYPE_DEFAULT);
+                if (IS_ERR(dpacl)) {
+                        error = PTR_ERR(dpacl);
+                        goto rel_pacl;
+                }
+
                 if (dpacl)
                         size += 2 * dpacl->a_count;
         }
 
-        *acl = nfs4_acl_new(size);
+        *acl = kmalloc(nfs4_acl_bytes(size), GFP_KERNEL);
         if (*acl == NULL) {
                 error = -ENOMEM;
                 goto out;
         }
+        (*acl)->naces = 0;
 
         _posix_to_nfsv4_one(pacl, *acl, flags & ~NFS4_ACL_TYPE_DEFAULT);
 
         if (dpacl)
                 _posix_to_nfsv4_one(dpacl, *acl, flags | NFS4_ACL_TYPE_DEFAULT);
 
- out:
-        posix_acl_release(pacl);
+out:
         posix_acl_release(dpacl);
+rel_pacl:
+        posix_acl_release(pacl);
         return error;
 }
 
@@ -872,16 +880,13 @@ ace2type(struct nfs4_ace *ace)
         return -1;
 }
 
-struct nfs4_acl *
-nfs4_acl_new(int n)
+/*
+ * return the size of the struct nfs4_acl required to represent an acl
+ * with @entries entries.
+ */
+int nfs4_acl_bytes(int entries)
 {
-        struct nfs4_acl *acl;
-
-        acl = kmalloc(sizeof(*acl) + n*sizeof(struct nfs4_ace), GFP_KERNEL);
-        if (acl == NULL)
-                return NULL;
-        acl->naces = 0;
-        return acl;
+        return sizeof(struct nfs4_acl) + entries * sizeof(struct nfs4_ace);
 }
 
 static struct {
@@ -935,5 +940,5 @@ __be32 nfs4_acl_write_who(struct xdr_stream *xdr, int who)
                 return 0;
         }
         WARN_ON_ONCE(1);
-        return -1;
+        return nfserr_serverfault;
 }
diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 2c73cae9899d..e0be57b0f79b 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -337,7 +337,7 @@ static void encode_cb_recall4args(struct xdr_stream *xdr,
         p = xdr_reserve_space(xdr, 4);
         *p++ = xdr_zero;                        /* truncate */
 
-        encode_nfs_fh4(xdr, &dp->dl_fh);
+        encode_nfs_fh4(xdr, &dp->dl_stid.sc_file->fi_fhandle);
 
         hdr->nops++;
 }
@@ -678,7 +678,7 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c
                                 (clp->cl_cred.cr_flavor >= RPC_AUTH_GSS_KRB5))
                         return -EINVAL;
                 args.client_name = clp->cl_cred.cr_principal;
-                args.prognumber = conn->cb_prog,
+                args.prognumber = conn->cb_prog;
                 args.protocol = XPRT_TRANSPORT_TCP;
                 args.authflavor = clp->cl_cred.cr_flavor;
                 clp->cl_cb_ident = conn->cb_ident;
@@ -689,7 +689,8 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c
                 clp->cl_cb_session = ses;
                 args.bc_xprt = conn->cb_xprt;
                 args.prognumber = clp->cl_cb_session->se_cb_prog;
-                args.protocol = XPRT_TRANSPORT_BC_TCP;
+                args.protocol = conn->cb_xprt->xpt_class->xcl_ident |
+                                XPRT_TRANSPORT_BC;
                 args.authflavor = ses->se_cb_sec.flavor;
         }
         /* Create RPC client */
@@ -904,7 +905,7 @@ static void nfsd4_cb_recall_release(void *calldata)
                 spin_lock(&clp->cl_lock);
                 list_del(&cb->cb_per_client);
                 spin_unlock(&clp->cl_lock);
-                nfs4_put_delegation(dp);
+                nfs4_put_stid(&dp->dl_stid);
         }
 }
 
@@ -933,7 +934,7 @@ void nfsd4_shutdown_callback(struct nfs4_client *clp)
         set_bit(NFSD4_CLIENT_CB_KILL, &clp->cl_flags);
         /*
          * Note this won't actually result in a null callback;
-         * instead, nfsd4_do_callback_rpc() will detect the killed
+         * instead, nfsd4_run_cb_null() will detect the killed
          * client, destroy the rpc client, and stop:
          */
         do_probe_callback(clp);
@@ -1011,9 +1012,9 @@ static void nfsd4_process_cb_update(struct nfsd4_callback *cb)
                 run_nfsd4_cb(cb);
 }
 
-static void nfsd4_do_callback_rpc(struct work_struct *w)
+static void
+nfsd4_run_callback_rpc(struct nfsd4_callback *cb)
 {
-        struct nfsd4_callback *cb = container_of(w, struct nfsd4_callback, cb_work);
         struct nfs4_client *clp = cb->cb_clp;
         struct rpc_clnt *clnt;
 
@@ -1031,9 +1032,22 @@ static void nfsd4_do_callback_rpc(struct work_struct *w)
                         cb->cb_ops, cb);
 }
 
-void nfsd4_init_callback(struct nfsd4_callback *cb)
+void
+nfsd4_run_cb_null(struct work_struct *w)
 {
-        INIT_WORK(&cb->cb_work, nfsd4_do_callback_rpc);
+        struct nfsd4_callback *cb = container_of(w, struct nfsd4_callback,
+                                                        cb_work);
+        nfsd4_run_callback_rpc(cb);
+}
+
+void
+nfsd4_run_cb_recall(struct work_struct *w)
+{
+        struct nfsd4_callback *cb = container_of(w, struct nfsd4_callback,
+                                                        cb_work);
+
+        nfsd4_prepare_cb_recall(cb->cb_op);
+        nfsd4_run_callback_rpc(cb);
 }
 
 void nfsd4_cb_recall(struct nfs4_delegation *dp)
diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 8f029db5d271..5e0dc528a0e8 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -177,7 +177,7 @@ fh_dup2(struct svc_fh *dst, struct svc_fh *src)
         fh_put(dst);
         dget(src->fh_dentry);
         if (src->fh_export)
-                cache_get(&src->fh_export->h);
+                exp_get(src->fh_export);
         *dst = *src;
 }
 
@@ -385,8 +385,6 @@ nfsd4_open(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         if (nfsd4_has_session(cstate))
                 copy_clientid(&open->op_clientid, cstate->session);
 
-        nfs4_lock_state();
-
         /* check seqid for replay. set nfs4_owner */
         resp = rqstp->rq_resp;
         status = nfsd4_process_open1(&resp->cstate, open, nn);
@@ -431,8 +429,7 @@ nfsd4_open(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
                         break;
                 case NFS4_OPEN_CLAIM_PREVIOUS:
                         status = nfs4_check_open_reclaim(&open->op_clientid,
-                                                         cstate->minorversion,
-                                                         nn);
+                                                         cstate, nn);
                         if (status)
                                 goto out;
                         open->op_openowner->oo_flags |= NFS4_OO_CONFIRMED;
@@ -461,19 +458,17 @@ nfsd4_open(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
          * set, (2) sets open->op_stateid, (3) sets open->op_delegation.
          */
         status = nfsd4_process_open2(rqstp, resfh, open);
-        WARN_ON(status && open->op_created);
+        WARN(status && open->op_created,
+             "nfsd4_process_open2 failed to open newly-created file! status=%u\n",
+             be32_to_cpu(status));
 out:
         if (resfh && resfh != &cstate->current_fh) {
                 fh_dup2(&cstate->current_fh, resfh);
                 fh_put(resfh);
                 kfree(resfh);
         }
-        nfsd4_cleanup_open_state(open, status);
-        if (open->op_openowner && !nfsd4_has_session(cstate))
-                cstate->replay_owner = &open->op_openowner->oo_owner;
+        nfsd4_cleanup_open_state(cstate, open, status);
         nfsd4_bump_seqid(cstate, status);
-        if (!cstate->replay_owner)
-                nfs4_unlock_state();
         return status;
 }
 
@@ -581,8 +576,12 @@ static void gen_boot_verifier(nfs4_verifier *verifier, struct net *net)
         __be32 verf[2];
         struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
-        verf[0] = (__be32)nn->nfssvc_boot.tv_sec;
-        verf[1] = (__be32)nn->nfssvc_boot.tv_usec;
+        /*
+         * This is opaque to client, so no need to byte-swap. Use
+         * __force to keep sparse happy
+         */
+        verf[0] = (__force __be32)nn->nfssvc_boot.tv_sec;
+        verf[1] = (__force __be32)nn->nfssvc_boot.tv_usec;
         memcpy(verifier->data, verf, sizeof(verifier->data));
 }
 
@@ -619,8 +618,7 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         case NF4LNK:
                 status = nfsd_symlink(rqstp, &cstate->current_fh,
                                       create->cr_name, create->cr_namelen,
-                                      create->cr_linkname, create->cr_linklen,
-                                      &resfh, &create->cr_iattr);
+                                      create->cr_data, &resfh);
                 break;
 
         case NF4BLK:
@@ -909,8 +907,8 @@ nfsd4_secinfo_no_name(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstat
         default:
                 return nfserr_inval;
         }
-        exp_get(cstate->current_fh.fh_export);
-        sin->sin_exp = cstate->current_fh.fh_export;
+
+        sin->sin_exp = exp_get(cstate->current_fh.fh_export);
         fh_put(&cstate->current_fh);
         return nfs_ok;
 }
@@ -1289,7 +1287,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp,
          * Don't use the deferral mechanism for NFSv4; compounds make it
          * too hard to avoid non-idempotency problems.
          */
-        rqstp->rq_usedeferral = 0;
+        rqstp->rq_usedeferral = false;
 
         /*
          * According to RFC3010, this takes precedence over all other errors.
@@ -1391,10 +1389,7 @@ encode_op:
                         args->ops, args->opcnt, resp->opcnt, op->opnum,
                         be32_to_cpu(status));
 
-                if (cstate->replay_owner) {
-                        nfs4_unlock_state();
-                        cstate->replay_owner = NULL;
-                }
+                nfsd4_cstate_clear_replay(cstate);
                 /* XXX Ugh, we need to get rid of this kind of special case: */
                 if (op->opnum == OP_READ && op->u.read.rd_filp)
                         fput(op->u.read.rd_filp);
@@ -1408,7 +1403,7 @@ encode_op:
         BUG_ON(cstate->replay_owner);
 out:
         /* Reset deferral mechanism for RPC deferrals */
-        rqstp->rq_usedeferral = 1;
+        rqstp->rq_usedeferral = true;
         dprintk("nfsv4 compound returned %d\n", ntohl(status));
         return status;
 }
@@ -1520,21 +1515,17 @@ static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
         u32 maxcount = 0, rlen = 0;
 
         maxcount = svc_max_payload(rqstp);
-        rlen = op->u.read.rd_length;
-
-        if (rlen > maxcount)
-                rlen = maxcount;
+        rlen = min(op->u.read.rd_length, maxcount);
 
         return (op_encode_hdr_size + 2 + XDR_QUADLEN(rlen)) * sizeof(__be32);
 }
 
 static inline u32 nfsd4_readdir_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 {
-        u32 maxcount = svc_max_payload(rqstp);
-        u32 rlen = op->u.readdir.rd_maxcount;
+        u32 maxcount = 0, rlen = 0;
 
-        if (rlen > maxcount)
-                rlen = maxcount;
+        maxcount = svc_max_payload(rqstp);
+        rlen = min(op->u.readdir.rd_maxcount, maxcount);
 
         return (op_encode_hdr_size + op_encode_verifier_maxsz +
                 XDR_QUADLEN(rlen)) * sizeof(__be32);
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 2204e1fe5725..2e80a59e7e91 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -70,13 +70,11 @@ static u64 current_sessionid = 1;
 #define CURRENT_STATEID(stateid) (!memcmp((stateid), &currentstateid, sizeof(stateid_t)))
 
 /* forward declarations */
-static int check_for_locks(struct nfs4_file *filp, struct nfs4_lockowner *lowner);
+static bool check_for_locks(struct nfs4_file *fp, struct nfs4_lockowner *lowner);
+static void nfs4_free_ol_stateid(struct nfs4_stid *stid);
 
 /* Locking: */
 
-/* Currently used for almost all code touching nfsv4 state: */
-static DEFINE_MUTEX(client_mutex);
-
 /*
  * Currently used for the del_recall_lru and file hash table.  In an
  * effort to decrease the scope of the client_mutex, this spinlock may
@@ -84,18 +82,18 @@ static DEFINE_MUTEX(client_mutex);
  */
 static DEFINE_SPINLOCK(state_lock);
 
+/*
+ * A waitqueue for all in-progress 4.0 CLOSE operations that are waiting for
+ * the refcount on the open stateid to drop.
+ */
+static DECLARE_WAIT_QUEUE_HEAD(close_wq);
+
 static struct kmem_cache *openowner_slab;
 static struct kmem_cache *lockowner_slab;
 static struct kmem_cache *file_slab;
 static struct kmem_cache *stateid_slab;
 static struct kmem_cache *deleg_slab;
 
-void
-nfs4_lock_state(void)
-{
-        mutex_lock(&client_mutex);
-}
-
 static void free_session(struct nfsd4_session *);
 
 static bool is_session_dead(struct nfsd4_session *ses)
@@ -103,12 +101,6 @@ static bool is_session_dead(struct nfsd4_session *ses)
         return ses->se_flags & NFS4_SESSION_DEAD;
 }
 
-void nfsd4_put_session(struct nfsd4_session *ses)
-{
-        if (atomic_dec_and_test(&ses->se_ref) && is_session_dead(ses))
-                free_session(ses);
-}
-
 static __be32 mark_session_dead_locked(struct nfsd4_session *ses, int ref_held_by_me)
 {
         if (atomic_read(&ses->se_ref) > ref_held_by_me)
@@ -117,46 +109,17 @@ static __be32 mark_session_dead_locked(struct nfsd4_session *ses, int ref_held_b
         return nfs_ok;
 }
 
-static __be32 nfsd4_get_session_locked(struct nfsd4_session *ses)
-{
-        if (is_session_dead(ses))
-                return nfserr_badsession;
-        atomic_inc(&ses->se_ref);
-        return nfs_ok;
-}
-
-void
-nfs4_unlock_state(void)
-{
-        mutex_unlock(&client_mutex);
-}
-
 static bool is_client_expired(struct nfs4_client *clp)
 {
         return clp->cl_time == 0;
 }
 
-static __be32 mark_client_expired_locked(struct nfs4_client *clp)
-{
-        if (atomic_read(&clp->cl_refcount))
-                return nfserr_jukebox;
-        clp->cl_time = 0;
-        return nfs_ok;
-}
-
-static __be32 mark_client_expired(struct nfs4_client *clp)
+static __be32 get_client_locked(struct nfs4_client *clp)
 {
         struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
-        __be32 ret;
 
-        spin_lock(&nn->client_lock);
-        ret = mark_client_expired_locked(clp);
-        spin_unlock(&nn->client_lock);
-        return ret;
-}
+        lockdep_assert_held(&nn->client_lock);
 
-static __be32 get_client_locked(struct nfs4_client *clp)
-{
         if (is_client_expired(clp))
                 return nfserr_expired;
         atomic_inc(&clp->cl_refcount);
@@ -197,13 +160,17 @@ renew_client(struct nfs4_client *clp)
 
 static void put_client_renew_locked(struct nfs4_client *clp)
 {
+        struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+
+        lockdep_assert_held(&nn->client_lock);
+
         if (!atomic_dec_and_test(&clp->cl_refcount))
                 return;
         if (!is_client_expired(clp))
                 renew_client_locked(clp);
 }
 
-void put_client_renew(struct nfs4_client *clp)
+static void put_client_renew(struct nfs4_client *clp)
 {
         struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
 
@@ -214,6 +181,79 @@ void put_client_renew(struct nfs4_client *clp)
         spin_unlock(&nn->client_lock);
 }
 
+static __be32 nfsd4_get_session_locked(struct nfsd4_session *ses)
+{
+        __be32 status;
+
+        if (is_session_dead(ses))
+                return nfserr_badsession;
+        status = get_client_locked(ses->se_client);
+        if (status)
+                return status;
+        atomic_inc(&ses->se_ref);
+        return nfs_ok;
+}
+
+static void nfsd4_put_session_locked(struct nfsd4_session *ses)
+{
+        struct nfs4_client *clp = ses->se_client;
+        struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+
+        lockdep_assert_held(&nn->client_lock);
+
+        if (atomic_dec_and_test(&ses->se_ref) && is_session_dead(ses))
+                free_session(ses);
+        put_client_renew_locked(clp);
+}
+
+static void nfsd4_put_session(struct nfsd4_session *ses)
+{
+        struct nfs4_client *clp = ses->se_client;
+        struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+
+        spin_lock(&nn->client_lock);
+        nfsd4_put_session_locked(ses);
+        spin_unlock(&nn->client_lock);
+}
+
+static int
+same_owner_str(struct nfs4_stateowner *sop, struct xdr_netobj *owner)
+{
+        return (sop->so_owner.len == owner->len) &&
+                0 == memcmp(sop->so_owner.data, owner->data, owner->len);
+}
+
+static struct nfs4_openowner *
+find_openstateowner_str_locked(unsigned int hashval, struct nfsd4_open *open,
+                        struct nfs4_client *clp)
+{
+        struct nfs4_stateowner *so;
+
+        lockdep_assert_held(&clp->cl_lock);
+
+        list_for_each_entry(so, &clp->cl_ownerstr_hashtbl[hashval],
+                            so_strhash) {
+                if (!so->so_is_open_owner)
+                        continue;
+                if (same_owner_str(so, &open->op_owner)) {
+                        atomic_inc(&so->so_count);
+                        return openowner(so);
+                }
+        }
+        return NULL;
+}
+
+static struct nfs4_openowner *
+find_openstateowner_str(unsigned int hashval, struct nfsd4_open *open,
+                        struct nfs4_client *clp)
+{
+        struct nfs4_openowner *oo;
+
+        spin_lock(&clp->cl_lock);
+        oo = find_openstateowner_str_locked(hashval, open, clp);
+        spin_unlock(&clp->cl_lock);
+        return oo;
+}
 
 static inline u32
 opaque_hashval(const void *ptr, int nbytes)
@@ -236,10 +276,11 @@ static void nfsd4_free_file(struct nfs4_file *f)
 static inline void
 put_nfs4_file(struct nfs4_file *fi)
 {
+        might_lock(&state_lock);
+
         if (atomic_dec_and_lock(&fi->fi_ref, &state_lock)) {
                 hlist_del(&fi->fi_hash);
                 spin_unlock(&state_lock);
-                iput(fi->fi_inode);
                 nfsd4_free_file(fi);
         }
 }
@@ -250,7 +291,80 @@ get_nfs4_file(struct nfs4_file *fi)
         atomic_inc(&fi->fi_ref);
 }
 
-static int num_delegations;
+static struct file *
+__nfs4_get_fd(struct nfs4_file *f, int oflag)
+{
+        if (f->fi_fds[oflag])
+                return get_file(f->fi_fds[oflag]);
+        return NULL;
+}
+
+static struct file *
+find_writeable_file_locked(struct nfs4_file *f)
+{
+        struct file *ret;
+
+        lockdep_assert_held(&f->fi_lock);
+
+        ret = __nfs4_get_fd(f, O_WRONLY);
+        if (!ret)
+                ret = __nfs4_get_fd(f, O_RDWR);
+        return ret;
+}
+
+static struct file *
+find_writeable_file(struct nfs4_file *f)
+{
+        struct file *ret;
+
+        spin_lock(&f->fi_lock);
+        ret = find_writeable_file_locked(f);
+        spin_unlock(&f->fi_lock);
+
+        return ret;
+}
+
+static struct file *find_readable_file_locked(struct nfs4_file *f)
+{
+        struct file *ret;
+
+        lockdep_assert_held(&f->fi_lock);
+
+        ret = __nfs4_get_fd(f, O_RDONLY);
+        if (!ret)
+                ret = __nfs4_get_fd(f, O_RDWR);
+        return ret;
+}
+
+static struct file *
+find_readable_file(struct nfs4_file *f)
+{
+        struct file *ret;
+
+        spin_lock(&f->fi_lock);
+        ret = find_readable_file_locked(f);
+        spin_unlock(&f->fi_lock);
+
+        return ret;
+}
+
+static struct file *
+find_any_file(struct nfs4_file *f)
+{
+        struct file *ret;
+
+        spin_lock(&f->fi_lock);
+        ret = __nfs4_get_fd(f, O_RDWR);
+        if (!ret) {
+                ret = __nfs4_get_fd(f, O_WRONLY);
+                if (!ret)
+                        ret = __nfs4_get_fd(f, O_RDONLY);
+        }
+        spin_unlock(&f->fi_lock);
+        return ret;
+}
+
+static atomic_long_t num_delegations;
 unsigned long max_delegations;
 
 /*
@@ -262,12 +376,11 @@ unsigned long max_delegations;
 #define OWNER_HASH_SIZE             (1 << OWNER_HASH_BITS)
 #define OWNER_HASH_MASK             (OWNER_HASH_SIZE - 1)
 
-static unsigned int ownerstr_hashval(u32 clientid, struct xdr_netobj *ownername)
+static unsigned int ownerstr_hashval(struct xdr_netobj *ownername)
 {
         unsigned int ret;
 
         ret = opaque_hashval(ownername->data, ownername->len);
-        ret += clientid;
         return ret & OWNER_HASH_MASK;
 }
 
@@ -275,75 +388,124 @@ static unsigned int ownerstr_hashval(u32 clientid, struct xdr_netobj *ownername)
 #define FILE_HASH_BITS                   8
 #define FILE_HASH_SIZE                  (1 << FILE_HASH_BITS)
 
-static unsigned int file_hashval(struct inode *ino)
+static unsigned int nfsd_fh_hashval(struct knfsd_fh *fh)
+{
+        return jhash2(fh->fh_base.fh_pad, XDR_QUADLEN(fh->fh_size), 0);
+}
+
+static unsigned int file_hashval(struct knfsd_fh *fh)
+{
+        return nfsd_fh_hashval(fh) & (FILE_HASH_SIZE - 1);
+}
+
+static bool nfsd_fh_match(struct knfsd_fh *fh1, struct knfsd_fh *fh2)
 {
-        /* XXX: why are we hashing on inode pointer, anyway? */
-        return hash_ptr(ino, FILE_HASH_BITS);
+        return fh1->fh_size == fh2->fh_size &&
+                !memcmp(fh1->fh_base.fh_pad,
+                                fh2->fh_base.fh_pad,
+                                fh1->fh_size);
 }
 
 static struct hlist_head file_hashtbl[FILE_HASH_SIZE];
 
-static void __nfs4_file_get_access(struct nfs4_file *fp, int oflag)
+static void
+__nfs4_file_get_access(struct nfs4_file *fp, u32 access)
 {
-        WARN_ON_ONCE(!(fp->fi_fds[oflag] || fp->fi_fds[O_RDWR]));
-        atomic_inc(&fp->fi_access[oflag]);
+        lockdep_assert_held(&fp->fi_lock);
+
+        if (access & NFS4_SHARE_ACCESS_WRITE)
+                atomic_inc(&fp->fi_access[O_WRONLY]);
+        if (access & NFS4_SHARE_ACCESS_READ)
+                atomic_inc(&fp->fi_access[O_RDONLY]);
 }
 
-static void nfs4_file_get_access(struct nfs4_file *fp, int oflag)
+static __be32
+nfs4_file_get_access(struct nfs4_file *fp, u32 access)
 {
-        if (oflag == O_RDWR) {
-                __nfs4_file_get_access(fp, O_RDONLY);
-                __nfs4_file_get_access(fp, O_WRONLY);
-        } else
-                __nfs4_file_get_access(fp, oflag);
+        lockdep_assert_held(&fp->fi_lock);
+
+        /* Does this access mode make sense? */
+        if (access & ~NFS4_SHARE_ACCESS_BOTH)
+                return nfserr_inval;
+
+        /* Does it conflict with a deny mode already set? */
+        if ((access & fp->fi_share_deny) != 0)
+                return nfserr_share_denied;
+
+        __nfs4_file_get_access(fp, access);
+        return nfs_ok;
 }
 
-static void nfs4_file_put_fd(struct nfs4_file *fp, int oflag)
+static __be32 nfs4_file_check_deny(struct nfs4_file *fp, u32 deny)
 {
-        if (fp->fi_fds[oflag]) {
-                fput(fp->fi_fds[oflag]);
-                fp->fi_fds[oflag] = NULL;
+        /* Common case is that there is no deny mode. */
+        if (deny) {
+                /* Does this deny mode make sense? */
+                if (deny & ~NFS4_SHARE_DENY_BOTH)
+                        return nfserr_inval;
+
+                if ((deny & NFS4_SHARE_DENY_READ) &&
+                    atomic_read(&fp->fi_access[O_RDONLY]))
+                        return nfserr_share_denied;
+
+                if ((deny & NFS4_SHARE_DENY_WRITE) &&
+                    atomic_read(&fp->fi_access[O_WRONLY]))
+                        return nfserr_share_denied;
         }
+        return nfs_ok;
 }
 
 static void __nfs4_file_put_access(struct nfs4_file *fp, int oflag)
 {
-        if (atomic_dec_and_test(&fp->fi_access[oflag])) {
-                nfs4_file_put_fd(fp, oflag);
+        might_lock(&fp->fi_lock);
+
+        if (atomic_dec_and_lock(&fp->fi_access[oflag], &fp->fi_lock)) {
+                struct file *f1 = NULL;
+                struct file *f2 = NULL;
+
+                swap(f1, fp->fi_fds[oflag]);
                 if (atomic_read(&fp->fi_access[1 - oflag]) == 0)
-                        nfs4_file_put_fd(fp, O_RDWR);
+                        swap(f2, fp->fi_fds[O_RDWR]);
+                spin_unlock(&fp->fi_lock);
+                if (f1)
+                        fput(f1);
+                if (f2)
+                        fput(f2);
         }
 }
 
-static void nfs4_file_put_access(struct nfs4_file *fp, int oflag)
+static void nfs4_file_put_access(struct nfs4_file *fp, u32 access)
 {
-        if (oflag == O_RDWR) {
-                __nfs4_file_put_access(fp, O_RDONLY);
+        WARN_ON_ONCE(access & ~NFS4_SHARE_ACCESS_BOTH);
+
+        if (access & NFS4_SHARE_ACCESS_WRITE)
                 __nfs4_file_put_access(fp, O_WRONLY);
-        } else
-                __nfs4_file_put_access(fp, oflag);
+        if (access & NFS4_SHARE_ACCESS_READ)
+                __nfs4_file_put_access(fp, O_RDONLY);
 }
 
-static struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct
-kmem_cache *slab)
+static struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl,
+                                         struct kmem_cache *slab)
 {
-        struct idr *stateids = &cl->cl_stateids;
         struct nfs4_stid *stid;
         int new_id;
 
-        stid = kmem_cache_alloc(slab, GFP_KERNEL);
+        stid = kmem_cache_zalloc(slab, GFP_KERNEL);
         if (!stid)
                 return NULL;
 
-        new_id = idr_alloc_cyclic(stateids, stid, 0, 0, GFP_KERNEL);
+        idr_preload(GFP_KERNEL);
+        spin_lock(&cl->cl_lock);
+        new_id = idr_alloc_cyclic(&cl->cl_stateids, stid, 0, 0, GFP_NOWAIT);
+        spin_unlock(&cl->cl_lock);
+        idr_preload_end();
         if (new_id < 0)
                 goto out_free;
         stid->sc_client = cl;
-        stid->sc_type = 0;
         stid->sc_stateid.si_opaque.so_id = new_id;
         stid->sc_stateid.si_opaque.so_clid = cl->cl_clientid;
         /* Will be incremented before return to client: */
-        stid->sc_stateid.si_generation = 0;
+        atomic_set(&stid->sc_count, 1);
 
         /*
          * It shouldn't be a problem to reuse an opaque stateid value.
@@ -360,9 +522,24 @@ out_free:
         return NULL;
 }
 
-static struct nfs4_ol_stateid * nfs4_alloc_stateid(struct nfs4_client *clp)
+static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
 {
-        return openlockstateid(nfs4_alloc_stid(clp, stateid_slab));
+        struct nfs4_stid *stid;
+        struct nfs4_ol_stateid *stp;
+
+        stid = nfs4_alloc_stid(clp, stateid_slab);
+        if (!stid)
+                return NULL;
+
+        stp = openlockstateid(stid);
+        stp->st_stid.sc_free = nfs4_free_ol_stateid;
+        return stp;
+}
+
+static void nfs4_free_deleg(struct nfs4_stid *stid)
+{
+        kmem_cache_free(deleg_slab, stid);
+        atomic_long_dec(&num_delegations);
 }
 
 /*
@@ -379,10 +556,11 @@ static struct nfs4_ol_stateid * nfs4_alloc_stateid(struct nfs4_client *clp)
  * Each filter is 256 bits.  We hash the filehandle to 32bit and use the
  * low 3 bytes as hash-table indices.
  *
- * 'state_lock', which is always held when block_delegations() is called,
+ * 'blocked_delegations_lock', which is always taken in block_delegations(),
  * is used to manage concurrent access.  Testing does not need the lock
  * except when swapping the two filters.
  */
+static DEFINE_SPINLOCK(blocked_delegations_lock);
 static struct bloom_pair {
         int     entries, old_entries;
         time_t  swap_time;
@@ -398,7 +576,7 @@ static int delegation_blocked(struct knfsd_fh *fh)
         if (bd->entries == 0)
                 return 0;
         if (seconds_since_boot() - bd->swap_time > 30) {
-                spin_lock(&state_lock);
+                spin_lock(&blocked_delegations_lock);
                 if (seconds_since_boot() - bd->swap_time > 30) {
                         bd->entries -= bd->old_entries;
                         bd->old_entries = bd->entries;
@@ -407,7 +585,7 @@ static int delegation_blocked(struct knfsd_fh *fh)
                         bd->new = 1-bd->new;
                         bd->swap_time = seconds_since_boot();
                 }
-                spin_unlock(&state_lock);
+                spin_unlock(&blocked_delegations_lock);
         }
         hash = arch_fast_hash(&fh->fh_base, fh->fh_size, 0);
         if (test_bit(hash&255, bd->set[0]) &&
@@ -430,69 +608,73 @@ static void block_delegations(struct knfsd_fh *fh)
 
         hash = arch_fast_hash(&fh->fh_base, fh->fh_size, 0);
 
+        spin_lock(&blocked_delegations_lock);
         __set_bit(hash&255, bd->set[bd->new]);
         __set_bit((hash>>8)&255, bd->set[bd->new]);
         __set_bit((hash>>16)&255, bd->set[bd->new]);
         if (bd->entries == 0)
                 bd->swap_time = seconds_since_boot();
         bd->entries += 1;
+        spin_unlock(&blocked_delegations_lock);
 }
 
 static struct nfs4_delegation *
-alloc_init_deleg(struct nfs4_client *clp, struct nfs4_ol_stateid *stp, struct svc_fh *current_fh)
+alloc_init_deleg(struct nfs4_client *clp, struct svc_fh *current_fh)
 {
         struct nfs4_delegation *dp;
+        long n;
 
         dprintk("NFSD alloc_init_deleg\n");
-        if (num_delegations > max_delegations)
-                return NULL;
+        n = atomic_long_inc_return(&num_delegations);
+        if (n < 0 || n > max_delegations)
+                goto out_dec;
         if (delegation_blocked(&current_fh->fh_handle))
-                return NULL;
+                goto out_dec;
         dp = delegstateid(nfs4_alloc_stid(clp, deleg_slab));
         if (dp == NULL)
-                return dp;
+                goto out_dec;
+
+        dp->dl_stid.sc_free = nfs4_free_deleg;
         /*
          * delegation seqid's are never incremented.  The 4.1 special
          * meaning of seqid 0 isn't meaningful, really, but let's avoid
          * 0 anyway just for consistency and use 1:
          */
         dp->dl_stid.sc_stateid.si_generation = 1;
-        num_delegations++;
         INIT_LIST_HEAD(&dp->dl_perfile);
         INIT_LIST_HEAD(&dp->dl_perclnt);
         INIT_LIST_HEAD(&dp->dl_recall_lru);
-        dp->dl_file = NULL;
         dp->dl_type = NFS4_OPEN_DELEGATE_READ;
-        fh_copy_shallow(&dp->dl_fh, &current_fh->fh_handle);
-        dp->dl_time = 0;
-        atomic_set(&dp->dl_count, 1);
-        nfsd4_init_callback(&dp->dl_recall);
+        INIT_WORK(&dp->dl_recall.cb_work, nfsd4_run_cb_recall);
         return dp;
+out_dec:
+        atomic_long_dec(&num_delegations);
+        return NULL;
 }
 
-static void remove_stid(struct nfs4_stid *s)
+void
+nfs4_put_stid(struct nfs4_stid *s)
 {
-        struct idr *stateids = &s->sc_client->cl_stateids;
+        struct nfs4_file *fp = s->sc_file;
+        struct nfs4_client *clp = s->sc_client;
 
-        idr_remove(stateids, s->sc_stateid.si_opaque.so_id);
-}
+        might_lock(&clp->cl_lock);
 
-static void nfs4_free_stid(struct kmem_cache *slab, struct nfs4_stid *s)
-{
-        kmem_cache_free(slab, s);
-}
-
-void
-nfs4_put_delegation(struct nfs4_delegation *dp)
-{
-        if (atomic_dec_and_test(&dp->dl_count)) {
-                nfs4_free_stid(deleg_slab, &dp->dl_stid);
-                num_delegations--;
+        if (!atomic_dec_and_lock(&s->sc_count, &clp->cl_lock)) {
+                wake_up_all(&close_wq);
+                return;
         }
+        idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
+        spin_unlock(&clp->cl_lock);
+        s->sc_free(s);
+        if (fp)
+                put_nfs4_file(fp);
 }
 
 static void nfs4_put_deleg_lease(struct nfs4_file *fp)
 {
+        lockdep_assert_held(&state_lock);
+
         if (!fp->fi_lease)
                 return;
         if (atomic_dec_and_test(&fp->fi_delegees)) {
@@ -512,54 +694,54 @@ static void
 hash_delegation_locked(struct nfs4_delegation *dp, struct nfs4_file *fp)
 {
         lockdep_assert_held(&state_lock);
+        lockdep_assert_held(&fp->fi_lock);
 
+        atomic_inc(&dp->dl_stid.sc_count);
         dp->dl_stid.sc_type = NFS4_DELEG_STID;
         list_add(&dp->dl_perfile, &fp->fi_delegations);
         list_add(&dp->dl_perclnt, &dp->dl_stid.sc_client->cl_delegations);
 }
 
-/* Called under the state lock. */
 static void
-unhash_delegation(struct nfs4_delegation *dp)
+unhash_delegation_locked(struct nfs4_delegation *dp)
 {
-        spin_lock(&state_lock);
-        list_del_init(&dp->dl_perclnt);
-        list_del_init(&dp->dl_perfile);
-        list_del_init(&dp->dl_recall_lru);
-        spin_unlock(&state_lock);
-        if (dp->dl_file) {
-                nfs4_put_deleg_lease(dp->dl_file);
-                put_nfs4_file(dp->dl_file);
-                dp->dl_file = NULL;
-        }
-}
-
+        struct nfs4_file *fp = dp->dl_stid.sc_file;
 
+        lockdep_assert_held(&state_lock);
 
-static void destroy_revoked_delegation(struct nfs4_delegation *dp)
-{
+        dp->dl_stid.sc_type = NFS4_CLOSED_DELEG_STID;
+        /* Ensure that deleg break won't try to requeue it */
+        ++dp->dl_time;
+        spin_lock(&fp->fi_lock);
+        list_del_init(&dp->dl_perclnt);
         list_del_init(&dp->dl_recall_lru);
-        remove_stid(&dp->dl_stid);
-        nfs4_put_delegation(dp);
+        list_del_init(&dp->dl_perfile);
+        spin_unlock(&fp->fi_lock);
+        if (fp)
+                nfs4_put_deleg_lease(fp);
 }
 
 static void destroy_delegation(struct nfs4_delegation *dp)
 {
-        unhash_delegation(dp);
-        remove_stid(&dp->dl_stid);
-        nfs4_put_delegation(dp);
+        spin_lock(&state_lock);
+        unhash_delegation_locked(dp);
+        spin_unlock(&state_lock);
+        nfs4_put_stid(&dp->dl_stid);
 }
 
 static void revoke_delegation(struct nfs4_delegation *dp)
 {
         struct nfs4_client *clp = dp->dl_stid.sc_client;
 
+        WARN_ON(!list_empty(&dp->dl_recall_lru));
+
         if (clp->cl_minorversion == 0)
-                destroy_delegation(dp);
+                nfs4_put_stid(&dp->dl_stid);
         else {
-                unhash_delegation(dp);
                 dp->dl_stid.sc_type = NFS4_REVOKED_DELEG_STID;
+                spin_lock(&clp->cl_lock);
                 list_add(&dp->dl_recall_lru, &clp->cl_revoked);
+                spin_unlock(&clp->cl_lock);
         }
 }
 
@@ -607,57 +789,62 @@ bmap_to_share_mode(unsigned long bmap) {
         return access;
 }
 
-static bool
-test_share(struct nfs4_ol_stateid *stp, struct nfsd4_open *open) {
-        unsigned int access, deny;
-
-        access = bmap_to_share_mode(stp->st_access_bmap);
-        deny = bmap_to_share_mode(stp->st_deny_bmap);
-        if ((access & open->op_share_deny) || (deny & open->op_share_access))
-                return false;
-        return true;
-}
-
 /* set share access for a given stateid */
 static inline void
 set_access(u32 access, struct nfs4_ol_stateid *stp)
 {
-        __set_bit(access, &stp->st_access_bmap);
+        unsigned char mask = 1 << access;
+
+        WARN_ON_ONCE(access > NFS4_SHARE_ACCESS_BOTH);
+        stp->st_access_bmap |= mask;
 }
 
 /* clear share access for a given stateid */
 static inline void
 clear_access(u32 access, struct nfs4_ol_stateid *stp)
 {
-        __clear_bit(access, &stp->st_access_bmap);
+        unsigned char mask = 1 << access;
+
+        WARN_ON_ONCE(access > NFS4_SHARE_ACCESS_BOTH);
+        stp->st_access_bmap &= ~mask;
 }
 
 /* test whether a given stateid has access */
 static inline bool
 test_access(u32 access, struct nfs4_ol_stateid *stp)
 {
-        return test_bit(access, &stp->st_access_bmap);
+        unsigned char mask = 1 << access;
+
+        return (bool)(stp->st_access_bmap & mask);
 }
 
 /* set share deny for a given stateid */
 static inline void
-set_deny(u32 access, struct nfs4_ol_stateid *stp)
+set_deny(u32 deny, struct nfs4_ol_stateid *stp)
 {
-        __set_bit(access, &stp->st_deny_bmap);
+        unsigned char mask = 1 << deny;
+
+        WARN_ON_ONCE(deny > NFS4_SHARE_DENY_BOTH);
+        stp->st_deny_bmap |= mask;
 }
 
 /* clear share deny for a given stateid */
 static inline void
-clear_deny(u32 access, struct nfs4_ol_stateid *stp)
+clear_deny(u32 deny, struct nfs4_ol_stateid *stp)
 {
-        __clear_bit(access, &stp->st_deny_bmap);
+        unsigned char mask = 1 << deny;
+
+        WARN_ON_ONCE(deny > NFS4_SHARE_DENY_BOTH);
+        stp->st_deny_bmap &= ~mask;
 }
 
 /* test whether a given stateid is denying specific access */
 static inline bool
-test_deny(u32 access, struct nfs4_ol_stateid *stp)
+test_deny(u32 deny, struct nfs4_ol_stateid *stp)
 {
-        return test_bit(access, &stp->st_deny_bmap);
+        unsigned char mask = 1 << deny;
+
+        return (bool)(stp->st_deny_bmap & mask);
 }
 
 static int nfs4_access_to_omode(u32 access)
@@ -674,138 +861,283 @@ static int nfs4_access_to_omode(u32 access)
         return O_RDONLY;
 }
 
+/*
+ * A stateid that had a deny mode associated with it is being released
+ * or downgraded. Recalculate the deny mode on the file.
+ */
+static void
+recalculate_deny_mode(struct nfs4_file *fp)
+{
+        struct nfs4_ol_stateid *stp;
+
+        spin_lock(&fp->fi_lock);
+        fp->fi_share_deny = 0;
+        list_for_each_entry(stp, &fp->fi_stateids, st_perfile)
+                fp->fi_share_deny |= bmap_to_share_mode(stp->st_deny_bmap);
+        spin_unlock(&fp->fi_lock);
+}
+
+static void
+reset_union_bmap_deny(u32 deny, struct nfs4_ol_stateid *stp)
+{
+        int i;
+        bool change = false;
+
+        for (i = 1; i < 4; i++) {
+                if ((i & deny) != i) {
+                        change = true;
+                        clear_deny(i, stp);
+                }
+        }
+
+        /* Recalculate per-file deny mode if there was a change */
+        if (change)
+                recalculate_deny_mode(stp->st_stid.sc_file);
+}
+
 /* release all access and file references for a given stateid */
 static void
 release_all_access(struct nfs4_ol_stateid *stp)
 {
         int i;
+        struct nfs4_file *fp = stp->st_stid.sc_file;
+
+        if (fp && stp->st_deny_bmap != 0)
+                recalculate_deny_mode(fp);
 
         for (i = 1; i < 4; i++) {
                 if (test_access(i, stp))
-                        nfs4_file_put_access(stp->st_file,
-                                             nfs4_access_to_omode(i));
+                        nfs4_file_put_access(stp->st_stid.sc_file, i);
                 clear_access(i, stp);
         }
 }
 
-static void unhash_generic_stateid(struct nfs4_ol_stateid *stp)
+static void nfs4_put_stateowner(struct nfs4_stateowner *sop)
 {
+        struct nfs4_client *clp = sop->so_client;
+
+        might_lock(&clp->cl_lock);
+
+        if (!atomic_dec_and_lock(&sop->so_count, &clp->cl_lock))
+                return;
+        sop->so_ops->so_unhash(sop);
+        spin_unlock(&clp->cl_lock);
+        kfree(sop->so_owner.data);
+        sop->so_ops->so_free(sop);
+}
+
+static void unhash_ol_stateid(struct nfs4_ol_stateid *stp)
+{
+        struct nfs4_file *fp = stp->st_stid.sc_file;
+
+        lockdep_assert_held(&stp->st_stateowner->so_client->cl_lock);
+
+        spin_lock(&fp->fi_lock);
         list_del(&stp->st_perfile);
+        spin_unlock(&fp->fi_lock);
         list_del(&stp->st_perstateowner);
 }
 
-static void close_generic_stateid(struct nfs4_ol_stateid *stp)
+static void nfs4_free_ol_stateid(struct nfs4_stid *stid)
 {
+        struct nfs4_ol_stateid *stp = openlockstateid(stid);
+
         release_all_access(stp);
-        put_nfs4_file(stp->st_file);
-        stp->st_file = NULL;
+        if (stp->st_stateowner)
+                nfs4_put_stateowner(stp->st_stateowner);
+        kmem_cache_free(stateid_slab, stid);
 }
 
-static void free_generic_stateid(struct nfs4_ol_stateid *stp)
+static void nfs4_free_lock_stateid(struct nfs4_stid *stid)
 {
-        remove_stid(&stp->st_stid);
-        nfs4_free_stid(stateid_slab, &stp->st_stid);
+        struct nfs4_ol_stateid *stp = openlockstateid(stid);
+        struct nfs4_lockowner *lo = lockowner(stp->st_stateowner);
+        struct file *file;
+
+        file = find_any_file(stp->st_stid.sc_file);
+        if (file)
+                filp_close(file, (fl_owner_t)lo);
+        nfs4_free_ol_stateid(stid);
 }
 
-static void release_lock_stateid(struct nfs4_ol_stateid *stp)
+/*
+ * Put the persistent reference to an already unhashed generic stateid, while
+ * holding the cl_lock. If it's the last reference, then put it onto the
+ * reaplist for later destruction.
+ */
+static void put_ol_stateid_locked(struct nfs4_ol_stateid *stp,
+                                       struct list_head *reaplist)
 {
-        struct file *file;
+        struct nfs4_stid *s = &stp->st_stid;
+        struct nfs4_client *clp = s->sc_client;
+
+        lockdep_assert_held(&clp->cl_lock);
 
-        unhash_generic_stateid(stp);
+        WARN_ON_ONCE(!list_empty(&stp->st_locks));
+
+        if (!atomic_dec_and_test(&s->sc_count)) {
+                wake_up_all(&close_wq);
+                return;
+        }
+
+        idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
+        list_add(&stp->st_locks, reaplist);
+}
+
+static void unhash_lock_stateid(struct nfs4_ol_stateid *stp)
+{
+        struct nfs4_openowner *oo = openowner(stp->st_openstp->st_stateowner);
+
+        lockdep_assert_held(&oo->oo_owner.so_client->cl_lock);
+
+        list_del_init(&stp->st_locks);
+        unhash_ol_stateid(stp);
         unhash_stid(&stp->st_stid);
-        file = find_any_file(stp->st_file);
-        if (file)
-                locks_remove_posix(file, (fl_owner_t)lockowner(stp->st_stateowner));
-        close_generic_stateid(stp);
-        free_generic_stateid(stp);
 }
 
-static void unhash_lockowner(struct nfs4_lockowner *lo)
+static void release_lock_stateid(struct nfs4_ol_stateid *stp)
 {
-        struct nfs4_ol_stateid *stp;
+        struct nfs4_openowner *oo = openowner(stp->st_openstp->st_stateowner);
 
-        list_del(&lo->lo_owner.so_strhash);
-        list_del(&lo->lo_perstateid);
-        list_del(&lo->lo_owner_ino_hash);
-        while (!list_empty(&lo->lo_owner.so_stateids)) {
-                stp = list_first_entry(&lo->lo_owner.so_stateids,
-                                struct nfs4_ol_stateid, st_perstateowner);
-                release_lock_stateid(stp);
-        }
+        spin_lock(&oo->oo_owner.so_client->cl_lock);
+        unhash_lock_stateid(stp);
+        spin_unlock(&oo->oo_owner.so_client->cl_lock);
+        nfs4_put_stid(&stp->st_stid);
 }
 
-static void nfs4_free_lockowner(struct nfs4_lockowner *lo)
+static void unhash_lockowner_locked(struct nfs4_lockowner *lo)
 {
-        kfree(lo->lo_owner.so_owner.data);
-        kmem_cache_free(lockowner_slab, lo);
+        struct nfs4_client *clp = lo->lo_owner.so_client;
+
+        lockdep_assert_held(&clp->cl_lock);
+
+        list_del_init(&lo->lo_owner.so_strhash);
+}
+
+/*
+ * Free a list of generic stateids that were collected earlier after being
+ * fully unhashed.
+ */
+static void
+free_ol_stateid_reaplist(struct list_head *reaplist)
+{
+        struct nfs4_ol_stateid *stp;
+        struct nfs4_file *fp;
+
+        might_sleep();
+
+        while (!list_empty(reaplist)) {
+                stp = list_first_entry(reaplist, struct nfs4_ol_stateid,
+                                       st_locks);
+                list_del(&stp->st_locks);
+                fp = stp->st_stid.sc_file;
+                stp->st_stid.sc_free(&stp->st_stid);
+                if (fp)
+                        put_nfs4_file(fp);
+        }
 }
 
 static void release_lockowner(struct nfs4_lockowner *lo)
 {
-        unhash_lockowner(lo);
-        nfs4_free_lockowner(lo);
+        struct nfs4_client *clp = lo->lo_owner.so_client;
+        struct nfs4_ol_stateid *stp;
+        struct list_head reaplist;
+
+        INIT_LIST_HEAD(&reaplist);
+
+        spin_lock(&clp->cl_lock);
+        unhash_lockowner_locked(lo);
+        while (!list_empty(&lo->lo_owner.so_stateids)) {
+                stp = list_first_entry(&lo->lo_owner.so_stateids,
+                                struct nfs4_ol_stateid, st_perstateowner);
+                unhash_lock_stateid(stp);
+                put_ol_stateid_locked(stp, &reaplist);
+        }
+        spin_unlock(&clp->cl_lock);
+        free_ol_stateid_reaplist(&reaplist);
+        nfs4_put_stateowner(&lo->lo_owner);
 }
 
-static void
-release_stateid_lockowners(struct nfs4_ol_stateid *open_stp)
+static void release_open_stateid_locks(struct nfs4_ol_stateid *open_stp,
+                                       struct list_head *reaplist)
 {
-        struct nfs4_lockowner *lo;
+        struct nfs4_ol_stateid *stp;
 
-        while (!list_empty(&open_stp->st_lockowners)) {
-                lo = list_entry(open_stp->st_lockowners.next,
-                                struct nfs4_lockowner, lo_perstateid);
-                release_lockowner(lo);
+        while (!list_empty(&open_stp->st_locks)) {
+                stp = list_entry(open_stp->st_locks.next,
+                                struct nfs4_ol_stateid, st_locks);
+                unhash_lock_stateid(stp);
+                put_ol_stateid_locked(stp, reaplist);
         }
 }
 
-static void unhash_open_stateid(struct nfs4_ol_stateid *stp)
+static void unhash_open_stateid(struct nfs4_ol_stateid *stp,
+                                struct list_head *reaplist)
 {
-        unhash_generic_stateid(stp);
-        release_stateid_lockowners(stp);
-        close_generic_stateid(stp);
+        lockdep_assert_held(&stp->st_stid.sc_client->cl_lock);
+
+        unhash_ol_stateid(stp);
+        release_open_stateid_locks(stp, reaplist);
 }
 
 static void release_open_stateid(struct nfs4_ol_stateid *stp)
 {
-        unhash_open_stateid(stp);
-        free_generic_stateid(stp);
+        LIST_HEAD(reaplist);
+
+        spin_lock(&stp->st_stid.sc_client->cl_lock);
+        unhash_open_stateid(stp, &reaplist);
+        put_ol_stateid_locked(stp, &reaplist);
+        spin_unlock(&stp->st_stid.sc_client->cl_lock);
+        free_ol_stateid_reaplist(&reaplist);
 }
 
-static void unhash_openowner(struct nfs4_openowner *oo)
+static void unhash_openowner_locked(struct nfs4_openowner *oo)
 {
-        struct nfs4_ol_stateid *stp;
+        struct nfs4_client *clp = oo->oo_owner.so_client;
 
-        list_del(&oo->oo_owner.so_strhash);
-        list_del(&oo->oo_perclient);
-        while (!list_empty(&oo->oo_owner.so_stateids)) {
-                stp = list_first_entry(&oo->oo_owner.so_stateids,
-                                struct nfs4_ol_stateid, st_perstateowner);
-                release_open_stateid(stp);
-        }
+        lockdep_assert_held(&clp->cl_lock);
+
+        list_del_init(&oo->oo_owner.so_strhash);
+        list_del_init(&oo->oo_perclient);
 }
 
 static void release_last_closed_stateid(struct nfs4_openowner *oo)
 {
-        struct nfs4_ol_stateid *s = oo->oo_last_closed_stid;
+        struct nfsd_net *nn = net_generic(oo->oo_owner.so_client->net,
+                                          nfsd_net_id);
+        struct nfs4_ol_stateid *s;
 
+        spin_lock(&nn->client_lock);
+        s = oo->oo_last_closed_stid;
         if (s) {
-                free_generic_stateid(s);
+                list_del_init(&oo->oo_close_lru);
                 oo->oo_last_closed_stid = NULL;
         }
-}
-
-static void nfs4_free_openowner(struct nfs4_openowner *oo)
-{
-        kfree(oo->oo_owner.so_owner.data);
-        kmem_cache_free(openowner_slab, oo);
+        spin_unlock(&nn->client_lock);
+        if (s)
+                nfs4_put_stid(&s->st_stid);
 }
 
 static void release_openowner(struct nfs4_openowner *oo)
 {
-        unhash_openowner(oo);
-        list_del(&oo->oo_close_lru);
+        struct nfs4_ol_stateid *stp;
+        struct nfs4_client *clp = oo->oo_owner.so_client;
+        struct list_head reaplist;
+
+        INIT_LIST_HEAD(&reaplist);
+
+        spin_lock(&clp->cl_lock);
+        unhash_openowner_locked(oo);
+        while (!list_empty(&oo->oo_owner.so_stateids)) {
+                stp = list_first_entry(&oo->oo_owner.so_stateids,
+                                struct nfs4_ol_stateid, st_perstateowner);
+                unhash_open_stateid(stp, &reaplist);
+                put_ol_stateid_locked(stp, &reaplist);
+        }
+        spin_unlock(&clp->cl_lock);
+        free_ol_stateid_reaplist(&reaplist);
         release_last_closed_stateid(oo);
-        nfs4_free_openowner(oo);
+        nfs4_put_stateowner(&oo->oo_owner);
 }
 
 static inline int
@@ -842,7 +1174,7 @@ void nfsd4_bump_seqid(struct nfsd4_compound_state *cstate, __be32 nfserr)
                 return;
 
         if (!seqid_mutating_err(ntohl(nfserr))) {
-                cstate->replay_owner = NULL;
+                nfsd4_cstate_clear_replay(cstate);
                 return;
         }
         if (!so)
@@ -1030,10 +1362,8 @@ static void nfsd4_init_conn(struct svc_rqst *rqstp, struct nfsd4_conn *conn, str
         if (ret)
                 /* oops; xprt is already down: */
                 nfsd4_conn_lost(&conn->cn_xpt_user);
-        if (conn->cn_flags & NFS4_CDFC4_BACK) {
-                /* callback channel may be back up */
-                nfsd4_probe_callback(ses->se_client);
-        }
+        /* We may have gained or lost a callback channel: */
+        nfsd4_probe_callback_sync(ses->se_client);
 }
 
 static struct nfsd4_conn *alloc_conn_from_crses(struct svc_rqst *rqstp, struct nfsd4_create_session *cses)
@@ -1073,9 +1403,6 @@ static void __free_session(struct nfsd4_session *ses)
 
 static void free_session(struct nfsd4_session *ses)
 {
-        struct nfsd_net *nn = net_generic(ses->se_client->net, nfsd_net_id);
-
-        lockdep_assert_held(&nn->client_lock);
         nfsd4_del_conns(ses);
         nfsd4_put_drc_mem(&ses->se_fchannel);
         __free_session(ses);
@@ -1097,12 +1424,10 @@ static void init_session(struct svc_rqst *rqstp, struct nfsd4_session *new, stru
         new->se_cb_sec = cses->cb_sec;
         atomic_set(&new->se_ref, 0);
         idx = hash_sessionid(&new->se_sessionid);
-        spin_lock(&nn->client_lock);
         list_add(&new->se_hash, &nn->sessionid_hashtbl[idx]);
         spin_lock(&clp->cl_lock);
         list_add(&new->se_perclnt, &clp->cl_sessions);
         spin_unlock(&clp->cl_lock);
-        spin_unlock(&nn->client_lock);
 
         if (cses->flags & SESSION4_BACK_CHAN) {
                 struct sockaddr *sa = svc_addr(rqstp);
@@ -1120,12 +1445,14 @@ static void init_session(struct svc_rqst *rqstp, struct nfsd4_session *new, stru
 
 /* caller must hold client_lock */
 static struct nfsd4_session *
-find_in_sessionid_hashtbl(struct nfs4_sessionid *sessionid, struct net *net)
+__find_in_sessionid_hashtbl(struct nfs4_sessionid *sessionid, struct net *net)
 {
         struct nfsd4_session *elem;
         int idx;
         struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
+        lockdep_assert_held(&nn->client_lock);
+
         dump_sessionid(__func__, sessionid);
         idx = hash_sessionid(sessionid);
         /* Search in the appropriate list */
@@ -1140,10 +1467,33 @@ find_in_sessionid_hashtbl(struct nfs4_sessionid *sessionid, struct net *net)
         return NULL;
 }
 
+static struct nfsd4_session *
+find_in_sessionid_hashtbl(struct nfs4_sessionid *sessionid, struct net *net,
+                __be32 *ret)
+{
+        struct nfsd4_session *session;
+        __be32 status = nfserr_badsession;
+
+        session = __find_in_sessionid_hashtbl(sessionid, net);
+        if (!session)
+                goto out;
+        status = nfsd4_get_session_locked(session);
+        if (status)
+                session = NULL;
+out:
+        *ret = status;
+        return session;
+}
+
 /* caller must hold client_lock */
 static void
 unhash_session(struct nfsd4_session *ses)
 {
+        struct nfs4_client *clp = ses->se_client;
+        struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+
+        lockdep_assert_held(&nn->client_lock);
+
         list_del(&ses->se_hash);
         spin_lock(&ses->se_client->cl_lock);
         list_del(&ses->se_perclnt);
@@ -1169,15 +1519,20 @@ STALE_CLIENTID(clientid_t *clid, struct nfsd_net *nn)
 static struct nfs4_client *alloc_client(struct xdr_netobj name)
 {
         struct nfs4_client *clp;
+        int i;
 
         clp = kzalloc(sizeof(struct nfs4_client), GFP_KERNEL);
         if (clp == NULL)
                 return NULL;
         clp->cl_name.data = kmemdup(name.data, name.len, GFP_KERNEL);
-        if (clp->cl_name.data == NULL) {
-                kfree(clp);
-                return NULL;
-        }
+        if (clp->cl_name.data == NULL)
+                goto err_no_name;
+        clp->cl_ownerstr_hashtbl = kmalloc(sizeof(struct list_head) *
+                        OWNER_HASH_SIZE, GFP_KERNEL);
+        if (!clp->cl_ownerstr_hashtbl)
+                goto err_no_hashtbl;
+        for (i = 0; i < OWNER_HASH_SIZE; i++)
+                INIT_LIST_HEAD(&clp->cl_ownerstr_hashtbl[i]);
         clp->cl_name.len = name.len;
         INIT_LIST_HEAD(&clp->cl_sessions);
         idr_init(&clp->cl_stateids);
@@ -1192,14 +1547,16 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
         spin_lock_init(&clp->cl_lock);
         rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table");
         return clp;
+err_no_hashtbl:
+        kfree(clp->cl_name.data);
+err_no_name:
+        kfree(clp);
+        return NULL;
 }
 
 static void
 free_client(struct nfs4_client *clp)
 {
-        struct nfsd_net __maybe_unused *nn = net_generic(clp->net, nfsd_net_id);
-
-        lockdep_assert_held(&nn->client_lock);
         while (!list_empty(&clp->cl_sessions)) {
                 struct nfsd4_session *ses;
                 ses = list_entry(clp->cl_sessions.next, struct nfsd4_session,
@@ -1210,18 +1567,32 @@ free_client(struct nfs4_client *clp)
         }
         rpc_destroy_wait_queue(&clp->cl_cb_waitq);
         free_svc_cred(&clp->cl_cred);
+        kfree(clp->cl_ownerstr_hashtbl);
         kfree(clp->cl_name.data);
         idr_destroy(&clp->cl_stateids);
         kfree(clp);
 }
 
 /* must be called under the client_lock */
-static inline void
+static void
 unhash_client_locked(struct nfs4_client *clp)
 {
+        struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
         struct nfsd4_session *ses;
 
-        list_del(&clp->cl_lru);
+        lockdep_assert_held(&nn->client_lock);
+
+        /* Mark the client as expired! */
+        clp->cl_time = 0;
+        /* Make it invisible */
+        if (!list_empty(&clp->cl_idhash)) {
+                list_del_init(&clp->cl_idhash);
+                if (test_bit(NFSD4_CLIENT_CONFIRMED, &clp->cl_flags))
+                        rb_erase(&clp->cl_namenode, &nn->conf_name_tree);
+                else
+                        rb_erase(&clp->cl_namenode, &nn->unconf_name_tree);
+        }
+        list_del_init(&clp->cl_lru);
         spin_lock(&clp->cl_lock);
         list_for_each_entry(ses, &clp->cl_sessions, se_perclnt)
                 list_del_init(&ses->se_hash);
@@ -1229,53 +1600,71 @@ unhash_client_locked(struct nfs4_client *clp)
 }
 
 static void
-destroy_client(struct nfs4_client *clp)
+unhash_client(struct nfs4_client *clp)
+{
+        struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+
+        spin_lock(&nn->client_lock);
+        unhash_client_locked(clp);
+        spin_unlock(&nn->client_lock);
+}
+
+static __be32 mark_client_expired_locked(struct nfs4_client *clp)
+{
+        if (atomic_read(&clp->cl_refcount))
+                return nfserr_jukebox;
+        unhash_client_locked(clp);
+        return nfs_ok;
+}
+
+static void
+__destroy_client(struct nfs4_client *clp)
 {
         struct nfs4_openowner *oo;
         struct nfs4_delegation *dp;
         struct list_head reaplist;
-        struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
 
         INIT_LIST_HEAD(&reaplist);
         spin_lock(&state_lock);
         while (!list_empty(&clp->cl_delegations)) {
                 dp = list_entry(clp->cl_delegations.next, struct nfs4_delegation, dl_perclnt);
-                list_del_init(&dp->dl_perclnt);
-                list_move(&dp->dl_recall_lru, &reaplist);
+                unhash_delegation_locked(dp);
+                list_add(&dp->dl_recall_lru, &reaplist);
         }
         spin_unlock(&state_lock);
         while (!list_empty(&reaplist)) {
                 dp = list_entry(reaplist.next, struct nfs4_delegation, dl_recall_lru);
-                destroy_delegation(dp);
+                list_del_init(&dp->dl_recall_lru);
+                nfs4_put_stid(&dp->dl_stid);
         }
-        list_splice_init(&clp->cl_revoked, &reaplist);
-        while (!list_empty(&reaplist)) {
+        while (!list_empty(&clp->cl_revoked)) {
                 dp = list_entry(reaplist.next, struct nfs4_delegation, dl_recall_lru);
-                destroy_revoked_delegation(dp);
+                list_del_init(&dp->dl_recall_lru);
+                nfs4_put_stid(&dp->dl_stid);
         }
         while (!list_empty(&clp->cl_openowners)) {
                 oo = list_entry(clp->cl_openowners.next, struct nfs4_openowner, oo_perclient);
+                atomic_inc(&oo->oo_owner.so_count);
                 release_openowner(oo);
         }
         nfsd4_shutdown_callback(clp);
         if (clp->cl_cb_conn.cb_xprt)
                 svc_xprt_put(clp->cl_cb_conn.cb_xprt);
-        list_del(&clp->cl_idhash);
-        if (test_bit(NFSD4_CLIENT_CONFIRMED, &clp->cl_flags))
-                rb_erase(&clp->cl_namenode, &nn->conf_name_tree);
-        else
-                rb_erase(&clp->cl_namenode, &nn->unconf_name_tree);
-        spin_lock(&nn->client_lock);
-        unhash_client_locked(clp);
-        WARN_ON_ONCE(atomic_read(&clp->cl_refcount));
         free_client(clp);
-        spin_unlock(&nn->client_lock);
+}
+
+static void
+destroy_client(struct nfs4_client *clp)
+{
+        unhash_client(clp);
+        __destroy_client(clp);
 }
 
 static void expire_client(struct nfs4_client *clp)
 {
+        unhash_client(clp);
         nfsd4_client_record_remove(clp);
-        destroy_client(clp);
+        __destroy_client(clp);
 }
 
 static void copy_verf(struct nfs4_client *target, nfs4_verifier *source)
@@ -1408,25 +1797,28 @@ static bool mach_creds_match(struct nfs4_client *cl, struct svc_rqst *rqstp)
         return 0 == strcmp(cl->cl_cred.cr_principal, cr->cr_principal);
 }
 
-static void gen_clid(struct nfs4_client *clp, struct nfsd_net *nn)
+static void gen_confirm(struct nfs4_client *clp, struct nfsd_net *nn)
 {
-        static u32 current_clientid = 1;
+        __be32 verf[2];
 
-        clp->cl_clientid.cl_boot = nn->boot_time;
-        clp->cl_clientid.cl_id = current_clientid++; 
+        /*
+         * This is opaque to client, so no need to byte-swap. Use
+         * __force to keep sparse happy
+         */
+        verf[0] = (__force __be32)get_seconds();
+        verf[1] = (__force __be32)nn->clientid_counter;
+        memcpy(clp->cl_confirm.data, verf, sizeof(clp->cl_confirm.data));
 }
 
-static void gen_confirm(struct nfs4_client *clp)
+static void gen_clid(struct nfs4_client *clp, struct nfsd_net *nn)
 {
-        __be32 verf[2];
-        static u32 i;
-
-        verf[0] = (__be32)get_seconds();
-        verf[1] = (__be32)i++;
-        memcpy(clp->cl_confirm.data, verf, sizeof(clp->cl_confirm.data));
+        clp->cl_clientid.cl_boot = nn->boot_time;
+        clp->cl_clientid.cl_id = nn->clientid_counter++;
+        gen_confirm(clp, nn);
 }
 
-static struct nfs4_stid *find_stateid(struct nfs4_client *cl, stateid_t *t)
+static struct nfs4_stid *
+find_stateid_locked(struct nfs4_client *cl, stateid_t *t)
 {
         struct nfs4_stid *ret;
 
@@ -1436,16 +1828,21 @@ static struct nfs4_stid *find_stateid(struct nfs4_client *cl, stateid_t *t)
         return ret;
 }
 
-static struct nfs4_stid *find_stateid_by_type(struct nfs4_client *cl, stateid_t *t, char typemask)
+static struct nfs4_stid *
+find_stateid_by_type(struct nfs4_client *cl, stateid_t *t, char typemask)
 {
         struct nfs4_stid *s;
 
-        s = find_stateid(cl, t);
-        if (!s)
-                return NULL;
-        if (typemask & s->sc_type)
-                return s;
-        return NULL;
+        spin_lock(&cl->cl_lock);
+        s = find_stateid_locked(cl, t);
+        if (s != NULL) {
+                if (typemask & s->sc_type)
+                        atomic_inc(&s->sc_count);
+                else
+                        s = NULL;
+        }
+        spin_unlock(&cl->cl_lock);
+        return s;
 }
 
 static struct nfs4_client *create_client(struct xdr_netobj name,
@@ -1455,7 +1852,6 @@ static struct nfs4_client *create_client(struct xdr_netobj name,
         struct sockaddr *sa = svc_addr(rqstp);
         int ret;
         struct net *net = SVC_NET(rqstp);
-        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
         clp = alloc_client(name);
         if (clp == NULL)
@@ -1463,17 +1859,14 @@ static struct nfs4_client *create_client(struct xdr_netobj name,
 
         ret = copy_cred(&clp->cl_cred, &rqstp->rq_cred);
         if (ret) {
-                spin_lock(&nn->client_lock);
                 free_client(clp);
-                spin_unlock(&nn->client_lock);
                 return NULL;
         }
-        nfsd4_init_callback(&clp->cl_cb_null);
+        INIT_WORK(&clp->cl_cb_null.cb_work, nfsd4_run_cb_null);
         clp->cl_time = get_seconds();
         clear_bit(0, &clp->cl_cb_slot_busy);
         copy_verf(clp, verf);
         rpc_copy_addr((struct sockaddr *) &clp->cl_addr, sa);
-        gen_confirm(clp);
         clp->cl_cb_session = NULL;
         clp->net = net;
         return clp;
@@ -1525,11 +1918,13 @@ add_to_unconfirmed(struct nfs4_client *clp)
         unsigned int idhashval;
         struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
 
+        lockdep_assert_held(&nn->client_lock);
+
         clear_bit(NFSD4_CLIENT_CONFIRMED, &clp->cl_flags);
         add_clp_to_name_tree(clp, &nn->unconf_name_tree);
         idhashval = clientid_hashval(clp->cl_clientid.cl_id);
         list_add(&clp->cl_idhash, &nn->unconf_id_hashtbl[idhashval]);
-        renew_client(clp);
+        renew_client_locked(clp);
 }
 
 static void
@@ -1538,12 +1933,14 @@ move_to_confirmed(struct nfs4_client *clp)
         unsigned int idhashval = clientid_hashval(clp->cl_clientid.cl_id);
         struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
 
+        lockdep_assert_held(&nn->client_lock);
+
         dprintk("NFSD: move_to_confirm nfs4_client %p\n", clp);
         list_move(&clp->cl_idhash, &nn->conf_id_hashtbl[idhashval]);
         rb_erase(&clp->cl_namenode, &nn->unconf_name_tree);
         add_clp_to_name_tree(clp, &nn->conf_name_tree);
         set_bit(NFSD4_CLIENT_CONFIRMED, &clp->cl_flags);
-        renew_client(clp);
+        renew_client_locked(clp);
 }
 
 static struct nfs4_client *
@@ -1556,7 +1953,7 @@ find_client_in_id_table(struct list_head *tbl, clientid_t *clid, bool sessions)
                 if (same_clid(&clp->cl_clientid, clid)) {
                         if ((bool)clp->cl_minorversion != sessions)
                                 return NULL;
-                        renew_client(clp);
+                        renew_client_locked(clp);
                         return clp;
                 }
         }
@@ -1568,6 +1965,7 @@ find_confirmed_client(clientid_t *clid, bool sessions, struct nfsd_net *nn)
 {
         struct list_head *tbl = nn->conf_id_hashtbl;
 
+        lockdep_assert_held(&nn->client_lock);
         return find_client_in_id_table(tbl, clid, sessions);
 }
 
@@ -1576,6 +1974,7 @@ find_unconfirmed_client(clientid_t *clid, bool sessions, struct nfsd_net *nn)
 {
         struct list_head *tbl = nn->unconf_id_hashtbl;
 
+        lockdep_assert_held(&nn->client_lock);
         return find_client_in_id_table(tbl, clid, sessions);
 }
 
@@ -1587,12 +1986,14 @@ static bool clp_used_exchangeid(struct nfs4_client *clp)
 static struct nfs4_client *
 find_confirmed_client_by_name(struct xdr_netobj *name, struct nfsd_net *nn)
 {
+        lockdep_assert_held(&nn->client_lock);
         return find_clp_in_name_tree(name, &nn->conf_name_tree);
 }
 
 static struct nfs4_client *
 find_unconfirmed_client_by_name(struct xdr_netobj *name, struct nfsd_net *nn)
 {
+        lockdep_assert_held(&nn->client_lock);
         return find_clp_in_name_tree(name, &nn->unconf_name_tree);
 }
 
@@ -1642,7 +2043,7 @@ out_err:
 /*
  * Cache a reply. nfsd4_check_resp_size() has bounded the cache size.
  */
-void
+static void
 nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 {
         struct xdr_buf *buf = resp->xdr.buf;
@@ -1758,7 +2159,8 @@ nfsd4_exchange_id(struct svc_rqst *rqstp,
                   struct nfsd4_compound_state *cstate,
                   struct nfsd4_exchange_id *exid)
 {
-        struct nfs4_client *unconf, *conf, *new;
+        struct nfs4_client *conf, *new;
+        struct nfs4_client *unconf = NULL;
         __be32 status;
         char                    addr_str[INET6_ADDRSTRLEN];
         nfs4_verifier           verf = exid->verifier;
@@ -1787,8 +2189,12 @@ nfsd4_exchange_id(struct svc_rqst *rqstp,
                 return nfserr_encr_alg_unsupp;
         }
 
+        new = create_client(exid->clname, rqstp, &verf);
+        if (new == NULL)
+                return nfserr_jukebox;
+
         /* Cases below refer to rfc 5661 section 18.35.4: */
-        nfs4_lock_state();
+        spin_lock(&nn->client_lock);
         conf = find_confirmed_client_by_name(&exid->clname, nn);
         if (conf) {
                 bool creds_match = same_creds(&conf->cl_cred, &rqstp->rq_cred);
@@ -1813,7 +2219,6 @@ nfsd4_exchange_id(struct svc_rqst *rqstp,
                         }
                         /* case 6 */
                         exid->flags |= EXCHGID4_FLAG_CONFIRMED_R;
-                        new = conf;
                         goto out_copy;
                 }
                 if (!creds_match) { /* case 3 */
@@ -1821,15 +2226,14 @@ nfsd4_exchange_id(struct svc_rqst *rqstp,
                                 status = nfserr_clid_inuse;
                                 goto out;
                         }
-                        expire_client(conf);
                         goto out_new;
                 }
                 if (verfs_match) { /* case 2 */
                         conf->cl_exchange_flags |= EXCHGID4_FLAG_CONFIRMED_R;
-                        new = conf;
                         goto out_copy;
                 }
                 /* case 5, client reboot */
+                conf = NULL;
                 goto out_new;
         }
 
@@ -1840,33 +2244,38 @@ nfsd4_exchange_id(struct svc_rqst *rqstp,
 
         unconf  = find_unconfirmed_client_by_name(&exid->clname, nn);
         if (unconf) /* case 4, possible retry or client restart */
-                expire_client(unconf);
+                unhash_client_locked(unconf);
 
         /* case 1 (normal case) */
 out_new:
-        new = create_client(exid->clname, rqstp, &verf);
-        if (new == NULL) {
-                status = nfserr_jukebox;
-                goto out;
+        if (conf) {
+                status = mark_client_expired_locked(conf);
+                if (status)
+                        goto out;
         }
         new->cl_minorversion = cstate->minorversion;
         new->cl_mach_cred = (exid->spa_how == SP4_MACH_CRED);
 
         gen_clid(new, nn);
         add_to_unconfirmed(new);
+        swap(new, conf);
 out_copy:
-        exid->clientid.cl_boot = new->cl_clientid.cl_boot;
-        exid->clientid.cl_id = new->cl_clientid.cl_id;
+        exid->clientid.cl_boot = conf->cl_clientid.cl_boot;
+        exid->clientid.cl_id = conf->cl_clientid.cl_id;
 
-        exid->seqid = new->cl_cs_slot.sl_seqid + 1;
-        nfsd4_set_ex_flags(new, exid);
+        exid->seqid = conf->cl_cs_slot.sl_seqid + 1;
+        nfsd4_set_ex_flags(conf, exid);
 
         dprintk("nfsd4_exchange_id seqid %d flags %x\n",
-                new->cl_cs_slot.sl_seqid, new->cl_exchange_flags);
+                conf->cl_cs_slot.sl_seqid, conf->cl_exchange_flags);
         status = nfs_ok;
 
 out:
-        nfs4_unlock_state();
+        spin_unlock(&nn->client_lock);
+        if (new)
+                expire_client(new);
+        if (unconf)
+                expire_client(unconf);
         return status;
 }
 
@@ -2010,6 +2419,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 {
         struct sockaddr *sa = svc_addr(rqstp);
         struct nfs4_client *conf, *unconf;
+        struct nfs4_client *old = NULL;
         struct nfsd4_session *new;
         struct nfsd4_conn *conn;
         struct nfsd4_clid_slot *cs_slot = NULL;
@@ -2035,7 +2445,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
         if (!conn)
                 goto out_free_session;
 
-        nfs4_lock_state();
+        spin_lock(&nn->client_lock);
         unconf = find_unconfirmed_client(&cr_ses->clientid, true, nn);
         conf = find_confirmed_client(&cr_ses->clientid, true, nn);
         WARN_ON_ONCE(conf && unconf);
@@ -2054,7 +2464,6 @@ nfsd4_create_session(struct svc_rqst *rqstp,
                         goto out_free_conn;
                 }
         } else if (unconf) {
-                struct nfs4_client *old;
                 if (!same_creds(&unconf->cl_cred, &rqstp->rq_cred) ||
                     !rpc_cmp_addr(sa, (struct sockaddr *) &unconf->cl_addr)) {
                         status = nfserr_clid_inuse;
@@ -2072,10 +2481,11 @@ nfsd4_create_session(struct svc_rqst *rqstp,
                 }
                 old = find_confirmed_client_by_name(&unconf->cl_name, nn);
                 if (old) {
-                        status = mark_client_expired(old);
-                        if (status)
+                        status = mark_client_expired_locked(old);
+                        if (status) {
+                                old = NULL;
                                 goto out_free_conn;
-                        expire_client(old);
+                        }
                 }
                 move_to_confirmed(unconf);
                 conf = unconf;
@@ -2091,20 +2501,27 @@ nfsd4_create_session(struct svc_rqst *rqstp,
         cr_ses->flags &= ~SESSION4_RDMA;
 
         init_session(rqstp, new, conf, cr_ses);
-        nfsd4_init_conn(rqstp, conn, new);
+        nfsd4_get_session_locked(new);
 
         memcpy(cr_ses->sessionid.data, new->se_sessionid.data,
                NFS4_MAX_SESSIONID_LEN);
         cs_slot->sl_seqid++;
         cr_ses->seqid = cs_slot->sl_seqid;
 
-        /* cache solo and embedded create sessions under the state lock */
+        /* cache solo and embedded create sessions under the client_lock */
         nfsd4_cache_create_session(cr_ses, cs_slot, status);
-        nfs4_unlock_state();
+        spin_unlock(&nn->client_lock);
+        /* init connection and backchannel */
+        nfsd4_init_conn(rqstp, conn, new);
+        nfsd4_put_session(new);
+        if (old)
+                expire_client(old);
         return status;
 out_free_conn:
-        nfs4_unlock_state();
+        spin_unlock(&nn->client_lock);
         free_conn(conn);
+        if (old)
+                expire_client(old);
 out_free_session:
         __free_session(new);
 out_release_drc_mem:
@@ -2152,17 +2569,16 @@ __be32 nfsd4_bind_conn_to_session(struct svc_rqst *rqstp,
         __be32 status;
         struct nfsd4_conn *conn;
         struct nfsd4_session *session;
-        struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+        struct net *net = SVC_NET(rqstp);
+        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
         if (!nfsd4_last_compound_op(rqstp))
                 return nfserr_not_only_op;
-        nfs4_lock_state();
         spin_lock(&nn->client_lock);
-        session = find_in_sessionid_hashtbl(&bcts->sessionid, SVC_NET(rqstp));
+        session = find_in_sessionid_hashtbl(&bcts->sessionid, net, &status);
         spin_unlock(&nn->client_lock);
-        status = nfserr_badsession;
         if (!session)
-                goto out;
+                goto out_no_session;
         status = nfserr_wrong_cred;
         if (!mach_creds_match(session->se_client, rqstp))
                 goto out;
@@ -2176,7 +2592,8 @@ __be32 nfsd4_bind_conn_to_session(struct svc_rqst *rqstp,
         nfsd4_init_conn(rqstp, conn, session);
         status = nfs_ok;
 out:
-        nfs4_unlock_state();
+        nfsd4_put_session(session);
+out_no_session:
         return status;
 }
 
@@ -2195,9 +2612,9 @@ nfsd4_destroy_session(struct svc_rqst *r,
         struct nfsd4_session *ses;
         __be32 status;
         int ref_held_by_me = 0;
-        struct nfsd_net *nn = net_generic(SVC_NET(r), nfsd_net_id);
+        struct net *net = SVC_NET(r);
+        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
-        nfs4_lock_state();
         status = nfserr_not_only_op;
         if (nfsd4_compound_in_session(cstate->session, &sessionid->sessionid)) {
                 if (!nfsd4_last_compound_op(r))
@@ -2206,14 +2623,12 @@ nfsd4_destroy_session(struct svc_rqst *r,
         }
         dump_sessionid(__func__, &sessionid->sessionid);
         spin_lock(&nn->client_lock);
-        ses = find_in_sessionid_hashtbl(&sessionid->sessionid, SVC_NET(r));
-        status = nfserr_badsession;
+        ses = find_in_sessionid_hashtbl(&sessionid->sessionid, net, &status);
         if (!ses)
                 goto out_client_lock;
         status = nfserr_wrong_cred;
         if (!mach_creds_match(ses->se_client, r))
-                goto out_client_lock;
-        nfsd4_get_session_locked(ses);
+                goto out_put_session;
         status = mark_session_dead_locked(ses, 1 + ref_held_by_me);
         if (status)
                 goto out_put_session;
@@ -2225,11 +2640,10 @@ nfsd4_destroy_session(struct svc_rqst *r,
         spin_lock(&nn->client_lock);
         status = nfs_ok;
 out_put_session:
-        nfsd4_put_session(ses);
+        nfsd4_put_session_locked(ses);
 out_client_lock:
         spin_unlock(&nn->client_lock);
 out:
-        nfs4_unlock_state();
         return status;
 }
 
@@ -2300,7 +2714,8 @@ nfsd4_sequence(struct svc_rqst *rqstp,
         struct nfsd4_conn *conn;
         __be32 status;
         int buflen;
-        struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+        struct net *net = SVC_NET(rqstp);
+        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
         if (resp->opcnt != 1)
                 return nfserr_sequence_pos;
@@ -2314,17 +2729,10 @@ nfsd4_sequence(struct svc_rqst *rqstp,
                 return nfserr_jukebox;
 
         spin_lock(&nn->client_lock);
-        status = nfserr_badsession;
-        session = find_in_sessionid_hashtbl(&seq->sessionid, SVC_NET(rqstp));
+        session = find_in_sessionid_hashtbl(&seq->sessionid, net, &status);
         if (!session)
                 goto out_no_session;
         clp = session->se_client;
-        status = get_client_locked(clp);
-        if (status)
-                goto out_no_session;
-        status = nfsd4_get_session_locked(session);
-        if (status)
-                goto out_put_client;
 
         status = nfserr_too_many_ops;
         if (nfsd4_session_too_many_ops(rqstp, session))
@@ -2354,6 +2762,7 @@ nfsd4_sequence(struct svc_rqst *rqstp,
                         goto out_put_session;
                 cstate->slot = slot;
                 cstate->session = session;
+                cstate->clp = clp;
                 /* Return the cached reply status and set cstate->status
                  * for nfsd4_proc_compound processing */
                 status = nfsd4_replay_cache_entry(resp, seq);
@@ -2388,6 +2797,7 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 
         cstate->slot = slot;
         cstate->session = session;
+        cstate->clp = clp;
 
 out:
         switch (clp->cl_cb_state) {
@@ -2408,31 +2818,48 @@ out_no_session:
         spin_unlock(&nn->client_lock);
         return status;
 out_put_session:
-        nfsd4_put_session(session);
-out_put_client:
-        put_client_renew_locked(clp);
+        nfsd4_put_session_locked(session);
         goto out_no_session;
 }
 
+void
+nfsd4_sequence_done(struct nfsd4_compoundres *resp)
+{
+        struct nfsd4_compound_state *cs = &resp->cstate;
+
+        if (nfsd4_has_session(cs)) {
+                if (cs->status != nfserr_replay_cache) {
+                        nfsd4_store_cache_entry(resp);
+                        cs->slot->sl_flags &= ~NFSD4_SLOT_INUSE;
+                }
+                /* Drop session reference that was taken in nfsd4_sequence() */
+                nfsd4_put_session(cs->session);
+        } else if (cs->clp)
+                put_client_renew(cs->clp);
+}
+
 __be32
 nfsd4_destroy_clientid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd4_destroy_clientid *dc)
 {
-        struct nfs4_client *conf, *unconf, *clp;
+        struct nfs4_client *conf, *unconf;
+        struct nfs4_client *clp = NULL;
         __be32 status = 0;
         struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
 
-        nfs4_lock_state();
+        spin_lock(&nn->client_lock);
         unconf = find_unconfirmed_client(&dc->clientid, true, nn);
         conf = find_confirmed_client(&dc->clientid, true, nn);
         WARN_ON_ONCE(conf && unconf);
 
         if (conf) {
-                clp = conf;
-
                 if (client_has_state(conf)) {
                         status = nfserr_clientid_busy;
                         goto out;
                 }
+                status = mark_client_expired_locked(conf);
+                if (status)
+                        goto out;
+                clp = conf;
         } else if (unconf)
                 clp = unconf;
         else {
@@ -2440,12 +2867,15 @@ nfsd4_destroy_clientid(struct svc_rqst *rqstp, struct nfsd4_compound_state *csta
                 goto out;
         }
         if (!mach_creds_match(clp, rqstp)) {
+                clp = NULL;
                 status = nfserr_wrong_cred;
                 goto out;
         }
-        expire_client(clp);
+        unhash_client_locked(clp);
 out:
-        nfs4_unlock_state();
+        spin_unlock(&nn->client_lock);
+        if (clp)
+                expire_client(clp);
         return status;
 }
 
@@ -2464,7 +2894,6 @@ nfsd4_reclaim_complete(struct svc_rqst *rqstp, struct nfsd4_compound_state *csta
                  return nfs_ok;
         }
 
-        nfs4_lock_state();
         status = nfserr_complete_already;
         if (test_and_set_bit(NFSD4_CLIENT_RECLAIM_COMPLETE,
                              &cstate->session->se_client->cl_flags))
@@ -2484,7 +2913,6 @@ nfsd4_reclaim_complete(struct svc_rqst *rqstp, struct nfsd4_compound_state *csta
         status = nfs_ok;
         nfsd4_client_record_create(cstate->session->se_client);
 out:
-        nfs4_unlock_state();
         return status;
 }
 
@@ -2494,12 +2922,16 @@ nfsd4_setclientid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 {
         struct xdr_netobj       clname = setclid->se_name;
         nfs4_verifier           clverifier = setclid->se_verf;
-        struct nfs4_client      *conf, *unconf, *new;
+        struct nfs4_client      *conf, *new;
+        struct nfs4_client      *unconf = NULL;
         __be32                  status;
         struct nfsd_net         *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
 
+        new = create_client(clname, rqstp, &clverifier);
+        if (new == NULL)
+                return nfserr_jukebox;
         /* Cases below refer to rfc 3530 section 14.2.33: */
-        nfs4_lock_state();
+        spin_lock(&nn->client_lock);
         conf = find_confirmed_client_by_name(&clname, nn);
         if (conf) {
                 /* case 0: */
@@ -2517,11 +2949,7 @@ nfsd4_setclientid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         }
         unconf = find_unconfirmed_client_by_name(&clname, nn);
         if (unconf)
-                expire_client(unconf);
-        status = nfserr_jukebox;
-        new = create_client(clname, rqstp, &clverifier);
-        if (new == NULL)
-                goto out;
+                unhash_client_locked(unconf);
         if (conf && same_verf(&conf->cl_verifier, &clverifier))
                 /* case 1: probable callback update */
                 copy_clid(new, conf);
@@ -2533,9 +2961,14 @@ nfsd4_setclientid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         setclid->se_clientid.cl_boot = new->cl_clientid.cl_boot;
         setclid->se_clientid.cl_id = new->cl_clientid.cl_id;
         memcpy(setclid->se_confirm.data, new->cl_confirm.data, sizeof(setclid->se_confirm.data));
+        new = NULL;
         status = nfs_ok;
 out:
-        nfs4_unlock_state();
+        spin_unlock(&nn->client_lock);
+        if (new)
+                free_client(new);
+        if (unconf)
+                expire_client(unconf);
         return status;
 }
 
@@ -2546,6 +2979,7 @@ nfsd4_setclientid_confirm(struct svc_rqst *rqstp,
                          struct nfsd4_setclientid_confirm *setclientid_confirm)
 {
         struct nfs4_client *conf, *unconf;
+        struct nfs4_client *old = NULL;
         nfs4_verifier confirm = setclientid_confirm->sc_confirm; 
         clientid_t * clid = &setclientid_confirm->sc_clientid;
         __be32 status;
@@ -2553,8 +2987,8 @@ nfsd4_setclientid_confirm(struct svc_rqst *rqstp,
 
         if (STALE_CLIENTID(clid, nn))
                 return nfserr_stale_clientid;
-        nfs4_lock_state();
 
+        spin_lock(&nn->client_lock);
         conf = find_confirmed_client(clid, false, nn);
         unconf = find_unconfirmed_client(clid, false, nn);
         /*
@@ -2578,22 +3012,30 @@ nfsd4_setclientid_confirm(struct svc_rqst *rqstp,
         }
         status = nfs_ok;
         if (conf) { /* case 1: callback update */
+                old = unconf;
+                unhash_client_locked(old);
                 nfsd4_change_callback(conf, &unconf->cl_cb_conn);
-                nfsd4_probe_callback(conf);
-                expire_client(unconf);
         } else { /* case 3: normal case; new or rebooted client */
-                conf = find_confirmed_client_by_name(&unconf->cl_name, nn);
-                if (conf) {
-                        status = mark_client_expired(conf);
-                        if (status)
+                old = find_confirmed_client_by_name(&unconf->cl_name, nn);
+                if (old) {
+                        status = mark_client_expired_locked(old);
+                        if (status) {
+                                old = NULL;
                                 goto out;
-                        expire_client(conf);
+                        }
                 }
                 move_to_confirmed(unconf);
-                nfsd4_probe_callback(unconf);
+                conf = unconf;
         }
+        get_client_locked(conf);
+        spin_unlock(&nn->client_lock);
+        nfsd4_probe_callback(conf);
+        spin_lock(&nn->client_lock);
+        put_client_renew_locked(conf);
 out:
-        nfs4_unlock_state();
+        spin_unlock(&nn->client_lock);
+        if (old)
+                expire_client(old);
         return status;
 }
 
@@ -2603,21 +3045,23 @@ static struct nfs4_file *nfsd4_alloc_file(void)
 }
 
 /* OPEN Share state helper functions */
-static void nfsd4_init_file(struct nfs4_file *fp, struct inode *ino)
+static void nfsd4_init_file(struct nfs4_file *fp, struct knfsd_fh *fh)
 {
-        unsigned int hashval = file_hashval(ino);
+        unsigned int hashval = file_hashval(fh);
+
+        lockdep_assert_held(&state_lock);
 
         atomic_set(&fp->fi_ref, 1);
+        spin_lock_init(&fp->fi_lock);
         INIT_LIST_HEAD(&fp->fi_stateids);
         INIT_LIST_HEAD(&fp->fi_delegations);
-        fp->fi_inode = igrab(ino);
+        fh_copy_shallow(&fp->fi_fhandle, fh);
         fp->fi_had_conflict = false;
         fp->fi_lease = NULL;
+        fp->fi_share_deny = 0;
         memset(fp->fi_fds, 0, sizeof(fp->fi_fds));
         memset(fp->fi_access, 0, sizeof(fp->fi_access));
-        spin_lock(&state_lock);
         hlist_add_head(&fp->fi_hash, &file_hashtbl[hashval]);
-        spin_unlock(&state_lock);
 }
 
 void
@@ -2673,6 +3117,28 @@ static void init_nfs4_replay(struct nfs4_replay *rp)
         rp->rp_status = nfserr_serverfault;
         rp->rp_buflen = 0;
         rp->rp_buf = rp->rp_ibuf;
+        mutex_init(&rp->rp_mutex);
+}
+
+static void nfsd4_cstate_assign_replay(struct nfsd4_compound_state *cstate,
+                struct nfs4_stateowner *so)
+{
+        if (!nfsd4_has_session(cstate)) {
+                mutex_lock(&so->so_replay.rp_mutex);
+                cstate->replay_owner = so;
+                atomic_inc(&so->so_count);
+        }
+}
+
+void nfsd4_cstate_clear_replay(struct nfsd4_compound_state *cstate)
+{
+        struct nfs4_stateowner *so = cstate->replay_owner;
+
+        if (so != NULL) {
+                cstate->replay_owner = NULL;
+                mutex_unlock(&so->so_replay.rp_mutex);
+                nfs4_put_stateowner(so);
+        }
 }
 
 static inline void *alloc_stateowner(struct kmem_cache *slab, struct xdr_netobj *owner, struct nfs4_client *clp)
@@ -2693,111 +3159,172 @@ static inline void *alloc_stateowner(struct kmem_cache *slab, struct xdr_netobj
         INIT_LIST_HEAD(&sop->so_stateids);
         sop->so_client = clp;
         init_nfs4_replay(&sop->so_replay);
+        atomic_set(&sop->so_count, 1);
         return sop;
 }
 
 static void hash_openowner(struct nfs4_openowner *oo, struct nfs4_client *clp, unsigned int strhashval)
 {
-        struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+        lockdep_assert_held(&clp->cl_lock);
 
-        list_add(&oo->oo_owner.so_strhash, &nn->ownerstr_hashtbl[strhashval]);
+        list_add(&oo->oo_owner.so_strhash,
+                 &clp->cl_ownerstr_hashtbl[strhashval]);
         list_add(&oo->oo_perclient, &clp->cl_openowners);
 }
 
+static void nfs4_unhash_openowner(struct nfs4_stateowner *so)
+{
+        unhash_openowner_locked(openowner(so));
+}
+
+static void nfs4_free_openowner(struct nfs4_stateowner *so)
+{
+        struct nfs4_openowner *oo = openowner(so);
+
+        kmem_cache_free(openowner_slab, oo);
+}
+
+static const struct nfs4_stateowner_operations openowner_ops = {
+        .so_unhash =    nfs4_unhash_openowner,
+        .so_free =      nfs4_free_openowner,
+};
+
 static struct nfs4_openowner *
-alloc_init_open_stateowner(unsigned int strhashval, struct nfs4_client *clp, struct nfsd4_open *open) {
-        struct nfs4_openowner *oo;
+alloc_init_open_stateowner(unsigned int strhashval, struct nfsd4_open *open,
+                           struct nfsd4_compound_state *cstate)
+{
+        struct nfs4_client *clp = cstate->clp;
+        struct nfs4_openowner *oo, *ret;
 
         oo = alloc_stateowner(openowner_slab, &open->op_owner, clp);
         if (!oo)
                 return NULL;
+        oo->oo_owner.so_ops = &openowner_ops;
         oo->oo_owner.so_is_open_owner = 1;
         oo->oo_owner.so_seqid = open->op_seqid;
-        oo->oo_flags = NFS4_OO_NEW;
+        oo->oo_flags = 0;
+        if (nfsd4_has_session(cstate))
+                oo->oo_flags |= NFS4_OO_CONFIRMED;
         oo->oo_time = 0;
         oo->oo_last_closed_stid = NULL;
         INIT_LIST_HEAD(&oo->oo_close_lru);
-        hash_openowner(oo, clp, strhashval);
+        spin_lock(&clp->cl_lock);
+        ret = find_openstateowner_str_locked(strhashval, open, clp);
+        if (ret == NULL) {
+                hash_openowner(oo, clp, strhashval);
+                ret = oo;
+        } else
+                nfs4_free_openowner(&oo->oo_owner);
+        spin_unlock(&clp->cl_lock);
         return oo;
 }
 
 static void init_open_stateid(struct nfs4_ol_stateid *stp, struct nfs4_file *fp, struct nfsd4_open *open) {
         struct nfs4_openowner *oo = open->op_openowner;
 
+        atomic_inc(&stp->st_stid.sc_count);
         stp->st_stid.sc_type = NFS4_OPEN_STID;
-        INIT_LIST_HEAD(&stp->st_lockowners);
-        list_add(&stp->st_perstateowner, &oo->oo_owner.so_stateids);
-        list_add(&stp->st_perfile, &fp->fi_stateids);
+        INIT_LIST_HEAD(&stp->st_locks);
         stp->st_stateowner = &oo->oo_owner;
+        atomic_inc(&stp->st_stateowner->so_count);
         get_nfs4_file(fp);
-        stp->st_file = fp;
+        stp->st_stid.sc_file = fp;
         stp->st_access_bmap = 0;
         stp->st_deny_bmap = 0;
-        set_access(open->op_share_access, stp);
-        set_deny(open->op_share_deny, stp);
         stp->st_openstp = NULL;
+        spin_lock(&oo->oo_owner.so_client->cl_lock);
+        list_add(&stp->st_perstateowner, &oo->oo_owner.so_stateids);
+        spin_lock(&fp->fi_lock);
+        list_add(&stp->st_perfile, &fp->fi_stateids);
+        spin_unlock(&fp->fi_lock);
+        spin_unlock(&oo->oo_owner.so_client->cl_lock);
 }
 
+/*
+ * In the 4.0 case we need to keep the owners around a little while to handle
+ * CLOSE replay. We still do need to release any file access that is held by
+ * them before returning however.
+ */
 static void
-move_to_close_lru(struct nfs4_openowner *oo, struct net *net)
+move_to_close_lru(struct nfs4_ol_stateid *s, struct net *net)
 {
-        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
+        struct nfs4_ol_stateid *last;
+        struct nfs4_openowner *oo = openowner(s->st_stateowner);
+        struct nfsd_net *nn = net_generic(s->st_stid.sc_client->net,
+                                                nfsd_net_id);
 
         dprintk("NFSD: move_to_close_lru nfs4_openowner %p\n", oo);
 
+        /*
+         * We know that we hold one reference via nfsd4_close, and another
+         * "persistent" reference for the client. If the refcount is higher
+         * than 2, then there are still calls in progress that are using this
+         * stateid. We can't put the sc_file reference until they are finished.
+         * Wait for the refcount to drop to 2. Since it has been unhashed,
+         * there should be no danger of the refcount going back up again at
+         * this point.
+         */
+        wait_event(close_wq, atomic_read(&s->st_stid.sc_count) == 2);
+
+        release_all_access(s);
+        if (s->st_stid.sc_file) {
+                put_nfs4_file(s->st_stid.sc_file);
+                s->st_stid.sc_file = NULL;
+        }
+
+        spin_lock(&nn->client_lock);
+        last = oo->oo_last_closed_stid;
+        oo->oo_last_closed_stid = s;
         list_move_tail(&oo->oo_close_lru, &nn->close_lru);
         oo->oo_time = get_seconds();
+        spin_unlock(&nn->client_lock);
+        if (last)
+                nfs4_put_stid(&last->st_stid);
 }
 
-static int
-same_owner_str(struct nfs4_stateowner *sop, struct xdr_netobj *owner,
-                                                        clientid_t *clid)
+/* search file_hashtbl[] for file */
+static struct nfs4_file *
+find_file_locked(struct knfsd_fh *fh)
 {
-        return (sop->so_owner.len == owner->len) &&
-                0 == memcmp(sop->so_owner.data, owner->data, owner->len) &&
-                (sop->so_client->cl_clientid.cl_id == clid->cl_id);
-}
+        unsigned int hashval = file_hashval(fh);
+        struct nfs4_file *fp;
 
-static struct nfs4_openowner *
-find_openstateowner_str(unsigned int hashval, struct nfsd4_open *open,
-                        bool sessions, struct nfsd_net *nn)
-{
-        struct nfs4_stateowner *so;
-        struct nfs4_openowner *oo;
-        struct nfs4_client *clp;
+        lockdep_assert_held(&state_lock);
 
-        list_for_each_entry(so, &nn->ownerstr_hashtbl[hashval], so_strhash) {
-                if (!so->so_is_open_owner)
-                        continue;
-                if (same_owner_str(so, &open->op_owner, &open->op_clientid)) {
-                        oo = openowner(so);
-                        clp = oo->oo_owner.so_client;
-                        if ((bool)clp->cl_minorversion != sessions)
-                                return NULL;
-                        renew_client(oo->oo_owner.so_client);
-                        return oo;
+        hlist_for_each_entry(fp, &file_hashtbl[hashval], fi_hash) {
+                if (nfsd_fh_match(&fp->fi_fhandle, fh)) {
+                        get_nfs4_file(fp);
+                        return fp;
                 }
         }
         return NULL;
 }
 
-/* search file_hashtbl[] for file */
 static struct nfs4_file *
-find_file(struct inode *ino)
+find_file(struct knfsd_fh *fh)
 {
-        unsigned int hashval = file_hashval(ino);
         struct nfs4_file *fp;
 
         spin_lock(&state_lock);
-        hlist_for_each_entry(fp, &file_hashtbl[hashval], fi_hash) {
-                if (fp->fi_inode == ino) {
-                        get_nfs4_file(fp);
-                        spin_unlock(&state_lock);
-                        return fp;
-                }
+        fp = find_file_locked(fh);
+        spin_unlock(&state_lock);
+        return fp;
+}
+
+static struct nfs4_file *
+find_or_add_file(struct nfs4_file *new, struct knfsd_fh *fh)
+{
+        struct nfs4_file *fp;
+
+        spin_lock(&state_lock);
+        fp = find_file_locked(fh);
+        if (fp == NULL) {
+                nfsd4_init_file(new, fh);
+                fp = new;
         }
         spin_unlock(&state_lock);
-        return NULL;
+
+        return fp;
 }
 
 /*
@@ -2807,47 +3334,53 @@ find_file(struct inode *ino)
 static __be32
 nfs4_share_conflict(struct svc_fh *current_fh, unsigned int deny_type)
 {
-        struct inode *ino = current_fh->fh_dentry->d_inode;
         struct nfs4_file *fp;
-        struct nfs4_ol_stateid *stp;
-        __be32 ret;
+        __be32 ret = nfs_ok;
 
-        fp = find_file(ino);
+        fp = find_file(&current_fh->fh_handle);
         if (!fp)
-                return nfs_ok;
-        ret = nfserr_locked;
-        /* Search for conflicting share reservations */
-        list_for_each_entry(stp, &fp->fi_stateids, st_perfile) {
-                if (test_deny(deny_type, stp) ||
-                    test_deny(NFS4_SHARE_DENY_BOTH, stp))
-                        goto out;
-        }
-        ret = nfs_ok;
-out:
+                return ret;
+        /* Check for conflicting share reservations */
+        spin_lock(&fp->fi_lock);
+        if (fp->fi_share_deny & deny_type)
+                ret = nfserr_locked;
+        spin_unlock(&fp->fi_lock);
         put_nfs4_file(fp);
         return ret;
 }
 
-static void nfsd_break_one_deleg(struct nfs4_delegation *dp)
+void nfsd4_prepare_cb_recall(struct nfs4_delegation *dp)
 {
-        struct nfs4_client *clp = dp->dl_stid.sc_client;
-        struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+        struct nfsd_net *nn = net_generic(dp->dl_stid.sc_client->net,
+                                          nfsd_net_id);
 
-        lockdep_assert_held(&state_lock);
-        /* We're assuming the state code never drops its reference
+        block_delegations(&dp->dl_stid.sc_file->fi_fhandle);
+
+        /*
+         * We can't do this in nfsd_break_deleg_cb because it is
+         * already holding inode->i_lock.
+         *
+         * If the dl_time != 0, then we know that it has already been
+         * queued for a lease break. Don't queue it again.
+         */
+        spin_lock(&state_lock);
+        if (dp->dl_time == 0) {
+                dp->dl_time = get_seconds();
+                list_add_tail(&dp->dl_recall_lru, &nn->del_recall_lru);
+        }
+        spin_unlock(&state_lock);
+}
+
+static void nfsd_break_one_deleg(struct nfs4_delegation *dp)
+{
+        /*
+         * We're assuming the state code never drops its reference
          * without first removing the lease.  Since we're in this lease
          * callback (and since the lease code is serialized by the kernel
          * lock) we know the server hasn't removed the lease yet, we know
-         * it's safe to take a reference: */
-        atomic_inc(&dp->dl_count);
-
-        list_add_tail(&dp->dl_recall_lru, &nn->del_recall_lru);
-
-        /* Only place dl_time is set; protected by i_lock: */
-        dp->dl_time = get_seconds();
-
-        block_delegations(&dp->dl_fh);
-
+         * it's safe to take a reference.
+         */
+        atomic_inc(&dp->dl_stid.sc_count);
         nfsd4_cb_recall(dp);
 }
 
@@ -2872,11 +3405,20 @@ static void nfsd_break_deleg_cb(struct file_lock *fl)
          */
         fl->fl_break_time = 0;
 
-        spin_lock(&state_lock);
+        spin_lock(&fp->fi_lock);
         fp->fi_had_conflict = true;
-        list_for_each_entry(dp, &fp->fi_delegations, dl_perfile)
-                nfsd_break_one_deleg(dp);
-        spin_unlock(&state_lock);
+        /*
+         * If there are no delegations on the list, then we can't count on this
+         * lease ever being cleaned up. Set the fl_break_time to jiffies so that
+         * time_out_leases will do it ASAP. The fact that fi_had_conflict is now
+         * true should keep any new delegations from being hashed.
+         */
+        if (list_empty(&fp->fi_delegations))
+                fl->fl_break_time = jiffies;
+        else
+                list_for_each_entry(dp, &fp->fi_delegations, dl_perfile)
+                        nfsd_break_one_deleg(dp);
+        spin_unlock(&fp->fi_lock);
 }
 
 static
@@ -2904,6 +3446,42 @@ static __be32 nfsd4_check_seqid(struct nfsd4_compound_state *cstate, struct nfs4
         return nfserr_bad_seqid;
 }
 
+static __be32 lookup_clientid(clientid_t *clid,
+                struct nfsd4_compound_state *cstate,
+                struct nfsd_net *nn)
+{
+        struct nfs4_client *found;
+
+        if (cstate->clp) {
+                found = cstate->clp;
+                if (!same_clid(&found->cl_clientid, clid))
+                        return nfserr_stale_clientid;
+                return nfs_ok;
+        }
+
+        if (STALE_CLIENTID(clid, nn))
+                return nfserr_stale_clientid;
+
+        /*
+         * For v4.1+ we get the client in the SEQUENCE op. If we don't have one
+         * cached already then we know this is for is for v4.0 and "sessions"
+         * will be false.
+         */
+        WARN_ON_ONCE(cstate->session);
+        spin_lock(&nn->client_lock);
+        found = find_confirmed_client(clid, false, nn);
+        if (!found) {
+                spin_unlock(&nn->client_lock);
+                return nfserr_expired;
+        }
+        atomic_inc(&found->cl_refcount);
+        spin_unlock(&nn->client_lock);
+
+        /* Cache the nfs4_client in cstate! */
+        cstate->clp = found;
+        return nfs_ok;
+}
+
 __be32
 nfsd4_process_open1(struct nfsd4_compound_state *cstate,
                     struct nfsd4_open *open, struct nfsd_net *nn)
@@ -2924,19 +3502,19 @@ nfsd4_process_open1(struct nfsd4_compound_state *cstate,
         if (open->op_file == NULL)
                 return nfserr_jukebox;
 
-        strhashval = ownerstr_hashval(clientid->cl_id, &open->op_owner);
-        oo = find_openstateowner_str(strhashval, open, cstate->minorversion, nn);
+        status = lookup_clientid(clientid, cstate, nn);
+        if (status)
+                return status;
+        clp = cstate->clp;
+
+        strhashval = ownerstr_hashval(&open->op_owner);
+        oo = find_openstateowner_str(strhashval, open, clp);
         open->op_openowner = oo;
         if (!oo) {
-                clp = find_confirmed_client(clientid, cstate->minorversion,
-                                            nn);
-                if (clp == NULL)
-                        return nfserr_expired;
                 goto new_owner;
         }
         if (!(oo->oo_flags & NFS4_OO_CONFIRMED)) {
                 /* Replace unconfirmed owners without checking for replay. */
-                clp = oo->oo_owner.so_client;
                 release_openowner(oo);
                 open->op_openowner = NULL;
                 goto new_owner;
@@ -2944,15 +3522,14 @@ nfsd4_process_open1(struct nfsd4_compound_state *cstate,
         status = nfsd4_check_seqid(cstate, &oo->oo_owner, open->op_seqid);
         if (status)
                 return status;
-        clp = oo->oo_owner.so_client;
         goto alloc_stateid;
 new_owner:
-        oo = alloc_init_open_stateowner(strhashval, clp, open);
+        oo = alloc_init_open_stateowner(strhashval, open, cstate);
         if (oo == NULL)
                 return nfserr_jukebox;
         open->op_openowner = oo;
 alloc_stateid:
-        open->op_stp = nfs4_alloc_stateid(clp);
+        open->op_stp = nfs4_alloc_open_stateid(clp);
         if (!open->op_stp)
                 return nfserr_jukebox;
         return nfs_ok;
@@ -2994,14 +3571,18 @@ nfs4_check_deleg(struct nfs4_client *cl, struct nfsd4_open *open,
 {
         int flags;
         __be32 status = nfserr_bad_stateid;
+        struct nfs4_delegation *deleg;
 
-        *dp = find_deleg_stateid(cl, &open->op_delegate_stateid);
-        if (*dp == NULL)
+        deleg = find_deleg_stateid(cl, &open->op_delegate_stateid);
+        if (deleg == NULL)
                 goto out;
         flags = share_access_to_flags(open->op_share_access);
-        status = nfs4_check_delegmode(*dp, flags);
-        if (status)
-                *dp = NULL;
+        status = nfs4_check_delegmode(deleg, flags);
+        if (status) {
+                nfs4_put_stid(&deleg->dl_stid);
+                goto out;
+        }
+        *dp = deleg;
 out:
         if (!nfsd4_is_deleg_cur(open))
                 return nfs_ok;
@@ -3011,24 +3592,25 @@ out:
         return nfs_ok;
 }
 
-static __be32
-nfs4_check_open(struct nfs4_file *fp, struct nfsd4_open *open, struct nfs4_ol_stateid **stpp)
+static struct nfs4_ol_stateid *
+nfsd4_find_existing_open(struct nfs4_file *fp, struct nfsd4_open *open)
 {
-        struct nfs4_ol_stateid *local;
+        struct nfs4_ol_stateid *local, *ret = NULL;
         struct nfs4_openowner *oo = open->op_openowner;
 
+        spin_lock(&fp->fi_lock);
         list_for_each_entry(local, &fp->fi_stateids, st_perfile) {
                 /* ignore lock owners */
                 if (local->st_stateowner->so_is_open_owner == 0)
                         continue;
-                /* remember if we have seen this open owner */
-                if (local->st_stateowner == &oo->oo_owner)
-                        *stpp = local;
-                /* check for conflicting share reservations */
-                if (!test_share(local, open))
-                        return nfserr_share_denied;
+                if (local->st_stateowner == &oo->oo_owner) {
+                        ret = local;
+                        atomic_inc(&ret->st_stid.sc_count);
+                        break;
+                }
         }
-        return nfs_ok;
+        spin_unlock(&fp->fi_lock);
+        return ret;
 }
 
 static inline int nfs4_access_to_access(u32 nfs4_access)
@@ -3042,24 +3624,6 @@ static inline int nfs4_access_to_access(u32 nfs4_access)
         return flags;
 }
 
-static __be32 nfs4_get_vfs_file(struct svc_rqst *rqstp, struct nfs4_file *fp,
-                struct svc_fh *cur_fh, struct nfsd4_open *open)
-{
-        __be32 status;
-        int oflag = nfs4_access_to_omode(open->op_share_access);
-        int access = nfs4_access_to_access(open->op_share_access);
-
-        if (!fp->fi_fds[oflag]) {
-                status = nfsd_open(rqstp, cur_fh, S_IFREG, access,
-                        &fp->fi_fds[oflag]);
-                if (status)
-                        return status;
-        }
-        nfs4_file_get_access(fp, oflag);
-
-        return nfs_ok;
-}
-
 static inline __be32
 nfsd4_truncate(struct svc_rqst *rqstp, struct svc_fh *fh,
                 struct nfsd4_open *open)
@@ -3075,34 +3639,99 @@ nfsd4_truncate(struct svc_rqst *rqstp, struct svc_fh *fh,
         return nfsd_setattr(rqstp, fh, &iattr, 0, (time_t)0);
 }
 
-static __be32
-nfs4_upgrade_open(struct svc_rqst *rqstp, struct nfs4_file *fp, struct svc_fh *cur_fh, struct nfs4_ol_stateid *stp, struct nfsd4_open *open)
+static __be32 nfs4_get_vfs_file(struct svc_rqst *rqstp, struct nfs4_file *fp,
+                struct svc_fh *cur_fh, struct nfs4_ol_stateid *stp,
+                struct nfsd4_open *open)
 {
-        u32 op_share_access = open->op_share_access;
-        bool new_access;
+        struct file *filp = NULL;
         __be32 status;
+        int oflag = nfs4_access_to_omode(open->op_share_access);
+        int access = nfs4_access_to_access(open->op_share_access);
+        unsigned char old_access_bmap, old_deny_bmap;
 
-        new_access = !test_access(op_share_access, stp);
-        if (new_access) {
-                status = nfs4_get_vfs_file(rqstp, fp, cur_fh, open);
-                if (status)
-                        return status;
+        spin_lock(&fp->fi_lock);
+
+        /*
+         * Are we trying to set a deny mode that would conflict with
+         * current access?
+         */
+        status = nfs4_file_check_deny(fp, open->op_share_deny);
+        if (status != nfs_ok) {
+                spin_unlock(&fp->fi_lock);
+                goto out;
         }
-        status = nfsd4_truncate(rqstp, cur_fh, open);
-        if (status) {
-                if (new_access) {
-                        int oflag = nfs4_access_to_omode(op_share_access);
-                        nfs4_file_put_access(fp, oflag);
-                }
-                return status;
+
+        /* set access to the file */
+        status = nfs4_file_get_access(fp, open->op_share_access);
+        if (status != nfs_ok) {
+                spin_unlock(&fp->fi_lock);
+                goto out;
         }
-        /* remember the open */
-        set_access(op_share_access, stp);
+
+        /* Set access bits in stateid */
+        old_access_bmap = stp->st_access_bmap;
+        set_access(open->op_share_access, stp);
+
+        /* Set new deny mask */
+        old_deny_bmap = stp->st_deny_bmap;
         set_deny(open->op_share_deny, stp);
+        fp->fi_share_deny |= (open->op_share_deny & NFS4_SHARE_DENY_BOTH);
 
-        return nfs_ok;
+        if (!fp->fi_fds[oflag]) {
+                spin_unlock(&fp->fi_lock);
+                status = nfsd_open(rqstp, cur_fh, S_IFREG, access, &filp);
+                if (status)
+                        goto out_put_access;
+                spin_lock(&fp->fi_lock);
+                if (!fp->fi_fds[oflag]) {
+                        fp->fi_fds[oflag] = filp;
+                        filp = NULL;
+                }
+        }
+        spin_unlock(&fp->fi_lock);
+        if (filp)
+                fput(filp);
+
+        status = nfsd4_truncate(rqstp, cur_fh, open);
+        if (status)
+                goto out_put_access;
+out:
+        return status;
+out_put_access:
+        stp->st_access_bmap = old_access_bmap;
+        nfs4_file_put_access(fp, open->op_share_access);
+        reset_union_bmap_deny(bmap_to_share_mode(old_deny_bmap), stp);
+        goto out;
 }
 
+static __be32
+nfs4_upgrade_open(struct svc_rqst *rqstp, struct nfs4_file *fp, struct svc_fh *cur_fh, struct nfs4_ol_stateid *stp, struct nfsd4_open *open)
+{
+        __be32 status;
+        unsigned char old_deny_bmap;
+
+        if (!test_access(open->op_share_access, stp))
+                return nfs4_get_vfs_file(rqstp, fp, cur_fh, stp, open);
+
+        /* test and set deny mode */
+        spin_lock(&fp->fi_lock);
+        status = nfs4_file_check_deny(fp, open->op_share_deny);
+        if (status == nfs_ok) {
+                old_deny_bmap = stp->st_deny_bmap;
+                set_deny(open->op_share_deny, stp);
+                fp->fi_share_deny |=
+                                (open->op_share_deny & NFS4_SHARE_DENY_BOTH);
+        }
+        spin_unlock(&fp->fi_lock);
+
+        if (status != nfs_ok)
+                return status;
+
+        status = nfsd4_truncate(rqstp, cur_fh, open);
+        if (status != nfs_ok)
+                reset_union_bmap_deny(old_deny_bmap, stp);
+        return status;
+}
 
 static void
 nfs4_set_claim_prev(struct nfsd4_open *open, bool has_session)
@@ -3123,7 +3752,7 @@ static bool nfsd4_cb_channel_good(struct nfs4_client *clp)
         return clp->cl_minorversion && clp->cl_cb_state == NFSD4_CB_UNKNOWN;
 }
 
-static struct file_lock *nfs4_alloc_init_lease(struct nfs4_delegation *dp, int flag)
+static struct file_lock *nfs4_alloc_init_lease(struct nfs4_file *fp, int flag)
 {
         struct file_lock *fl;
 
@@ -3135,53 +3764,101 @@ static struct file_lock *nfs4_alloc_init_lease(struct nfs4_delegation *dp, int f
         fl->fl_flags = FL_DELEG;
         fl->fl_type = flag == NFS4_OPEN_DELEGATE_READ? F_RDLCK: F_WRLCK;
         fl->fl_end = OFFSET_MAX;
-        fl->fl_owner = (fl_owner_t)(dp->dl_file);
+        fl->fl_owner = (fl_owner_t)fp;
         fl->fl_pid = current->tgid;
         return fl;
 }
 
 static int nfs4_setlease(struct nfs4_delegation *dp)
 {
-        struct nfs4_file *fp = dp->dl_file;
+        struct nfs4_file *fp = dp->dl_stid.sc_file;
         struct file_lock *fl;
-        int status;
+        struct file *filp;
+        int status = 0;
 
-        fl = nfs4_alloc_init_lease(dp, NFS4_OPEN_DELEGATE_READ);
+        fl = nfs4_alloc_init_lease(fp, NFS4_OPEN_DELEGATE_READ);
         if (!fl)
                 return -ENOMEM;
-        fl->fl_file = find_readable_file(fp);
-        status = vfs_setlease(fl->fl_file, fl->fl_type, &fl);
-        if (status)
-                goto out_free;
+        filp = find_readable_file(fp);
+        if (!filp) {
+                /* We should always have a readable file here */
+                WARN_ON_ONCE(1);
+                return -EBADF;
+        }
+        fl->fl_file = filp;
+        status = vfs_setlease(filp, fl->fl_type, &fl);
+        if (status) {
+                locks_free_lock(fl);
+                goto out_fput;
+        }
+        spin_lock(&state_lock);
+        spin_lock(&fp->fi_lock);
+        /* Did the lease get broken before we took the lock? */
+        status = -EAGAIN;
+        if (fp->fi_had_conflict)
+                goto out_unlock;
+        /* Race breaker */
+        if (fp->fi_lease) {
+                status = 0;
+                atomic_inc(&fp->fi_delegees);
+                hash_delegation_locked(dp, fp);
+                goto out_unlock;
+        }
         fp->fi_lease = fl;
-        fp->fi_deleg_file = get_file(fl->fl_file);
+        fp->fi_deleg_file = filp;
         atomic_set(&fp->fi_delegees, 1);
-        spin_lock(&state_lock);
         hash_delegation_locked(dp, fp);
+        spin_unlock(&fp->fi_lock);
         spin_unlock(&state_lock);
         return 0;
-out_free:
-        locks_free_lock(fl);
+out_unlock:
+        spin_unlock(&fp->fi_lock);
+        spin_unlock(&state_lock);
+out_fput:
+        fput(filp);
         return status;
 }
 
-static int nfs4_set_delegation(struct nfs4_delegation *dp, struct nfs4_file *fp)
+static struct nfs4_delegation *
+nfs4_set_delegation(struct nfs4_client *clp, struct svc_fh *fh,
+                    struct nfs4_file *fp)
 {
+        int status;
+        struct nfs4_delegation *dp;
+
         if (fp->fi_had_conflict)
-                return -EAGAIN;
+                return ERR_PTR(-EAGAIN);
+
+        dp = alloc_init_deleg(clp, fh);
+        if (!dp)
+                return ERR_PTR(-ENOMEM);
+
         get_nfs4_file(fp);
-        dp->dl_file = fp;
-        if (!fp->fi_lease)
-                return nfs4_setlease(dp);
         spin_lock(&state_lock);
+        spin_lock(&fp->fi_lock);
+        dp->dl_stid.sc_file = fp;
+        if (!fp->fi_lease) {
+                spin_unlock(&fp->fi_lock);
+                spin_unlock(&state_lock);
+                status = nfs4_setlease(dp);
+                goto out;
+        }
         atomic_inc(&fp->fi_delegees);
         if (fp->fi_had_conflict) {
-                spin_unlock(&state_lock);
-                return -EAGAIN;
+                status = -EAGAIN;
+                goto out_unlock;
         }
         hash_delegation_locked(dp, fp);
+        status = 0;
+out_unlock:
+        spin_unlock(&fp->fi_lock);
         spin_unlock(&state_lock);
-        return 0;
+out:
+        if (status) {
+                nfs4_put_stid(&dp->dl_stid);
+                return ERR_PTR(status);
+        }
+        return dp;
 }
 
 static void nfsd4_open_deleg_none_ext(struct nfsd4_open *open, int status)
@@ -3212,11 +3889,12 @@ static void nfsd4_open_deleg_none_ext(struct nfsd4_open *open, int status)
  * proper support for them.
  */
 static void
-nfs4_open_delegation(struct net *net, struct svc_fh *fh,
-                     struct nfsd4_open *open, struct nfs4_ol_stateid *stp)
+nfs4_open_delegation(struct svc_fh *fh, struct nfsd4_open *open,
+                        struct nfs4_ol_stateid *stp)
 {
         struct nfs4_delegation *dp;
-        struct nfs4_openowner *oo = container_of(stp->st_stateowner, struct nfs4_openowner, oo_owner);
+        struct nfs4_openowner *oo = openowner(stp->st_stateowner);
+        struct nfs4_client *clp = stp->st_stid.sc_client;
         int cb_up;
         int status = 0;
 
@@ -3235,7 +3913,7 @@ nfs4_open_delegation(struct net *net, struct svc_fh *fh,
                          * Let's not give out any delegations till everyone's
                          * had the chance to reclaim theirs....
                          */
-                        if (locks_in_grace(net))
+                        if (locks_in_grace(clp->net))
                                 goto out_no_deleg;
                         if (!cb_up || !(oo->oo_flags & NFS4_OO_CONFIRMED))
                                 goto out_no_deleg;
@@ -3254,21 +3932,17 @@ nfs4_open_delegation(struct net *net, struct svc_fh *fh,
                 default:
                         goto out_no_deleg;
         }
-        dp = alloc_init_deleg(oo->oo_owner.so_client, stp, fh);
-        if (dp == NULL)
+        dp = nfs4_set_delegation(clp, fh, stp->st_stid.sc_file);
+        if (IS_ERR(dp))
                 goto out_no_deleg;
-        status = nfs4_set_delegation(dp, stp->st_file);
-        if (status)
-                goto out_free;
 
         memcpy(&open->op_delegate_stateid, &dp->dl_stid.sc_stateid, sizeof(dp->dl_stid.sc_stateid));
 
         dprintk("NFSD: delegation stateid=" STATEID_FMT "\n",
                 STATEID_VAL(&dp->dl_stid.sc_stateid));
         open->op_delegate_type = NFS4_OPEN_DELEGATE_READ;
+        nfs4_put_stid(&dp->dl_stid);
         return;
-out_free:
-        destroy_delegation(dp);
 out_no_deleg:
         open->op_delegate_type = NFS4_OPEN_DELEGATE_NONE;
         if (open->op_claim_type == NFS4_OPEN_CLAIM_PREVIOUS &&
@@ -3301,16 +3975,12 @@ static void nfsd4_deleg_xgrade_none_ext(struct nfsd4_open *open,
          */
 }
 
-/*
- * called with nfs4_lock_state() held.
- */
 __be32
 nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfsd4_open *open)
 {
         struct nfsd4_compoundres *resp = rqstp->rq_resp;
         struct nfs4_client *cl = open->op_openowner->oo_owner.so_client;
         struct nfs4_file *fp = NULL;
-        struct inode *ino = current_fh->fh_dentry->d_inode;
         struct nfs4_ol_stateid *stp = NULL;
         struct nfs4_delegation *dp = NULL;
         __be32 status;
@@ -3320,21 +3990,18 @@ nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nf
          * and check for delegations in the process of being recalled.
          * If not found, create the nfs4_file struct
          */
-        fp = find_file(ino);
-        if (fp) {
-                if ((status = nfs4_check_open(fp, open, &stp)))
-                        goto out;
+        fp = find_or_add_file(open->op_file, &current_fh->fh_handle);
+        if (fp != open->op_file) {
                 status = nfs4_check_deleg(cl, open, &dp);
                 if (status)
                         goto out;
+                stp = nfsd4_find_existing_open(fp, open);
         } else {
+                open->op_file = NULL;
                 status = nfserr_bad_stateid;
                 if (nfsd4_is_deleg_cur(open))
                         goto out;
                 status = nfserr_jukebox;
-                fp = open->op_file;
-                open->op_file = NULL;
-                nfsd4_init_file(fp, ino);
         }
 
         /*
@@ -3347,22 +4014,19 @@ nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nf
                 if (status)
                         goto out;
         } else {
-                status = nfs4_get_vfs_file(rqstp, fp, current_fh, open);
-                if (status)
-                        goto out;
-                status = nfsd4_truncate(rqstp, current_fh, open);
-                if (status)
-                        goto out;
                 stp = open->op_stp;
                 open->op_stp = NULL;
                 init_open_stateid(stp, fp, open);
+                status = nfs4_get_vfs_file(rqstp, fp, current_fh, stp, open);
+                if (status) {
+                        release_open_stateid(stp);
+                        goto out;
+                }
         }
         update_stateid(&stp->st_stid.sc_stateid);
         memcpy(&open->op_stateid, &stp->st_stid.sc_stateid, sizeof(stateid_t));
 
         if (nfsd4_has_session(&resp->cstate)) {
-                open->op_openowner->oo_flags |= NFS4_OO_CONFIRMED;
-
                 if (open->op_deleg_want & NFS4_SHARE_WANT_NO_DELEG) {
                         open->op_delegate_type = NFS4_OPEN_DELEGATE_NONE_EXT;
                         open->op_why_no_deleg = WND4_NOT_WANTED;
@@ -3374,7 +4038,7 @@ nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nf
         * Attempt to hand out a delegation. No error return, because the
         * OPEN succeeds even if we fail.
         */
-        nfs4_open_delegation(SVC_NET(rqstp), current_fh, open, stp);
+        nfs4_open_delegation(current_fh, open, stp);
 nodeleg:
         status = nfs_ok;
 
@@ -3397,41 +4061,27 @@ out:
         if (!(open->op_openowner->oo_flags & NFS4_OO_CONFIRMED) &&
             !nfsd4_has_session(&resp->cstate))
                 open->op_rflags |= NFS4_OPEN_RESULT_CONFIRM;
+        if (dp)
+                nfs4_put_stid(&dp->dl_stid);
+        if (stp)
+                nfs4_put_stid(&stp->st_stid);
 
         return status;
 }
 
-void nfsd4_cleanup_open_state(struct nfsd4_open *open, __be32 status)
+void nfsd4_cleanup_open_state(struct nfsd4_compound_state *cstate,
+                              struct nfsd4_open *open, __be32 status)
 {
         if (open->op_openowner) {
-                struct nfs4_openowner *oo = open->op_openowner;
+                struct nfs4_stateowner *so = &open->op_openowner->oo_owner;
 
-                if (!list_empty(&oo->oo_owner.so_stateids))
-                        list_del_init(&oo->oo_close_lru);
-                if (oo->oo_flags & NFS4_OO_NEW) {
-                        if (status) {
-                                release_openowner(oo);
-                                open->op_openowner = NULL;
-                        } else
-                                oo->oo_flags &= ~NFS4_OO_NEW;
-                }
+                nfsd4_cstate_assign_replay(cstate, so);
+                nfs4_put_stateowner(so);
         }
         if (open->op_file)
                 nfsd4_free_file(open->op_file);
         if (open->op_stp)
-                free_generic_stateid(open->op_stp);
-}
-
-static __be32 lookup_clientid(clientid_t *clid, bool session, struct nfsd_net *nn, struct nfs4_client **clp)
-{
-        struct nfs4_client *found;
-
-        if (STALE_CLIENTID(clid, nn))
-                return nfserr_stale_clientid;
-        found = find_confirmed_client(clid, session, nn);
-        if (clp)
-                *clp = found;
-        return found ? nfs_ok : nfserr_expired;
+                nfs4_put_stid(&open->op_stp->st_stid);
 }
 
 __be32
@@ -3442,19 +4092,18 @@ nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         __be32 status;
         struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
 
-        nfs4_lock_state();
         dprintk("process_renew(%08x/%08x): starting\n", 
                         clid->cl_boot, clid->cl_id);
-        status = lookup_clientid(clid, cstate->minorversion, nn, &clp);
+        status = lookup_clientid(clid, cstate, nn);
         if (status)
                 goto out;
+        clp = cstate->clp;
         status = nfserr_cb_path_down;
         if (!list_empty(&clp->cl_delegations)
                         && clp->cl_cb_state != NFSD4_CB_UP)
                 goto out;
         status = nfs_ok;
 out:
-        nfs4_unlock_state();
         return status;
 }
 
@@ -3483,12 +4132,11 @@ nfs4_laundromat(struct nfsd_net *nn)
         struct nfs4_client *clp;
         struct nfs4_openowner *oo;
         struct nfs4_delegation *dp;
+        struct nfs4_ol_stateid *stp;
         struct list_head *pos, *next, reaplist;
         time_t cutoff = get_seconds() - nn->nfsd4_lease;
         time_t t, new_timeo = nn->nfsd4_lease;
 
-        nfs4_lock_state();
-
         dprintk("NFSD: laundromat service - starting\n");
         nfsd4_end_grace(nn);
         INIT_LIST_HEAD(&reaplist);
@@ -3505,13 +4153,14 @@ nfs4_laundromat(struct nfsd_net *nn)
                                 clp->cl_clientid.cl_id);
                         continue;
                 }
-                list_move(&clp->cl_lru, &reaplist);
+                list_add(&clp->cl_lru, &reaplist);
         }
         spin_unlock(&nn->client_lock);
         list_for_each_safe(pos, next, &reaplist) {
                 clp = list_entry(pos, struct nfs4_client, cl_lru);
                 dprintk("NFSD: purging unused client (clientid %08x)\n",
                         clp->cl_clientid.cl_id);
+                list_del_init(&clp->cl_lru);
                 expire_client(clp);
         }
         spin_lock(&state_lock);
@@ -3524,24 +4173,37 @@ nfs4_laundromat(struct nfsd_net *nn)
                         new_timeo = min(new_timeo, t);
                         break;
                 }
-                list_move(&dp->dl_recall_lru, &reaplist);
+                unhash_delegation_locked(dp);
+                list_add(&dp->dl_recall_lru, &reaplist);
         }
         spin_unlock(&state_lock);
-        list_for_each_safe(pos, next, &reaplist) {
-                dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru);
+        while (!list_empty(&reaplist)) {
+                dp = list_first_entry(&reaplist, struct nfs4_delegation,
+                                        dl_recall_lru);
+                list_del_init(&dp->dl_recall_lru);
                 revoke_delegation(dp);
         }
-        list_for_each_safe(pos, next, &nn->close_lru) {
-                oo = container_of(pos, struct nfs4_openowner, oo_close_lru);
-                if (time_after((unsigned long)oo->oo_time, (unsigned long)cutoff)) {
+
+        spin_lock(&nn->client_lock);
+        while (!list_empty(&nn->close_lru)) {
+                oo = list_first_entry(&nn->close_lru, struct nfs4_openowner,
+                                        oo_close_lru);
+                if (time_after((unsigned long)oo->oo_time,
+                               (unsigned long)cutoff)) {
                         t = oo->oo_time - cutoff;
                         new_timeo = min(new_timeo, t);
                         break;
                 }
-                release_openowner(oo);
+                list_del_init(&oo->oo_close_lru);
+                stp = oo->oo_last_closed_stid;
+                oo->oo_last_closed_stid = NULL;
+                spin_unlock(&nn->client_lock);
+                nfs4_put_stid(&stp->st_stid);
+                spin_lock(&nn->client_lock);
         }
+        spin_unlock(&nn->client_lock);
+
         new_timeo = max_t(time_t, new_timeo, NFSD_LAUNDROMAT_MINTIMEOUT);
-        nfs4_unlock_state();
         return new_timeo;
 }
 
@@ -3564,7 +4226,7 @@ laundromat_main(struct work_struct *laundry)
 
 static inline __be32 nfs4_check_fh(struct svc_fh *fhp, struct nfs4_ol_stateid *stp)
 {
-        if (fhp->fh_dentry->d_inode != stp->st_file->fi_inode)
+        if (!nfsd_fh_match(&fhp->fh_handle, &stp->st_stid.sc_file->fi_fhandle))
                 return nfserr_bad_stateid;
         return nfs_ok;
 }
@@ -3666,10 +4328,10 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
 {
         struct nfs4_stid *s;
         struct nfs4_ol_stateid *ols;
-        __be32 status;
+        __be32 status = nfserr_bad_stateid;
 
         if (ZERO_STATEID(stateid) || ONE_STATEID(stateid))
-                return nfserr_bad_stateid;
+                return status;
         /* Client debugging aid. */
         if (!same_clid(&stateid->si_opaque.so_clid, &cl->cl_clientid)) {
                 char addr_str[INET6_ADDRSTRLEN];
@@ -3677,53 +4339,62 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
                                  sizeof(addr_str));
                 pr_warn_ratelimited("NFSD: client %s testing state ID "
                                         "with incorrect client ID\n", addr_str);
-                return nfserr_bad_stateid;
+                return status;
         }
-        s = find_stateid(cl, stateid);
+        spin_lock(&cl->cl_lock);
+        s = find_stateid_locked(cl, stateid);
         if (!s)
-                return nfserr_bad_stateid;
+                goto out_unlock;
         status = check_stateid_generation(stateid, &s->sc_stateid, 1);
         if (status)
-                return status;
+                goto out_unlock;
         switch (s->sc_type) {
         case NFS4_DELEG_STID:
-                return nfs_ok;
+                status = nfs_ok;
+                break;
         case NFS4_REVOKED_DELEG_STID:
-                return nfserr_deleg_revoked;
+                status = nfserr_deleg_revoked;
+                break;
         case NFS4_OPEN_STID:
         case NFS4_LOCK_STID:
                 ols = openlockstateid(s);
                 if (ols->st_stateowner->so_is_open_owner
                                 && !(openowner(ols->st_stateowner)->oo_flags
                                                 & NFS4_OO_CONFIRMED))
-                        return nfserr_bad_stateid;
-                return nfs_ok;
+                        status = nfserr_bad_stateid;
+                else
+                        status = nfs_ok;
+                break;
         default:
                 printk("unknown stateid type %x\n", s->sc_type);
+                /* Fallthrough */
         case NFS4_CLOSED_STID:
-                return nfserr_bad_stateid;
+        case NFS4_CLOSED_DELEG_STID:
+                status = nfserr_bad_stateid;
         }
+out_unlock:
+        spin_unlock(&cl->cl_lock);
+        return status;
 }
 
-static __be32 nfsd4_lookup_stateid(stateid_t *stateid, unsigned char typemask,
-                                   struct nfs4_stid **s, bool sessions,
-                                   struct nfsd_net *nn)
+static __be32
+nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
+                     stateid_t *stateid, unsigned char typemask,
+                     struct nfs4_stid **s, struct nfsd_net *nn)
 {
-        struct nfs4_client *cl;
         __be32 status;
 
         if (ZERO_STATEID(stateid) || ONE_STATEID(stateid))
                 return nfserr_bad_stateid;
-        status = lookup_clientid(&stateid->si_opaque.so_clid, sessions,
-                                                        nn, &cl);
+        status = lookup_clientid(&stateid->si_opaque.so_clid, cstate, nn);
         if (status == nfserr_stale_clientid) {
-                if (sessions)
+                if (cstate->session)
                         return nfserr_bad_stateid;
                 return nfserr_stale_stateid;
         }
         if (status)
                 return status;
-        *s = find_stateid_by_type(cl, stateid, typemask);
+        *s = find_stateid_by_type(cstate->clp, stateid, typemask);
         if (!*s)
                 return nfserr_bad_stateid;
         return nfs_ok;
@@ -3754,12 +4425,11 @@ nfs4_preprocess_stateid_op(struct net *net, struct nfsd4_compound_state *cstate,
         if (ZERO_STATEID(stateid) || ONE_STATEID(stateid))
                 return check_special_stateids(net, current_fh, stateid, flags);
 
-        nfs4_lock_state();
-
-        status = nfsd4_lookup_stateid(stateid, NFS4_DELEG_STID|NFS4_OPEN_STID|NFS4_LOCK_STID,
-                                      &s, cstate->minorversion, nn);
+        status = nfsd4_lookup_stateid(cstate, stateid,
+                                NFS4_DELEG_STID|NFS4_OPEN_STID|NFS4_LOCK_STID,
+                                &s, nn);
         if (status)
-                goto out;
+                return status;
         status = check_stateid_generation(stateid, &s->sc_stateid, nfsd4_has_session(cstate));
         if (status)
                 goto out;
@@ -3770,12 +4440,13 @@ nfs4_preprocess_stateid_op(struct net *net, struct nfsd4_compound_state *cstate,
                 if (status)
                         goto out;
                 if (filpp) {
-                        file = dp->dl_file->fi_deleg_file;
+                        file = dp->dl_stid.sc_file->fi_deleg_file;
                         if (!file) {
                                 WARN_ON_ONCE(1);
                                 status = nfserr_serverfault;
                                 goto out;
                         }
+                        get_file(file);
                 }
                 break;
         case NFS4_OPEN_STID:
@@ -3791,10 +4462,12 @@ nfs4_preprocess_stateid_op(struct net *net, struct nfsd4_compound_state *cstate,
                 if (status)
                         goto out;
                 if (filpp) {
+                        struct nfs4_file *fp = stp->st_stid.sc_file;
+
                         if (flags & RD_STATE)
-                                file = find_readable_file(stp->st_file);
+                                file = find_readable_file(fp);
                         else
-                                file = find_writeable_file(stp->st_file);
+                                file = find_writeable_file(fp);
                 }
                 break;
         default:
@@ -3803,28 +4476,12 @@ nfs4_preprocess_stateid_op(struct net *net, struct nfsd4_compound_state *cstate,
         }
         status = nfs_ok;
         if (file)
-                *filpp = get_file(file);
+                *filpp = file;
 out:
-        nfs4_unlock_state();
+        nfs4_put_stid(s);
         return status;
 }
 
-static __be32
-nfsd4_free_lock_stateid(struct nfs4_ol_stateid *stp)
-{
-        struct nfs4_lockowner *lo = lockowner(stp->st_stateowner);
-
-        if (check_for_locks(stp->st_file, lo))
-                return nfserr_locks_held;
-        /*
-         * Currently there's a 1-1 lock stateid<->lockowner
-         * correspondance, and we have to delete the lockowner when we
-         * delete the lock stateid:
-         */
-        release_lockowner(lo);
-        return nfs_ok;
-}
-
 /*
  * Test if the stateid is valid
  */
@@ -3835,11 +4492,9 @@ nfsd4_test_stateid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         struct nfsd4_test_stateid_id *stateid;
         struct nfs4_client *cl = cstate->session->se_client;
 
-        nfs4_lock_state();
         list_for_each_entry(stateid, &test_stateid->ts_stateid_list, ts_id_list)
                 stateid->ts_id_status =
                         nfsd4_validate_stateid(cl, &stateid->ts_id_stateid);
-        nfs4_unlock_state();
 
         return nfs_ok;
 }
@@ -3851,37 +4506,50 @@ nfsd4_free_stateid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         stateid_t *stateid = &free_stateid->fr_stateid;
         struct nfs4_stid *s;
         struct nfs4_delegation *dp;
+        struct nfs4_ol_stateid *stp;
         struct nfs4_client *cl = cstate->session->se_client;
         __be32 ret = nfserr_bad_stateid;
 
-        nfs4_lock_state();
-        s = find_stateid(cl, stateid);
+        spin_lock(&cl->cl_lock);
+        s = find_stateid_locked(cl, stateid);
         if (!s)
-                goto out;
+                goto out_unlock;
         switch (s->sc_type) {
         case NFS4_DELEG_STID:
                 ret = nfserr_locks_held;
-                goto out;
+                break;
         case NFS4_OPEN_STID:
-        case NFS4_LOCK_STID:
                 ret = check_stateid_generation(stateid, &s->sc_stateid, 1);
                 if (ret)
-                        goto out;
-                if (s->sc_type == NFS4_LOCK_STID)
-                        ret = nfsd4_free_lock_stateid(openlockstateid(s));
-                else
-                        ret = nfserr_locks_held;
+                        break;
+                ret = nfserr_locks_held;
                 break;
+        case NFS4_LOCK_STID:
+                ret = check_stateid_generation(stateid, &s->sc_stateid, 1);
+                if (ret)
+                        break;
+                stp = openlockstateid(s);
+                ret = nfserr_locks_held;
+                if (check_for_locks(stp->st_stid.sc_file,
+                                    lockowner(stp->st_stateowner)))
+                        break;
+                unhash_lock_stateid(stp);
+                spin_unlock(&cl->cl_lock);
+                nfs4_put_stid(s);
+                ret = nfs_ok;
+                goto out;
         case NFS4_REVOKED_DELEG_STID:
                 dp = delegstateid(s);
-                destroy_revoked_delegation(dp);
+                list_del_init(&dp->dl_recall_lru);
+                spin_unlock(&cl->cl_lock);
+                nfs4_put_stid(s);
                 ret = nfs_ok;
-                break;
-        default:
-                ret = nfserr_bad_stateid;
+                goto out;
+        /* Default falls through and returns nfserr_bad_stateid */
         }
+out_unlock:
+        spin_unlock(&cl->cl_lock);
 out:
-        nfs4_unlock_state();
         return ret;
 }
 
@@ -3926,20 +4594,24 @@ nfs4_preprocess_seqid_op(struct nfsd4_compound_state *cstate, u32 seqid,
 {
         __be32 status;
         struct nfs4_stid *s;
+        struct nfs4_ol_stateid *stp = NULL;
 
         dprintk("NFSD: %s: seqid=%d stateid = " STATEID_FMT "\n", __func__,
                 seqid, STATEID_VAL(stateid));
 
         *stpp = NULL;
-        status = nfsd4_lookup_stateid(stateid, typemask, &s,
-                                      cstate->minorversion, nn);
+        status = nfsd4_lookup_stateid(cstate, stateid, typemask, &s, nn);
         if (status)
                 return status;
-        *stpp = openlockstateid(s);
-        if (!nfsd4_has_session(cstate))
-                cstate->replay_owner = (*stpp)->st_stateowner;
+        stp = openlockstateid(s);
+        nfsd4_cstate_assign_replay(cstate, stp->st_stateowner);
 
-        return nfs4_seqid_op_checks(cstate, stateid, seqid, *stpp);
+        status = nfs4_seqid_op_checks(cstate, stateid, seqid, stp);
+        if (!status)
+                *stpp = stp;
+        else
+                nfs4_put_stid(&stp->st_stid);
+        return status;
 }
 
 static __be32 nfs4_preprocess_confirmed_seqid_op(struct nfsd4_compound_state *cstate, u32 seqid,
@@ -3947,14 +4619,18 @@ static __be32 nfs4_preprocess_confirmed_seqid_op(struct nfsd4_compound_state *cs
 {
         __be32 status;
         struct nfs4_openowner *oo;
+        struct nfs4_ol_stateid *stp;
 
         status = nfs4_preprocess_seqid_op(cstate, seqid, stateid,
-                                                NFS4_OPEN_STID, stpp, nn);
+                                                NFS4_OPEN_STID, &stp, nn);
         if (status)
                 return status;
-        oo = openowner((*stpp)->st_stateowner);
-        if (!(oo->oo_flags & NFS4_OO_CONFIRMED))
+        oo = openowner(stp->st_stateowner);
+        if (!(oo->oo_flags & NFS4_OO_CONFIRMED)) {
+                nfs4_put_stid(&stp->st_stid);
                 return nfserr_bad_stateid;
+        }
+        *stpp = stp;
         return nfs_ok;
 }
 
@@ -3974,8 +4650,6 @@ nfsd4_open_confirm(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         if (status)
                 return status;
 
-        nfs4_lock_state();
-
         status = nfs4_preprocess_seqid_op(cstate,
                                         oc->oc_seqid, &oc->oc_req_stateid,
                                         NFS4_OPEN_STID, &stp, nn);
@@ -3984,7 +4658,7 @@ nfsd4_open_confirm(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         oo = openowner(stp->st_stateowner);
         status = nfserr_bad_stateid;
         if (oo->oo_flags & NFS4_OO_CONFIRMED)
-                goto out;
+                goto put_stateid;
         oo->oo_flags |= NFS4_OO_CONFIRMED;
         update_stateid(&stp->st_stid.sc_stateid);
         memcpy(&oc->oc_resp_stateid, &stp->st_stid.sc_stateid, sizeof(stateid_t));
@@ -3993,10 +4667,10 @@ nfsd4_open_confirm(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 
         nfsd4_client_record_create(oo->oo_owner.so_client);
         status = nfs_ok;
+put_stateid:
+        nfs4_put_stid(&stp->st_stid);
 out:
         nfsd4_bump_seqid(cstate, status);
-        if (!cstate->replay_owner)
-                nfs4_unlock_state();
         return status;
 }
 
@@ -4004,7 +4678,7 @@ static inline void nfs4_stateid_downgrade_bit(struct nfs4_ol_stateid *stp, u32 a
 {
         if (!test_access(access, stp))
                 return;
-        nfs4_file_put_access(stp->st_file, nfs4_access_to_omode(access));
+        nfs4_file_put_access(stp->st_stid.sc_file, access);
         clear_access(access, stp);
 }
 
@@ -4026,16 +4700,6 @@ static inline void nfs4_stateid_downgrade(struct nfs4_ol_stateid *stp, u32 to_ac
         }
 }
 
-static void
-reset_union_bmap_deny(unsigned long deny, struct nfs4_ol_stateid *stp)
-{
-        int i;
-        for (i = 0; i < 4; i++) {
-                if ((i & deny) != i)
-                        clear_deny(i, stp);
-        }
-}
-
 __be32
 nfsd4_open_downgrade(struct svc_rqst *rqstp,
                      struct nfsd4_compound_state *cstate,
@@ -4053,21 +4717,20 @@ nfsd4_open_downgrade(struct svc_rqst *rqstp,
                 dprintk("NFSD: %s: od_deleg_want=0x%x ignored\n", __func__,
                         od->od_deleg_want);
 
-        nfs4_lock_state();
         status = nfs4_preprocess_confirmed_seqid_op(cstate, od->od_seqid,
                                         &od->od_stateid, &stp, nn);
         if (status)
                 goto out; 
         status = nfserr_inval;
         if (!test_access(od->od_share_access, stp)) {
-                dprintk("NFSD: access not a subset current bitmap: 0x%lx, input access=%08x\n",
+                dprintk("NFSD: access not a subset of current bitmap: 0x%hhx, input access=%08x\n",
                         stp->st_access_bmap, od->od_share_access);
-                goto out;
+                goto put_stateid;
         }
         if (!test_deny(od->od_share_deny, stp)) {
-                dprintk("NFSD:deny not a subset current bitmap: 0x%lx, input deny=%08x\n",
+                dprintk("NFSD: deny not a subset of current bitmap: 0x%hhx, input deny=%08x\n",
                         stp->st_deny_bmap, od->od_share_deny);
-                goto out;
+                goto put_stateid;
         }
         nfs4_stateid_downgrade(stp, od->od_share_access);
 
@@ -4076,17 +4739,31 @@ nfsd4_open_downgrade(struct svc_rqst *rqstp,
         update_stateid(&stp->st_stid.sc_stateid);
         memcpy(&od->od_stateid, &stp->st_stid.sc_stateid, sizeof(stateid_t));
         status = nfs_ok;
+put_stateid:
+        nfs4_put_stid(&stp->st_stid);
 out:
         nfsd4_bump_seqid(cstate, status);
-        if (!cstate->replay_owner)
-                nfs4_unlock_state();
         return status;
 }
 
 static void nfsd4_close_open_stateid(struct nfs4_ol_stateid *s)
 {
-        unhash_open_stateid(s);
+        struct nfs4_client *clp = s->st_stid.sc_client;
+        LIST_HEAD(reaplist);
+
         s->st_stid.sc_type = NFS4_CLOSED_STID;
+        spin_lock(&clp->cl_lock);
+        unhash_open_stateid(s, &reaplist);
+
+        if (clp->cl_minorversion) {
+                put_ol_stateid_locked(s, &reaplist);
+                spin_unlock(&clp->cl_lock);
+                free_ol_stateid_reaplist(&reaplist);
+        } else {
+                spin_unlock(&clp->cl_lock);
+                free_ol_stateid_reaplist(&reaplist);
+                move_to_close_lru(s, clp->net);
+        }
 }
 
 /*
@@ -4097,7 +4774,6 @@ nfsd4_close(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
             struct nfsd4_close *close)
 {
         __be32 status;
-        struct nfs4_openowner *oo;
         struct nfs4_ol_stateid *stp;
         struct net *net = SVC_NET(rqstp);
         struct nfsd_net *nn = net_generic(net, nfsd_net_id);
@@ -4105,7 +4781,6 @@ nfsd4_close(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         dprintk("NFSD: nfsd4_close on file %pd\n", 
                         cstate->current_fh.fh_dentry);
 
-        nfs4_lock_state();
         status = nfs4_preprocess_seqid_op(cstate, close->cl_seqid,
                                         &close->cl_stateid,
                                         NFS4_OPEN_STID|NFS4_CLOSED_STID,
@@ -4113,31 +4788,14 @@ nfsd4_close(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         nfsd4_bump_seqid(cstate, status);
         if (status)
                 goto out; 
-        oo = openowner(stp->st_stateowner);
         update_stateid(&stp->st_stid.sc_stateid);
         memcpy(&close->cl_stateid, &stp->st_stid.sc_stateid, sizeof(stateid_t));
 
         nfsd4_close_open_stateid(stp);
 
-        if (cstate->minorversion)
-                free_generic_stateid(stp);
-        else
-                oo->oo_last_closed_stid = stp;
-
-        if (list_empty(&oo->oo_owner.so_stateids)) {
-                if (cstate->minorversion)
-                        release_openowner(oo);
-                else {
-                        /*
-                         * In the 4.0 case we need to keep the owners around a
-                         * little while to handle CLOSE replay.
-                         */
-                        move_to_close_lru(oo, SVC_NET(rqstp));
-                }
-        }
+        /* put reference from nfs4_preprocess_seqid_op */
+        nfs4_put_stid(&stp->st_stid);
 out:
-        if (!cstate->replay_owner)
-                nfs4_unlock_state();
         return status;
 }
 
@@ -4154,28 +4812,24 @@ nfsd4_delegreturn(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         if ((status = fh_verify(rqstp, &cstate->current_fh, S_IFREG, 0)))
                 return status;
 
-        nfs4_lock_state();
-        status = nfsd4_lookup_stateid(stateid, NFS4_DELEG_STID, &s,
-                                      cstate->minorversion, nn);
+        status = nfsd4_lookup_stateid(cstate, stateid, NFS4_DELEG_STID, &s, nn);
         if (status)
                 goto out;
         dp = delegstateid(s);
         status = check_stateid_generation(stateid, &dp->dl_stid.sc_stateid, nfsd4_has_session(cstate));
         if (status)
-                goto out;
+                goto put_stateid;
 
         destroy_delegation(dp);
+put_stateid:
+        nfs4_put_stid(&dp->dl_stid);
 out:
-        nfs4_unlock_state();
-
         return status;
 }
 
 
 #define LOFF_OVERFLOW(start, len)      ((u64)(len) > ~(u64)(start))
 
-#define LOCKOWNER_INO_HASH_MASK (LOCKOWNER_INO_HASH_SIZE - 1)
-
 static inline u64
 end_offset(u64 start, u64 len)
 {
@@ -4196,13 +4850,6 @@ last_byte_offset(u64 start, u64 len)
         return end > start ? end - 1: NFS4_MAX_UINT64;
 }
 
-static unsigned int lockowner_ino_hashval(struct inode *inode, u32 cl_id, struct xdr_netobj *ownername)
-{
-        return (file_hashval(inode) + cl_id
-                        + opaque_hashval(ownername->data, ownername->len))
-                & LOCKOWNER_INO_HASH_MASK;
-}
-
 /*
  * TODO: Linux file offsets are _signed_ 64-bit quantities, which means that
  * we can't properly handle lock requests that go beyond the (2^63 - 1)-th
@@ -4255,47 +4902,56 @@ nevermind:
                 deny->ld_type = NFS4_WRITE_LT;
 }
 
-static bool same_lockowner_ino(struct nfs4_lockowner *lo, struct inode *inode, clientid_t *clid, struct xdr_netobj *owner)
+static struct nfs4_lockowner *
+find_lockowner_str_locked(clientid_t *clid, struct xdr_netobj *owner,
+                struct nfs4_client *clp)
 {
-        struct nfs4_ol_stateid *lst;
+        unsigned int strhashval = ownerstr_hashval(owner);
+        struct nfs4_stateowner *so;
 
-        if (!same_owner_str(&lo->lo_owner, owner, clid))
-                return false;
-        if (list_empty(&lo->lo_owner.so_stateids)) {
-                WARN_ON_ONCE(1);
-                return false;
+        lockdep_assert_held(&clp->cl_lock);
+
+        list_for_each_entry(so, &clp->cl_ownerstr_hashtbl[strhashval],
+                            so_strhash) {
+                if (so->so_is_open_owner)
+                        continue;
+                if (!same_owner_str(so, owner))
+                        continue;
+                atomic_inc(&so->so_count);
+                return lockowner(so);
         }
-        lst = list_first_entry(&lo->lo_owner.so_stateids,
-                               struct nfs4_ol_stateid, st_perstateowner);
-        return lst->st_file->fi_inode == inode;
+        return NULL;
 }
 
 static struct nfs4_lockowner *
-find_lockowner_str(struct inode *inode, clientid_t *clid,
-                   struct xdr_netobj *owner, struct nfsd_net *nn)
+find_lockowner_str(clientid_t *clid, struct xdr_netobj *owner,
+                struct nfs4_client *clp)
 {
-        unsigned int hashval = lockowner_ino_hashval(inode, clid->cl_id, owner);
         struct nfs4_lockowner *lo;
 
-        list_for_each_entry(lo, &nn->lockowner_ino_hashtbl[hashval], lo_owner_ino_hash) {
-                if (same_lockowner_ino(lo, inode, clid, owner))
-                        return lo;
-        }
-        return NULL;
+        spin_lock(&clp->cl_lock);
+        lo = find_lockowner_str_locked(clid, owner, clp);
+        spin_unlock(&clp->cl_lock);
+        return lo;
 }
 
-static void hash_lockowner(struct nfs4_lockowner *lo, unsigned int strhashval, struct nfs4_client *clp, struct nfs4_ol_stateid *open_stp)
+static void nfs4_unhash_lockowner(struct nfs4_stateowner *sop)
 {
-        struct inode *inode = open_stp->st_file->fi_inode;
-        unsigned int inohash = lockowner_ino_hashval(inode,
-                        clp->cl_clientid.cl_id, &lo->lo_owner.so_owner);
-        struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+        unhash_lockowner_locked(lockowner(sop));
+}
+
+static void nfs4_free_lockowner(struct nfs4_stateowner *sop)
+{
+        struct nfs4_lockowner *lo = lockowner(sop);
 
-        list_add(&lo->lo_owner.so_strhash, &nn->ownerstr_hashtbl[strhashval]);
-        list_add(&lo->lo_owner_ino_hash, &nn->lockowner_ino_hashtbl[inohash]);
-        list_add(&lo->lo_perstateid, &open_stp->st_lockowners);
+        kmem_cache_free(lockowner_slab, lo);
 }
 
+static const struct nfs4_stateowner_operations lockowner_ops = {
+        .so_unhash =    nfs4_unhash_lockowner,
+        .so_free =      nfs4_free_lockowner,
+};
+
 /*
  * Alloc a lock owner structure.
  * Called in nfsd4_lock - therefore, OPEN and OPEN_CONFIRM (if needed) has 
@@ -4303,42 +4959,107 @@ static void hash_lockowner(struct nfs4_lockowner *lo, unsigned int strhashval, s
  *
  * strhashval = ownerstr_hashval
  */
-
 static struct nfs4_lockowner *
-alloc_init_lock_stateowner(unsigned int strhashval, struct nfs4_client *clp, struct nfs4_ol_stateid *open_stp, struct nfsd4_lock *lock) {
-        struct nfs4_lockowner *lo;
+alloc_init_lock_stateowner(unsigned int strhashval, struct nfs4_client *clp,
+                           struct nfs4_ol_stateid *open_stp,
+                           struct nfsd4_lock *lock)
+{
+        struct nfs4_lockowner *lo, *ret;
 
         lo = alloc_stateowner(lockowner_slab, &lock->lk_new_owner, clp);
         if (!lo)
                 return NULL;
         INIT_LIST_HEAD(&lo->lo_owner.so_stateids);
         lo->lo_owner.so_is_open_owner = 0;
-        /* It is the openowner seqid that will be incremented in encode in the
-         * case of new lockowners; so increment the lock seqid manually: */
-        lo->lo_owner.so_seqid = lock->lk_new_lock_seqid + 1;
-        hash_lockowner(lo, strhashval, clp, open_stp);
+        lo->lo_owner.so_seqid = lock->lk_new_lock_seqid;
+        lo->lo_owner.so_ops = &lockowner_ops;
+        spin_lock(&clp->cl_lock);
+        ret = find_lockowner_str_locked(&clp->cl_clientid,
+                        &lock->lk_new_owner, clp);
+        if (ret == NULL) {
+                list_add(&lo->lo_owner.so_strhash,
+                         &clp->cl_ownerstr_hashtbl[strhashval]);
+                ret = lo;
+        } else
+                nfs4_free_lockowner(&lo->lo_owner);
+        spin_unlock(&clp->cl_lock);
         return lo;
 }
 
-static struct nfs4_ol_stateid *
-alloc_init_lock_stateid(struct nfs4_lockowner *lo, struct nfs4_file *fp, struct nfs4_ol_stateid *open_stp)
+static void
+init_lock_stateid(struct nfs4_ol_stateid *stp, struct nfs4_lockowner *lo,
+                  struct nfs4_file *fp, struct inode *inode,
+                  struct nfs4_ol_stateid *open_stp)
 {
-        struct nfs4_ol_stateid *stp;
         struct nfs4_client *clp = lo->lo_owner.so_client;
 
-        stp = nfs4_alloc_stateid(clp);
-        if (stp == NULL)
-                return NULL;
+        lockdep_assert_held(&clp->cl_lock);
+
+        atomic_inc(&stp->st_stid.sc_count);
         stp->st_stid.sc_type = NFS4_LOCK_STID;
-        list_add(&stp->st_perfile, &fp->fi_stateids);
-        list_add(&stp->st_perstateowner, &lo->lo_owner.so_stateids);
         stp->st_stateowner = &lo->lo_owner;
+        atomic_inc(&lo->lo_owner.so_count);
         get_nfs4_file(fp);
-        stp->st_file = fp;
+        stp->st_stid.sc_file = fp;
+        stp->st_stid.sc_free = nfs4_free_lock_stateid;
         stp->st_access_bmap = 0;
         stp->st_deny_bmap = open_stp->st_deny_bmap;
         stp->st_openstp = open_stp;
-        return stp;
+        list_add(&stp->st_locks, &open_stp->st_locks);
+        list_add(&stp->st_perstateowner, &lo->lo_owner.so_stateids);
+        spin_lock(&fp->fi_lock);
+        list_add(&stp->st_perfile, &fp->fi_stateids);
+        spin_unlock(&fp->fi_lock);
+}
+
+static struct nfs4_ol_stateid *
+find_lock_stateid(struct nfs4_lockowner *lo, struct nfs4_file *fp)
+{
+        struct nfs4_ol_stateid *lst;
+        struct nfs4_client *clp = lo->lo_owner.so_client;
+
+        lockdep_assert_held(&clp->cl_lock);
+
+        list_for_each_entry(lst, &lo->lo_owner.so_stateids, st_perstateowner) {
+                if (lst->st_stid.sc_file == fp) {
+                        atomic_inc(&lst->st_stid.sc_count);
+                        return lst;
+                }
+        }
+        return NULL;
+}
+
+static struct nfs4_ol_stateid *
+find_or_create_lock_stateid(struct nfs4_lockowner *lo, struct nfs4_file *fi,
+                            struct inode *inode, struct nfs4_ol_stateid *ost,
+                            bool *new)
+{
+        struct nfs4_stid *ns = NULL;
+        struct nfs4_ol_stateid *lst;
+        struct nfs4_openowner *oo = openowner(ost->st_stateowner);
+        struct nfs4_client *clp = oo->oo_owner.so_client;
+
+        spin_lock(&clp->cl_lock);
+        lst = find_lock_stateid(lo, fi);
+        if (lst == NULL) {
+                spin_unlock(&clp->cl_lock);
+                ns = nfs4_alloc_stid(clp, stateid_slab);
+                if (ns == NULL)
+                        return NULL;
+
+                spin_lock(&clp->cl_lock);
+                lst = find_lock_stateid(lo, fi);
+                if (likely(!lst)) {
+                        lst = openlockstateid(ns);
+                        init_lock_stateid(lst, lo, fi, inode, ost);
+                        ns = NULL;
+                        *new = true;
+                }
+        }
+        spin_unlock(&clp->cl_lock);
+        if (ns)
+                nfs4_put_stid(ns);
+        return lst;
 }
 
 static int
@@ -4350,46 +5071,53 @@ check_lock_length(u64 offset, u64 length)
 
 static void get_lock_access(struct nfs4_ol_stateid *lock_stp, u32 access)
 {
-        struct nfs4_file *fp = lock_stp->st_file;
-        int oflag = nfs4_access_to_omode(access);
+        struct nfs4_file *fp = lock_stp->st_stid.sc_file;
+
+        lockdep_assert_held(&fp->fi_lock);
 
         if (test_access(access, lock_stp))
                 return;
-        nfs4_file_get_access(fp, oflag);
+        __nfs4_file_get_access(fp, access);
         set_access(access, lock_stp);
 }
 
-static __be32 lookup_or_create_lock_state(struct nfsd4_compound_state *cstate, struct nfs4_ol_stateid *ost, struct nfsd4_lock *lock, struct nfs4_ol_stateid **lst, bool *new)
+static __be32
+lookup_or_create_lock_state(struct nfsd4_compound_state *cstate,
+                            struct nfs4_ol_stateid *ost,
+                            struct nfsd4_lock *lock,
+                            struct nfs4_ol_stateid **lst, bool *new)
 {
-        struct nfs4_file *fi = ost->st_file;
+        __be32 status;
+        struct nfs4_file *fi = ost->st_stid.sc_file;
         struct nfs4_openowner *oo = openowner(ost->st_stateowner);
         struct nfs4_client *cl = oo->oo_owner.so_client;
+        struct inode *inode = cstate->current_fh.fh_dentry->d_inode;
         struct nfs4_lockowner *lo;
         unsigned int strhashval;
-        struct nfsd_net *nn = net_generic(cl->net, nfsd_net_id);
-
-        lo = find_lockowner_str(fi->fi_inode, &cl->cl_clientid,
-                                &lock->v.new.owner, nn);
-        if (lo) {
-                if (!cstate->minorversion)
-                        return nfserr_bad_seqid;
-                /* XXX: a lockowner always has exactly one stateid: */
-                *lst = list_first_entry(&lo->lo_owner.so_stateids,
-                                struct nfs4_ol_stateid, st_perstateowner);
-                return nfs_ok;
+
+        lo = find_lockowner_str(&cl->cl_clientid, &lock->v.new.owner, cl);
+        if (!lo) {
+                strhashval = ownerstr_hashval(&lock->v.new.owner);
+                lo = alloc_init_lock_stateowner(strhashval, cl, ost, lock);
+                if (lo == NULL)
+                        return nfserr_jukebox;
+        } else {
+                /* with an existing lockowner, seqids must be the same */
+                status = nfserr_bad_seqid;
+                if (!cstate->minorversion &&
+                    lock->lk_new_lock_seqid != lo->lo_owner.so_seqid)
+                        goto out;
         }
-        strhashval = ownerstr_hashval(cl->cl_clientid.cl_id,
-                        &lock->v.new.owner);
-        lo = alloc_init_lock_stateowner(strhashval, cl, ost, lock);
-        if (lo == NULL)
-                return nfserr_jukebox;
-        *lst = alloc_init_lock_stateid(lo, fi, ost);
+
+        *lst = find_or_create_lock_stateid(lo, fi, inode, ost, new);
         if (*lst == NULL) {
-                release_lockowner(lo);
-                return nfserr_jukebox;
+                status = nfserr_jukebox;
+                goto out;
         }
-        *new = true;
-        return nfs_ok;
+        status = nfs_ok;
+out:
+        nfs4_put_stateowner(&lo->lo_owner);
+        return status;
 }
 
 /*
@@ -4401,14 +5129,16 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 {
         struct nfs4_openowner *open_sop = NULL;
         struct nfs4_lockowner *lock_sop = NULL;
-        struct nfs4_ol_stateid *lock_stp;
+        struct nfs4_ol_stateid *lock_stp = NULL;
+        struct nfs4_ol_stateid *open_stp = NULL;
+        struct nfs4_file *fp;
         struct file *filp = NULL;
         struct file_lock *file_lock = NULL;
         struct file_lock *conflock = NULL;
         __be32 status = 0;
-        bool new_state = false;
         int lkflg;
         int err;
+        bool new = false;
         struct net *net = SVC_NET(rqstp);
         struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
@@ -4425,11 +5155,7 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
                 return status;
         }
 
-        nfs4_lock_state();
-
         if (lock->lk_is_new) {
-                struct nfs4_ol_stateid *open_stp = NULL;
-
                 if (nfsd4_has_session(cstate))
                         /* See rfc 5661 18.10.3: given clientid is ignored: */
                         memcpy(&lock->v.new.clientid,
@@ -4453,12 +5179,13 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
                                                 &lock->v.new.clientid))
                         goto out;
                 status = lookup_or_create_lock_state(cstate, open_stp, lock,
-                                                        &lock_stp, &new_state);
-        } else
+                                                        &lock_stp, &new);
+        } else {
                 status = nfs4_preprocess_seqid_op(cstate,
                                        lock->lk_old_lock_seqid,
                                        &lock->lk_old_lock_stateid,
                                        NFS4_LOCK_STID, &lock_stp, nn);
+        }
         if (status)
                 goto out;
         lock_sop = lockowner(lock_stp->st_stateowner);
@@ -4482,20 +5209,25 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
                 goto out;
         }
 
+        fp = lock_stp->st_stid.sc_file;
         locks_init_lock(file_lock);
         switch (lock->lk_type) {
                 case NFS4_READ_LT:
                 case NFS4_READW_LT:
-                        filp = find_readable_file(lock_stp->st_file);
+                        spin_lock(&fp->fi_lock);
+                        filp = find_readable_file_locked(fp);
                         if (filp)
                                 get_lock_access(lock_stp, NFS4_SHARE_ACCESS_READ);
+                        spin_unlock(&fp->fi_lock);
                         file_lock->fl_type = F_RDLCK;
                         break;
                 case NFS4_WRITE_LT:
                 case NFS4_WRITEW_LT:
-                        filp = find_writeable_file(lock_stp->st_file);
+                        spin_lock(&fp->fi_lock);
+                        filp = find_writeable_file_locked(fp);
                         if (filp)
                                 get_lock_access(lock_stp, NFS4_SHARE_ACCESS_WRITE);
+                        spin_unlock(&fp->fi_lock);
                         file_lock->fl_type = F_WRLCK;
                         break;
                 default:
@@ -4544,11 +5276,27 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
                 break;
         }
 out:
-        if (status && new_state)
-                release_lockowner(lock_sop);
+        if (filp)
+                fput(filp);
+        if (lock_stp) {
+                /* Bump seqid manually if the 4.0 replay owner is openowner */
+                if (cstate->replay_owner &&
+                    cstate->replay_owner != &lock_sop->lo_owner &&
+                    seqid_mutating_err(ntohl(status)))
+                        lock_sop->lo_owner.so_seqid++;
+
+                /*
+                 * If this is a new, never-before-used stateid, and we are
+                 * returning an error, then just go ahead and release it.
+                 */
+                if (status && new)
+                        release_lock_stateid(lock_stp);
+
+                nfs4_put_stid(&lock_stp->st_stid);
+        }
+        if (open_stp)
+                nfs4_put_stid(&open_stp->st_stid);
         nfsd4_bump_seqid(cstate, status);
-        if (!cstate->replay_owner)
-                nfs4_unlock_state();
         if (file_lock)
                 locks_free_lock(file_lock);
         if (conflock)
@@ -4580,9 +5328,8 @@ __be32
 nfsd4_lockt(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
             struct nfsd4_lockt *lockt)
 {
-        struct inode *inode;
         struct file_lock *file_lock = NULL;
-        struct nfs4_lockowner *lo;
+        struct nfs4_lockowner *lo = NULL;
         __be32 status;
         struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
 
@@ -4592,10 +5339,8 @@ nfsd4_lockt(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         if (check_lock_length(lockt->lt_offset, lockt->lt_length))
                  return nfserr_inval;
 
-        nfs4_lock_state();
-
         if (!nfsd4_has_session(cstate)) {
-                status = lookup_clientid(&lockt->lt_clientid, false, nn, NULL);
+                status = lookup_clientid(&lockt->lt_clientid, cstate, nn);
                 if (status)
                         goto out;
         }
@@ -4603,7 +5348,6 @@ nfsd4_lockt(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         if ((status = fh_verify(rqstp, &cstate->current_fh, S_IFREG, 0)))
                 goto out;
 
-        inode = cstate->current_fh.fh_dentry->d_inode;
         file_lock = locks_alloc_lock();
         if (!file_lock) {
                 dprintk("NFSD: %s: unable to allocate lock!\n", __func__);
@@ -4626,7 +5370,8 @@ nfsd4_lockt(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
                 goto out;
         }
 
-        lo = find_lockowner_str(inode, &lockt->lt_clientid, &lockt->lt_owner, nn);
+        lo = find_lockowner_str(&lockt->lt_clientid, &lockt->lt_owner,
+                                cstate->clp);
         if (lo)
                 file_lock->fl_owner = (fl_owner_t)lo;
         file_lock->fl_pid = current->tgid;
@@ -4646,7 +5391,8 @@ nfsd4_lockt(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
                 nfs4_set_lock_denied(file_lock, &lockt->lt_denied);
         }
 out:
-        nfs4_unlock_state();
+        if (lo)
+                nfs4_put_stateowner(&lo->lo_owner);
         if (file_lock)
                 locks_free_lock(file_lock);
         return status;
@@ -4670,23 +5416,21 @@ nfsd4_locku(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         if (check_lock_length(locku->lu_offset, locku->lu_length))
                  return nfserr_inval;
 
-        nfs4_lock_state();
-                                                                                
         status = nfs4_preprocess_seqid_op(cstate, locku->lu_seqid,
                                         &locku->lu_stateid, NFS4_LOCK_STID,
                                         &stp, nn);
         if (status)
                 goto out;
-        filp = find_any_file(stp->st_file);
+        filp = find_any_file(stp->st_stid.sc_file);
         if (!filp) {
                 status = nfserr_lock_range;
-                goto out;
+                goto put_stateid;
         }
         file_lock = locks_alloc_lock();
         if (!file_lock) {
                 dprintk("NFSD: %s: unable to allocate lock!\n", __func__);
                 status = nfserr_jukebox;
-                goto out;
+                goto fput;
         }
         locks_init_lock(file_lock);
         file_lock->fl_type = F_UNLCK;
@@ -4708,41 +5452,51 @@ nfsd4_locku(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
         }
         update_stateid(&stp->st_stid.sc_stateid);
         memcpy(&locku->lu_stateid, &stp->st_stid.sc_stateid, sizeof(stateid_t));
-
+fput:
+        fput(filp);
+put_stateid:
+        nfs4_put_stid(&stp->st_stid);
 out:
         nfsd4_bump_seqid(cstate, status);
-        if (!cstate->replay_owner)
-                nfs4_unlock_state();
         if (file_lock)
                 locks_free_lock(file_lock);
         return status;
 
 out_nfserr:
         status = nfserrno(err);
-        goto out;
+        goto fput;
 }
 
 /*
  * returns
- *      1: locks held by lockowner
- *      0: no locks held by lockowner
+ *      true:  locks held by lockowner
+ *      false: no locks held by lockowner
  */
-static int
-check_for_locks(struct nfs4_file *filp, struct nfs4_lockowner *lowner)
+static bool
+check_for_locks(struct nfs4_file *fp, struct nfs4_lockowner *lowner)
 {
         struct file_lock **flpp;
-        struct inode *inode = filp->fi_inode;
-        int status = 0;
+        int status = false;
+        struct file *filp = find_any_file(fp);
+        struct inode *inode;
+
+        if (!filp) {
+                /* Any valid lock stateid should have some sort of access */
+                WARN_ON_ONCE(1);
+                return status;
+        }
+
+        inode = file_inode(filp);
 
         spin_lock(&inode->i_lock);
         for (flpp = &inode->i_flock; *flpp != NULL; flpp = &(*flpp)->fl_next) {
                 if ((*flpp)->fl_owner == (fl_owner_t)lowner) {
-                        status = 1;
-                        goto out;
+                        status = true;
+                        break;
                 }
         }
-out:
         spin_unlock(&inode->i_lock);
+        fput(filp);
         return status;
 }
 
@@ -4753,53 +5507,46 @@ nfsd4_release_lockowner(struct svc_rqst *rqstp,
 {
         clientid_t *clid = &rlockowner->rl_clientid;
         struct nfs4_stateowner *sop;
-        struct nfs4_lockowner *lo;
+        struct nfs4_lockowner *lo = NULL;
         struct nfs4_ol_stateid *stp;
         struct xdr_netobj *owner = &rlockowner->rl_owner;
-        struct list_head matches;
-        unsigned int hashval = ownerstr_hashval(clid->cl_id, owner);
+        unsigned int hashval = ownerstr_hashval(owner);
         __be32 status;
         struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+        struct nfs4_client *clp;
 
         dprintk("nfsd4_release_lockowner clientid: (%08x/%08x):\n",
                 clid->cl_boot, clid->cl_id);
 
-        nfs4_lock_state();
-
-        status = lookup_clientid(clid, cstate->minorversion, nn, NULL);
+        status = lookup_clientid(clid, cstate, nn);
         if (status)
-                goto out;
+                return status;
 
-        status = nfserr_locks_held;
-        INIT_LIST_HEAD(&matches);
+        clp = cstate->clp;
+        /* Find the matching lock stateowner */
+        spin_lock(&clp->cl_lock);
+        list_for_each_entry(sop, &clp->cl_ownerstr_hashtbl[hashval],
+                            so_strhash) {
 
-        list_for_each_entry(sop, &nn->ownerstr_hashtbl[hashval], so_strhash) {
-                if (sop->so_is_open_owner)
+                if (sop->so_is_open_owner || !same_owner_str(sop, owner))
                         continue;
-                if (!same_owner_str(sop, owner, clid))
-                        continue;
-                list_for_each_entry(stp, &sop->so_stateids,
-                                st_perstateowner) {
-                        lo = lockowner(sop);
-                        if (check_for_locks(stp->st_file, lo))
-                                goto out;
-                        list_add(&lo->lo_list, &matches);
+
+                /* see if there are still any locks associated with it */
+                lo = lockowner(sop);
+                list_for_each_entry(stp, &sop->so_stateids, st_perstateowner) {
+                        if (check_for_locks(stp->st_stid.sc_file, lo)) {
+                                status = nfserr_locks_held;
+                                spin_unlock(&clp->cl_lock);
+                                return status;
+                        }
                 }
+
+                atomic_inc(&sop->so_count);
+                break;
         }
-        /* Clients probably won't expect us to return with some (but not all)
-         * of the lockowner state released; so don't release any until all
-         * have been checked. */
-        status = nfs_ok;
-        while (!list_empty(&matches)) {
-                lo = list_entry(matches.next, struct nfs4_lockowner,
-                                                                lo_list);
-                /* unhash_stateowner deletes so_perclient only
-                 * for openowners. */
-                list_del(&lo->lo_list);
+        spin_unlock(&clp->cl_lock);
+        if (lo)
                 release_lockowner(lo);
-        }
-out:
-        nfs4_unlock_state();
         return status;
 }
 
@@ -4887,34 +5634,123 @@ nfsd4_find_reclaim_client(const char *recdir, struct nfsd_net *nn)
 * Called from OPEN. Look for clientid in reclaim list.
 */
 __be32
-nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct nfsd_net *nn)
+nfs4_check_open_reclaim(clientid_t *clid,
+                struct nfsd4_compound_state *cstate,
+                struct nfsd_net *nn)
 {
-        struct nfs4_client *clp;
+        __be32 status;
 
         /* find clientid in conf_id_hashtbl */
-        clp = find_confirmed_client(clid, sessions, nn);
-        if (clp == NULL)
+        status = lookup_clientid(clid, cstate, nn);
+        if (status)
                 return nfserr_reclaim_bad;
 
-        return nfsd4_client_record_check(clp) ? nfserr_reclaim_bad : nfs_ok;
+        if (nfsd4_client_record_check(cstate->clp))
+                return nfserr_reclaim_bad;
+
+        return nfs_ok;
 }
 
 #ifdef CONFIG_NFSD_FAULT_INJECTION
+static inline void
+put_client(struct nfs4_client *clp)
+{
+        atomic_dec(&clp->cl_refcount);
+}
 
-u64 nfsd_forget_client(struct nfs4_client *clp, u64 max)
+static struct nfs4_client *
+nfsd_find_client(struct sockaddr_storage *addr, size_t addr_size)
 {
-        if (mark_client_expired(clp))
-                return 0;
-        expire_client(clp);
-        return 1;
+        struct nfs4_client *clp;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                          nfsd_net_id);
+
+        if (!nfsd_netns_ready(nn))
+                return NULL;
+
+        list_for_each_entry(clp, &nn->client_lru, cl_lru) {
+                if (memcmp(&clp->cl_addr, addr, addr_size) == 0)
+                        return clp;
+        }
+        return NULL;
 }
 
-u64 nfsd_print_client(struct nfs4_client *clp, u64 num)
+u64
+nfsd_inject_print_clients(void)
 {
+        struct nfs4_client *clp;
+        u64 count = 0;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                          nfsd_net_id);
         char buf[INET6_ADDRSTRLEN];
-        rpc_ntop((struct sockaddr *)&clp->cl_addr, buf, sizeof(buf));
-        printk(KERN_INFO "NFS Client: %s\n", buf);
-        return 1;
+
+        if (!nfsd_netns_ready(nn))
+                return 0;
+
+        spin_lock(&nn->client_lock);
+        list_for_each_entry(clp, &nn->client_lru, cl_lru) {
+                rpc_ntop((struct sockaddr *)&clp->cl_addr, buf, sizeof(buf));
+                pr_info("NFS Client: %s\n", buf);
+                ++count;
+        }
+        spin_unlock(&nn->client_lock);
+
+        return count;
+}
+
+u64
+nfsd_inject_forget_client(struct sockaddr_storage *addr, size_t addr_size)
+{
+        u64 count = 0;
+        struct nfs4_client *clp;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                          nfsd_net_id);
+
+        if (!nfsd_netns_ready(nn))
+                return count;
+
+        spin_lock(&nn->client_lock);
+        clp = nfsd_find_client(addr, addr_size);
+        if (clp) {
+                if (mark_client_expired_locked(clp) == nfs_ok)
+                        ++count;
+                else
+                        clp = NULL;
+        }
+        spin_unlock(&nn->client_lock);
+
+        if (clp)
+                expire_client(clp);
+
+        return count;
+}
+
+u64
+nfsd_inject_forget_clients(u64 max)
+{
+        u64 count = 0;
+        struct nfs4_client *clp, *next;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
+        LIST_HEAD(reaplist);
+
+        if (!nfsd_netns_ready(nn))
+                return count;
+
+        spin_lock(&nn->client_lock);
+        list_for_each_entry_safe(clp, next, &nn->client_lru, cl_lru) {
+                if (mark_client_expired_locked(clp) == nfs_ok) {
+                        list_add(&clp->cl_lru, &reaplist);
+                        if (max != 0 && ++count >= max)
+                                break;
+                }
+        }
+        spin_unlock(&nn->client_lock);
+
+        list_for_each_entry_safe(clp, next, &reaplist, cl_lru)
+                expire_client(clp);
+
+        return count;
 }
 
 static void nfsd_print_count(struct nfs4_client *clp, unsigned int count,
@@ -4925,158 +5761,484 @@ static void nfsd_print_count(struct nfs4_client *clp, unsigned int count,
         printk(KERN_INFO "NFS Client: %s has %u %s\n", buf, count, type);
 }
 
-static u64 nfsd_foreach_client_lock(struct nfs4_client *clp, u64 max, void (*func)(struct nfs4_lockowner *))
+static void
+nfsd_inject_add_lock_to_list(struct nfs4_ol_stateid *lst,
+                             struct list_head *collect)
+{
+        struct nfs4_client *clp = lst->st_stid.sc_client;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                          nfsd_net_id);
+
+        if (!collect)
+                return;
+
+        lockdep_assert_held(&nn->client_lock);
+        atomic_inc(&clp->cl_refcount);
+        list_add(&lst->st_locks, collect);
+}
+
+static u64 nfsd_foreach_client_lock(struct nfs4_client *clp, u64 max,
+                                    struct list_head *collect,
+                                    void (*func)(struct nfs4_ol_stateid *))
 {
         struct nfs4_openowner *oop;
-        struct nfs4_lockowner *lop, *lo_next;
         struct nfs4_ol_stateid *stp, *st_next;
+        struct nfs4_ol_stateid *lst, *lst_next;
         u64 count = 0;
 
+        spin_lock(&clp->cl_lock);
         list_for_each_entry(oop, &clp->cl_openowners, oo_perclient) {
-                list_for_each_entry_safe(stp, st_next, &oop->oo_owner.so_stateids, st_perstateowner) {
-                        list_for_each_entry_safe(lop, lo_next, &stp->st_lockowners, lo_perstateid) {
-                                if (func)
-                                        func(lop);
-                                if (++count == max)
-                                        return count;
+                list_for_each_entry_safe(stp, st_next,
+                                &oop->oo_owner.so_stateids, st_perstateowner) {
+                        list_for_each_entry_safe(lst, lst_next,
+                                        &stp->st_locks, st_locks) {
+                                if (func) {
+                                        func(lst);
+                                        nfsd_inject_add_lock_to_list(lst,
+                                                                collect);
+                                }
+                                ++count;
+                                /*
+                                 * Despite the fact that these functions deal
+                                 * with 64-bit integers for "count", we must
+                                 * ensure that it doesn't blow up the
+                                 * clp->cl_refcount. Throw a warning if we
+                                 * start to approach INT_MAX here.
+                                 */
+                                WARN_ON_ONCE(count == (INT_MAX / 2));
+                                if (count == max)
+                                        goto out;
                         }
                 }
         }
+out:
+        spin_unlock(&clp->cl_lock);
 
         return count;
 }
 
-u64 nfsd_forget_client_locks(struct nfs4_client *clp, u64 max)
+static u64
+nfsd_collect_client_locks(struct nfs4_client *clp, struct list_head *collect,
+                          u64 max)
 {
-        return nfsd_foreach_client_lock(clp, max, release_lockowner);
+        return nfsd_foreach_client_lock(clp, max, collect, unhash_lock_stateid);
 }
 
-u64 nfsd_print_client_locks(struct nfs4_client *clp, u64 max)
+static u64
+nfsd_print_client_locks(struct nfs4_client *clp)
 {
-        u64 count = nfsd_foreach_client_lock(clp, max, NULL);
+        u64 count = nfsd_foreach_client_lock(clp, 0, NULL, NULL);
         nfsd_print_count(clp, count, "locked files");
         return count;
 }
 
-static u64 nfsd_foreach_client_open(struct nfs4_client *clp, u64 max, void (*func)(struct nfs4_openowner *))
+u64
+nfsd_inject_print_locks(void)
+{
+        struct nfs4_client *clp;
+        u64 count = 0;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
+
+        if (!nfsd_netns_ready(nn))
+                return 0;
+
+        spin_lock(&nn->client_lock);
+        list_for_each_entry(clp, &nn->client_lru, cl_lru)
+                count += nfsd_print_client_locks(clp);
+        spin_unlock(&nn->client_lock);
+
+        return count;
+}
+
+static void
+nfsd_reap_locks(struct list_head *reaplist)
+{
+        struct nfs4_client *clp;
+        struct nfs4_ol_stateid *stp, *next;
+
+        list_for_each_entry_safe(stp, next, reaplist, st_locks) {
+                list_del_init(&stp->st_locks);
+                clp = stp->st_stid.sc_client;
+                nfs4_put_stid(&stp->st_stid);
+                put_client(clp);
+        }
+}
+
+u64
+nfsd_inject_forget_client_locks(struct sockaddr_storage *addr, size_t addr_size)
+{
+        unsigned int count = 0;
+        struct nfs4_client *clp;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
+        LIST_HEAD(reaplist);
+
+        if (!nfsd_netns_ready(nn))
+                return count;
+
+        spin_lock(&nn->client_lock);
+        clp = nfsd_find_client(addr, addr_size);
+        if (clp)
+                count = nfsd_collect_client_locks(clp, &reaplist, 0);
+        spin_unlock(&nn->client_lock);
+        nfsd_reap_locks(&reaplist);
+        return count;
+}
+
+u64
+nfsd_inject_forget_locks(u64 max)
+{
+        u64 count = 0;
+        struct nfs4_client *clp;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
+        LIST_HEAD(reaplist);
+
+        if (!nfsd_netns_ready(nn))
+                return count;
+
+        spin_lock(&nn->client_lock);
+        list_for_each_entry(clp, &nn->client_lru, cl_lru) {
+                count += nfsd_collect_client_locks(clp, &reaplist, max - count);
+                if (max != 0 && count >= max)
+                        break;
+        }
+        spin_unlock(&nn->client_lock);
+        nfsd_reap_locks(&reaplist);
+        return count;
+}
+
+static u64
+nfsd_foreach_client_openowner(struct nfs4_client *clp, u64 max,
+                              struct list_head *collect,
+                              void (*func)(struct nfs4_openowner *))
 {
         struct nfs4_openowner *oop, *next;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
         u64 count = 0;
 
+        lockdep_assert_held(&nn->client_lock);
+
+        spin_lock(&clp->cl_lock);
         list_for_each_entry_safe(oop, next, &clp->cl_openowners, oo_perclient) {
-                if (func)
+                if (func) {
                         func(oop);
-                if (++count == max)
+                        if (collect) {
+                                atomic_inc(&clp->cl_refcount);
+                                list_add(&oop->oo_perclient, collect);
+                        }
+                }
+                ++count;
+                /*
+                 * Despite the fact that these functions deal with
+                 * 64-bit integers for "count", we must ensure that
+                 * it doesn't blow up the clp->cl_refcount. Throw a
+                 * warning if we start to approach INT_MAX here.
+                 */
+                WARN_ON_ONCE(count == (INT_MAX / 2));
+                if (count == max)
                         break;
         }
+        spin_unlock(&clp->cl_lock);
 
         return count;
 }
 
-u64 nfsd_forget_client_openowners(struct nfs4_client *clp, u64 max)
+static u64
+nfsd_print_client_openowners(struct nfs4_client *clp)
 {
-        return nfsd_foreach_client_open(clp, max, release_openowner);
+        u64 count = nfsd_foreach_client_openowner(clp, 0, NULL, NULL);
+
+        nfsd_print_count(clp, count, "openowners");
+        return count;
 }
 
-u64 nfsd_print_client_openowners(struct nfs4_client *clp, u64 max)
+static u64
+nfsd_collect_client_openowners(struct nfs4_client *clp,
+                               struct list_head *collect, u64 max)
 {
-        u64 count = nfsd_foreach_client_open(clp, max, NULL);
-        nfsd_print_count(clp, count, "open files");
-        return count;
+        return nfsd_foreach_client_openowner(clp, max, collect,
+                                                unhash_openowner_locked);
 }
 
-static u64 nfsd_find_all_delegations(struct nfs4_client *clp, u64 max,
-                                     struct list_head *victims)
+u64
+nfsd_inject_print_openowners(void)
 {
-        struct nfs4_delegation *dp, *next;
+        struct nfs4_client *clp;
         u64 count = 0;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
+
+        if (!nfsd_netns_ready(nn))
+                return 0;
+
+        spin_lock(&nn->client_lock);
+        list_for_each_entry(clp, &nn->client_lru, cl_lru)
+                count += nfsd_print_client_openowners(clp);
+        spin_unlock(&nn->client_lock);
 
-        lockdep_assert_held(&state_lock);
-        list_for_each_entry_safe(dp, next, &clp->cl_delegations, dl_perclnt) {
-                if (victims)
-                        list_move(&dp->dl_recall_lru, victims);
-                if (++count == max)
-                        break;
-        }
         return count;
 }
 
-u64 nfsd_forget_client_delegations(struct nfs4_client *clp, u64 max)
+static void
+nfsd_reap_openowners(struct list_head *reaplist)
 {
-        struct nfs4_delegation *dp, *next;
-        LIST_HEAD(victims);
-        u64 count;
+        struct nfs4_client *clp;
+        struct nfs4_openowner *oop, *next;
 
-        spin_lock(&state_lock);
-        count = nfsd_find_all_delegations(clp, max, &victims);
-        spin_unlock(&state_lock);
+        list_for_each_entry_safe(oop, next, reaplist, oo_perclient) {
+                list_del_init(&oop->oo_perclient);
+                clp = oop->oo_owner.so_client;
+                release_openowner(oop);
+                put_client(clp);
+        }
+}
 
-        list_for_each_entry_safe(dp, next, &victims, dl_recall_lru)
-                revoke_delegation(dp);
+u64
+nfsd_inject_forget_client_openowners(struct sockaddr_storage *addr,
+                                     size_t addr_size)
+{
+        unsigned int count = 0;
+        struct nfs4_client *clp;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
+        LIST_HEAD(reaplist);
 
+        if (!nfsd_netns_ready(nn))
+                return count;
+
+        spin_lock(&nn->client_lock);
+        clp = nfsd_find_client(addr, addr_size);
+        if (clp)
+                count = nfsd_collect_client_openowners(clp, &reaplist, 0);
+        spin_unlock(&nn->client_lock);
+        nfsd_reap_openowners(&reaplist);
         return count;
 }
 
-u64 nfsd_recall_client_delegations(struct nfs4_client *clp, u64 max)
+u64
+nfsd_inject_forget_openowners(u64 max)
 {
-        struct nfs4_delegation *dp, *next;
-        LIST_HEAD(victims);
-        u64 count;
+        u64 count = 0;
+        struct nfs4_client *clp;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
+        LIST_HEAD(reaplist);
 
-        spin_lock(&state_lock);
-        count = nfsd_find_all_delegations(clp, max, &victims);
-        list_for_each_entry_safe(dp, next, &victims, dl_recall_lru)
-                nfsd_break_one_deleg(dp);
-        spin_unlock(&state_lock);
+        if (!nfsd_netns_ready(nn))
+                return count;
 
+        spin_lock(&nn->client_lock);
+        list_for_each_entry(clp, &nn->client_lru, cl_lru) {
+                count += nfsd_collect_client_openowners(clp, &reaplist,
+                                                        max - count);
+                if (max != 0 && count >= max)
+                        break;
+        }
+        spin_unlock(&nn->client_lock);
+        nfsd_reap_openowners(&reaplist);
         return count;
 }
 
-u64 nfsd_print_client_delegations(struct nfs4_client *clp, u64 max)
+static u64 nfsd_find_all_delegations(struct nfs4_client *clp, u64 max,
+                                     struct list_head *victims)
 {
+        struct nfs4_delegation *dp, *next;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
         u64 count = 0;
 
+        lockdep_assert_held(&nn->client_lock);
+
         spin_lock(&state_lock);
-        count = nfsd_find_all_delegations(clp, max, NULL);
+        list_for_each_entry_safe(dp, next, &clp->cl_delegations, dl_perclnt) {
+                if (victims) {
+                        /*
+                         * It's not safe to mess with delegations that have a
+                         * non-zero dl_time. They might have already been broken
+                         * and could be processed by the laundromat outside of
+                         * the state_lock. Just leave them be.
+                         */
+                        if (dp->dl_time != 0)
+                                continue;
+
+                        atomic_inc(&clp->cl_refcount);
+                        unhash_delegation_locked(dp);
+                        list_add(&dp->dl_recall_lru, victims);
+                }
+                ++count;
+                /*
+                 * Despite the fact that these functions deal with
+                 * 64-bit integers for "count", we must ensure that
+                 * it doesn't blow up the clp->cl_refcount. Throw a
+                 * warning if we start to approach INT_MAX here.
+                 */
+                WARN_ON_ONCE(count == (INT_MAX / 2));
+                if (count == max)
+                        break;
+        }
         spin_unlock(&state_lock);
+        return count;
+}
+
+static u64
+nfsd_print_client_delegations(struct nfs4_client *clp)
+{
+        u64 count = nfsd_find_all_delegations(clp, 0, NULL);
 
         nfsd_print_count(clp, count, "delegations");
         return count;
 }
 
-u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfs4_client *, u64))
+u64
+nfsd_inject_print_delegations(void)
 {
-        struct nfs4_client *clp, *next;
+        struct nfs4_client *clp;
         u64 count = 0;
-        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns, nfsd_net_id);
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
 
         if (!nfsd_netns_ready(nn))
                 return 0;
 
-        list_for_each_entry_safe(clp, next, &nn->client_lru, cl_lru) {
-                count += func(clp, max - count);
-                if ((max != 0) && (count >= max))
-                        break;
+        spin_lock(&nn->client_lock);
+        list_for_each_entry(clp, &nn->client_lru, cl_lru)
+                count += nfsd_print_client_delegations(clp);
+        spin_unlock(&nn->client_lock);
+
+        return count;
+}
+
+static void
+nfsd_forget_delegations(struct list_head *reaplist)
+{
+        struct nfs4_client *clp;
+        struct nfs4_delegation *dp, *next;
+
+        list_for_each_entry_safe(dp, next, reaplist, dl_recall_lru) {
+                list_del_init(&dp->dl_recall_lru);
+                clp = dp->dl_stid.sc_client;
+                revoke_delegation(dp);
+                put_client(clp);
         }
+}
 
+u64
+nfsd_inject_forget_client_delegations(struct sockaddr_storage *addr,
+                                      size_t addr_size)
+{
+        u64 count = 0;
+        struct nfs4_client *clp;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
+        LIST_HEAD(reaplist);
+
+        if (!nfsd_netns_ready(nn))
+                return count;
+
+        spin_lock(&nn->client_lock);
+        clp = nfsd_find_client(addr, addr_size);
+        if (clp)
+                count = nfsd_find_all_delegations(clp, 0, &reaplist);
+        spin_unlock(&nn->client_lock);
+
+        nfsd_forget_delegations(&reaplist);
         return count;
 }
 
-struct nfs4_client *nfsd_find_client(struct sockaddr_storage *addr, size_t addr_size)
+u64
+nfsd_inject_forget_delegations(u64 max)
 {
+        u64 count = 0;
         struct nfs4_client *clp;
-        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns, nfsd_net_id);
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
+        LIST_HEAD(reaplist);
 
         if (!nfsd_netns_ready(nn))
-                return NULL;
+                return count;
 
+        spin_lock(&nn->client_lock);
         list_for_each_entry(clp, &nn->client_lru, cl_lru) {
-                if (memcmp(&clp->cl_addr, addr, addr_size) == 0)
-                        return clp;
+                count += nfsd_find_all_delegations(clp, max - count, &reaplist);
+                if (max != 0 && count >= max)
+                        break;
         }
-        return NULL;
+        spin_unlock(&nn->client_lock);
+        nfsd_forget_delegations(&reaplist);
+        return count;
 }
 
+static void
+nfsd_recall_delegations(struct list_head *reaplist)
+{
+        struct nfs4_client *clp;
+        struct nfs4_delegation *dp, *next;
+
+        list_for_each_entry_safe(dp, next, reaplist, dl_recall_lru) {
+                list_del_init(&dp->dl_recall_lru);
+                clp = dp->dl_stid.sc_client;
+                /*
+                 * We skipped all entries that had a zero dl_time before,
+                 * so we can now reset the dl_time back to 0. If a delegation
+                 * break comes in now, then it won't make any difference since
+                 * we're recalling it either way.
+                 */
+                spin_lock(&state_lock);
+                dp->dl_time = 0;
+                spin_unlock(&state_lock);
+                nfsd_break_one_deleg(dp);
+                put_client(clp);
+        }
+}
+
+u64
+nfsd_inject_recall_client_delegations(struct sockaddr_storage *addr,
+                                      size_t addr_size)
+{
+        u64 count = 0;
+        struct nfs4_client *clp;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
+        LIST_HEAD(reaplist);
+
+        if (!nfsd_netns_ready(nn))
+                return count;
+
+        spin_lock(&nn->client_lock);
+        clp = nfsd_find_client(addr, addr_size);
+        if (clp)
+                count = nfsd_find_all_delegations(clp, 0, &reaplist);
+        spin_unlock(&nn->client_lock);
+
+        nfsd_recall_delegations(&reaplist);
+        return count;
+}
+
+u64
+nfsd_inject_recall_delegations(u64 max)
+{
+        u64 count = 0;
+        struct nfs4_client *clp, *next;
+        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
+                                                nfsd_net_id);
+        LIST_HEAD(reaplist);
+
+        if (!nfsd_netns_ready(nn))
+                return count;
+
+        spin_lock(&nn->client_lock);
+        list_for_each_entry_safe(clp, next, &nn->client_lru, cl_lru) {
+                count += nfsd_find_all_delegations(clp, max - count, &reaplist);
+                if (max != 0 && ++count >= max)
+                        break;
+        }
+        spin_unlock(&nn->client_lock);
+        nfsd_recall_delegations(&reaplist);
+        return count;
+}
 #endif /* CONFIG_NFSD_FAULT_INJECTION */
 
 /*
@@ -5113,14 +6275,6 @@ static int nfs4_state_create_net(struct net *net)
                         CLIENT_HASH_SIZE, GFP_KERNEL);
         if (!nn->unconf_id_hashtbl)
                 goto err_unconf_id;
-        nn->ownerstr_hashtbl = kmalloc(sizeof(struct list_head) *
-                        OWNER_HASH_SIZE, GFP_KERNEL);
-        if (!nn->ownerstr_hashtbl)
-                goto err_ownerstr;
-        nn->lockowner_ino_hashtbl = kmalloc(sizeof(struct list_head) *
-                        LOCKOWNER_INO_HASH_SIZE, GFP_KERNEL);
-        if (!nn->lockowner_ino_hashtbl)
-                goto err_lockowner_ino;
         nn->sessionid_hashtbl = kmalloc(sizeof(struct list_head) *
                         SESSION_HASH_SIZE, GFP_KERNEL);
         if (!nn->sessionid_hashtbl)
@@ -5130,10 +6284,6 @@ static int nfs4_state_create_net(struct net *net)
                 INIT_LIST_HEAD(&nn->conf_id_hashtbl[i]);
                 INIT_LIST_HEAD(&nn->unconf_id_hashtbl[i]);
         }
-        for (i = 0; i < OWNER_HASH_SIZE; i++)
-                INIT_LIST_HEAD(&nn->ownerstr_hashtbl[i]);
-        for (i = 0; i < LOCKOWNER_INO_HASH_SIZE; i++)
-                INIT_LIST_HEAD(&nn->lockowner_ino_hashtbl[i]);
         for (i = 0; i < SESSION_HASH_SIZE; i++)
                 INIT_LIST_HEAD(&nn->sessionid_hashtbl[i]);
         nn->conf_name_tree = RB_ROOT;
@@ -5149,10 +6299,6 @@ static int nfs4_state_create_net(struct net *net)
         return 0;
 
 err_sessionid:
-        kfree(nn->lockowner_ino_hashtbl);
-err_lockowner_ino:
-        kfree(nn->ownerstr_hashtbl);
-err_ownerstr:
         kfree(nn->unconf_id_hashtbl);
 err_unconf_id:
         kfree(nn->conf_id_hashtbl);
@@ -5182,8 +6328,6 @@ nfs4_state_destroy_net(struct net *net)
         }
 
         kfree(nn->sessionid_hashtbl);
-        kfree(nn->lockowner_ino_hashtbl);
-        kfree(nn->ownerstr_hashtbl);
         kfree(nn->unconf_id_hashtbl);
         kfree(nn->conf_id_hashtbl);
         put_net(net);
@@ -5247,22 +6391,22 @@ nfs4_state_shutdown_net(struct net *net)
         cancel_delayed_work_sync(&nn->laundromat_work);
         locks_end_grace(&nn->nfsd4_manager);
 
-        nfs4_lock_state();
         INIT_LIST_HEAD(&reaplist);
         spin_lock(&state_lock);
         list_for_each_safe(pos, next, &nn->del_recall_lru) {
                 dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru);
-                list_move(&dp->dl_recall_lru, &reaplist);
+                unhash_delegation_locked(dp);
+                list_add(&dp->dl_recall_lru, &reaplist);
         }
         spin_unlock(&state_lock);
         list_for_each_safe(pos, next, &reaplist) {
                 dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru);
-                destroy_delegation(dp);
+                list_del_init(&dp->dl_recall_lru);
+                nfs4_put_stid(&dp->dl_stid);
         }
 
         nfsd4_client_tracking_exit(net);
         nfs4_state_destroy_net(net);
-        nfs4_unlock_state();
 }
 
 void
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 944275c8f56d..f9821ce6658a 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -181,28 +181,43 @@ static int zero_clientid(clientid_t *clid)
 }
 
 /**
- * defer_free - mark an allocation as deferred freed
- * @argp: NFSv4 compound argument structure to be freed with
- * @release: release callback to free @p, typically kfree()
- * @p: pointer to be freed
+ * svcxdr_tmpalloc - allocate memory to be freed after compound processing
+ * @argp: NFSv4 compound argument structure
+ * @p: pointer to be freed (with kfree())
  *
  * Marks @p to be freed when processing the compound operation
  * described in @argp finishes.
  */
-static int
-defer_free(struct nfsd4_compoundargs *argp,
-                void (*release)(const void *), void *p)
+static void *
+svcxdr_tmpalloc(struct nfsd4_compoundargs *argp, u32 len)
 {
-        struct tmpbuf *tb;
+        struct svcxdr_tmpbuf *tb;
 
-        tb = kmalloc(sizeof(*tb), GFP_KERNEL);
+        tb = kmalloc(sizeof(*tb) + len, GFP_KERNEL);
         if (!tb)
-                return -ENOMEM;
-        tb->buf = p;
-        tb->release = release;
+                return NULL;
         tb->next = argp->to_free;
         argp->to_free = tb;
-        return 0;
+        return tb->buf;
+}
+
+/*
+ * For xdr strings that need to be passed to other kernel api's
+ * as null-terminated strings.
+ *
+ * Note null-terminating in place usually isn't safe since the
+ * buffer might end on a page boundary.
+ */
+static char *
+svcxdr_dupstr(struct nfsd4_compoundargs *argp, void *buf, u32 len)
+{
+        char *p = svcxdr_tmpalloc(argp, len + 1);
+
+        if (!p)
+                return NULL;
+        memcpy(p, buf, len);
+        p[len] = '\0';
+        return p;
 }
 
 /**
@@ -217,19 +232,13 @@ defer_free(struct nfsd4_compoundargs *argp,
  */
 static char *savemem(struct nfsd4_compoundargs *argp, __be32 *p, int nbytes)
 {
-        if (p == argp->tmp) {
-                p = kmemdup(argp->tmp, nbytes, GFP_KERNEL);
-                if (!p)
-                        return NULL;
-        } else {
-                BUG_ON(p != argp->tmpp);
-                argp->tmpp = NULL;
-        }
-        if (defer_free(argp, kfree, p)) {
-                kfree(p);
+        void *ret;
+
+        ret = svcxdr_tmpalloc(argp, nbytes);
+        if (!ret)
                 return NULL;
-        } else
-                return (char *)p;
+        memcpy(ret, p, nbytes);
+        return ret;
 }
 
 static __be32
@@ -292,12 +301,10 @@ nfsd4_decode_fattr(struct nfsd4_compoundargs *argp, u32 *bmval,
                 if (nace > NFS4_ACL_MAX)
                         return nfserr_fbig;
 
-                *acl = nfs4_acl_new(nace);
+                *acl = svcxdr_tmpalloc(argp, nfs4_acl_bytes(nace));
                 if (*acl == NULL)
                         return nfserr_jukebox;
 
-                defer_free(argp, kfree, *acl);
-
                 (*acl)->naces = nace;
                 for (ace = (*acl)->aces; ace < (*acl)->aces + nace; ace++) {
                         READ_BUF(16); len += 16;
@@ -418,12 +425,10 @@ nfsd4_decode_fattr(struct nfsd4_compoundargs *argp, u32 *bmval,
                         return nfserr_badlabel;
                 len += (XDR_QUADLEN(dummy32) << 2);
                 READMEM(buf, dummy32);
-                label->data = kzalloc(dummy32 + 1, GFP_KERNEL);
+                label->len = dummy32;
+                label->data = svcxdr_dupstr(argp, buf, dummy32);
                 if (!label->data)
                         return nfserr_jukebox;
-                label->len = dummy32;
-                defer_free(argp, kfree, label->data);
-                memcpy(label->data, buf, dummy32);
         }
 #endif
 
@@ -598,20 +603,11 @@ nfsd4_decode_create(struct nfsd4_compoundargs *argp, struct nfsd4_create *create
         switch (create->cr_type) {
         case NF4LNK:
                 READ_BUF(4);
-                create->cr_linklen = be32_to_cpup(p++);
-                READ_BUF(create->cr_linklen);
-                /*
-                 * The VFS will want a null-terminated string, and
-                 * null-terminating in place isn't safe since this might
-                 * end on a page boundary:
-                 */
-                create->cr_linkname =
-                                kmalloc(create->cr_linklen + 1, GFP_KERNEL);
-                if (!create->cr_linkname)
+                create->cr_datalen = be32_to_cpup(p++);
+                READ_BUF(create->cr_datalen);
+                create->cr_data = svcxdr_dupstr(argp, p, create->cr_datalen);
+                if (!create->cr_data)
                         return nfserr_jukebox;
-                memcpy(create->cr_linkname, p, create->cr_linklen);
-                create->cr_linkname[create->cr_linklen] = '\0';
-                defer_free(argp, kfree, create->cr_linkname);
                 break;
         case NF4BLK:
         case NF4CHR:
@@ -1481,13 +1477,12 @@ nfsd4_decode_test_stateid(struct nfsd4_compoundargs *argp, struct nfsd4_test_sta
         INIT_LIST_HEAD(&test_stateid->ts_stateid_list);
 
         for (i = 0; i < test_stateid->ts_num_ids; i++) {
-                stateid = kmalloc(sizeof(struct nfsd4_test_stateid_id), GFP_KERNEL);
+                stateid = svcxdr_tmpalloc(argp, sizeof(*stateid));
                 if (!stateid) {
                         status = nfserrno(-ENOMEM);
                         goto out;
                 }
 
-                defer_free(argp, kfree, stateid);
                 INIT_LIST_HEAD(&stateid->ts_id_list);
                 list_add_tail(&stateid->ts_id_list, &test_stateid->ts_stateid_list);
 
@@ -1640,7 +1635,7 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp)
                 goto xdr_error;
 
         if (argp->opcnt > ARRAY_SIZE(argp->iops)) {
-                argp->ops = kmalloc(argp->opcnt * sizeof(*argp->ops), GFP_KERNEL);
+                argp->ops = kzalloc(argp->opcnt * sizeof(*argp->ops), GFP_KERNEL);
                 if (!argp->ops) {
                         argp->ops = argp->iops;
                         dprintk("nfsd: couldn't allocate room for COMPOUND\n");
@@ -3077,11 +3072,8 @@ static __be32 nfsd4_encode_splice_read(
         __be32 nfserr;
         __be32 *p = xdr->p - 2;
 
-        /*
-         * Don't inline pages unless we know there's room for eof,
-         * count, and possible padding:
-         */
-        if (xdr->end - xdr->p < 3)
+        /* Make sure there will be room for padding if needed */
+        if (xdr->end - xdr->p < 1)
                 return nfserr_resource;
 
         nfserr = nfsd_splice_read(read->rd_rqstp, file,
@@ -3147,9 +3139,7 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
         len = maxcount;
         v = 0;
 
-        thislen = (void *)xdr->end - (void *)xdr->p;
-        if (len < thislen)
-                thislen = len;
+        thislen = min_t(long, len, ((void *)xdr->end - (void *)xdr->p));
         p = xdr_reserve_space(xdr, (thislen+3)&~3);
         WARN_ON_ONCE(!p);
         resp->rqstp->rq_vec[v].iov_base = p;
@@ -3216,10 +3206,8 @@ nfsd4_encode_read(struct nfsd4_compoundres *resp, __be32 nfserr,
         xdr_commit_encode(xdr);
 
         maxcount = svc_max_payload(resp->rqstp);
-        if (maxcount > xdr->buf->buflen - xdr->buf->len)
-                maxcount = xdr->buf->buflen - xdr->buf->len;
-        if (maxcount > read->rd_length)
-                maxcount = read->rd_length;
+        maxcount = min_t(unsigned long, maxcount, (xdr->buf->buflen - xdr->buf->len));
+        maxcount = min_t(unsigned long, maxcount, read->rd_length);
 
         if (!read->rd_filp) {
                 err = nfsd_get_tmp_read_open(resp->rqstp, read->rd_fhp,
@@ -3937,8 +3925,6 @@ status:
  * 
  * XDR note: do not encode rp->rp_buflen: the buffer contains the
  * previously sent already encoded operation.
- *
- * called with nfs4_lock_state() held
  */
 void
 nfsd4_encode_replay(struct xdr_stream *xdr, struct nfsd4_op *op)
@@ -3977,9 +3963,8 @@ int nfsd4_release_compoundargs(void *rq, __be32 *p, void *resp)
         kfree(args->tmpp);
         args->tmpp = NULL;
         while (args->to_free) {
-                struct tmpbuf *tb = args->to_free;
+                struct svcxdr_tmpbuf *tb = args->to_free;
                 args->to_free = tb->next;
-                tb->release(tb->buf);
                 kfree(tb);
         }
         return 1;
@@ -4012,7 +3997,6 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
         /*
          * All that remains is to write the tag and operation count...
          */
-        struct nfsd4_compound_state *cs = &resp->cstate;
         struct xdr_buf *buf = resp->xdr.buf;
 
         WARN_ON_ONCE(buf->len != buf->head[0].iov_len + buf->page_len +
@@ -4026,19 +4010,7 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
         p += XDR_QUADLEN(resp->taglen);
         *p++ = htonl(resp->opcnt);
 
-        if (nfsd4_has_session(cs)) {
-                struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
-                struct nfs4_client *clp = cs->session->se_client;
-                if (cs->status != nfserr_replay_cache) {
-                        nfsd4_store_cache_entry(resp);
-                        cs->slot->sl_flags &= ~NFSD4_SLOT_INUSE;
-                }
-                /* Renew the clientid on success and on replay */
-                spin_lock(&nn->client_lock);
-                nfsd4_put_session(cs->session);
-                spin_unlock(&nn->client_lock);
-                put_client_renew(clp);
-        }
+        nfsd4_sequence_done(resp);
         return 1;
 }
 
diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c
index 6040da8830ff..ff9567633245 100644
--- a/fs/nfsd/nfscache.c
+++ b/fs/nfsd/nfscache.c
@@ -221,7 +221,12 @@ static void
 hash_refile(struct svc_cacherep *rp)
 {
         hlist_del_init(&rp->c_hash);
-        hlist_add_head(&rp->c_hash, cache_hash + hash_32(rp->c_xid, maskbits));
+        /*
+         * No point in byte swapping c_xid since we're just using it to pick
+         * a hash bucket.
+         */
+        hlist_add_head(&rp->c_hash, cache_hash +
+                        hash_32((__force u32)rp->c_xid, maskbits));
 }
 
 /*
@@ -356,7 +361,11 @@ nfsd_cache_search(struct svc_rqst *rqstp, __wsum csum)
         struct hlist_head       *rh;
         unsigned int            entries = 0;
 
-        rh = &cache_hash[hash_32(rqstp->rq_xid, maskbits)];
+        /*
+         * No point in byte swapping rq_xid since we're just using it to pick
+         * a hash bucket.
+         */
+        rh = &cache_hash[hash_32((__force u32)rqstp->rq_xid, maskbits)];
         hlist_for_each_entry(rp, rh, c_hash) {
                 ++entries;
                 if (nfsd_cache_match(rqstp, csum, rp)) {
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index 51844048937f..4e042105fb6e 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -39,6 +39,7 @@ enum {
         NFSD_Versions,
         NFSD_Ports,
         NFSD_MaxBlkSize,
+        NFSD_MaxConnections,
         NFSD_SupportedEnctypes,
         /*
          * The below MUST come last.  Otherwise we leave a hole in nfsd_files[]
@@ -62,6 +63,7 @@ static ssize_t write_pool_threads(struct file *file, char *buf, size_t size);
 static ssize_t write_versions(struct file *file, char *buf, size_t size);
 static ssize_t write_ports(struct file *file, char *buf, size_t size);
 static ssize_t write_maxblksize(struct file *file, char *buf, size_t size);
+static ssize_t write_maxconn(struct file *file, char *buf, size_t size);
 #ifdef CONFIG_NFSD_V4
 static ssize_t write_leasetime(struct file *file, char *buf, size_t size);
 static ssize_t write_gracetime(struct file *file, char *buf, size_t size);
@@ -77,6 +79,7 @@ static ssize_t (*write_op[])(struct file *, char *, size_t) = {
         [NFSD_Versions] = write_versions,
         [NFSD_Ports] = write_ports,
         [NFSD_MaxBlkSize] = write_maxblksize,
+        [NFSD_MaxConnections] = write_maxconn,
 #ifdef CONFIG_NFSD_V4
         [NFSD_Leasetime] = write_leasetime,
         [NFSD_Gracetime] = write_gracetime,
@@ -369,8 +372,7 @@ static ssize_t write_filehandle(struct file *file, char *buf, size_t size)
 
         if (maxsize < NFS_FHSIZE)
                 return -EINVAL;
-        if (maxsize > NFS3_FHSIZE)
-                maxsize = NFS3_FHSIZE;
+        maxsize = min(maxsize, NFS3_FHSIZE);
 
         if (qword_get(&mesg, mesg, size)>0)
                 return -EINVAL;
@@ -871,10 +873,8 @@ static ssize_t write_maxblksize(struct file *file, char *buf, size_t size)
                 /* force bsize into allowed range and
                  * required alignment.
                  */
-                if (bsize < 1024)
-                        bsize = 1024;
-                if (bsize > NFSSVC_MAXBLKSIZE)
-                        bsize = NFSSVC_MAXBLKSIZE;
+                bsize = max_t(int, bsize, 1024);
+                bsize = min_t(int, bsize, NFSSVC_MAXBLKSIZE);
                 bsize &= ~(1024-1);
                 mutex_lock(&nfsd_mutex);
                 if (nn->nfsd_serv) {
@@ -889,6 +889,44 @@ static ssize_t write_maxblksize(struct file *file, char *buf, size_t size)
                                                         nfsd_max_blksize);
 }
 
+/**
+ * write_maxconn - Set or report the current max number of connections
+ *
+ * Input:
+ *                      buf:            ignored
+ *                      size:           zero
+ * OR
+ *
+ * Input:
+ *                      buf:            C string containing an unsigned
+ *                                      integer value representing the new
+ *                                      number of max connections
+ *                      size:           non-zero length of C string in @buf
+ * Output:
+ *      On success:     passed-in buffer filled with '\n'-terminated C string
+ *                      containing numeric value of max_connections setting
+ *                      for this net namespace;
+ *                      return code is the size in bytes of the string
+ *      On error:       return code is zero or a negative errno value
+ */
+static ssize_t write_maxconn(struct file *file, char *buf, size_t size)
+{
+        char *mesg = buf;
+        struct net *net = file->f_dentry->d_sb->s_fs_info;
+        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
+        unsigned int maxconn = nn->max_connections;
+
+        if (size > 0) {
+                int rv = get_uint(&mesg, &maxconn);
+
+                if (rv)
+                        return rv;
+                nn->max_connections = maxconn;
+        }
+
+        return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%u\n", maxconn);
+}
+
 #ifdef CONFIG_NFSD_V4
 static ssize_t __nfsd4_write_time(struct file *file, char *buf, size_t size,
                                   time_t *time, struct nfsd_net *nn)
@@ -1064,6 +1102,7 @@ static int nfsd_fill_super(struct super_block * sb, void * data, int silent)
                 [NFSD_Versions] = {"versions", &transaction_ops, S_IWUSR|S_IRUSR},
                 [NFSD_Ports] = {"portlist", &transaction_ops, S_IWUSR|S_IRUGO},
                 [NFSD_MaxBlkSize] = {"max_block_size", &transaction_ops, S_IWUSR|S_IRUGO},
+                [NFSD_MaxConnections] = {"max_connections", &transaction_ops, S_IWUSR|S_IRUGO},
 #if defined(CONFIG_SUNRPC_GSS) || defined(CONFIG_SUNRPC_GSS_MODULE)
                 [NFSD_SupportedEnctypes] = {"supported_krb5_enctypes", &supported_enctypes_ops, S_IRUGO},
 #endif /* CONFIG_SUNRPC_GSS or CONFIG_SUNRPC_GSS_MODULE */
diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c
index ec8393418154..e883a5868be6 100644
--- a/fs/nfsd/nfsfh.c
+++ b/fs/nfsd/nfsfh.c
@@ -162,7 +162,14 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp)
                         /* deprecated, convert to type 3 */
                         len = key_len(FSID_ENCODE_DEV)/4;
                         fh->fh_fsid_type = FSID_ENCODE_DEV;
-                        fh->fh_fsid[0] = new_encode_dev(MKDEV(ntohl(fh->fh_fsid[0]), ntohl(fh->fh_fsid[1])));
+                        /*
+                         * struct knfsd_fh uses host-endian fields, which are
+                         * sometimes used to hold net-endian values. This
+                         * confuses sparse, so we must use __force here to
+                         * keep it from complaining.
+                         */
+                        fh->fh_fsid[0] = new_encode_dev(MKDEV(ntohl((__force __be32)fh->fh_fsid[0]),
+                                                        ntohl((__force __be32)fh->fh_fsid[1])));
                         fh->fh_fsid[1] = fh->fh_fsid[2];
                 }
                 data_left -= len;
@@ -539,8 +546,7 @@ fh_compose(struct svc_fh *fhp, struct svc_export *exp, struct dentry *dentry,
                        dentry);
 
         fhp->fh_dentry = dget(dentry); /* our internal copy */
-        fhp->fh_export = exp;
-        cache_get(&exp->h);
+        fhp->fh_export = exp_get(exp);
 
         if (fhp->fh_handle.fh_version == 0xca) {
                 /* old style filehandle please */
diff --git a/fs/nfsd/nfsfh.h b/fs/nfsd/nfsfh.h
index 2e89e70ac15c..08236d70c667 100644
--- a/fs/nfsd/nfsfh.h
+++ b/fs/nfsd/nfsfh.h
@@ -73,8 +73,15 @@ enum fsid_source {
 extern enum fsid_source fsid_source(struct svc_fh *fhp);
 
 
-/* This might look a little large to "inline" but in all calls except
+/*
+ * This might look a little large to "inline" but in all calls except
  * one, 'vers' is constant so moste of the function disappears.
+ *
+ * In some cases the values are considered to be host endian and in
+ * others, net endian. fsidv is always considered to be u32 as the
+ * callers don't know which it will be. So we must use __force to keep
+ * sparse from complaining. Since these values are opaque to the
+ * client, that shouldn't be a problem.
  */
 static inline void mk_fsid(int vers, u32 *fsidv, dev_t dev, ino_t ino,
                            u32 fsid, unsigned char *uuid)
@@ -82,7 +89,7 @@ static inline void mk_fsid(int vers, u32 *fsidv, dev_t dev, ino_t ino,
         u32 *up;
         switch(vers) {
         case FSID_DEV:
-                fsidv[0] = htonl((MAJOR(dev)<<16) |
+                fsidv[0] = (__force __u32)htonl((MAJOR(dev)<<16) |
                                  MINOR(dev));
                 fsidv[1] = ino_t_to_u32(ino);
                 break;
@@ -90,8 +97,8 @@ static inline void mk_fsid(int vers, u32 *fsidv, dev_t dev, ino_t ino,
                 fsidv[0] = fsid;
                 break;
         case FSID_MAJOR_MINOR:
-                fsidv[0] = htonl(MAJOR(dev));
-                fsidv[1] = htonl(MINOR(dev));
+                fsidv[0] = (__force __u32)htonl(MAJOR(dev));
+                fsidv[1] = (__force __u32)htonl(MINOR(dev));
                 fsidv[2] = ino_t_to_u32(ino);
                 break;
 
diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c
index 54c6b3d3cc79..b8680738f588 100644
--- a/fs/nfsd/nfsproc.c
+++ b/fs/nfsd/nfsproc.c
@@ -403,12 +403,13 @@ nfsd_proc_symlink(struct svc_rqst *rqstp, struct nfsd_symlinkargs *argp,
 
         fh_init(&newfh, NFS_FHSIZE);
         /*
-         * Create the link, look up new file and set attrs.
+         * Crazy hack: the request fits in a page, and already-decoded
+         * attributes follow argp->tname, so it's safe to just write a
+         * null to ensure it's null-terminated:
          */
+        argp->tname[argp->tlen] = '\0';
         nfserr = nfsd_symlink(rqstp, &argp->ffh, argp->fname, argp->flen,
-                                                 argp->tname, argp->tlen,
-                                                 &newfh, &argp->attrs);
-
+                                                 argp->tname, &newfh);
 
         fh_put(&argp->ffh);
         fh_put(&newfh);
@@ -716,6 +717,7 @@ nfserrno (int errno)
                 { nfserr_noent, -ENOENT },
                 { nfserr_io, -EIO },
                 { nfserr_nxio, -ENXIO },
+                { nfserr_fbig, -E2BIG },
                 { nfserr_acces, -EACCES },
                 { nfserr_exist, -EEXIST },
                 { nfserr_xdev, -EXDEV },
@@ -743,6 +745,7 @@ nfserrno (int errno)
                 { nfserr_notsupp, -EOPNOTSUPP },
                 { nfserr_toosmall, -ETOOSMALL },
                 { nfserr_serverfault, -ESERVERFAULT },
+                { nfserr_serverfault, -ENFILE },
         };
         int     i;
 
@@ -750,7 +753,7 @@ nfserrno (int errno)
                 if (nfs_errtbl[i].syserr == errno)
                         return nfs_errtbl[i].nfserr;
         }
-        printk (KERN_INFO "nfsd: non-standard errno: %d\n", errno);
+        WARN(1, "nfsd: non-standard errno: %d\n", errno);
         return nfserr_io;
 }
 
diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index 1879e43f2868..752d56bbe0ba 100644
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -221,7 +221,8 @@ static int nfsd_startup_generic(int nrservs)
          */
         ret = nfsd_racache_init(2*nrservs);
         if (ret)
-                return ret;
+                goto dec_users;
+
         ret = nfs4_state_start();
         if (ret)
                 goto out_racache;
@@ -229,6 +230,8 @@ static int nfsd_startup_generic(int nrservs)
 
 out_racache:
         nfsd_racache_shutdown();
+dec_users:
+        nfsd_users--;
         return ret;
 }
 
@@ -405,6 +408,7 @@ int nfsd_create_serv(struct net *net)
         if (nn->nfsd_serv == NULL)
                 return -ENOMEM;
 
+        nn->nfsd_serv->sv_maxconn = nn->max_connections;
         error = svc_bind(nn->nfsd_serv, net);
         if (error < 0) {
                 svc_destroy(nn->nfsd_serv);
@@ -469,8 +473,7 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net)
         /* enforce a global maximum number of threads */
         tot = 0;
         for (i = 0; i < n; i++) {
-                if (nthreads[i] > NFSD_MAXSERVS)
-                        nthreads[i] = NFSD_MAXSERVS;
+                nthreads[i] = min(nthreads[i], NFSD_MAXSERVS);
                 tot += nthreads[i];
         }
         if (tot > NFSD_MAXSERVS) {
@@ -519,11 +522,11 @@ nfsd_svc(int nrservs, struct net *net)
 
         mutex_lock(&nfsd_mutex);
         dprintk("nfsd: creating service\n");
-        if (nrservs <= 0)
-                nrservs = 0;
-        if (nrservs > NFSD_MAXSERVS)
-                nrservs = NFSD_MAXSERVS;
+
+        nrservs = max(nrservs, 0);
+        nrservs = min(nrservs, NFSD_MAXSERVS);
         error = 0;
+
         if (nrservs == 0 && nn->nfsd_serv == NULL)
                 goto out;
 
@@ -564,6 +567,7 @@ nfsd(void *vrqstp)
         struct svc_rqst *rqstp = (struct svc_rqst *) vrqstp;
         struct svc_xprt *perm_sock = list_entry(rqstp->rq_server->sv_permsocks.next, typeof(struct svc_xprt), xpt_list);
         struct net *net = perm_sock->xpt_net;
+        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
         int err;
 
         /* Lock module and set up kernel thread */
@@ -597,6 +601,9 @@ nfsd(void *vrqstp)
          * The main request loop
          */
         for (;;) {
+                /* Update sv_maxconn if it has changed */
+                rqstp->rq_server->sv_maxconn = nn->max_connections;
+
                 /*
                  * Find a socket with data available and call its
                  * recvfrom routine.
diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c
index 1ac306b769df..412d7061f9e5 100644
--- a/fs/nfsd/nfsxdr.c
+++ b/fs/nfsd/nfsxdr.c
@@ -257,8 +257,7 @@ nfssvc_decode_readargs(struct svc_rqst *rqstp, __be32 *p,
         len = args->count     = ntohl(*p++);
         p++; /* totalcount - unused */
 
-        if (len > NFSSVC_MAXBLKSIZE_V2)
-                len = NFSSVC_MAXBLKSIZE_V2;
+        len = min_t(unsigned int, len, NFSSVC_MAXBLKSIZE_V2);
 
         /* set up somewhere to store response.
          * We take pages, put them on reslist and include in iovec
@@ -268,7 +267,7 @@ nfssvc_decode_readargs(struct svc_rqst *rqstp, __be32 *p,
                 struct page *p = *(rqstp->rq_next_page++);
 
                 rqstp->rq_vec[v].iov_base = page_address(p);
-                rqstp->rq_vec[v].iov_len = len < PAGE_SIZE?len:PAGE_SIZE;
+                rqstp->rq_vec[v].iov_len = min_t(unsigned int, len, PAGE_SIZE);
                 len -= rqstp->rq_vec[v].iov_len;
                 v++;
         }
@@ -400,9 +399,7 @@ nfssvc_decode_readdirargs(struct svc_rqst *rqstp, __be32 *p,
                 return 0;
         args->cookie = ntohl(*p++);
         args->count  = ntohl(*p++);
-        if (args->count > PAGE_SIZE)
-                args->count = PAGE_SIZE;
-
+        args->count  = min_t(u32, args->count, PAGE_SIZE);
         args->buffer = page_address(*(rqstp->rq_next_page++));
 
         return xdr_argsize_check(rqstp, p);
@@ -516,10 +513,11 @@ nfssvc_encode_entry(void *ccdv, const char *name,
         }
         if (cd->offset)
                 *cd->offset = htonl(offset);
-        if (namlen > NFS2_MAXNAMLEN)
-                namlen = NFS2_MAXNAMLEN;/* truncate filename */
 
+        /* truncate filename */
+        namlen = min(namlen, NFS2_MAXNAMLEN);
         slen = XDR_QUADLEN(namlen);
+
         if ((buflen = cd->buflen - slen - 4) < 0) {
                 cd->common.err = nfserr_toosmall;
                 return -EINVAL;
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 374c66283ac5..4a89e00d7461 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -72,7 +72,13 @@ struct nfsd4_callback {
         bool cb_done;
 };
 
+/*
+ * A core object that represents a "common" stateid. These are generally
+ * embedded within the different (more specific) stateid objects and contain
+ * fields that are of general use to any stateid.
+ */
 struct nfs4_stid {
+        atomic_t sc_count;
 #define NFS4_OPEN_STID 1
 #define NFS4_LOCK_STID 2
 #define NFS4_DELEG_STID 4
@@ -80,22 +86,43 @@ struct nfs4_stid {
 #define NFS4_CLOSED_STID 8
 /* For a deleg stateid kept around only to process free_stateid's: */
 #define NFS4_REVOKED_DELEG_STID 16
+#define NFS4_CLOSED_DELEG_STID 32
         unsigned char sc_type;
         stateid_t sc_stateid;
         struct nfs4_client *sc_client;
+        struct nfs4_file *sc_file;
+        void (*sc_free)(struct nfs4_stid *);
 };
 
+/*
+ * Represents a delegation stateid. The nfs4_client holds references to these
+ * and they are put when it is being destroyed or when the delegation is
+ * returned by the client:
+ *
+ * o 1 reference as long as a delegation is still in force (taken when it's
+ *   alloc'd, put when it's returned or revoked)
+ *
+ * o 1 reference as long as a recall rpc is in progress (taken when the lease
+ *   is broken, put when the rpc exits)
+ *
+ * o 1 more ephemeral reference for each nfsd thread currently doing something
+ *   with that delegation without holding the cl_lock
+ *
+ * If the server attempts to recall a delegation and the client doesn't do so
+ * before a timeout, the server may also revoke the delegation. In that case,
+ * the object will either be destroyed (v4.0) or moved to a per-client list of
+ * revoked delegations (v4.1+).
+ *
+ * This object is a superset of the nfs4_stid.
+ */
 struct nfs4_delegation {
         struct nfs4_stid        dl_stid; /* must be first field */
         struct list_head        dl_perfile;
         struct list_head        dl_perclnt;
         struct list_head        dl_recall_lru;  /* delegation recalled */
-        atomic_t                dl_count;       /* ref count */
-        struct nfs4_file        *dl_file;
         u32                     dl_type;
         time_t                  dl_time;
 /* For recall: */
-        struct knfsd_fh         dl_fh;
         int                     dl_retries;
         struct nfsd4_callback   dl_recall;
 };
@@ -194,6 +221,11 @@ struct nfsd4_conn {
         unsigned char cn_flags;
 };
 
+/*
+ * Representation of a v4.1+ session. These are refcounted in a similar fashion
+ * to the nfs4_client. References are only taken when the server is actively
+ * working on the object (primarily during the processing of compounds).
+ */
 struct nfsd4_session {
         atomic_t                se_ref;
         struct list_head        se_hash;        /* hash by sessionid */
@@ -212,8 +244,6 @@ struct nfsd4_session {
         struct nfsd4_slot       *se_slots[];    /* forward channel slots */
 };
 
-extern void nfsd4_put_session(struct nfsd4_session *ses);
-
 /* formatted contents of nfs4_sessionid */
 struct nfsd4_sessionid {
         clientid_t      clientid;
@@ -225,17 +255,35 @@ struct nfsd4_sessionid {
 
 /*
  * struct nfs4_client - one per client.  Clientids live here.
- *      o Each nfs4_client is hashed by clientid.
  *
- *      o Each nfs4_clients is also hashed by name 
- *        (the opaque quantity initially sent by the client to identify itself).
+ * The initial object created by an NFS client using SETCLIENTID (for NFSv4.0)
+ * or EXCHANGE_ID (for NFSv4.1+). These objects are refcounted and timestamped.
+ * Each nfsd_net_ns object contains a set of these and they are tracked via
+ * short and long form clientid. They are hashed and searched for under the
+ * per-nfsd_net client_lock spinlock.
+ *
+ * References to it are only held during the processing of compounds, and in
+ * certain other operations. In their "resting state" they have a refcount of
+ * 0. If they are not renewed within a lease period, they become eligible for
+ * destruction by the laundromat.
+ *
+ * These objects can also be destroyed prematurely by the fault injection code,
+ * or if the client sends certain forms of SETCLIENTID or EXCHANGE_ID updates.
+ * Care is taken *not* to do this however when the objects have an elevated
+ * refcount.
+ *
+ * o Each nfs4_client is hashed by clientid
+ *
+ * o Each nfs4_clients is also hashed by name (the opaque quantity initially
+ *   sent by the client to identify itself).
  *        
- *      o cl_perclient list is used to ensure no dangling stateowner references
- *        when we expire the nfs4_client
+ * o cl_perclient list is used to ensure no dangling stateowner references
+ *   when we expire the nfs4_client
  */
 struct nfs4_client {
         struct list_head        cl_idhash;      /* hash by cl_clientid.id */
         struct rb_node          cl_namenode;    /* link into by-name trees */
+        struct list_head        *cl_ownerstr_hashtbl;
         struct list_head        cl_openowners;
         struct idr              cl_stateids;    /* stateid lookup */
         struct list_head        cl_delegations;
@@ -329,21 +377,43 @@ struct nfs4_replay {
         unsigned int            rp_buflen;
         char                    *rp_buf;
         struct knfsd_fh         rp_openfh;
+        struct mutex            rp_mutex;
         char                    rp_ibuf[NFSD4_REPLAY_ISIZE];
 };
 
+struct nfs4_stateowner;
+
+struct nfs4_stateowner_operations {
+        void (*so_unhash)(struct nfs4_stateowner *);
+        void (*so_free)(struct nfs4_stateowner *);
+};
+
+/*
+ * A core object that represents either an open or lock owner. The object and
+ * lock owner objects have one of these embedded within them. Refcounts and
+ * other fields common to both owner types are contained within these
+ * structures.
+ */
 struct nfs4_stateowner {
-        struct list_head        so_strhash;   /* hash by op_name */
-        struct list_head        so_stateids;
-        struct nfs4_client *    so_client;
-        /* after increment in ENCODE_SEQID_OP_TAIL, represents the next
+        struct list_head                        so_strhash;
+        struct list_head                        so_stateids;
+        struct nfs4_client                      *so_client;
+        const struct nfs4_stateowner_operations *so_ops;
+        /* after increment in nfsd4_bump_seqid, represents the next
          * sequence id expected from the client: */
-        u32                     so_seqid;
-        struct xdr_netobj       so_owner;     /* open owner name */
-        struct nfs4_replay      so_replay;
-        bool                    so_is_open_owner;
+        atomic_t                                so_count;
+        u32                                     so_seqid;
+        struct xdr_netobj                       so_owner; /* open owner name */
+        struct nfs4_replay                      so_replay;
+        bool                                    so_is_open_owner;
 };
 
+/*
+ * When a file is opened, the client provides an open state owner opaque string
+ * that indicates the "owner" of that open. These objects are refcounted.
+ * References to it are held by each open state associated with it. This object
+ * is a superset of the nfs4_stateowner struct.
+ */
 struct nfs4_openowner {
         struct nfs4_stateowner  oo_owner; /* must be first field */
         struct list_head        oo_perclient;
@@ -358,15 +428,17 @@ struct nfs4_openowner {
         struct nfs4_ol_stateid *oo_last_closed_stid;
         time_t                  oo_time; /* time of placement on so_close_lru */
 #define NFS4_OO_CONFIRMED   1
-#define NFS4_OO_NEW         4
         unsigned char           oo_flags;
 };
 
+/*
+ * Represents a generic "lockowner". Similar to an openowner. References to it
+ * are held by the lock stateids that are created on its behalf. This object is
+ * a superset of the nfs4_stateowner struct (or would be if it needed any extra
+ * fields).
+ */
 struct nfs4_lockowner {
         struct nfs4_stateowner  lo_owner; /* must be first element */
-        struct list_head        lo_owner_ino_hash; /* hash by owner,file */
-        struct list_head        lo_perstateid;
-        struct list_head        lo_list; /* for temporary uses */
 };
 
 static inline struct nfs4_openowner * openowner(struct nfs4_stateowner *so)
@@ -379,9 +451,17 @@ static inline struct nfs4_lockowner * lockowner(struct nfs4_stateowner *so)
         return container_of(so, struct nfs4_lockowner, lo_owner);
 }
 
-/* nfs4_file: a file opened by some number of (open) nfs4_stateowners. */
+/*
+ * nfs4_file: a file opened by some number of (open) nfs4_stateowners.
+ *
+ * These objects are global. nfsd only keeps one instance of a nfs4_file per
+ * inode (though it may keep multiple file descriptors open per inode). These
+ * are tracked in the file_hashtbl which is protected by the state_lock
+ * spinlock.
+ */
 struct nfs4_file {
         atomic_t                fi_ref;
+        spinlock_t              fi_lock;
         struct hlist_node       fi_hash;    /* hash by "struct inode *" */
         struct list_head        fi_stateids;
         struct list_head        fi_delegations;
@@ -395,49 +475,36 @@ struct nfs4_file {
          *   + 1 to both of the above if NFS4_SHARE_ACCESS_BOTH is set.
          */
         atomic_t                fi_access[2];
+        u32                     fi_share_deny;
         struct file             *fi_deleg_file;
         struct file_lock        *fi_lease;
         atomic_t                fi_delegees;
-        struct inode            *fi_inode;
+        struct knfsd_fh         fi_fhandle;
         bool                    fi_had_conflict;
 };
 
-/* XXX: for first cut may fall back on returning file that doesn't work
- * at all? */
-static inline struct file *find_writeable_file(struct nfs4_file *f)
-{
-        if (f->fi_fds[O_WRONLY])
-                return f->fi_fds[O_WRONLY];
-        return f->fi_fds[O_RDWR];
-}
-
-static inline struct file *find_readable_file(struct nfs4_file *f)
-{
-        if (f->fi_fds[O_RDONLY])
-                return f->fi_fds[O_RDONLY];
-        return f->fi_fds[O_RDWR];
-}
-
-static inline struct file *find_any_file(struct nfs4_file *f)
-{
-        if (f->fi_fds[O_RDWR])
-                return f->fi_fds[O_RDWR];
-        else if (f->fi_fds[O_WRONLY])
-                return f->fi_fds[O_WRONLY];
-        else
-                return f->fi_fds[O_RDONLY];
-}
-
-/* "ol" stands for "Open or Lock".  Better suggestions welcome. */
+/*
+ * A generic struct representing either a open or lock stateid. The nfs4_client
+ * holds a reference to each of these objects, and they in turn hold a
+ * reference to their respective stateowners. The client's reference is
+ * released in response to a close or unlock (depending on whether it's an open
+ * or lock stateid) or when the client is being destroyed.
+ *
+ * In the case of v4.0 open stateids, these objects are preserved for a little
+ * while after close in order to handle CLOSE replays. Those are eventually
+ * reclaimed via a LRU scheme by the laundromat.
+ *
+ * This object is a superset of the nfs4_stid. "ol" stands for "Open or Lock".
+ * Better suggestions welcome.
+ */
 struct nfs4_ol_stateid {
         struct nfs4_stid    st_stid; /* must be first field */
         struct list_head              st_perfile;
         struct list_head              st_perstateowner;
-        struct list_head              st_lockowners;
+        struct list_head              st_locks;
         struct nfs4_stateowner      * st_stateowner;
-        struct nfs4_file            * st_file;
-        unsigned long                 st_access_bmap;
-        unsigned long                 st_deny_bmap;
+        unsigned char                 st_access_bmap;
+        unsigned char                 st_deny_bmap;
         struct nfs4_ol_stateid         * st_openstp;
 };
 
@@ -456,15 +523,16 @@ struct nfsd_net;
 extern __be32 nfs4_preprocess_stateid_op(struct net *net,
                 struct nfsd4_compound_state *cstate,
                 stateid_t *stateid, int flags, struct file **filp);
-extern void nfs4_lock_state(void);
-extern void nfs4_unlock_state(void);
+void nfs4_put_stid(struct nfs4_stid *s);
 void nfs4_remove_reclaim_record(struct nfs4_client_reclaim *, struct nfsd_net *);
 extern void nfs4_release_reclaim(struct nfsd_net *);
 extern struct nfs4_client_reclaim *nfsd4_find_reclaim_client(const char *recdir,
                                                         struct nfsd_net *nn);
-extern __be32 nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct nfsd_net *nn);
+extern __be32 nfs4_check_open_reclaim(clientid_t *clid,
+                struct nfsd4_compound_state *cstate, struct nfsd_net *nn);
 extern int set_callback_cred(void);
-extern void nfsd4_init_callback(struct nfsd4_callback *);
+void nfsd4_run_cb_null(struct work_struct *w);
+void nfsd4_run_cb_recall(struct work_struct *w);
 extern void nfsd4_probe_callback(struct nfs4_client *clp);
 extern void nfsd4_probe_callback_sync(struct nfs4_client *clp);
 extern void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *);
@@ -472,11 +540,10 @@ extern void nfsd4_cb_recall(struct nfs4_delegation *dp);
 extern int nfsd4_create_callback_queue(void);
 extern void nfsd4_destroy_callback_queue(void);
 extern void nfsd4_shutdown_callback(struct nfs4_client *);
-extern void nfs4_put_delegation(struct nfs4_delegation *dp);
+extern void nfsd4_prepare_cb_recall(struct nfs4_delegation *dp);
 extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *name,
                                                         struct nfsd_net *nn);
 extern bool nfs4_has_reclaimed_state(const char *name, struct nfsd_net *nn);
-extern void put_client_renew(struct nfs4_client *clp);
 
 /* nfs4recover operations */
 extern int nfsd4_client_tracking_init(struct net *net);
@@ -490,19 +557,24 @@ extern void nfsd4_record_grace_done(struct nfsd_net *nn, time_t boot_time);
 #ifdef CONFIG_NFSD_FAULT_INJECTION
 int nfsd_fault_inject_init(void);
 void nfsd_fault_inject_cleanup(void);
-u64 nfsd_for_n_state(u64, u64 (*)(struct nfs4_client *, u64));
-struct nfs4_client *nfsd_find_client(struct sockaddr_storage *, size_t);
-
-u64 nfsd_forget_client(struct nfs4_client *, u64);
-u64 nfsd_forget_client_locks(struct nfs4_client*, u64);
-u64 nfsd_forget_client_openowners(struct nfs4_client *, u64);
-u64 nfsd_forget_client_delegations(struct nfs4_client *, u64);
-u64 nfsd_recall_client_delegations(struct nfs4_client *, u64);
-
-u64 nfsd_print_client(struct nfs4_client *, u64);
-u64 nfsd_print_client_locks(struct nfs4_client *, u64);
-u64 nfsd_print_client_openowners(struct nfs4_client *, u64);
-u64 nfsd_print_client_delegations(struct nfs4_client *, u64);
+
+u64 nfsd_inject_print_clients(void);
+u64 nfsd_inject_forget_client(struct sockaddr_storage *, size_t);
+u64 nfsd_inject_forget_clients(u64);
+
+u64 nfsd_inject_print_locks(void);
+u64 nfsd_inject_forget_client_locks(struct sockaddr_storage *, size_t);
+u64 nfsd_inject_forget_locks(u64);
+
+u64 nfsd_inject_print_openowners(void);
+u64 nfsd_inject_forget_client_openowners(struct sockaddr_storage *, size_t);
+u64 nfsd_inject_forget_openowners(u64);
+
+u64 nfsd_inject_print_delegations(void);
+u64 nfsd_inject_forget_client_delegations(struct sockaddr_storage *, size_t);
+u64 nfsd_inject_forget_delegations(u64);
+u64 nfsd_inject_recall_client_delegations(struct sockaddr_storage *, size_t);
+u64 nfsd_inject_recall_delegations(u64);
 #else /* CONFIG_NFSD_FAULT_INJECTION */
 static inline int nfsd_fault_inject_init(void) { return 0; }
 static inline void nfsd_fault_inject_cleanup(void) {}
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index 140c496f612c..f501a9b5c9df 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -189,8 +189,7 @@ nfsd_lookup_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp,
         dprintk("nfsd: nfsd_lookup(fh %s, %.*s)\n", SVCFH_fmt(fhp), len,name);
 
         dparent = fhp->fh_dentry;
-        exp  = fhp->fh_export;
-        exp_get(exp);
+        exp = exp_get(fhp->fh_export);
 
         /* Lookup the name, but don't follow links */
         if (isdotent(name, len)) {
@@ -464,7 +463,7 @@ out_put_write_access:
         if (size_change)
                 put_write_access(inode);
         if (!err)
-                commit_metadata(fhp);
+                err = nfserrno(commit_metadata(fhp));
 out:
         return err;
 }
@@ -820,7 +819,8 @@ static int nfsd_direct_splice_actor(struct pipe_inode_info *pipe,
         return __splice_from_pipe(pipe, sd, nfsd_splice_actor);
 }
 
-__be32 nfsd_finish_read(struct file *file, unsigned long *count, int host_err)
+static __be32
+nfsd_finish_read(struct file *file, unsigned long *count, int host_err)
 {
         if (host_err >= 0) {
                 nfsdstats.io_read += host_err;
@@ -831,7 +831,7 @@ __be32 nfsd_finish_read(struct file *file, unsigned long *count, int host_err)
                 return nfserrno(host_err);
 }
 
-int nfsd_splice_read(struct svc_rqst *rqstp,
+__be32 nfsd_splice_read(struct svc_rqst *rqstp,
                      struct file *file, loff_t offset, unsigned long *count)
 {
         struct splice_desc sd = {
@@ -847,7 +847,7 @@ int nfsd_splice_read(struct svc_rqst *rqstp,
         return nfsd_finish_read(file, count, host_err);
 }
 
-int nfsd_readv(struct file *file, loff_t offset, struct kvec *vec, int vlen,
+__be32 nfsd_readv(struct file *file, loff_t offset, struct kvec *vec, int vlen,
                 unsigned long *count)
 {
         mm_segment_t oldfs;
@@ -1121,7 +1121,8 @@ nfsd_create_setattr(struct svc_rqst *rqstp, struct svc_fh *resfhp,
                 iap->ia_valid &= ~(ATTR_UID|ATTR_GID);
         if (iap->ia_valid)
                 return nfsd_setattr(rqstp, resfhp, iap, 0, (time_t)0);
-        return 0;
+        /* Callers expect file metadata to be committed here */
+        return nfserrno(commit_metadata(resfhp));
 }
 
 /* HPUX client sometimes creates a file in mode 000, and sets size to 0.
@@ -1253,9 +1254,10 @@ nfsd_create(struct svc_rqst *rqstp, struct svc_fh *fhp,
         err = nfsd_create_setattr(rqstp, resfhp, iap);
 
         /*
-         * nfsd_setattr already committed the child.  Transactional filesystems
-         * had a chance to commit changes for both parent and child
-         * simultaneously making the following commit_metadata a noop.
+         * nfsd_create_setattr already committed the child.  Transactional
+         * filesystems had a chance to commit changes for both parent and
+         * child * simultaneously making the following commit_metadata a
+         * noop.
          */
         err2 = nfserrno(commit_metadata(fhp));
         if (err2)
@@ -1426,7 +1428,8 @@ do_nfsd_create(struct svc_rqst *rqstp, struct svc_fh *fhp,
         err = nfsd_create_setattr(rqstp, resfhp, iap);
 
         /*
-         * nfsd_setattr already committed the child (and possibly also the parent).
+         * nfsd_create_setattr already committed the child
+         * (and possibly also the parent).
          */
         if (!err)
                 err = nfserrno(commit_metadata(fhp));
@@ -1504,16 +1507,15 @@ out_nfserr:
 __be32
 nfsd_symlink(struct svc_rqst *rqstp, struct svc_fh *fhp,
                                 char *fname, int flen,
-                                char *path,  int plen,
-                                struct svc_fh *resfhp,
-                                struct iattr *iap)
+                                char *path,
+                                struct svc_fh *resfhp)
 {
         struct dentry   *dentry, *dnew;
         __be32          err, cerr;
         int             host_err;
 
         err = nfserr_noent;
-        if (!flen || !plen)
+        if (!flen || path[0] == '\0')
                 goto out;
         err = nfserr_exist;
         if (isdotent(fname, flen))
@@ -1534,18 +1536,7 @@ nfsd_symlink(struct svc_rqst *rqstp, struct svc_fh *fhp,
         if (IS_ERR(dnew))
                 goto out_nfserr;
 
-        if (unlikely(path[plen] != 0)) {
-                char *path_alloced = kmalloc(plen+1, GFP_KERNEL);
-                if (path_alloced == NULL)
-                        host_err = -ENOMEM;
-                else {
-                        strncpy(path_alloced, path, plen);
-                        path_alloced[plen] = 0;
-                        host_err = vfs_symlink(dentry->d_inode, dnew, path_alloced);
-                        kfree(path_alloced);
-                }
-        } else
-                host_err = vfs_symlink(dentry->d_inode, dnew, path);
+        host_err = vfs_symlink(dentry->d_inode, dnew, path);
         err = nfserrno(host_err);
         if (!err)
                 err = nfserrno(commit_metadata(fhp));
@@ -2093,8 +2084,7 @@ nfsd_racache_init(int cache_size)
         if (raparm_hash[0].pb_head)
                 return 0;
         nperbucket = DIV_ROUND_UP(cache_size, RAPARM_HASH_SIZE);
-        if (nperbucket < 2)
-                nperbucket = 2;
+        nperbucket = max(2, nperbucket);
         cache_size = nperbucket * RAPARM_HASH_SIZE;
 
         dprintk("nfsd: allocating %d readahead buffers.\n", cache_size);
diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h
index 91b6ae3f658b..c2ff3f14e5f6 100644
--- a/fs/nfsd/vfs.h
+++ b/fs/nfsd/vfs.h
@@ -74,9 +74,9 @@ struct raparms;
 __be32          nfsd_get_tmp_read_open(struct svc_rqst *, struct svc_fh *,
                                 struct file **, struct raparms **);
 void            nfsd_put_tmp_read_open(struct file *, struct raparms *);
-int             nfsd_splice_read(struct svc_rqst *,
+__be32          nfsd_splice_read(struct svc_rqst *,
                                 struct file *, loff_t, unsigned long *);
-int             nfsd_readv(struct file *, loff_t, struct kvec *, int,
+__be32          nfsd_readv(struct file *, loff_t, struct kvec *, int,
                                 unsigned long *);
 __be32          nfsd_read(struct svc_rqst *, struct svc_fh *,
                                 loff_t, struct kvec *, int, unsigned long *);
@@ -85,8 +85,8 @@ __be32 		nfsd_write(struct svc_rqst *, struct svc_fh *,struct file *,
 __be32          nfsd_readlink(struct svc_rqst *, struct svc_fh *,
                                 char *, int *);
 __be32          nfsd_symlink(struct svc_rqst *, struct svc_fh *,
-                                char *name, int len, char *path, int plen,
-                                struct svc_fh *res, struct iattr *);
+                                char *name, int len, char *path,
+                                struct svc_fh *res);
 __be32          nfsd_link(struct svc_rqst *, struct svc_fh *,
                                 char *, int, struct svc_fh *);
 __be32          nfsd_rename(struct svc_rqst *,
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 18cbb6d9c8a9..465e7799742a 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -55,6 +55,7 @@ struct nfsd4_compound_state {
         struct svc_fh           current_fh;
         struct svc_fh           save_fh;
         struct nfs4_stateowner  *replay_owner;
+        struct nfs4_client      *clp;
         /* For sessions DRC */
         struct nfsd4_session    *session;
         struct nfsd4_slot       *slot;
@@ -107,8 +108,8 @@ struct nfsd4_create {
         u32             cr_type;            /* request */
         union {                             /* request */
                 struct {
-                        u32 namelen;
-                        char *name;
+                        u32 datalen;
+                        char *data;
                 } link;   /* NF4LNK */
                 struct {
                         u32 specdata1;
@@ -121,8 +122,8 @@ struct nfsd4_create {
         struct nfs4_acl *cr_acl;
         struct xdr_netobj cr_label;
 };
-#define cr_linklen      u.link.namelen
-#define cr_linkname     u.link.name
+#define cr_datalen      u.link.datalen
+#define cr_data         u.link.data
 #define cr_specdata1    u.dev.specdata1
 #define cr_specdata2    u.dev.specdata2
 
@@ -478,6 +479,14 @@ struct nfsd4_op {
 
 bool nfsd4_cache_this_op(struct nfsd4_op *);
 
+/*
+ * Memory needed just for the duration of processing one compound:
+ */
+struct svcxdr_tmpbuf {
+        struct svcxdr_tmpbuf *next;
+        char buf[];
+};
+
 struct nfsd4_compoundargs {
         /* scratch variables for XDR decode */
         __be32 *                        p;
@@ -486,11 +495,7 @@ struct nfsd4_compoundargs {
         int                             pagelen;
         __be32                          tmp[8];
         __be32 *                        tmpp;
-        struct tmpbuf {
-                struct tmpbuf *next;
-                void (*release)(const void *);
-                void *buf;
-        }                               *to_free;
+        struct svcxdr_tmpbuf            *to_free;
 
         struct svc_rqst                 *rqstp;
 
@@ -574,7 +579,6 @@ extern __be32 nfsd4_setclientid(struct svc_rqst *rqstp,
 extern __be32 nfsd4_setclientid_confirm(struct svc_rqst *rqstp,
                 struct nfsd4_compound_state *,
                 struct nfsd4_setclientid_confirm *setclientid_confirm);
-extern void nfsd4_store_cache_entry(struct nfsd4_compoundres *resp);
 extern __be32 nfsd4_exchange_id(struct svc_rqst *rqstp,
                 struct nfsd4_compound_state *, struct nfsd4_exchange_id *);
 extern __be32 nfsd4_backchannel_ctl(struct svc_rqst *, struct nfsd4_compound_state *, struct nfsd4_backchannel_ctl *);
@@ -585,6 +589,7 @@ extern __be32 nfsd4_create_session(struct svc_rqst *,
 extern __be32 nfsd4_sequence(struct svc_rqst *,
                 struct nfsd4_compound_state *,
                 struct nfsd4_sequence *);
+extern void nfsd4_sequence_done(struct nfsd4_compoundres *resp);
 extern __be32 nfsd4_destroy_session(struct svc_rqst *,
                 struct nfsd4_compound_state *,
                 struct nfsd4_destroy_session *);
@@ -594,7 +599,9 @@ extern __be32 nfsd4_process_open1(struct nfsd4_compound_state *,
                 struct nfsd4_open *open, struct nfsd_net *nn);
 extern __be32 nfsd4_process_open2(struct svc_rqst *rqstp,
                 struct svc_fh *current_fh, struct nfsd4_open *open);
-extern void nfsd4_cleanup_open_state(struct nfsd4_open *open, __be32 status);
+extern void nfsd4_cstate_clear_replay(struct nfsd4_compound_state *cstate);
+extern void nfsd4_cleanup_open_state(struct nfsd4_compound_state *cstate,
+                struct nfsd4_open *open, __be32 status);
 extern __be32 nfsd4_open_confirm(struct svc_rqst *rqstp,
                 struct nfsd4_compound_state *, struct nfsd4_open_confirm *oc);
 extern __be32 nfsd4_close(struct svc_rqst *rqstp,
@@ -625,6 +632,7 @@ extern __be32 nfsd4_test_stateid(struct svc_rqst *rqstp,
 extern __be32 nfsd4_free_stateid(struct svc_rqst *rqstp,
                 struct nfsd4_compound_state *, struct nfsd4_free_stateid *free_stateid);
 extern void nfsd4_bump_seqid(struct nfsd4_compound_state *, __be32 nfserr);
+
 #endif
 
 /*
diff --git a/fs/proc/Makefile b/fs/proc/Makefile
index 239493ec718e..7151ea428041 100644
--- a/fs/proc/Makefile
+++ b/fs/proc/Makefile
@@ -23,6 +23,7 @@ proc-y	+= version.o
 proc-y  += softirqs.o
 proc-y  += namespaces.o
 proc-y  += self.o
+proc-y  += thread_self.o
 proc-$(CONFIG_PROC_SYSCTL)      += proc_sysctl.o
 proc-$(CONFIG_NET)              += proc_net.o
 proc-$(CONFIG_PROC_KCORE)       += kcore.o
diff --git a/fs/proc/base.c b/fs/proc/base.c
index 043c83cb51f9..baf852b648ad 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -2814,7 +2814,7 @@ retry:
         return iter;
 }
 
-#define TGID_OFFSET (FIRST_PROCESS_ENTRY + 1)
+#define TGID_OFFSET (FIRST_PROCESS_ENTRY + 2)
 
 /* for the /proc/ directory itself, after non-process stuff has been done */
 int proc_pid_readdir(struct file *file, struct dir_context *ctx)
@@ -2826,14 +2826,19 @@ int proc_pid_readdir(struct file *file, struct dir_context *ctx)
         if (pos >= PID_MAX_LIMIT + TGID_OFFSET)
                 return 0;
 
-        if (pos == TGID_OFFSET - 1) {
+        if (pos == TGID_OFFSET - 2) {
                 struct inode *inode = ns->proc_self->d_inode;
                 if (!dir_emit(ctx, "self", 4, inode->i_ino, DT_LNK))
                         return 0;
-                iter.tgid = 0;
-        } else {
-                iter.tgid = pos - TGID_OFFSET;
+                ctx->pos = pos = pos + 1;
+        }
+        if (pos == TGID_OFFSET - 1) {
+                struct inode *inode = ns->proc_thread_self->d_inode;
+                if (!dir_emit(ctx, "thread-self", 11, inode->i_ino, DT_LNK))
+                        return 0;
+                ctx->pos = pos = pos + 1;
         }
+        iter.tgid = pos - TGID_OFFSET;
         iter.task = NULL;
         for (iter = next_tgid(ns, iter);
              iter.task;
@@ -2862,6 +2867,9 @@ static const struct pid_entry tid_base_stuff[] = {
         DIR("fd",        S_IRUSR|S_IXUSR, proc_fd_inode_operations, proc_fd_operations),
         DIR("fdinfo",    S_IRUSR|S_IXUSR, proc_fdinfo_inode_operations, proc_fdinfo_operations),
         DIR("ns",        S_IRUSR|S_IXUGO, proc_ns_dir_inode_operations, proc_ns_dir_operations),
+#ifdef CONFIG_NET
+        DIR("net",        S_IRUGO|S_IXUGO, proc_net_inode_operations, proc_net_operations),
+#endif
         REG("environ",   S_IRUSR, proc_environ_operations),
         ONE("auxv",      S_IRUSR, proc_pid_auxv),
         ONE("status",    S_IRUGO, proc_pid_status),
diff --git a/fs/proc/inode.c b/fs/proc/inode.c
index 0adbc02d60e3..333080d7a671 100644
--- a/fs/proc/inode.c
+++ b/fs/proc/inode.c
@@ -442,6 +442,7 @@ struct inode *proc_get_inode(struct super_block *sb, struct proc_dir_entry *de)
 int proc_fill_super(struct super_block *s)
 {
         struct inode *root_inode;
+        int ret;
 
         s->s_flags |= MS_NODIRATIME | MS_NOSUID | MS_NOEXEC;
         s->s_blocksize = 1024;
@@ -463,5 +464,9 @@ int proc_fill_super(struct super_block *s)
                 return -ENOMEM;
         }
 
-        return proc_setup_self(s);
+        ret = proc_setup_self(s);
+        if (ret) {
+                return ret;
+        }
+        return proc_setup_thread_self(s);
 }
diff --git a/fs/proc/internal.h b/fs/proc/internal.h
index a024cf7b260f..7da13e49128a 100644
--- a/fs/proc/internal.h
+++ b/fs/proc/internal.h
@@ -231,6 +231,12 @@ static inline int proc_net_init(void) { return 0; }
 extern int proc_setup_self(struct super_block *);
 
 /*
+ * proc_thread_self.c
+ */
+extern int proc_setup_thread_self(struct super_block *);
+extern void proc_thread_self_init(void);
+
+/*
  * proc_sysctl.c
  */
 #ifdef CONFIG_PROC_SYSCTL
diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c
index 4677bb7dc7c2..39481028ec08 100644
--- a/fs/proc/proc_net.c
+++ b/fs/proc/proc_net.c
@@ -113,9 +113,11 @@ static struct net *get_proc_task_net(struct inode *dir)
         rcu_read_lock();
         task = pid_task(proc_pid(dir), PIDTYPE_PID);
         if (task != NULL) {
-                ns = task_nsproxy(task);
+                task_lock(task);
+                ns = task->nsproxy;
                 if (ns != NULL)
                         net = get_net(ns->net_ns);
+                task_unlock(task);
         }
         rcu_read_unlock();
 
@@ -224,7 +226,7 @@ static struct pernet_operations __net_initdata proc_net_ns_ops = {
 
 int __init proc_net_init(void)
 {
-        proc_symlink("net", NULL, "self/net");
+        proc_symlink("net", NULL, "thread-self/net");
 
         return register_pernet_subsys(&proc_net_ns_ops);
 }
diff --git a/fs/proc/root.c b/fs/proc/root.c
index 574bafc41f0b..6296c7626963 100644
--- a/fs/proc/root.c
+++ b/fs/proc/root.c
@@ -149,6 +149,8 @@ static void proc_kill_sb(struct super_block *sb)
         ns = (struct pid_namespace *)sb->s_fs_info;
         if (ns->proc_self)
                 dput(ns->proc_self);
+        if (ns->proc_thread_self)
+                dput(ns->proc_thread_self);
         kill_anon_super(sb);
         put_pid_ns(ns);
 }
@@ -170,7 +172,8 @@ void __init proc_root_init(void)
                 return;
 
         proc_self_init();
-        proc_symlink("mounts", NULL, "self/mounts");
+        proc_thread_self_init();
+        proc_symlink("mounts", NULL, "thread-self/mounts");
 
         proc_net_init();
 
diff --git a/fs/proc/thread_self.c b/fs/proc/thread_self.c
new file mode 100644
index 000000000000..59075b509df3
--- /dev/null
+++ b/fs/proc/thread_self.c
@@ -0,0 +1,85 @@
+#include <linux/sched.h>
+#include <linux/namei.h>
+#include <linux/slab.h>
+#include <linux/pid_namespace.h>
+#include "internal.h"
+
+/*
+ * /proc/thread_self:
+ */
+static int proc_thread_self_readlink(struct dentry *dentry, char __user *buffer,
+                              int buflen)
+{
+        struct pid_namespace *ns = dentry->d_sb->s_fs_info;
+        pid_t tgid = task_tgid_nr_ns(current, ns);
+        pid_t pid = task_pid_nr_ns(current, ns);
+        char tmp[PROC_NUMBUF + 6 + PROC_NUMBUF];
+        if (!pid)
+                return -ENOENT;
+        sprintf(tmp, "%d/task/%d", tgid, pid);
+        return readlink_copy(buffer, buflen, tmp);
+}
+
+static void *proc_thread_self_follow_link(struct dentry *dentry, struct nameidata *nd)
+{
+        struct pid_namespace *ns = dentry->d_sb->s_fs_info;
+        pid_t tgid = task_tgid_nr_ns(current, ns);
+        pid_t pid = task_pid_nr_ns(current, ns);
+        char *name = ERR_PTR(-ENOENT);
+        if (pid) {
+                name = kmalloc(PROC_NUMBUF + 6 + PROC_NUMBUF, GFP_KERNEL);
+                if (!name)
+                        name = ERR_PTR(-ENOMEM);
+                else
+                        sprintf(name, "%d/task/%d", tgid, pid);
+        }
+        nd_set_link(nd, name);
+        return NULL;
+}
+
+static const struct inode_operations proc_thread_self_inode_operations = {
+        .readlink       = proc_thread_self_readlink,
+        .follow_link    = proc_thread_self_follow_link,
+        .put_link       = kfree_put_link,
+};
+
+static unsigned thread_self_inum;
+
+int proc_setup_thread_self(struct super_block *s)
+{
+        struct inode *root_inode = s->s_root->d_inode;
+        struct pid_namespace *ns = s->s_fs_info;
+        struct dentry *thread_self;
+
+        mutex_lock(&root_inode->i_mutex);
+        thread_self = d_alloc_name(s->s_root, "thread-self");
+        if (thread_self) {
+                struct inode *inode = new_inode_pseudo(s);
+                if (inode) {
+                        inode->i_ino = thread_self_inum;
+                        inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME;
+                        inode->i_mode = S_IFLNK | S_IRWXUGO;
+                        inode->i_uid = GLOBAL_ROOT_UID;
+                        inode->i_gid = GLOBAL_ROOT_GID;
+                        inode->i_op = &proc_thread_self_inode_operations;
+                        d_add(thread_self, inode);
+                } else {
+                        dput(thread_self);
+                        thread_self = ERR_PTR(-ENOMEM);
+                }
+        } else {
+                thread_self = ERR_PTR(-ENOMEM);
+        }
+        mutex_unlock(&root_inode->i_mutex);
+        if (IS_ERR(thread_self)) {
+                pr_err("proc_fill_super: can't allocate /proc/thread_self\n");
+                return PTR_ERR(thread_self);
+        }
+        ns->proc_thread_self = thread_self;
+        return 0;
+}
+
+void __init proc_thread_self_init(void)
+{
+        proc_alloc_inum(&thread_self_inum);
+}
diff --git a/fs/proc_namespace.c b/fs/proc_namespace.c
index 1a81373947f3..73ca1740d839 100644
--- a/fs/proc_namespace.c
+++ b/fs/proc_namespace.c
@@ -232,17 +232,15 @@ static int mounts_open_common(struct inode *inode, struct file *file,
         if (!task)
                 goto err;
 
-        rcu_read_lock();
-        nsp = task_nsproxy(task);
+        task_lock(task);
+        nsp = task->nsproxy;
         if (!nsp || !nsp->mnt_ns) {
-                rcu_read_unlock();
+                task_unlock(task);
                 put_task_struct(task);
                 goto err;
         }
         ns = nsp->mnt_ns;
         get_mnt_ns(ns);
-        rcu_read_unlock();
-        task_lock(task);
         if (!task->fs) {
                 task_unlock(task);
                 put_task_struct(task);
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 202f0a7171e8..1d9f0f1ff52d 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -133,6 +133,7 @@ struct ttm_tt {
  * struct ttm_dma_tt
  *
  * @ttm: Base ttm_tt struct.
+ * @cpu_address: The CPU address of the pages
  * @dma_address: The DMA (bus) addresses of the pages
  * @pages_list: used by some page allocation backend
  *
@@ -142,6 +143,7 @@ struct ttm_tt {
  */
 struct ttm_dma_tt {
         struct ttm_tt ttm;
+        void **cpu_address;
         dma_addr_t *dma_address;
         struct list_head pages_list;
 };
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index 5320153c311b..807cbc46d73e 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -364,6 +364,17 @@ extern bool osc_sb_apei_support_acked;
 #define OSC_PCI_EXPRESS_CAPABILITY_CONTROL      0x00000010
 #define OSC_PCI_CONTROL_MASKS                   0x0000001f
 
+#define ACPI_GSB_ACCESS_ATTRIB_QUICK            0x00000002
+#define ACPI_GSB_ACCESS_ATTRIB_SEND_RCV         0x00000004
+#define ACPI_GSB_ACCESS_ATTRIB_BYTE             0x00000006
+#define ACPI_GSB_ACCESS_ATTRIB_WORD             0x00000008
+#define ACPI_GSB_ACCESS_ATTRIB_BLOCK            0x0000000A
+#define ACPI_GSB_ACCESS_ATTRIB_MULTIBYTE        0x0000000B
+#define ACPI_GSB_ACCESS_ATTRIB_WORD_CALL        0x0000000C
+#define ACPI_GSB_ACCESS_ATTRIB_BLOCK_CALL       0x0000000D
+#define ACPI_GSB_ACCESS_ATTRIB_RAW_BYTES        0x0000000E
+#define ACPI_GSB_ACCESS_ATTRIB_RAW_PROCESS      0x0000000F
+
 extern acpi_status acpi_pci_osc_control_set(acpi_handle handle,
                                              u32 *mask, u32 req);
 
diff --git a/include/linux/cred.h b/include/linux/cred.h
index f61d6c8f5ef3..b2d0820837c4 100644
--- a/include/linux/cred.h
+++ b/include/linux/cred.h
@@ -259,6 +259,15 @@ static inline void put_cred(const struct cred *_cred)
         rcu_dereference_protected(current->cred, 1)
 
 /**
+ * current_real_cred - Access the current task's objective credentials
+ *
+ * Access the objective credentials of the current task.  RCU-safe,
+ * since nobody else can modify it.
+ */
+#define current_real_cred() \
+        rcu_dereference_protected(current->real_cred, 1)
+
+/**
  * __task_cred - Access a task's objective credentials
  * @task: The task to query
  *
diff --git a/include/linux/ftrace_event.h b/include/linux/ftrace_event.h
index 06c6faa9e5cc..28672e87e910 100644
--- a/include/linux/ftrace_event.h
+++ b/include/linux/ftrace_event.h
@@ -571,40 +571,6 @@ do {									\
                 __trace_printk(ip, fmt, ##args);                        \
 } while (0)
 
-/**
- * tracepoint_string - register constant persistent string to trace system
- * @str - a constant persistent string that will be referenced in tracepoints
- *
- * If constant strings are being used in tracepoints, it is faster and
- * more efficient to just save the pointer to the string and reference
- * that with a printf "%s" instead of saving the string in the ring buffer
- * and wasting space and time.
- *
- * The problem with the above approach is that userspace tools that read
- * the binary output of the trace buffers do not have access to the string.
- * Instead they just show the address of the string which is not very
- * useful to users.
- *
- * With tracepoint_string(), the string will be registered to the tracing
- * system and exported to userspace via the debugfs/tracing/printk_formats
- * file that maps the string address to the string text. This way userspace
- * tools that read the binary buffers have a way to map the pointers to
- * the ASCII strings they represent.
- *
- * The @str used must be a constant string and persistent as it would not
- * make sense to show a string that no longer exists. But it is still fine
- * to be used with modules, because when modules are unloaded, if they
- * had tracepoints, the ring buffers are cleared too. As long as the string
- * does not change during the life of the module, it is fine to use
- * tracepoint_string() within a module.
- */
-#define tracepoint_string(str)                                          \
-        ({                                                              \
-                static const char *___tp_str __tracepoint_string = str; \
-                ___tp_str;                                              \
-        })
-#define __tracepoint_string     __attribute__((section("__tracepoint_str")))
-
 #ifdef CONFIG_PERF_EVENTS
 struct perf_event;
 
diff --git a/include/linux/i2c.h b/include/linux/i2c.h
index b556e0ab946f..ea507665896c 100644
--- a/include/linux/i2c.h
+++ b/include/linux/i2c.h
@@ -577,4 +577,16 @@ static inline struct i2c_adapter *of_find_i2c_adapter_by_node(struct device_node
 }
 #endif /* CONFIG_OF */
 
+#ifdef CONFIG_I2C_ACPI
+int acpi_i2c_install_space_handler(struct i2c_adapter *adapter);
+void acpi_i2c_remove_space_handler(struct i2c_adapter *adapter);
+void acpi_i2c_register_devices(struct i2c_adapter *adap);
+#else
+static inline void acpi_i2c_register_devices(struct i2c_adapter *adap) { }
+static inline void acpi_i2c_remove_space_handler(struct i2c_adapter *adapter)
+{ }
+static inline int acpi_i2c_install_space_handler(struct i2c_adapter *adapter)
+{ return 0; }
+#endif
+
 #endif /* _LINUX_I2C_H */
diff --git a/include/linux/i2c/s6000.h b/include/linux/i2c/s6000.h
deleted file mode 100644
index d9b34bfdae76..000000000000
--- a/include/linux/i2c/s6000.h
+++ /dev/null
@@ -1,10 +0,0 @@
-#ifndef __LINUX_I2C_S6000_H
-#define __LINUX_I2C_S6000_H
-
-struct s6_i2c_platform_data {
-        const char *clock; /* the clock to use */
-        int bus_num; /* the bus number to register */
-};
-
-#endif
-
diff --git a/include/linux/mount.h b/include/linux/mount.h
index 839bac270904..b0c1e6574e7f 100644
--- a/include/linux/mount.h
+++ b/include/linux/mount.h
@@ -42,13 +42,20 @@ struct mnt_namespace;
  * flag, consider how it interacts with shared mounts.
  */
 #define MNT_SHARED_MASK (MNT_UNBINDABLE)
-#define MNT_PROPAGATION_MASK    (MNT_SHARED | MNT_UNBINDABLE)
+#define MNT_USER_SETTABLE_MASK  (MNT_NOSUID | MNT_NODEV | MNT_NOEXEC \
+                                 | MNT_NOATIME | MNT_NODIRATIME | MNT_RELATIME \
+                                 | MNT_READONLY)
+#define MNT_ATIME_MASK (MNT_NOATIME | MNT_NODIRATIME | MNT_RELATIME )
 
 #define MNT_INTERNAL_FLAGS (MNT_SHARED | MNT_WRITE_HOLD | MNT_INTERNAL | \
                             MNT_DOOMED | MNT_SYNC_UMOUNT | MNT_MARKED)
 
 #define MNT_INTERNAL    0x4000
 
+#define MNT_LOCK_ATIME          0x040000
+#define MNT_LOCK_NOEXEC         0x080000
+#define MNT_LOCK_NOSUID         0x100000
+#define MNT_LOCK_NODEV          0x200000
 #define MNT_LOCK_READONLY       0x400000
 #define MNT_LOCKED              0x800000
 #define MNT_DOOMED              0x1000000
diff --git a/include/linux/nsproxy.h b/include/linux/nsproxy.h
index b4ec59d159ac..35fa08fd7739 100644
--- a/include/linux/nsproxy.h
+++ b/include/linux/nsproxy.h
@@ -40,32 +40,28 @@ extern struct nsproxy init_nsproxy;
  * the namespaces access rules are:
  *
  *  1. only current task is allowed to change tsk->nsproxy pointer or
- *     any pointer on the nsproxy itself
+ *     any pointer on the nsproxy itself.  Current must hold the task_lock
+ *     when changing tsk->nsproxy.
  *
  *  2. when accessing (i.e. reading) current task's namespaces - no
  *     precautions should be taken - just dereference the pointers
  *
  *  3. the access to other task namespaces is performed like this
- *     rcu_read_lock();
- *     nsproxy = task_nsproxy(tsk);
+ *     task_lock(task);
+ *     nsproxy = task->nsproxy;
  *     if (nsproxy != NULL) {
  *             / *
  *               * work with the namespaces here
  *               * e.g. get the reference on one of them
  *               * /
  *     } / *
- *         * NULL task_nsproxy() means that this task is
+ *         * NULL task->nsproxy means that this task is
  *         * almost dead (zombie)
  *         * /
- *     rcu_read_unlock();
+ *     task_unlock(task);
  *
  */
 
-static inline struct nsproxy *task_nsproxy(struct task_struct *tsk)
-{
-        return rcu_dereference(tsk->nsproxy);
-}
-
 int copy_namespaces(unsigned long flags, struct task_struct *tsk);
 void exit_task_namespaces(struct task_struct *tsk);
 void switch_task_namespaces(struct task_struct *tsk, struct nsproxy *new);
diff --git a/include/linux/pid_namespace.h b/include/linux/pid_namespace.h
index 7246ef3d4455..1997ffc295a7 100644
--- a/include/linux/pid_namespace.h
+++ b/include/linux/pid_namespace.h
@@ -33,6 +33,7 @@ struct pid_namespace {
 #ifdef CONFIG_PROC_FS
         struct vfsmount *proc_mnt;
         struct dentry *proc_self;
+        struct dentry *proc_thread_self;
 #endif
 #ifdef CONFIG_BSD_PROCESS_ACCT
         struct bsd_acct_struct *bacct;
diff --git a/include/linux/sched.h b/include/linux/sched.h
index db2f6474e95e..857ba40426ba 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2360,8 +2360,10 @@ static inline int on_sig_stack(unsigned long sp)
 
 static inline int sas_ss_flags(unsigned long sp)
 {
-        return (current->sas_ss_size == 0 ? SS_DISABLE
-                : on_sig_stack(sp) ? SS_ONSTACK : 0);
+        if (!current->sas_ss_size)
+                return SS_DISABLE;
+
+        return on_sig_stack(sp) ? SS_ONSTACK : 0;
 }
 
 static inline unsigned long sigsp(unsigned long sp, struct ksignal *ksig)
diff --git a/include/linux/signal.h b/include/linux/signal.h
index c9e65360c49a..750196fcc0a5 100644
--- a/include/linux/signal.h
+++ b/include/linux/signal.h
@@ -280,9 +280,8 @@ struct ksignal {
         int sig;
 };
 
-extern int get_signal_to_deliver(siginfo_t *info, struct k_sigaction *return_ka, struct pt_regs *regs, void *cookie);
+extern int get_signal(struct ksignal *ksig);
 extern void signal_setup_done(int failed, struct ksignal *ksig, int stepping);
-extern void signal_delivered(int sig, siginfo_t *info, struct k_sigaction *ka, struct pt_regs *regs, int stepping);
 extern void exit_signals(struct task_struct *tsk);
 extern void kernel_sigaction(int, __sighandler_t);
 
@@ -301,18 +300,6 @@ static inline void disallow_signal(int sig)
         kernel_sigaction(sig, SIG_IGN);
 }
 
-/*
- * Eventually that'll replace get_signal_to_deliver(); macro for now,
- * to avoid nastiness with include order.
- */
-#define get_signal(ksig)                                        \
-({                                                              \
-        struct ksignal *p = (ksig);                             \
-        p->sig = get_signal_to_deliver(&p->info, &p->ka,        \
-                                        signal_pt_regs(), NULL);\
-        p->sig > 0;                                             \
-})
-
 extern struct kmem_cache *sighand_cachep;
 
 int unhandled_signal(struct task_struct *tsk, int sig);
diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index 1bc7cd05b22e..cf61ecd148e0 100644
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -236,7 +236,7 @@ struct svc_rqst {
         struct svc_cred         rq_cred;        /* auth info */
         void *                  rq_xprt_ctxt;   /* transport specific context ptr */
         struct svc_deferred_req*rq_deferred;    /* deferred request we are replaying */
-        int                     rq_usedeferral; /* use deferral */
+        bool                    rq_usedeferral; /* use deferral */
 
         size_t                  rq_xprt_hlen;   /* xprt header len */
         struct xdr_buf          rq_arg;
@@ -277,7 +277,7 @@ struct svc_rqst {
         struct auth_domain *    rq_gssclient;   /* "gss/"-style peer info */
         int                     rq_cachetype;
         struct svc_cacherep *   rq_cacherep;    /* cache info */
-        int                     rq_splice_ok;   /* turned off in gss privacy
+        bool                    rq_splice_ok;   /* turned off in gss privacy
                                                  * to prevent encrypting page
                                                  * cache pages */
         wait_queue_head_t       rq_wait;        /* synchronization */
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index 5cf99a016368..975da754c778 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -174,8 +174,7 @@ struct svcxprt_rdma {
  * page size of 4k, or 32k * 2 ops / 4k = 16 outstanding RDMA_READ.  */
 #define RPCRDMA_ORD             (64/4)
 #define RPCRDMA_SQ_DEPTH_MULT   8
-#define RPCRDMA_MAX_THREADS     16
-#define RPCRDMA_MAX_REQUESTS    16
+#define RPCRDMA_MAX_REQUESTS    32
 #define RPCRDMA_MAX_REQ_SIZE    4096
 
 /* svc_rdma_marshal.c */
diff --git a/include/linux/sunrpc/svc_xprt.h b/include/linux/sunrpc/svc_xprt.h
index 7235040a19b2..ce6e4182a5b2 100644
--- a/include/linux/sunrpc/svc_xprt.h
+++ b/include/linux/sunrpc/svc_xprt.h
@@ -25,6 +25,7 @@ struct svc_xprt_ops {
         void            (*xpo_detach)(struct svc_xprt *);
         void            (*xpo_free)(struct svc_xprt *);
         int             (*xpo_secure_port)(struct svc_rqst *);
+        void            (*xpo_adjust_wspace)(struct svc_xprt *);
 };
 
 struct svc_xprt_class {
@@ -33,6 +34,7 @@ struct svc_xprt_class {
         struct svc_xprt_ops     *xcl_ops;
         struct list_head        xcl_list;
         u32                     xcl_max_payload;
+        int                     xcl_ident;
 };
 
 /*
diff --git a/include/linux/tracehook.h b/include/linux/tracehook.h
index 6f8ab7da27c4..84d497297c5f 100644
--- a/include/linux/tracehook.h
+++ b/include/linux/tracehook.h
@@ -133,10 +133,6 @@ static inline void tracehook_report_syscall_exit(struct pt_regs *regs, int step)
 
 /**
  * tracehook_signal_handler - signal handler setup is complete
- * @sig:                number of signal being delivered
- * @info:               siginfo_t of signal being delivered
- * @ka:                 sigaction setting that chose the handler
- * @regs:               user register state
  * @stepping:           nonzero if debugger single-step or block-step in use
  *
  * Called by the arch code after a signal handler has been set up.
@@ -146,9 +142,7 @@ static inline void tracehook_report_syscall_exit(struct pt_regs *regs, int step)
  * Called without locks, shortly before returning to user mode
  * (or handling more signals).
  */
-static inline void tracehook_signal_handler(int sig, siginfo_t *info,
-                                            const struct k_sigaction *ka,
-                                            struct pt_regs *regs, int stepping)
+static inline void tracehook_signal_handler(int stepping)
 {
         if (stepping)
                 ptrace_notify(SIGTRAP);
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index 2e2a5f7717e5..b1293f15f592 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -249,6 +249,50 @@ extern void syscall_unregfunc(void);
 
 #endif /* CONFIG_TRACEPOINTS */
 
+#ifdef CONFIG_TRACING
+/**
+ * tracepoint_string - register constant persistent string to trace system
+ * @str - a constant persistent string that will be referenced in tracepoints
+ *
+ * If constant strings are being used in tracepoints, it is faster and
+ * more efficient to just save the pointer to the string and reference
+ * that with a printf "%s" instead of saving the string in the ring buffer
+ * and wasting space and time.
+ *
+ * The problem with the above approach is that userspace tools that read
+ * the binary output of the trace buffers do not have access to the string.
+ * Instead they just show the address of the string which is not very
+ * useful to users.
+ *
+ * With tracepoint_string(), the string will be registered to the tracing
+ * system and exported to userspace via the debugfs/tracing/printk_formats
+ * file that maps the string address to the string text. This way userspace
+ * tools that read the binary buffers have a way to map the pointers to
+ * the ASCII strings they represent.
+ *
+ * The @str used must be a constant string and persistent as it would not
+ * make sense to show a string that no longer exists. But it is still fine
+ * to be used with modules, because when modules are unloaded, if they
+ * had tracepoints, the ring buffers are cleared too. As long as the string
+ * does not change during the life of the module, it is fine to use
+ * tracepoint_string() within a module.
+ */
+#define tracepoint_string(str)                                          \
+        ({                                                              \
+                static const char *___tp_str __tracepoint_string = str; \
+                ___tp_str;                                              \
+        })
+#define __tracepoint_string     __attribute__((section("__tracepoint_str")))
+#else
+/*
+ * tracepoint_string() is used to save the string address for userspace
+ * tracing tools. When tracing isn't configured, there's no need to save
+ * anything.
+ */
+# define tracepoint_string(str) str
+# define __tracepoint_string
+#endif
+
 /*
  * The need for the DECLARE_TRACE_NOARGS() is to handle the prototype
  * (void). "void" is a special value in a function prototype and can
diff --git a/include/net/netlabel.h b/include/net/netlabel.h
index a4fc39bb3e4f..7b5a300de7f5 100644
--- a/include/net/netlabel.h
+++ b/include/net/netlabel.h
@@ -524,10 +524,10 @@ static inline int netlbl_catmap_setrng(struct netlbl_lsm_catmap **catmap,
 {
         return 0;
 }
-static int netlbl_catmap_setlong(struct netlbl_lsm_catmap **catmap,
-                                 u32 offset,
-                                 unsigned long bitmap,
-                                 gfp_t flags)
+static inline int netlbl_catmap_setlong(struct netlbl_lsm_catmap **catmap,
+                                        u32 offset,
+                                        unsigned long bitmap,
+                                        gfp_t flags)
 {
         return 0;
 }
diff --git a/include/trace/events/ipi.h b/include/trace/events/ipi.h
new file mode 100644
index 000000000000..834a7362a610
--- /dev/null
+++ b/include/trace/events/ipi.h
@@ -0,0 +1,89 @@
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM ipi
+
+#if !defined(_TRACE_IPI_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_IPI_H
+
+#include <linux/tracepoint.h>
+
+/**
+ * ipi_raise - called when a smp cross call is made
+ *
+ * @mask: mask of recipient CPUs for the IPI
+ * @reason: string identifying the IPI purpose
+ *
+ * It is necessary for @reason to be a static string declared with
+ * __tracepoint_string.
+ */
+TRACE_EVENT(ipi_raise,
+
+        TP_PROTO(const struct cpumask *mask, const char *reason),
+
+        TP_ARGS(mask, reason),
+
+        TP_STRUCT__entry(
+                __bitmask(target_cpus, nr_cpumask_bits)
+                __field(const char *, reason)
+        ),
+
+        TP_fast_assign(
+                __assign_bitmask(target_cpus, cpumask_bits(mask), nr_cpumask_bits);
+                __entry->reason = reason;
+        ),
+
+        TP_printk("target_mask=%s (%s)", __get_bitmask(target_cpus), __entry->reason)
+);
+
+DECLARE_EVENT_CLASS(ipi_handler,
+
+        TP_PROTO(const char *reason),
+
+        TP_ARGS(reason),
+
+        TP_STRUCT__entry(
+                __field(const char *, reason)
+        ),
+
+        TP_fast_assign(
+                __entry->reason = reason;
+        ),
+
+        TP_printk("(%s)", __entry->reason)
+);
+
+/**
+ * ipi_entry - called immediately before the IPI handler
+ *
+ * @reason: string identifying the IPI purpose
+ *
+ * It is necessary for @reason to be a static string declared with
+ * __tracepoint_string, ideally the same as used with trace_ipi_raise
+ * for that IPI.
+ */
+DEFINE_EVENT(ipi_handler, ipi_entry,
+
+        TP_PROTO(const char *reason),
+
+        TP_ARGS(reason)
+);
+
+/**
+ * ipi_exit - called immediately after the IPI handler returns
+ *
+ * @reason: string identifying the IPI purpose
+ *
+ * It is necessary for @reason to be a static string declared with
+ * __tracepoint_string, ideally the same as used with trace_ipi_raise for
+ * that IPI.
+ */
+DEFINE_EVENT(ipi_handler, ipi_exit,
+
+        TP_PROTO(const char *reason),
+
+        TP_ARGS(reason)
+);
+
+#endif /* _TRACE_IPI_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/uapi/drm/nouveau_drm.h b/include/uapi/drm/nouveau_drm.h
index 2a5769fdf8ba..0d7608dc1a34 100644
--- a/include/uapi/drm/nouveau_drm.h
+++ b/include/uapi/drm/nouveau_drm.h
@@ -25,6 +25,16 @@
 #ifndef __NOUVEAU_DRM_H__
 #define __NOUVEAU_DRM_H__
 
+#define DRM_NOUVEAU_EVENT_NVIF                                       0x80000000
+
+/* reserved object handles when using deprecated object APIs - these
+ * are here so that libdrm can allow interoperability with the new
+ * object APIs
+ */
+#define NOUVEAU_ABI16_CLIENT   0xffffffff
+#define NOUVEAU_ABI16_DEVICE   0xdddddddd
+#define NOUVEAU_ABI16_CHAN(n) (0xcccc0000 | (n))
+
 #define NOUVEAU_GEM_DOMAIN_CPU       (1 << 0)
 #define NOUVEAU_GEM_DOMAIN_VRAM      (1 << 1)
 #define NOUVEAU_GEM_DOMAIN_GART      (1 << 2)
@@ -123,6 +133,7 @@ struct drm_nouveau_gem_cpu_fini {
 #define DRM_NOUVEAU_GROBJ_ALLOC        0x04 /* deprecated */
 #define DRM_NOUVEAU_NOTIFIEROBJ_ALLOC  0x05 /* deprecated */
 #define DRM_NOUVEAU_GPUOBJ_FREE        0x06 /* deprecated */
+#define DRM_NOUVEAU_NVIF               0x07
 #define DRM_NOUVEAU_GEM_NEW            0x40
 #define DRM_NOUVEAU_GEM_PUSHBUF        0x41
 #define DRM_NOUVEAU_GEM_CPU_PREP       0x42
diff --git a/ipc/namespace.c b/ipc/namespace.c
index 59451c1e214d..b54468e48e32 100644
--- a/ipc/namespace.c
+++ b/ipc/namespace.c
@@ -154,11 +154,11 @@ static void *ipcns_get(struct task_struct *task)
         struct ipc_namespace *ns = NULL;
         struct nsproxy *nsproxy;
 
-        rcu_read_lock();
-        nsproxy = task_nsproxy(task);
+        task_lock(task);
+        nsproxy = task->nsproxy;
         if (nsproxy)
                 ns = get_ipc_ns(nsproxy->ipc_ns);
-        rcu_read_unlock();
+        task_unlock(task);
 
         return ns;
 }
diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c
index 8e7811086b82..ef42d0ab3115 100644
--- a/kernel/nsproxy.c
+++ b/kernel/nsproxy.c
@@ -204,20 +204,13 @@ void switch_task_namespaces(struct task_struct *p, struct nsproxy *new)
 
         might_sleep();
 
+        task_lock(p);
         ns = p->nsproxy;
+        p->nsproxy = new;
+        task_unlock(p);
 
-        rcu_assign_pointer(p->nsproxy, new);
-
-        if (ns && atomic_dec_and_test(&ns->count)) {
-                /*
-                 * wait for others to get what they want from this nsproxy.
-                 *
-                 * cannot release this nsproxy via the call_rcu() since
-                 * put_mnt_ns() will want to sleep
-                 */
-                synchronize_rcu();
+        if (ns && atomic_dec_and_test(&ns->count))
                 free_nsproxy(ns);
-        }
 }
 
 void exit_task_namespaces(struct task_struct *p)
diff --git a/kernel/signal.c b/kernel/signal.c
index 40b76e351e64..8f0876f9f6dd 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -2170,8 +2170,7 @@ static int ptrace_signal(int signr, siginfo_t *info)
         return signr;
 }
 
-int get_signal_to_deliver(siginfo_t *info, struct k_sigaction *return_ka,
-                          struct pt_regs *regs, void *cookie)
+int get_signal(struct ksignal *ksig)
 {
         struct sighand_struct *sighand = current->sighand;
         struct signal_struct *signal = current->signal;
@@ -2241,13 +2240,13 @@ relock:
                         goto relock;
                 }
 
-                signr = dequeue_signal(current, &current->blocked, info);
+                signr = dequeue_signal(current, &current->blocked, &ksig->info);
 
                 if (!signr)
                         break; /* will return 0 */
 
                 if (unlikely(current->ptrace) && signr != SIGKILL) {
-                        signr = ptrace_signal(signr, info);
+                        signr = ptrace_signal(signr, &ksig->info);
                         if (!signr)
                                 continue;
                 }
@@ -2255,13 +2254,13 @@ relock:
                 ka = &sighand->action[signr-1];
 
                 /* Trace actually delivered signals. */
-                trace_signal_deliver(signr, info, ka);
+                trace_signal_deliver(signr, &ksig->info, ka);
 
                 if (ka->sa.sa_handler == SIG_IGN) /* Do nothing.  */
                         continue;
                 if (ka->sa.sa_handler != SIG_DFL) {
                         /* Run the handler.  */
-                        *return_ka = *ka;
+                        ksig->ka = *ka;
 
                         if (ka->sa.sa_flags & SA_ONESHOT)
                                 ka->sa.sa_handler = SIG_DFL;
@@ -2311,7 +2310,7 @@ relock:
                                 spin_lock_irq(&sighand->siglock);
                         }
 
-                        if (likely(do_signal_stop(info->si_signo))) {
+                        if (likely(do_signal_stop(ksig->info.si_signo))) {
                                 /* It released the siglock.  */
                                 goto relock;
                         }
@@ -2332,7 +2331,7 @@ relock:
 
                 if (sig_kernel_coredump(signr)) {
                         if (print_fatal_signals)
-                                print_fatal_signal(info->si_signo);
+                                print_fatal_signal(ksig->info.si_signo);
                         proc_coredump_connector(current);
                         /*
                          * If it was able to dump core, this kills all
@@ -2342,34 +2341,32 @@ relock:
                          * first and our do_group_exit call below will use
                          * that value and ignore the one we pass it.
                          */
-                        do_coredump(info);
+                        do_coredump(&ksig->info);
                 }
 
                 /*
                  * Death signals, no core dump.
                  */
-                do_group_exit(info->si_signo);
+                do_group_exit(ksig->info.si_signo);
                 /* NOTREACHED */
         }
         spin_unlock_irq(&sighand->siglock);
-        return signr;
+
+        ksig->sig = signr;
+        return ksig->sig > 0;
 }
 
 /**
  * signal_delivered - 
- * @sig:                number of signal being delivered
- * @info:               siginfo_t of signal being delivered
- * @ka:                 sigaction setting that chose the handler
- * @regs:               user register state
+ * @ksig:               kernel signal struct
  * @stepping:           nonzero if debugger single-step or block-step in use
  *
  * This function should be called when a signal has successfully been
- * delivered. It updates the blocked signals accordingly (@ka->sa.sa_mask
+ * delivered. It updates the blocked signals accordingly (@ksig->ka.sa.sa_mask
  * is always blocked, and the signal itself is blocked unless %SA_NODEFER
- * is set in @ka->sa.sa_flags.  Tracing is notified.
+ * is set in @ksig->ka.sa.sa_flags.  Tracing is notified.
  */
-void signal_delivered(int sig, siginfo_t *info, struct k_sigaction *ka,
-                        struct pt_regs *regs, int stepping)
+static void signal_delivered(struct ksignal *ksig, int stepping)
 {
         sigset_t blocked;
 
@@ -2379,11 +2376,11 @@ void signal_delivered(int sig, siginfo_t *info, struct k_sigaction *ka,
            simply clear the restore sigmask flag.  */
         clear_restore_sigmask();
 
-        sigorsets(&blocked, &current->blocked, &ka->sa.sa_mask);
-        if (!(ka->sa.sa_flags & SA_NODEFER))
-                sigaddset(&blocked, sig);
+        sigorsets(&blocked, &current->blocked, &ksig->ka.sa.sa_mask);
+        if (!(ksig->ka.sa.sa_flags & SA_NODEFER))
+                sigaddset(&blocked, ksig->sig);
         set_current_blocked(&blocked);
-        tracehook_signal_handler(sig, info, ka, regs, stepping);
+        tracehook_signal_handler(stepping);
 }
 
 void signal_setup_done(int failed, struct ksignal *ksig, int stepping)
@@ -2391,8 +2388,7 @@ void signal_setup_done(int failed, struct ksignal *ksig, int stepping)
         if (failed)
                 force_sigsegv(ksig->sig, current);
         else
-                signal_delivered(ksig->sig, &ksig->info, &ksig->ka,
-                        signal_pt_regs(), stepping);
+                signal_delivered(ksig, stepping);
 }
 
 /*
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 925f629658d6..afb04b9b818a 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -1968,7 +1968,7 @@ rb_add_time_stamp(struct ring_buffer_event *event, u64 delta)
 
 /**
  * rb_update_event - update event type and data
- * @event: the even to update
+ * @event: the event to update
  * @type: the type of event
  * @length: the size of the event field in the ring buffer
  *
@@ -3341,21 +3341,16 @@ static void rb_iter_reset(struct ring_buffer_iter *iter)
         struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
 
         /* Iterator usage is expected to have record disabled */
-        if (list_empty(&cpu_buffer->reader_page->list)) {
-                iter->head_page = rb_set_head_page(cpu_buffer);
-                if (unlikely(!iter->head_page))
-                        return;
-                iter->head = iter->head_page->read;
-        } else {
-                iter->head_page = cpu_buffer->reader_page;
-                iter->head = cpu_buffer->reader_page->read;
-        }
+        iter->head_page = cpu_buffer->reader_page;
+        iter->head = cpu_buffer->reader_page->read;
+
+        iter->cache_reader_page = iter->head_page;
+        iter->cache_read = iter->head;
+
         if (iter->head)
                 iter->read_stamp = cpu_buffer->read_stamp;
         else
                 iter->read_stamp = iter->head_page->page->time_stamp;
-        iter->cache_reader_page = cpu_buffer->reader_page;
-        iter->cache_read = cpu_buffer->read;
 }
 
 /**
@@ -3748,12 +3743,14 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
                 return NULL;
 
         /*
-         * We repeat when a time extend is encountered.
-         * Since the time extend is always attached to a data event,
-         * we should never loop more than once.
-         * (We never hit the following condition more than twice).
+         * We repeat when a time extend is encountered or we hit
+         * the end of the page. Since the time extend is always attached
+         * to a data event, we should never loop more than three times.
+         * Once for going to next page, once on time extend, and
+         * finally once to get the event.
+         * (We never hit the following condition more than thrice).
          */
-        if (RB_WARN_ON(cpu_buffer, ++nr_loops > 2))
+        if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3))
                 return NULL;
 
         if (rb_per_cpu_empty(cpu_buffer))
diff --git a/kernel/utsname.c b/kernel/utsname.c
index fd393124e507..883aaaa7de8a 100644
--- a/kernel/utsname.c
+++ b/kernel/utsname.c
@@ -93,13 +93,13 @@ static void *utsns_get(struct task_struct *task)
         struct uts_namespace *ns = NULL;
         struct nsproxy *nsproxy;
 
-        rcu_read_lock();
-        nsproxy = task_nsproxy(task);
+        task_lock(task);
+        nsproxy = task->nsproxy;
         if (nsproxy) {
                 ns = nsproxy->uts_ns;
                 get_uts_ns(ns);
         }
-        rcu_read_unlock();
+        task_unlock(task);
 
         return ns;
 }
diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c
index 85b62691f4f2..7c6b51a58968 100644
--- a/net/core/net_namespace.c
+++ b/net/core/net_namespace.c
@@ -373,9 +373,11 @@ struct net *get_net_ns_by_pid(pid_t pid)
         tsk = find_task_by_vpid(pid);
         if (tsk) {
                 struct nsproxy *nsproxy;
-                nsproxy = task_nsproxy(tsk);
+                task_lock(tsk);
+                nsproxy = tsk->nsproxy;
                 if (nsproxy)
                         net = get_net(nsproxy->net_ns);
+                task_unlock(tsk);
         }
         rcu_read_unlock();
         return net;
@@ -632,11 +634,11 @@ static void *netns_get(struct task_struct *task)
         struct net *net = NULL;
         struct nsproxy *nsproxy;
 
-        rcu_read_lock();
-        nsproxy = task_nsproxy(task);
+        task_lock(task);
+        nsproxy = task->nsproxy;
         if (nsproxy)
                 net = get_net(nsproxy->net_ns);
-        rcu_read_unlock();
+        task_unlock(task);
 
         return net;
 }
diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c
index 4ce5eccec1f6..c548ab213f76 100644
--- a/net/sunrpc/auth_gss/svcauth_gss.c
+++ b/net/sunrpc/auth_gss/svcauth_gss.c
@@ -886,7 +886,7 @@ unwrap_priv_data(struct svc_rqst *rqstp, struct xdr_buf *buf, u32 seq, struct gs
         u32 priv_len, maj_stat;
         int pad, saved_len, remaining_len, offset;
 
-        rqstp->rq_splice_ok = 0;
+        rqstp->rq_splice_ok = false;
 
         priv_len = svc_getnl(&buf->head[0]);
         if (rqstp->rq_deferred) {
diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 5de6801cd924..1db5007ddbce 100644
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -1086,9 +1086,9 @@ svc_process_common(struct svc_rqst *rqstp, struct kvec *argv, struct kvec *resv)
                 goto err_short_len;
 
         /* Will be turned off only in gss privacy case: */
-        rqstp->rq_splice_ok = 1;
+        rqstp->rq_splice_ok = true;
         /* Will be turned off only when NFSv4 Sessions are used */
-        rqstp->rq_usedeferral = 1;
+        rqstp->rq_usedeferral = true;
         rqstp->rq_dropme = false;
 
         /* Setup reply header */
diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
index b4737fbdec13..6666c6745858 100644
--- a/net/sunrpc/svc_xprt.c
+++ b/net/sunrpc/svc_xprt.c
@@ -23,6 +23,7 @@ static int svc_deferred_recv(struct svc_rqst *rqstp);
 static struct cache_deferred_req *svc_defer(struct cache_req *req);
 static void svc_age_temp_xprts(unsigned long closure);
 static void svc_delete_xprt(struct svc_xprt *xprt);
+static void svc_xprt_do_enqueue(struct svc_xprt *xprt);
 
 /* apparently the "standard" is that clients close
  * idle connections after 5 minutes, servers after
@@ -222,11 +223,12 @@ static void svc_xprt_received(struct svc_xprt *xprt)
         if (!test_bit(XPT_BUSY, &xprt->xpt_flags))
                 return;
         /* As soon as we clear busy, the xprt could be closed and
-         * 'put', so we need a reference to call svc_xprt_enqueue with:
+         * 'put', so we need a reference to call svc_xprt_do_enqueue with:
          */
         svc_xprt_get(xprt);
+        smp_mb__before_atomic();
         clear_bit(XPT_BUSY, &xprt->xpt_flags);
-        svc_xprt_enqueue(xprt);
+        svc_xprt_do_enqueue(xprt);
         svc_xprt_put(xprt);
 }
 
@@ -335,12 +337,7 @@ static bool svc_xprt_has_something_to_do(struct svc_xprt *xprt)
         return false;
 }
 
-/*
- * Queue up a transport with data pending. If there are idle nfsd
- * processes, wake 'em up.
- *
- */
-void svc_xprt_enqueue(struct svc_xprt *xprt)
+static void svc_xprt_do_enqueue(struct svc_xprt *xprt)
 {
         struct svc_pool *pool;
         struct svc_rqst *rqstp;
@@ -398,6 +395,18 @@ void svc_xprt_enqueue(struct svc_xprt *xprt)
 out_unlock:
         spin_unlock_bh(&pool->sp_lock);
 }
+
+/*
+ * Queue up a transport with data pending. If there are idle nfsd
+ * processes, wake 'em up.
+ *
+ */
+void svc_xprt_enqueue(struct svc_xprt *xprt)
+{
+        if (test_bit(XPT_BUSY, &xprt->xpt_flags))
+                return;
+        svc_xprt_do_enqueue(xprt);
+}
 EXPORT_SYMBOL_GPL(svc_xprt_enqueue);
 
 /*
@@ -439,6 +448,8 @@ void svc_reserve(struct svc_rqst *rqstp, int space)
                 atomic_sub((rqstp->rq_reserved - space), &xprt->xpt_reserved);
                 rqstp->rq_reserved = space;
 
+                if (xprt->xpt_ops->xpo_adjust_wspace)
+                        xprt->xpt_ops->xpo_adjust_wspace(xprt);
                 svc_xprt_enqueue(xprt);
         }
 }
diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c
index b507cd327d9b..c24a8ff33f8f 100644
--- a/net/sunrpc/svcsock.c
+++ b/net/sunrpc/svcsock.c
@@ -446,15 +446,43 @@ static void svc_write_space(struct sock *sk)
         }
 }
 
+static int svc_tcp_has_wspace(struct svc_xprt *xprt)
+{
+        struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
+        struct svc_serv *serv = svsk->sk_xprt.xpt_server;
+        int required;
+
+        if (test_bit(XPT_LISTENER, &xprt->xpt_flags))
+                return 1;
+        required = atomic_read(&xprt->xpt_reserved) + serv->sv_max_mesg;
+        if (sk_stream_wspace(svsk->sk_sk) >= required ||
+            (sk_stream_min_wspace(svsk->sk_sk) == 0 &&
+             atomic_read(&xprt->xpt_reserved) == 0))
+                return 1;
+        set_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);
+        return 0;
+}
+
 static void svc_tcp_write_space(struct sock *sk)
 {
+        struct svc_sock *svsk = (struct svc_sock *)(sk->sk_user_data);
         struct socket *sock = sk->sk_socket;
 
-        if (sk_stream_is_writeable(sk) && sock)
+        if (!sk_stream_is_writeable(sk) || !sock)
+                return;
+        if (!svsk || svc_tcp_has_wspace(&svsk->sk_xprt))
                 clear_bit(SOCK_NOSPACE, &sock->flags);
         svc_write_space(sk);
 }
 
+static void svc_tcp_adjust_wspace(struct svc_xprt *xprt)
+{
+        struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
+
+        if (svc_tcp_has_wspace(xprt))
+                clear_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);
+}
+
 /*
  * See net/ipv6/ip_sockglue.c : ip_cmsg_recv_pktinfo
  */
@@ -692,6 +720,7 @@ static struct svc_xprt_class svc_udp_class = {
         .xcl_owner = THIS_MODULE,
         .xcl_ops = &svc_udp_ops,
         .xcl_max_payload = RPCSVC_MAXPAYLOAD_UDP,
+        .xcl_ident = XPRT_TRANSPORT_UDP,
 };
 
 static void svc_udp_init(struct svc_sock *svsk, struct svc_serv *serv)
@@ -1197,23 +1226,6 @@ static void svc_tcp_prep_reply_hdr(struct svc_rqst *rqstp)
         svc_putnl(resv, 0);
 }
 
-static int svc_tcp_has_wspace(struct svc_xprt *xprt)
-{
-        struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);
-        struct svc_serv *serv = svsk->sk_xprt.xpt_server;
-        int required;
-
-        if (test_bit(XPT_LISTENER, &xprt->xpt_flags))
-                return 1;
-        required = atomic_read(&xprt->xpt_reserved) + serv->sv_max_mesg;
-        if (sk_stream_wspace(svsk->sk_sk) >= required ||
-            (sk_stream_min_wspace(svsk->sk_sk) == 0 &&
-             atomic_read(&xprt->xpt_reserved) == 0))
-                return 1;
-        set_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);
-        return 0;
-}
-
 static struct svc_xprt *svc_tcp_create(struct svc_serv *serv,
                                        struct net *net,
                                        struct sockaddr *sa, int salen,
@@ -1285,6 +1297,7 @@ static struct svc_xprt_ops svc_tcp_ops = {
         .xpo_has_wspace = svc_tcp_has_wspace,
         .xpo_accept = svc_tcp_accept,
         .xpo_secure_port = svc_sock_secure_port,
+        .xpo_adjust_wspace = svc_tcp_adjust_wspace,
 };
 
 static struct svc_xprt_class svc_tcp_class = {
@@ -1292,6 +1305,7 @@ static struct svc_xprt_class svc_tcp_class = {
         .xcl_owner = THIS_MODULE,
         .xcl_ops = &svc_tcp_ops,
         .xcl_max_payload = RPCSVC_MAXPAYLOAD_TCP,
+        .xcl_ident = XPRT_TRANSPORT_TCP,
 };
 
 void svc_init_xprt_sock(void)
diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c
index 23fb4e75e245..290af97bf6f9 100644
--- a/net/sunrpc/xdr.c
+++ b/net/sunrpc/xdr.c
@@ -509,7 +509,8 @@ void xdr_commit_encode(struct xdr_stream *xdr)
 }
 EXPORT_SYMBOL_GPL(xdr_commit_encode);
 
-__be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, size_t nbytes)
+static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,
+                size_t nbytes)
 {
         static __be32 *p;
         int space_left;
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index c3b2b3369e52..51c63165073c 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -1306,7 +1306,7 @@ struct rpc_xprt *xprt_create_transport(struct xprt_create *args)
                 }
         }
         spin_unlock(&xprt_list_lock);
-        printk(KERN_ERR "RPC: transport (%d) not supported\n", args->ident);
+        dprintk("RPC: transport (%d) not supported\n", args->ident);
         return ERR_PTR(-EIO);
 
 found:
diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
index 8f92a61ee2df..e0110270d650 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
@@ -43,6 +43,7 @@
 #include <linux/sunrpc/debug.h>
 #include <linux/sunrpc/rpc_rdma.h>
 #include <linux/spinlock.h>
+#include <linux/highmem.h>
 #include <asm/unaligned.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/rdma_cm.h>
@@ -435,6 +436,32 @@ static int rdma_read_chunks(struct svcxprt_rdma *xprt,
         return ret;
 }
 
+/*
+ * To avoid a separate RDMA READ just for a handful of zero bytes,
+ * RFC 5666 section 3.7 allows the client to omit the XDR zero pad
+ * in chunk lists.
+ */
+static void
+rdma_fix_xdr_pad(struct xdr_buf *buf)
+{
+        unsigned int page_len = buf->page_len;
+        unsigned int size = (XDR_QUADLEN(page_len) << 2) - page_len;
+        unsigned int offset, pg_no;
+        char *p;
+
+        if (size == 0)
+                return;
+
+        pg_no = page_len >> PAGE_SHIFT;
+        offset = page_len & ~PAGE_MASK;
+        p = page_address(buf->pages[pg_no]);
+        memset(p + offset, 0, size);
+
+        buf->page_len += size;
+        buf->buflen += size;
+        buf->len += size;
+}
+
 static int rdma_read_complete(struct svc_rqst *rqstp,
                               struct svc_rdma_op_ctxt *head)
 {
@@ -449,6 +476,7 @@ static int rdma_read_complete(struct svc_rqst *rqstp,
                 rqstp->rq_pages[page_no] = head->pages[page_no];
         }
         /* Point rq_arg.pages past header */
+        rdma_fix_xdr_pad(&head->arg);
         rqstp->rq_arg.pages = &rqstp->rq_pages[head->hdr_count];
         rqstp->rq_arg.page_len = head->arg.page_len;
         rqstp->rq_arg.page_base = head->arg.page_base;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
index 49fd21a5c215..9f1b50689c0f 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
@@ -192,6 +192,8 @@ static int send_write(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp,
                 xdr_sge_no++;
                 BUG_ON(xdr_sge_no > vec->count);
                 bc -= sge_bytes;
+                if (sge_no == xprt->sc_max_sge)
+                        break;
         }
 
         /* Prepare WRITE WR */
@@ -209,7 +211,7 @@ static int send_write(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp,
         atomic_inc(&rdma_stat_write);
         if (svc_rdma_send(xprt, &write_wr))
                 goto err;
-        return 0;
+        return write_len - bc;
  err:
         svc_rdma_unmap_dma(ctxt);
         svc_rdma_put_context(ctxt, 0);
@@ -225,7 +227,6 @@ static int send_write_chunks(struct svcxprt_rdma *xprt,
 {
         u32 xfer_len = rqstp->rq_res.page_len + rqstp->rq_res.tail[0].iov_len;
         int write_len;
-        int max_write;
         u32 xdr_off;
         int chunk_off;
         int chunk_no;
@@ -239,8 +240,6 @@ static int send_write_chunks(struct svcxprt_rdma *xprt,
         res_ary = (struct rpcrdma_write_array *)
                 &rdma_resp->rm_body.rm_chunks[1];
 
-        max_write = xprt->sc_max_sge * PAGE_SIZE;
-
         /* Write chunks start at the pagelist */
         for (xdr_off = rqstp->rq_res.head[0].iov_len, chunk_no = 0;
              xfer_len && chunk_no < arg_ary->wc_nchunks;
@@ -260,23 +259,21 @@ static int send_write_chunks(struct svcxprt_rdma *xprt,
                                                 write_len);
                 chunk_off = 0;
                 while (write_len) {
-                        int this_write;
-                        this_write = min(write_len, max_write);
                         ret = send_write(xprt, rqstp,
                                          ntohl(arg_ch->rs_handle),
                                          rs_offset + chunk_off,
                                          xdr_off,
-                                         this_write,
+                                         write_len,
                                          vec);
-                        if (ret) {
+                        if (ret <= 0) {
                                 dprintk("svcrdma: RDMA_WRITE failed, ret=%d\n",
                                         ret);
                                 return -EIO;
                         }
-                        chunk_off += this_write;
-                        xdr_off += this_write;
-                        xfer_len -= this_write;
-                        write_len -= this_write;
+                        chunk_off += ret;
+                        xdr_off += ret;
+                        xfer_len -= ret;
+                        write_len -= ret;
                 }
         }
         /* Update the req with the number of chunks actually used */
@@ -293,7 +290,6 @@ static int send_reply_chunks(struct svcxprt_rdma *xprt,
 {
         u32 xfer_len = rqstp->rq_res.len;
         int write_len;
-        int max_write;
         u32 xdr_off;
         int chunk_no;
         int chunk_off;
@@ -311,8 +307,6 @@ static int send_reply_chunks(struct svcxprt_rdma *xprt,
         res_ary = (struct rpcrdma_write_array *)
                 &rdma_resp->rm_body.rm_chunks[2];
 
-        max_write = xprt->sc_max_sge * PAGE_SIZE;
-
         /* xdr offset starts at RPC message */
         nchunks = ntohl(arg_ary->wc_nchunks);
         for (xdr_off = 0, chunk_no = 0;
@@ -330,24 +324,21 @@ static int send_reply_chunks(struct svcxprt_rdma *xprt,
                                                 write_len);
                 chunk_off = 0;
                 while (write_len) {
-                        int this_write;
-
-                        this_write = min(write_len, max_write);
                         ret = send_write(xprt, rqstp,
                                          ntohl(ch->rs_handle),
                                          rs_offset + chunk_off,
                                          xdr_off,
-                                         this_write,
+                                         write_len,
                                          vec);
-                        if (ret) {
+                        if (ret <= 0) {
                                 dprintk("svcrdma: RDMA_WRITE failed, ret=%d\n",
                                         ret);
                                 return -EIO;
                         }
-                        chunk_off += this_write;
-                        xdr_off += this_write;
-                        xfer_len -= this_write;
-                        write_len -= this_write;
+                        chunk_off += ret;
+                        xdr_off += ret;
+                        xfer_len -= ret;
+                        write_len -= ret;
                 }
         }
         /* Update the req with the number of chunks actually used */
diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index e7323fbbd348..374feb44afea 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -92,6 +92,7 @@ struct svc_xprt_class svc_rdma_class = {
         .xcl_owner = THIS_MODULE,
         .xcl_ops = &svc_rdma_ops,
         .xcl_max_payload = RPCSVC_MAXPAYLOAD_TCP,
+        .xcl_ident = XPRT_TRANSPORT_RDMA,
 };
 
 struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *xprt)
@@ -942,23 +943,8 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
 
         ret = rdma_create_qp(newxprt->sc_cm_id, newxprt->sc_pd, &qp_attr);
         if (ret) {
-                /*
-                 * XXX: This is a hack. We need a xx_request_qp interface
-                 * that will adjust the qp_attr's with a best-effort
-                 * number
-                 */
-                qp_attr.cap.max_send_sge -= 2;
-                qp_attr.cap.max_recv_sge -= 2;
-                ret = rdma_create_qp(newxprt->sc_cm_id, newxprt->sc_pd,
-                                     &qp_attr);
-                if (ret) {
-                        dprintk("svcrdma: failed to create QP, ret=%d\n", ret);
-                        goto errout;
-                }
-                newxprt->sc_max_sge = qp_attr.cap.max_send_sge;
-                newxprt->sc_max_sge = qp_attr.cap.max_recv_sge;
-                newxprt->sc_sq_depth = qp_attr.cap.max_send_wr;
-                newxprt->sc_max_requests = qp_attr.cap.max_recv_wr;
+                dprintk("svcrdma: failed to create QP, ret=%d\n", ret);
+                goto errout;
         }
         newxprt->sc_qp = newxprt->sc_cm_id->qp;
 
diff --git a/security/selinux/netif.c b/security/selinux/netif.c
index 3c3de4ca0ebc..50ce177d71a0 100644
--- a/security/selinux/netif.c
+++ b/security/selinux/netif.c
@@ -272,7 +272,7 @@ static struct notifier_block sel_netif_netdev_notifier = {
 
 static __init int sel_netif_init(void)
 {
-        int i, err;
+        int i;
 
         if (!selinux_enabled)
                 return 0;
@@ -282,7 +282,7 @@ static __init int sel_netif_init(void)
 
         register_netdevice_notifier(&sel_netif_netdev_notifier);
 
-        return err;
+        return 0;
 }
 
 __initcall(sel_netif_init);
diff --git a/security/selinux/netnode.c b/security/selinux/netnode.c
index ddf315260839..da923f89d2a9 100644
--- a/security/selinux/netnode.c
+++ b/security/selinux/netnode.c
@@ -303,7 +303,6 @@ void sel_netnode_flush(void)
 static __init int sel_netnode_init(void)
 {
         int iter;
-        int ret;
 
         if (!selinux_enabled)
                 return 0;
@@ -313,7 +312,7 @@ static __init int sel_netnode_init(void)
                 sel_netnode_hash[iter].size = 0;
         }
 
-        return ret;
+        return 0;
 }
 
 __initcall(sel_netnode_init);
diff --git a/security/selinux/netport.c b/security/selinux/netport.c
index 73ac6784d091..3311cc393cb4 100644
--- a/security/selinux/netport.c
+++ b/security/selinux/netport.c
@@ -237,7 +237,6 @@ void sel_netport_flush(void)
 static __init int sel_netport_init(void)
 {
         int iter;
-        int ret;
 
         if (!selinux_enabled)
                 return 0;
@@ -247,7 +246,7 @@ static __init int sel_netport_init(void)
                 sel_netport_hash[iter].size = 0;
         }
 
-        return ret;
+        return 0;
 }
 
 __initcall(sel_netport_init);
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 6fd2a4402069..36ff2e4c7b6f 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -5,6 +5,7 @@ TARGETS += kcmp
 TARGETS += memfd
 TARGETS += memory-hotplug
 TARGETS += mqueue
+TARGETS += mount
 TARGETS += net
 TARGETS += ptrace
 TARGETS += timers
diff --git a/tools/testing/selftests/mount/Makefile b/tools/testing/selftests/mount/Makefile
new file mode 100644
index 000000000000..337d853c2b72
--- /dev/null
+++ b/tools/testing/selftests/mount/Makefile
@@ -0,0 +1,17 @@
+# Makefile for mount selftests.
+
+all: unprivileged-remount-test
+
+unprivileged-remount-test: unprivileged-remount-test.c
+        gcc -Wall -O2 unprivileged-remount-test.c -o unprivileged-remount-test
+
+# Allow specific tests to be selected.
+test_unprivileged_remount: unprivileged-remount-test
+        @if [ -f /proc/self/uid_map ] ; then ./unprivileged-remount-test ; fi
+
+run_tests: all test_unprivileged_remount
+
+clean:
+        rm -f unprivileged-remount-test
+
+.PHONY: all test_unprivileged_remount
diff --git a/tools/testing/selftests/mount/unprivileged-remount-test.c b/tools/testing/selftests/mount/unprivileged-remount-test.c
new file mode 100644
index 000000000000..1b3ff2fda4d0
--- /dev/null
+++ b/tools/testing/selftests/mount/unprivileged-remount-test.c
@@ -0,0 +1,242 @@
+#define _GNU_SOURCE
+#include <sched.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/mount.h>
+#include <sys/wait.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <grp.h>
+#include <stdbool.h>
+#include <stdarg.h>
+
+#ifndef CLONE_NEWNS
+# define CLONE_NEWNS 0x00020000
+#endif
+#ifndef CLONE_NEWUTS
+# define CLONE_NEWUTS 0x04000000
+#endif
+#ifndef CLONE_NEWIPC
+# define CLONE_NEWIPC 0x08000000
+#endif
+#ifndef CLONE_NEWNET
+# define CLONE_NEWNET 0x40000000
+#endif
+#ifndef CLONE_NEWUSER
+# define CLONE_NEWUSER 0x10000000
+#endif
+#ifndef CLONE_NEWPID
+# define CLONE_NEWPID 0x20000000
+#endif
+
+#ifndef MS_RELATIME
+#define MS_RELATIME (1 << 21)
+#endif
+#ifndef MS_STRICTATIME
+#define MS_STRICTATIME (1 << 24)
+#endif
+
+static void die(char *fmt, ...)
+{
+        va_list ap;
+        va_start(ap, fmt);
+        vfprintf(stderr, fmt, ap);
+        va_end(ap);
+        exit(EXIT_FAILURE);
+}
+
+static void write_file(char *filename, char *fmt, ...)
+{
+        char buf[4096];
+        int fd;
+        ssize_t written;
+        int buf_len;
+        va_list ap;
+
+        va_start(ap, fmt);
+        buf_len = vsnprintf(buf, sizeof(buf), fmt, ap);
+        va_end(ap);
+        if (buf_len < 0) {
+                die("vsnprintf failed: %s\n",
+                    strerror(errno));
+        }
+        if (buf_len >= sizeof(buf)) {
+                die("vsnprintf output truncated\n");
+        }
+
+        fd = open(filename, O_WRONLY);
+        if (fd < 0) {
+                die("open of %s failed: %s\n",
+                    filename, strerror(errno));
+        }
+        written = write(fd, buf, buf_len);
+        if (written != buf_len) {
+                if (written >= 0) {
+                        die("short write to %s\n", filename);
+                } else {
+                        die("write to %s failed: %s\n",
+                                filename, strerror(errno));
+                }
+        }
+        if (close(fd) != 0) {
+                die("close of %s failed: %s\n",
+                        filename, strerror(errno));
+        }
+}
+
+static void create_and_enter_userns(void)
+{
+        uid_t uid;
+        gid_t gid;
+
+        uid = getuid();
+        gid = getgid();
+
+        if (unshare(CLONE_NEWUSER) !=0) {
+                die("unshare(CLONE_NEWUSER) failed: %s\n",
+                        strerror(errno));
+        }
+
+        write_file("/proc/self/uid_map", "0 %d 1", uid);
+        write_file("/proc/self/gid_map", "0 %d 1", gid);
+
+        if (setgroups(0, NULL) != 0) {
+                die("setgroups failed: %s\n",
+                        strerror(errno));
+        }
+        if (setgid(0) != 0) {
+                die ("setgid(0) failed %s\n",
+                        strerror(errno));
+        }
+        if (setuid(0) != 0) {
+                die("setuid(0) failed %s\n",
+                        strerror(errno));
+        }
+}
+
+static
+bool test_unpriv_remount(int mount_flags, int remount_flags, int invalid_flags)
+{
+        pid_t child;
+
+        child = fork();
+        if (child == -1) {
+                die("fork failed: %s\n",
+                        strerror(errno));
+        }
+        if (child != 0) { /* parent */
+                pid_t pid;
+                int status;
+                pid = waitpid(child, &status, 0);
+                if (pid == -1) {
+                        die("waitpid failed: %s\n",
+                                strerror(errno));
+                }
+                if (pid != child) {
+                        die("waited for %d got %d\n",
+                                child, pid);
+                }
+                if (!WIFEXITED(status)) {
+                        die("child did not terminate cleanly\n");
+                }
+                return WEXITSTATUS(status) == EXIT_SUCCESS ? true : false;
+        }
+
+        create_and_enter_userns();
+        if (unshare(CLONE_NEWNS) != 0) {
+                die("unshare(CLONE_NEWNS) failed: %s\n",
+                        strerror(errno));
+        }
+
+        if (mount("testing", "/tmp", "ramfs", mount_flags, NULL) != 0) {
+                die("mount of /tmp failed: %s\n",
+                        strerror(errno));
+        }
+
+        create_and_enter_userns();
+
+        if (unshare(CLONE_NEWNS) != 0) {
+                die("unshare(CLONE_NEWNS) failed: %s\n",
+                        strerror(errno));
+        }
+
+        if (mount("/tmp", "/tmp", "none",
+                  MS_REMOUNT | MS_BIND | remount_flags, NULL) != 0) {
+                /* system("cat /proc/self/mounts"); */
+                die("remount of /tmp failed: %s\n",
+                    strerror(errno));
+        }
+
+        if (mount("/tmp", "/tmp", "none",
+                  MS_REMOUNT | MS_BIND | invalid_flags, NULL) == 0) {
+                /* system("cat /proc/self/mounts"); */
+                die("remount of /tmp with invalid flags "
+                    "succeeded unexpectedly\n");
+        }
+        exit(EXIT_SUCCESS);
+}
+
+static bool test_unpriv_remount_simple(int mount_flags)
+{
+        return test_unpriv_remount(mount_flags, mount_flags, 0);
+}
+
+static bool test_unpriv_remount_atime(int mount_flags, int invalid_flags)
+{
+        return test_unpriv_remount(mount_flags, mount_flags, invalid_flags);
+}
+
+int main(int argc, char **argv)
+{
+        if (!test_unpriv_remount_simple(MS_RDONLY|MS_NODEV)) {
+                die("MS_RDONLY malfunctions\n");
+        }
+        if (!test_unpriv_remount_simple(MS_NODEV)) {
+                die("MS_NODEV malfunctions\n");
+        }
+        if (!test_unpriv_remount_simple(MS_NOSUID|MS_NODEV)) {
+                die("MS_NOSUID malfunctions\n");
+        }
+        if (!test_unpriv_remount_simple(MS_NOEXEC|MS_NODEV)) {
+                die("MS_NOEXEC malfunctions\n");
+        }
+        if (!test_unpriv_remount_atime(MS_RELATIME|MS_NODEV,
+                                       MS_NOATIME|MS_NODEV))
+        {
+                die("MS_RELATIME malfunctions\n");
+        }
+        if (!test_unpriv_remount_atime(MS_STRICTATIME|MS_NODEV,
+                                       MS_NOATIME|MS_NODEV))
+        {
+                die("MS_STRICTATIME malfunctions\n");
+        }
+        if (!test_unpriv_remount_atime(MS_NOATIME|MS_NODEV,
+                                       MS_STRICTATIME|MS_NODEV))
+        {
+                die("MS_RELATIME malfunctions\n");
+        }
+        if (!test_unpriv_remount_atime(MS_RELATIME|MS_NODIRATIME|MS_NODEV,
+                                       MS_NOATIME|MS_NODEV))
+        {
+                die("MS_RELATIME malfunctions\n");
+        }
+        if (!test_unpriv_remount_atime(MS_STRICTATIME|MS_NODIRATIME|MS_NODEV,
+                                       MS_NOATIME|MS_NODEV))
+        {
+                die("MS_RELATIME malfunctions\n");
+        }
+        if (!test_unpriv_remount_atime(MS_NOATIME|MS_NODIRATIME|MS_NODEV,
+                                       MS_STRICTATIME|MS_NODEV))
+        {
+                die("MS_RELATIME malfunctions\n");
+        }
+        if (!test_unpriv_remount(MS_STRICTATIME|MS_NODEV, MS_NODEV,
+                                 MS_NOATIME|MS_NODEV))
+        {
+                die("Default atime malfunctions\n");
+        }
+        return EXIT_SUCCESS;
+}