aboutsummaryrefslogtreecommitdiffstats
BranchCommit messageAuthorAge
archive/unc-master-3.0P-FP: fix BUG_ON releated to priority inheritanceBjoern Brandenburg13 years
archived-2013.1uncachedev: mmap memory that is not cached by CPUsGlenn Elliott12 years
archived-private-masterMerge branch 'wip-2.6.34' into old-private-masterAndrea Bastoni15 years
archived-semi-partMerge branch 'wip-semi-part' of ssh://cvs/cvs/proj/litmus/repo/litmus2010 int...Andrea Bastoni15 years
demoFurther refinementsJonathan Herman14 years
ecrts-pgm-finalMerge branch 'wip-ecrts14-pgm' of ssh://rtsrv.cs.unc.edu/home/litmus/litmus-r...Glenn Elliott12 years
ecrts14-pgm-finalMerge branch 'wip-ecrts14-pgm' of ssh://rtsrv.cs.unc.edu/home/litmus/litmus-r...Glenn Elliott12 years
gpusync-rtss12Final GPUSync implementation.Glenn Elliott12 years
gpusync/stagingRename IKGLP R2DGLP.Glenn Elliott12 years
linux-tipMerge branch 'slab/urgent' of git://git.kernel.org/pub/scm/linux/kernel/git/p...Linus Torvalds15 years
litmus2008-patch-seriesadd i386 feather-trace implementationBjoern B. Brandenburg16 years
masterPSN-EDF: use inferred_sporadic_job_release_atBjoern Brandenburg9 years
pgmmake it compileGlenn Elliott12 years
prop/litmus-signalsInfrastructure for Litmus signals.Glenn Elliott13 years
prop/robust-tie-breakFixed bug in edf_higher_prio().Glenn Elliott13 years
stagingFix tracepoint compilation errorFelipe Cerqueira13 years
test9/23/2016Namhoon Kim9 years
tracing-develTest kernel tracing events capabilitiesAndrea Bastoni16 years
v2.6.34-with-arm-patchessmsc911x: Add spinlocks around registers accessCatalin Marinas15 years
v2015.1Add ARM syscall def for get_current_budgetBjoern Brandenburg10 years
wip-2011.2-bbbLitmus core: simplify np-section protocolBjoern B. Brandenburg14 years
wip-2011.2-bbb-traceRefactor sched_trace_log_message() -> debug_trace_log_message()Andrea Bastoni14 years
wip-2012.3-gpuSOBLIV draining support for C-EDF.Glenn Elliott12 years
wip-2012.3-gpu-preportpick up last C-RM fileGlenn Elliott12 years
wip-2012.3-gpu-rtss13Fix critical bug in GPU tracker.Glenn Elliott12 years
wip-2012.3-gpu-sobliv-budget-w-ksharkProper sobliv draining and many bug fixes.Glenn Elliott12 years
wip-aedzl-finalMake it easier to compile AEDZL interfaces in liblitmus.Glenn Elliott15 years
wip-aedzl-revisedAdd sched_trace data for Apative EDZLGlenn Elliott15 years
wip-arbit-deadlineFix compilation bug.Glenn Elliott13 years
wip-aux-tasksDescription of refined aux task inheritance.Glenn Elliott13 years
wip-bbbGSN-EDF & Core: improve debug TRACE'ing for NP sectionsBjoern B. Brandenburg14 years
wip-bbb-prio-donuse correct timestampBjoern B. Brandenburg14 years
wip-better-breakImplement hash-based EDF tie-breaking.Glenn Elliott13 years
wip-binary-heapMake C-EDF work with simplified binheap_deleteGlenn Elliott13 years
wip-budgetAdded support for choices in budget policy enforcement.Glenn Elliott15 years
wip-colorSummarize schedulability with final recordJonathan Herman13 years
wip-color-jlhsched_color: Fixed two bugs causing crashing on experiment restart and a rare...Jonathan Herman13 years
wip-d10-hz1000Enable HZ=1000 on District 10Bjoern B. Brandenburg15 years
wip-default-clusteringFeature: Make default C-EDF clustering compile-time configurable.Glenn Elliott15 years
wip-dissipation-jericksoUpdate from 2.6.36 to 2.6.36.4Jeremy Erickson11 years
wip-dissipation2-jericksoUpdate 2.6.36 to 2.6.36.4Jeremy Erickson11 years
wip-ecrts14-pgmMerge branch 'wip-ecrts14-pgm' of ssh://rtsrv.cs.unc.edu/home/litmus/litmus-r...Glenn Elliott12 years
wip-edf-hsblast tested versionJonathan Herman14 years
wip-edf-osLookup table EDF-osJeremy Erickson12 years
wip-edf-tie-breakMerge branch 'wip-edf-tie-break' of ssh://rtsrv.cs.unc.edu/home/litmus/litmus...Glenn Elliott13 years
wip-edzl-critiqueUse hr_timer's active checks instead of having own flag.Glenn Elliott15 years
wip-edzl-finalImplementation of the EDZL scheduler.Glenn Elliott15 years
wip-edzl-revisedClean up comments.Glenn Elliott15 years
wip-eventsAdded support for tracing arbitrary actions.Jonathan Herman15 years
wip-extra-debugDBG: add additional tracingBjoern B. Brandenburg15 years
wip-fix-switch-jericksoAttempt to fix race condition with plugin switchingJeremy Erickson15 years
wip-fix3sched: show length of runqueue clock deactivation in /proc/sched_debugBjoern B. Brandenburg15 years
wip-fmlp-dequeueImprove FMLP queue management.Glenn Elliott14 years
wip-ft-irq-flagFeather-Trace: keep track of interrupt-related interference.Bjoern B. Brandenburg14 years
wip-gpu-cleanupEnable sched_trace log injection from userspaceGlenn Elliott13 years
wip-gpu-interruptsRemove option for threading of all softirqs.Glenn Elliott14 years
wip-gpu-rtas12Generalized GPU cost predictors + EWMA. (untested)Glenn Elliott13 years
wip-gpu-rtss12Final GPUSync implementation.Glenn Elliott13 years
wip-gpu-rtss12-srpexperimental changes to support GPUs under SRPGlenn Elliott13 years
wip-gpusync-mergeCleanup priority tracking for budget enforcement.Glenn Elliott11 years
wip-ikglpMove RSM and IKGLP imp. to own .c filesGlenn Elliott13 years
wip-k-fmlpMerge branch 'mpi-master' into wip-k-fmlpGlenn Elliott14 years
wip-kernel-coloringAdded recolor syscallNamhoon Kim7 years
wip-kernthreadsKludge work-queue processing into klitirqd.Glenn Elliott15 years
wip-klmirqd-to-auxAllow klmirqd threads to be given names.Glenn Elliott13 years
wip-ksharkMerge branch 'mpi-staging' into wip-ksharkJonathan Herman13 years
wip-litmus-3.2Merge commit 'v3.2' into litmus-stagingAndrea Bastoni13 years
wip-litmus2011.2Cleanup: Coding conformance for affinity stuff.Glenn Elliott14 years
wip-litmus3.0-2011.2Feather-Trace: keep track of interrupt-related interference.Bjoern B. Brandenburg14 years
wip-master-2.6.33-rtAvoid deadlock when switching task policy to BACKGROUND (ugly)Andrea Bastoni15 years
wip-mcRemoved ARM-specific hacks which disabled less common mixed-criticality featu...Jonathan Herman12 years
wip-mc-bipasaMC-EDF addedbipasa chattopadhyay13 years
wip-mc-jericksoSplit C/D queuesJeremy Erickson15 years
wip-mc2-cache-slackManually patched mc^2 related codeMing Yang10 years
wip-mcrit-maccosmeticMac Mollison15 years
wip-merge-3.0Prevent Linux to send IPI and queue tasks on remote CPUs.Andrea Bastoni14 years
wip-merge-v3.0Prevent Linux to send IPI and queue tasks on remote CPUs.Andrea Bastoni14 years
wip-migration-affinityNULL affinity dereference in C-EDF.Glenn Elliott14 years
wip-mmap-uncacheshare branch with othersGlenn Elliott13 years
wip-modechangeRTSS 2017 submissionNamhoon Kim8 years
wip-nested-lockingAppears to be working.Bryan Ward12 years
wip-omlp-gedfFirst implementation of G-OMLP.Glenn Elliott15 years
wip-paiSome cleanup of PAIGlenn Elliott14 years
wip-percore-lib9/21/2016Namhoon Kim9 years
wip-performanceCONFIG_DONT_PREEMPT_ON_TIE: Don't preeempt a scheduled task on priority tie.Glenn Elliott14 years
wip-pgmAdd PGM support to C-FLGlenn Elliott12 years
wip-pgm-splitFirst draft of C-FL-splitNamhoon Kim12 years
wip-pm-ovdAdd preemption-and-migration overhead tracing supportAndrea Bastoni15 years
wip-prio-inhP-EDF updated to use the generic pi framework.Glenn Elliott15 years
wip-prioq-dglBUG FIX: Support DGLs with PRIOQ_MUTEXGlenn Elliott13 years
wip-refactored-gedfGeneralizd architecture for GEDF-style scheduelrs to reduce code redundancy.Glenn Elliott15 years
wip-release-master-fixbugfix: release master CPU must signal task was pickedBjoern B. Brandenburg14 years
wip-robust-tie-breakEDF priority tie-breaks.Glenn Elliott13 years
wip-rt-ksharkMove task time accounting into the complete_job method.Jonathan Herman13 years
wip-rtas12-pgmScheduling of PGM jobs.Glenn Elliott13 years
wip-semi-partFix compile error with newer GCCJeremy Erickson12 years
wip-semi-part-edfos-jericksoUse initial CPU set by clientJeremy Erickson12 years
wip-shared-libTODO: Fix condition checks in replicate_page_move_mapping()Namhoon Kim9 years
wip-shared-lib2RTAS 2017 Submission ver.Namhoon Kim9 years
wip-shared-memInitial commit for shared libraryNamhoon Kim9 years
wip-splitting-jericksoFix release behaviorJeremy Erickson13 years
wip-splitting-omlp-jericksoBjoern's Dissertation Code with Priority DonationJeremy Erickson13 years
wip-stage-binheapAn efficient binary heap implementation.Glenn Elliott13 years
wip-sun-portDynamic memory allocation and clean exit for FeatherTraceChristopher Kenna15 years
wip-timer-tracebugfix: C-EDF, clear scheduled field of the correct CPU upon task_exitAndrea Bastoni15 years
wip-tracepointsAdd kernel-style events for sched_trace_XXX() functionsAndrea Bastoni14 years
 
TagDownloadAuthorAge
2015.1commit 8e51b37822...Bjoern Brandenburg10 years
2013.1commit bcaacec1ca...Glenn Elliott12 years
2012.3commit c158b5fbe4...Jonathan Herman13 years
2012.2commit b53c479a0f...Glenn Elliott13 years
2012.1commit 83b11ea1c6...Bjoern B. Brandenburg14 years
rtas12-mc-beta-expcommit 8e236ee20f...Christopher Kenna14 years
2011.1commit d11808b5c6...Christopher Kenna15 years
v2.6.37-rc4commit e8a7e48bb2...Linus Torvalds15 years
v2.6.37-rc3commit 3561d43fd2...Linus Torvalds15 years
v2.6.37-rc2commit e53beacd23...Linus Torvalds15 years
v2.6.37-rc1commit c8ddb2713c...Linus Torvalds15 years
v2.6.36commit f6f94e2ab1...Linus Torvalds15 years
2010.2commit 5c5456402d...Bjoern B. Brandenburg15 years
v2.6.36-rc8commit cd07202cc8...Linus Torvalds15 years
v2.6.36-rc7commit cb655d0f3d...Linus Torvalds15 years
v2.6.36-rc6commit 899611ee7d...Linus Torvalds15 years
v2.6.36-rc5commit b30a3f6257...Linus Torvalds15 years
v2.6.36-rc4commit 49553c2ef8...Linus Torvalds15 years
v2.6.36-rc3commit 2bfc96a127...Linus Torvalds15 years
v2.6.36-rc2commit 76be97c1fc...Linus Torvalds15 years
v2.6.36-rc1commit da5cabf80e...Linus Torvalds15 years
v2.6.35commit 9fe6206f40...Linus Torvalds15 years
v2.6.35-rc6commit b37fa16e78...Linus Torvalds15 years
v2.6.35-rc5commit 1c5474a65b...Linus Torvalds15 years
v2.6.35-rc4commit 815c4163b6...Linus Torvalds15 years
v2.6.35-rc3commit 7e27d6e778...Linus Torvalds15 years
v2.6.35-rc2commit e44a21b726...Linus Torvalds15 years
v2.6.35-rc1commit 67a3e12b05...Linus Torvalds15 years
2010.1commit 7c1ff4c544...Andrea Bastoni15 years
v2.6.34commit e40152ee1e...Linus Torvalds15 years
v2.6.33.4commit 4640b4e7d9...Greg Kroah-Hartman15 years
v2.6.34-rc7commit b57f95a382...Linus Torvalds15 years
v2.6.34-rc6commit 66f41d4c5c...Linus Torvalds15 years
v2.6.33.3commit 3e7ad8ed97...Greg Kroah-Hartman15 years
v2.6.34-rc5commit 01bf0b6457...Linus Torvalds15 years
v2.6.34-rc4commit 0d0fb0f9c5...Linus Torvalds15 years
v2.6.33.2commit 19f00f070c...Greg Kroah-Hartman15 years
v2.6.34-rc3commit 2eaa9cfdf3...Linus Torvalds15 years
v2.6.34-rc2commit 220bf991b0...Linus Torvalds15 years
v2.6.33.1commit dbdafe5ccf...Greg Kroah-Hartman15 years
v2.6.34-rc1commit 57d54889cd...Linus Torvalds16 years
v2.6.33commit 60b341b778...Linus Torvalds16 years
v2.6.33-rc8commit 724e6d3fe8...Linus Torvalds16 years
v2.6.33-rc7commit 29275254ca...Linus Torvalds16 years
v2.6.33-rc6commit abe94c756c...Linus Torvalds16 years
v2.6.33-rc5commit 92dcffb916...Linus Torvalds16 years
v2.6.33-rc4commit 7284ce6c9f...Linus Torvalds16 years
v2.6.33-rc3commit 74d2e4f8d7...Linus Torvalds16 years
v2.6.33-rc2commit 6b7b284958...Linus Torvalds16 years
v2.6.33-rc1commit 55639353a0...Linus Torvalds16 years
v2.6.32commit 22763c5cf3...Linus Torvalds16 years
v2.6.32-rc8commit 648f4e3e50...Linus Torvalds16 years
v2.6.32-rc7commit 156171c71a...Linus Torvalds16 years
v2.6.32-rc6commit b419148e56...Linus Torvalds16 years
v2.6.32-rc5commit 012abeea66...Linus Torvalds16 years
v2.6.32-rc4commit 161291396e...Linus Torvalds16 years
v2.6.32-rc3commit 374576a8b6...Linus Torvalds16 years
v2.6.32-rc1commit 17d857be64...Linus Torvalds16 years
v2.6.32-rc2commit 17d857be64...Linus Torvalds16 years
v2.6.31commit 74fca6a428...Linus Torvalds16 years
v2.6.31-rc9commit e07cccf404...Linus Torvalds16 years
v2.6.31-rc8commit 326ba5010a...Linus Torvalds16 years
v2.6.31-rc7commit 422bef879e...Linus Torvalds16 years
v2.6.31-rc6commit 64f1607ffb...Linus Torvalds16 years
v2.6.31-rc5commit ed680c4ad4...Linus Torvalds16 years
v2.6.31-rc4commit 4be3bd7849...Linus Torvalds16 years
v2.6.31-rc3commit 6847e154e3...Linus Torvalds16 years
v2.6.31-rc2commit 8e4a718ff3...Linus Torvalds16 years
v2.6.31-rc1commit 28d0325ce6...Linus Torvalds16 years
v2.6.30commit 07a2039b8e...Linus Torvalds16 years
v2.6.30-rc8commit 9fa7eb283c...Linus Torvalds16 years
v2.6.30-rc7commit 59a3759d0f...Linus Torvalds16 years
v2.6.30-rc6commit 1406de8e11...Linus Torvalds16 years
v2.6.30-rc5commit 091bf7624d...Linus Torvalds16 years
v2.6.30-rc4commit 091438dd56...Linus Torvalds16 years
v2.6.30-rc3commit 0910697403...Linus Torvalds16 years
v2.6.30-rc2commit 0882e8dd3a...Linus Torvalds16 years
v2.6.30-rc1commit 577c9c456f...Linus Torvalds16 years
v2.6.29commit 8e0ee43bc2...Linus Torvalds16 years
v2.6.29-rc8commit 041b62374c...Linus Torvalds17 years
v2.6.29-rc7commit fec6c6fec3...Linus Torvalds17 years
v2.6.29-rc6commit 20f4d6c3a2...Linus Torvalds17 years
v2.6.29-rc5commit d2f8d7ee1a...Linus Torvalds17 years
v2.6.29-rc4commit 8e4921515c...Linus Torvalds17 years
v2.6.29-rc3commit 18e352e4a7...Linus Torvalds17 years
v2.6.29-rc2commit 1de9e8e70f...Linus Torvalds17 years
v2.6.29-rc1commit c59765042f...Linus Torvalds17 years
v2.6.28commit 4a6908a3a0...Linus Torvalds17 years
v2.6.28-rc9commit 929096fe9f...Linus Torvalds17 years
v2.6.28-rc8commit 8b1fae4e42...Linus Torvalds17 years
v2.6.28-rc7commit 061e41fdb5...Linus Torvalds17 years
v2.6.28-rc6commit 13d428afc0...Linus Torvalds17 years
v2.6.28-rc5commit 9bf1a2445f...Linus Torvalds17 years
v2.6.28-rc4commit f7160c7573...Linus Torvalds17 years
v2.6.28-rc3commit 45beca08dd...Linus Torvalds17 years
v2.6.28-rc2commit 0173a3265b...Linus Torvalds17 years
v2.6.28-rc1commit 57f8f7b60d...Linus Torvalds17 years
v2.6.27commit 3fa8749e58...Linus Torvalds17 years
v2.6.27-rc9commit 4330ed8ed4...Linus Torvalds17 years
v2.6.27-rc8commit 94aca1dac6...Linus Torvalds17 years
v2.6.27-rc7commit 72d31053f6...Linus Torvalds17 years
v2.6.27-rc6commit adee14b2e1...Linus Torvalds17 years
v2.6.27-rc5commit 24342c34a0...Linus Torvalds17 years
v2.6.27-rc4commit 6a55617ed5...Linus Torvalds17 years
v2.6.27-rc3commit 30a2f3c60a...Linus Torvalds17 years
v2.6.27-rc2commit 0967d61ea0...Linus Torvalds17 years
v2.6.27-rc1commit 6e86841d05...Linus Torvalds17 years
v2.6.26commit bce7f793da...Linus Torvalds17 years
v2.6.26-rc9commit b7279469d6...Linus Torvalds17 years
v2.6.26-rc8commit 543cf4cb3f...Linus Torvalds17 years
v2.6.26-rc7commit d70ac829b7...Linus Torvalds17 years
v2.6.26-rc6commit 5dd34572ad...Linus Torvalds17 years
v2.6.26-rc5commit 53c8ba9540...Linus Torvalds17 years
v2.6.26-rc4commit e490517a03...Linus Torvalds17 years
v2.6.26-rc3commit b8291ad07a...Linus Torvalds17 years
v2.6.26-rc2commit 492c2e476e...Linus Torvalds17 years
v2.6.26-rc1commit 2ddcca36c8...Linus Torvalds17 years
v2.6.25commit 4b119e21d0...Linus Torvalds17 years
v2.6.25-rc9commit 120dd64cac...Linus Torvalds17 years
v2.6.25-rc8commit 0e81a8ae37...Linus Torvalds17 years
v2.6.25-rc7commit 05dda977f2...Linus Torvalds17 years
v2.6.25-rc6commit a978b30af3...Linus Torvalds17 years
v2.6.25-rc5commit cdeeeae056...Linus Torvalds18 years
v2.6.25-rc4commit 29e8c3c304...Linus Torvalds18 years
v2.6.25-rc3commit bfa274e243...Linus Torvalds18 years
v2.6.25-rc2commit 101142c37b...Linus Torvalds18 years
v2.6.25-rc1commit 19af35546d...Linus Torvalds18 years
v2.6.24commit 49914084e7...Linus Torvalds18 years
v2.6.24-rc8commit cbd9c88369...Linus Torvalds18 years
v2.6.24-rc7commit 3ce5445046...Linus Torvalds18 years
v2.6.24-rc6commit ea67db4cdb...Linus Torvalds18 years
v2.6.24-rc5commit 82d29bf6dc...Linus Torvalds18 years
v2.6.24-rc4commit 09b56adc98...Linus Torvalds18 years
v2.6.24-rc3commit d9f8bcbf67...Linus Torvalds18 years
v2.6.24-rc2commit dbeeb816e8...Linus Torvalds18 years
v2.6.24-rc1commit c9927c2bf4...Linus Torvalds18 years
v2.6.23commit bbf25010f1...Linus Torvalds18 years
v2.6.23-rc9commit 3146b39c18...Linus Torvalds18 years
v2.6.23-rc8commit 4942de4a0e...Linus Torvalds18 years
v2.6.23-rc7commit 81cfe79b9c...Linus Torvalds18 years
v2.6.23-rc6commit 0d4cbb5e7f...Linus Torvalds18 years
v2.6.23-rc5commit 40ffbfad6b...Linus Torvalds18 years
v2.6.23-rc4commit b07d68b5ca...Linus Torvalds18 years
v2.6.23-rc3commit 39d3520c92...Linus Torvalds18 years
v2.6.23-rc2commit d4ac2477fa...Linus Torvalds18 years
v2.6.23-rc1commit f695baf2df...Linus Torvalds18 years
v2.6.22commit 7dcca30a32...Linus Torvalds18 years
v2.6.22-rc7commit a38d6181ff...Linus Torvalds18 years
v2.6.22-rc6commit 189548642c...Linus Torvalds18 years
v2.6.22-rc5commit 188e1f81ba...Linus Torvalds18 years
v2.6.22-rc4commit 5ecd3100e6...Linus Torvalds18 years
v2.6.22-rc3commit c420bc9f09...Linus Torvalds18 years
v2.6.22-rc2commit 55b637c6a0...Linus Torvalds18 years
v2.6.22-rc1commit 39403865d2...Linus Torvalds18 years
v2.6.21commit de46c33745...Linus Torvalds18 years
v2.6.21-rc7commit 94a05509a9...Linus Torvalds18 years
v2.6.21-rc6commit a21bd69e15...Linus Torvalds18 years
v2.6.21-rc5commit e0f2e3a06b...Linus Torvalds18 years
v2.6.21-rc4commit db98e0b434...Linus Torvalds19 years
v2.6.21-rc3commit 08e15e81a4...Linus Torvalds19 years
v2.6.21-rc2commit 606135a308...Linus Torvalds19 years
v2.6.21-rc1commit c8f71b01a5...Linus Torvalds19 years
v2.6.20commit 62d0cfcb27...Linus Torvalds19 years
v2.6.20-rc7commit f56df2f4db...Linus Torvalds19 years
v2.6.20-rc6commit 99abfeafb5...Linus Torvalds19 years
v2.6.20-rc5commit a8b3485287...Linus Torvalds19 years
v2.6.20-rc4commit bf81b46482...Linus Torvalds19 years
v2.6.20-rc3commit 669df1b478...Linus Torvalds19 years
v2.6.20-rc2commit 3bf8ba38f3...Linus Torvalds19 years
v2.6.20-rc1commit cc016448b0...Linus Torvalds19 years
v2.6.19commit 0215ffb08c...Linus Torvalds19 years
v2.6.19-rc6commit 44597f65f6...Linus Torvalds19 years
v2.6.19-rc5commit 80c2188127...Linus Torvalds19 years
v2.6.19-rc4commit ae99a78af3...Linus Torvalds19 years
v2.6.19-rc3commit 7059abedd2...Linus Torvalds19 years
v2.6.19-rc2commit b4bd8c6643...Linus Torvalds19 years
v2.6.19-rc1commit d223a60106...Linus Torvalds19 years
v2.6.18commit e478bec0ba...Linus Torvalds19 years
v2.6.18-rc7commit 95064a75eb...Linus Torvalds19 years
v2.6.18-rc6commit c336923b66...Linus Torvalds19 years
v2.6.18-rc5commit 60d4684068...Linus Torvalds19 years
v2.6.18-rc4commit 9f737633e6...Linus Torvalds19 years
v2.6.18-rc3commit b6ff50833a...Linus Torvalds19 years
v2.6.18-rc2commit 82d6897fef...Linus Torvalds19 years
v2.6.18-rc1commit 120bda20c6...Linus Torvalds19 years
v2.6.17commit 427abfa28a...Linus Torvalds19 years
v2.6.17-rc6commit 1def630a6a...Linus Torvalds19 years
v2.6.17-rc5commit a8bd60705a...Linus Torvalds19 years
v2.6.17-rc4commit d8c3291c73...Linus Torvalds19 years
v2.6.17-rc3commit 2be4d50295...Linus Torvalds19 years
v2.6.17-rc2commit 8bbde0e6d5...Linus Torvalds19 years
v2.6.17-rc1commit 6246b6128b...Linus Torvalds19 years
v2.6.16commit 7705a8792b...Linus Torvalds19 years
v2.6.16-rc6commit 535744878e...Linus Torvalds20 years
v2.6.16-rc5commit b9a33cebac...Linus Torvalds20 years
v2.6.16-rc4commit bd71c2b174...Linus Torvalds20 years
v2.6.16-rc3commit e9bb4c9929...Linus Torvalds20 years
v2.6.16-rc2commit 826eeb53a6...Linus Torvalds20 years
v2.6.16-rc1commit 2664b25051...Linus Torvalds20 years
v2.6.15commit 88026842b0...Linus Torvalds20 years
v2.6.15-rc7commit f89f5948fc...Linus Torvalds20 years
v2.6.15-rc6commit df7addbb45...Linus Torvalds20 years
v2.6.15-rc5commit 436b0f76f2...Linus Torvalds20 years
v2.6.15-rc4commit 5666c0947e...Linus Torvalds20 years
v2.6.15-rc3commit 624f54be20...Linus Torvalds20 years
v2.6.15-rc2commit 3bedff1d73...Linus Torvalds20 years
v2.6.15-rc1commit cd52d1ee9a...Linus Torvalds20 years
v2.6.14commit 741b2252a5...Linus Torvalds20 years
v2.6.14-rc5commit 93918e9afc...Linus Torvalds20 years
v2.6.14-rc4commit 907a426179...Linus Torvalds20 years
v2.6.14-rc3commit 1c9426e8a5...Linus Torvalds20 years
v2.6.14-rc2commit 676d55ae30...Linus Torvalds20 years
v2.6.14-rc1commit 2f4ba45a75...Linus Torvalds20 years
v2.6.13commit 02b3e4e2d7...Linus Torvalds20 years
v2.6.13-rc7commit 0572e3da3f...Linus Torvalds20 years
v2.6.13-rc6commit 6fc32179de...Linus Torvalds20 years
v2.6.13-rc5commit 9a351e30d7...Linus Torvalds20 years
v2.6.13-rc4commit 6395352334...Linus Torvalds20 years
v2.6.11tree c39ae07f39...
v2.6.11-treetree c39ae07f39...
v2.6.12commit 9ee1c939d1...
v2.6.12-rc2commit 1da177e4c3...
v2.6.12-rc3commit a2755a80f4...
v2.6.12-rc4commit 88d7bd8cb9...
v2.6.12-rc5commit 2a24ab628a...
v2.6.12-rc6commit 7cef5677ef...
v2.6.13-rc1commit 4c91aedb75...
v2.6.13-rc2commit a18bcb7450...
v2.6.13-rc3commit c32511e271...
opt">.iova & page_mask)) >> page_shift; if (mr->umem->writable) access_mask |= ODP_WRITE_ALLOWED_BIT; current_seq = READ_ONCE(odp->notifiers_seq); /* * Ensure the sequence number is valid for some time before we call * gup. */ smp_rmb(); ret = ib_umem_odp_map_dma_pages(to_ib_umem_odp(mr->umem), io_virt, size, access_mask, current_seq); if (ret < 0) goto out; np = ret; mutex_lock(&odp->umem_mutex); if (!ib_umem_mmu_notifier_retry(to_ib_umem_odp(mr->umem), current_seq)) { /* * No need to check whether the MTTs really belong to * this MR, since ib_umem_odp_map_dma_pages already * checks this. */ ret = mlx5_ib_update_xlt(mr, start_idx, np, page_shift, MLX5_IB_UPD_XLT_ATOMIC); } else { ret = -EAGAIN; } mutex_unlock(&odp->umem_mutex); if (ret < 0) { if (ret != -EAGAIN) mlx5_ib_err(dev, "Failed to update mkey page tables\n"); goto out; } if (bytes_mapped) { u32 new_mappings = (np << page_shift) - (io_virt - round_down(io_virt, 1 << page_shift)); *bytes_mapped += min_t(u32, new_mappings, size); } npages += np << (page_shift - PAGE_SHIFT); bcnt -= size; if (unlikely(bcnt)) { struct ib_umem_odp *next; io_virt += size; next = odp_next(odp); if (unlikely(!next || next->umem.address != io_virt)) { mlx5_ib_dbg(dev, "next implicit leaf removed at 0x%llx. got %p\n", io_virt, next); return -EAGAIN; } odp = next; mr = odp->private; goto next_mr; } return npages; out: if (ret == -EAGAIN) { if (mr->parent || !odp->dying) { unsigned long timeout = msecs_to_jiffies(MMU_NOTIFIER_TIMEOUT); if (!wait_for_completion_timeout( &odp->notifier_completion, timeout)) { mlx5_ib_warn(dev, "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n", current_seq, odp->notifiers_seq, odp->notifiers_count); } } else { /* The MR is being killed, kill the QP as well. */ ret = -EFAULT; } } return ret; } struct pf_frame { struct pf_frame *next; u32 key; u64 io_virt; size_t bcnt; int depth; }; /* * Handle a single data segment in a page-fault WQE or RDMA region. * * Returns number of OS pages retrieved on success. The caller may continue to * the next data segment. * Can return the following error codes: * -EAGAIN to designate a temporary error. The caller will abort handling the * page fault and resolve it. * -EFAULT when there's an error mapping the requested pages. The caller will * abort the page fault handling. */ static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, u32 key, u64 io_virt, size_t bcnt, u32 *bytes_committed, u32 *bytes_mapped) { int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0; struct pf_frame *head = NULL, *frame; struct mlx5_core_mkey *mmkey; struct mlx5_ib_mw *mw; struct mlx5_ib_mr *mr; struct mlx5_klm *pklm; u32 *out = NULL; size_t offset; srcu_key = srcu_read_lock(&dev->mr_srcu); io_virt += *bytes_committed; bcnt -= *bytes_committed; next_mr: mmkey = __mlx5_mr_lookup(dev->mdev, mlx5_base_mkey(key)); if (!mmkey || mmkey->key != key) { mlx5_ib_dbg(dev, "failed to find mkey %x\n", key); ret = -EFAULT; goto srcu_unlock; } switch (mmkey->type) { case MLX5_MKEY_MR: mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); if (!mr->live || !mr->ibmr.pd) { mlx5_ib_dbg(dev, "got dead MR\n"); ret = -EFAULT; goto srcu_unlock; } if (!mr->umem->is_odp) { mlx5_ib_dbg(dev, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", key); if (bytes_mapped) *bytes_mapped += bcnt; ret = 0; goto srcu_unlock; } ret = pagefault_mr(dev, mr, io_virt, bcnt, bytes_mapped); if (ret < 0) goto srcu_unlock; npages += ret; ret = 0; break; case MLX5_MKEY_MW: mw = container_of(mmkey, struct mlx5_ib_mw, mmkey); if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) { mlx5_ib_dbg(dev, "indirection level exceeded\n"); ret = -EFAULT; goto srcu_unlock; } outlen = MLX5_ST_SZ_BYTES(query_mkey_out) + sizeof(*pklm) * (mw->ndescs - 2); if (outlen > cur_outlen) { kfree(out); out = kzalloc(outlen, GFP_KERNEL); if (!out) { ret = -ENOMEM; goto srcu_unlock; } cur_outlen = outlen; } pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out, bsf0_klm0_pas_mtt0_1); ret = mlx5_core_query_mkey(dev->mdev, &mw->mmkey, out, outlen); if (ret) goto srcu_unlock; offset = io_virt - MLX5_GET64(query_mkey_out, out, memory_key_mkey_entry.start_addr); for (i = 0; bcnt && i < mw->ndescs; i++, pklm++) { if (offset >= be32_to_cpu(pklm->bcount)) { offset -= be32_to_cpu(pklm->bcount); continue; } frame = kzalloc(sizeof(*frame), GFP_KERNEL); if (!frame) { ret = -ENOMEM; goto srcu_unlock; } frame->key = be32_to_cpu(pklm->key); frame->io_virt = be64_to_cpu(pklm->va) + offset; frame->bcnt = min_t(size_t, bcnt, be32_to_cpu(pklm->bcount) - offset); frame->depth = depth + 1; frame->next = head; head = frame; bcnt -= frame->bcnt; offset = 0; } break; default: mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type); ret = -EFAULT; goto srcu_unlock; } if (head) { frame = head; head = frame->next; key = frame->key; io_virt = frame->io_virt; bcnt = frame->bcnt; depth = frame->depth; kfree(frame); goto next_mr; } srcu_unlock: while (head) { frame = head; head = frame->next; kfree(frame); } kfree(out); srcu_read_unlock(&dev->mr_srcu, srcu_key); *bytes_committed = 0; return ret ? ret : npages; } /** * Parse a series of data segments for page fault handling. * * @qp the QP on which the fault occurred. * @pfault contains page fault information. * @wqe points at the first data segment in the WQE. * @wqe_end points after the end of the WQE. * @bytes_mapped receives the number of bytes that the function was able to * map. This allows the caller to decide intelligently whether * enough memory was mapped to resolve the page fault * successfully (e.g. enough for the next MTU, or the entire * WQE). * @total_wqe_bytes receives the total data size of this WQE in bytes (minus * the committed bytes). * * Returns the number of pages loaded if positive, zero for an empty WQE, or a * negative error code. */ static int pagefault_data_segments(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, struct mlx5_ib_qp *qp, void *wqe, void *wqe_end, u32 *bytes_mapped, u32 *total_wqe_bytes, int receive_queue) { int ret = 0, npages = 0; u64 io_virt; u32 key; u32 byte_count; size_t bcnt; int inline_segment; /* Skip SRQ next-WQE segment. */ if (receive_queue && qp->ibqp.srq) wqe += sizeof(struct mlx5_wqe_srq_next_seg); if (bytes_mapped) *bytes_mapped = 0; if (total_wqe_bytes) *total_wqe_bytes = 0; while (wqe < wqe_end) { struct mlx5_wqe_data_seg *dseg = wqe; io_virt = be64_to_cpu(dseg->addr); key = be32_to_cpu(dseg->lkey); byte_count = be32_to_cpu(dseg->byte_count); inline_segment = !!(byte_count & MLX5_INLINE_SEG); bcnt = byte_count & ~MLX5_INLINE_SEG; if (inline_segment) { bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK; wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt, 16); } else { wqe += sizeof(*dseg); } /* receive WQE end of sg list. */ if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY && io_virt == 0) break; if (!inline_segment && total_wqe_bytes) { *total_wqe_bytes += bcnt - min_t(size_t, bcnt, pfault->bytes_committed); } /* A zero length data segment designates a length of 2GB. */ if (bcnt == 0) bcnt = 1U << 31; if (inline_segment || bcnt <= pfault->bytes_committed) { pfault->bytes_committed -= min_t(size_t, bcnt, pfault->bytes_committed); continue; } ret = pagefault_single_data_segment(dev, key, io_virt, bcnt, &pfault->bytes_committed, bytes_mapped); if (ret < 0) break; npages += ret; } return ret < 0 ? ret : npages; } static const u32 mlx5_ib_odp_opcode_cap[] = { [MLX5_OPCODE_SEND] = IB_ODP_SUPPORT_SEND, [MLX5_OPCODE_SEND_IMM] = IB_ODP_SUPPORT_SEND, [MLX5_OPCODE_SEND_INVAL] = IB_ODP_SUPPORT_SEND, [MLX5_OPCODE_RDMA_WRITE] = IB_ODP_SUPPORT_WRITE, [MLX5_OPCODE_RDMA_WRITE_IMM] = IB_ODP_SUPPORT_WRITE, [MLX5_OPCODE_RDMA_READ] = IB_ODP_SUPPORT_READ, [MLX5_OPCODE_ATOMIC_CS] = IB_ODP_SUPPORT_ATOMIC, [MLX5_OPCODE_ATOMIC_FA] = IB_ODP_SUPPORT_ATOMIC, }; /* * Parse initiator WQE. Advances the wqe pointer to point at the * scatter-gather list, and set wqe_end to the end of the WQE. */ static int mlx5_ib_mr_initiator_pfault_handler( struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) { struct mlx5_wqe_ctrl_seg *ctrl = *wqe; u16 wqe_index = pfault->wqe.wqe_index; u32 transport_caps; struct mlx5_base_av *av; unsigned ds, opcode; #if defined(DEBUG) u32 ctrl_wqe_index, ctrl_qpn; #endif u32 qpn = qp->trans_qp.base.mqp.qpn; ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; if (ds * MLX5_WQE_DS_UNITS > wqe_length) { mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n", ds, wqe_length); return -EFAULT; } if (ds == 0) { mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n", wqe_index, qpn); return -EFAULT; } #if defined(DEBUG) ctrl_wqe_index = (be32_to_cpu(ctrl->opmod_idx_opcode) & MLX5_WQE_CTRL_WQE_INDEX_MASK) >> MLX5_WQE_CTRL_WQE_INDEX_SHIFT; if (wqe_index != ctrl_wqe_index) { mlx5_ib_err(dev, "Got WQE with invalid wqe_index. wqe_index=0x%x, qpn=0x%x ctrl->wqe_index=0x%x\n", wqe_index, qpn, ctrl_wqe_index); return -EFAULT; } ctrl_qpn = (be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_QPN_MASK) >> MLX5_WQE_CTRL_QPN_SHIFT; if (qpn != ctrl_qpn) { mlx5_ib_err(dev, "Got WQE with incorrect QP number. wqe_index=0x%x, qpn=0x%x ctrl->qpn=0x%x\n", wqe_index, qpn, ctrl_qpn); return -EFAULT; } #endif /* DEBUG */ *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS; *wqe += sizeof(*ctrl); opcode = be32_to_cpu(ctrl->opmod_idx_opcode) & MLX5_WQE_CTRL_OPCODE_MASK; switch (qp->ibqp.qp_type) { case IB_QPT_RC: transport_caps = dev->odp_caps.per_transport_caps.rc_odp_caps; break; case IB_QPT_UD: transport_caps = dev->odp_caps.per_transport_caps.ud_odp_caps; break; default: mlx5_ib_err(dev, "ODP fault on QP of an unsupported transport 0x%x\n", qp->ibqp.qp_type); return -EFAULT; } if (unlikely(opcode >= ARRAY_SIZE(mlx5_ib_odp_opcode_cap) || !(transport_caps & mlx5_ib_odp_opcode_cap[opcode]))) { mlx5_ib_err(dev, "ODP fault on QP of an unsupported opcode 0x%x\n", opcode); return -EFAULT; } if (qp->ibqp.qp_type != IB_QPT_RC) { av = *wqe; if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV)) *wqe += sizeof(struct mlx5_av); else *wqe += sizeof(struct mlx5_base_av); } switch (opcode) { case MLX5_OPCODE_RDMA_WRITE: case MLX5_OPCODE_RDMA_WRITE_IMM: case MLX5_OPCODE_RDMA_READ: *wqe += sizeof(struct mlx5_wqe_raddr_seg); break; case MLX5_OPCODE_ATOMIC_CS: case MLX5_OPCODE_ATOMIC_FA: *wqe += sizeof(struct mlx5_wqe_raddr_seg); *wqe += sizeof(struct mlx5_wqe_atomic_seg); break; } return 0; } /* * Parse responder WQE. Advances the wqe pointer to point at the * scatter-gather list, and set wqe_end to the end of the WQE. */ static int mlx5_ib_mr_responder_pfault_handler( struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) { struct mlx5_ib_wq *wq = &qp->rq; int wqe_size = 1 << wq->wqe_shift; if (qp->ibqp.srq) { mlx5_ib_err(dev, "ODP fault on SRQ is not supported\n"); return -EFAULT; } if (qp->wq_sig) { mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n"); return -EFAULT; } if (wqe_size > wqe_length) { mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); return -EFAULT; } switch (qp->ibqp.qp_type) { case IB_QPT_RC: if (!(dev->odp_caps.per_transport_caps.rc_odp_caps & IB_ODP_SUPPORT_RECV)) goto invalid_transport_or_opcode; break; default: invalid_transport_or_opcode: mlx5_ib_err(dev, "ODP fault on QP of an unsupported transport. transport: 0x%x\n", qp->ibqp.qp_type); return -EFAULT; } *wqe_end = *wqe + wqe_size; return 0; } static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev, u32 wq_num, int pf_type) { enum mlx5_res_type res_type; switch (pf_type) { case MLX5_WQE_PF_TYPE_RMP: res_type = MLX5_RES_SRQ; break; case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE: case MLX5_WQE_PF_TYPE_RESP: case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC: res_type = MLX5_RES_QP; break; default: return NULL; } return mlx5_core_res_hold(dev->mdev, wq_num, res_type); } static inline struct mlx5_ib_qp *res_to_qp(struct mlx5_core_rsc_common *res) { struct mlx5_core_qp *mqp = (struct mlx5_core_qp *)res; return to_mibqp(mqp); } static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault) { int ret; void *wqe, *wqe_end; u32 bytes_mapped, total_wqe_bytes; char *buffer = NULL; int resume_with_error = 1; u16 wqe_index = pfault->wqe.wqe_index; int requestor = pfault->type & MLX5_PFAULT_REQUESTOR; struct mlx5_core_rsc_common *res; struct mlx5_ib_qp *qp; res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type); if (!res) { mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num); return; } switch (res->res) { case MLX5_RES_QP: qp = res_to_qp(res); break; default: mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n", pfault->type); goto resolve_page_fault; } buffer = (char *)__get_free_page(GFP_KERNEL); if (!buffer) { mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n"); goto resolve_page_fault; } ret = mlx5_ib_read_user_wqe(qp, requestor, wqe_index, buffer, PAGE_SIZE, &qp->trans_qp.base); if (ret < 0) { mlx5_ib_err(dev, "Failed reading a WQE following page fault, error=%d, wqe_index=%x, qpn=%x\n", ret, wqe_index, pfault->token); goto resolve_page_fault; } wqe = buffer; if (requestor) ret = mlx5_ib_mr_initiator_pfault_handler(dev, pfault, qp, &wqe, &wqe_end, ret); else ret = mlx5_ib_mr_responder_pfault_handler(dev, pfault, qp, &wqe, &wqe_end, ret); if (ret < 0) goto resolve_page_fault; if (wqe >= wqe_end) { mlx5_ib_err(dev, "ODP fault on invalid WQE.\n"); goto resolve_page_fault; } ret = pagefault_data_segments(dev, pfault, qp, wqe, wqe_end, &bytes_mapped, &total_wqe_bytes, !requestor); if (ret == -EAGAIN) { resume_with_error = 0; goto resolve_page_fault; } else if (ret < 0 || total_wqe_bytes > bytes_mapped) { goto resolve_page_fault; } resume_with_error = 0; resolve_page_fault: mlx5_ib_page_fault_resume(dev, pfault, resume_with_error); mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n", pfault->wqe.wq_num, resume_with_error, pfault->type); mlx5_core_res_put(res); free_page((unsigned long)buffer); } static int pages_in_range(u64 address, u32 length) { return (ALIGN(address + length, PAGE_SIZE) - (address & PAGE_MASK)) >> PAGE_SHIFT; } static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault) { u64 address; u32 length; u32 prefetch_len = pfault->bytes_committed; int prefetch_activated = 0; u32 rkey = pfault->rdma.r_key; int ret; /* The RDMA responder handler handles the page fault in two parts. * First it brings the necessary pages for the current packet * (and uses the pfault context), and then (after resuming the QP) * prefetches more pages. The second operation cannot use the pfault * context and therefore uses the dummy_pfault context allocated on * the stack */ pfault->rdma.rdma_va += pfault->bytes_committed; pfault->rdma.rdma_op_len -= min(pfault->bytes_committed, pfault->rdma.rdma_op_len); pfault->bytes_committed = 0; address = pfault->rdma.rdma_va; length = pfault->rdma.rdma_op_len; /* For some operations, the hardware cannot tell the exact message * length, and in those cases it reports zero. Use prefetch * logic. */ if (length == 0) { prefetch_activated = 1; length = pfault->rdma.packet_size; prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len); } ret = pagefault_single_data_segment(dev, rkey, address, length, &pfault->bytes_committed, NULL); if (ret == -EAGAIN) { /* We're racing with an invalidation, don't prefetch */ prefetch_activated = 0; } else if (ret < 0 || pages_in_range(address, length) > ret) { mlx5_ib_page_fault_resume(dev, pfault, 1); if (ret != -ENOENT) mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n", ret, pfault->token, pfault->type); return; } mlx5_ib_page_fault_resume(dev, pfault, 0); mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n", pfault->token, pfault->type, prefetch_activated); /* At this point, there might be a new pagefault already arriving in * the eq, switch to the dummy pagefault for the rest of the * processing. We're still OK with the objects being alive as the * work-queue is being fenced. */ if (prefetch_activated) { u32 bytes_committed = 0; ret = pagefault_single_data_segment(dev, rkey, address, prefetch_len, &bytes_committed, NULL); if (ret < 0 && ret != -EAGAIN) { mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n", ret, pfault->token, address, prefetch_len); } } } static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault) { u8 event_subtype = pfault->event_subtype; switch (event_subtype) { case MLX5_PFAULT_SUBTYPE_WQE: mlx5_ib_mr_wqe_pfault_handler(dev, pfault); break; case MLX5_PFAULT_SUBTYPE_RDMA: mlx5_ib_mr_rdma_pfault_handler(dev, pfault); break; default: mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n", event_subtype); mlx5_ib_page_fault_resume(dev, pfault, 1); } } static void mlx5_ib_eqe_pf_action(struct work_struct *work) { struct mlx5_pagefault *pfault = container_of(work, struct mlx5_pagefault, work); struct mlx5_ib_pf_eq *eq = pfault->eq; mlx5_ib_pfault(eq->dev, pfault); mempool_free(pfault, eq->pool); } static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq) { struct mlx5_eqe_page_fault *pf_eqe; struct mlx5_pagefault *pfault; struct mlx5_eqe *eqe; int cc = 0; while ((eqe = mlx5_eq_get_eqe(eq->core, cc))) { pfault = mempool_alloc(eq->pool, GFP_ATOMIC); if (!pfault) { schedule_work(&eq->work); break; } pf_eqe = &eqe->data.page_fault; pfault->event_subtype = eqe->sub_type; pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed); mlx5_ib_dbg(eq->dev, "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n", eqe->sub_type, pfault->bytes_committed); switch (eqe->sub_type) { case MLX5_PFAULT_SUBTYPE_RDMA: /* RDMA based event */ pfault->type = be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24; pfault->token = be32_to_cpu(pf_eqe->rdma.pftype_token) & MLX5_24BIT_MASK; pfault->rdma.r_key = be32_to_cpu(pf_eqe->rdma.r_key); pfault->rdma.packet_size = be16_to_cpu(pf_eqe->rdma.packet_length); pfault->rdma.rdma_op_len = be32_to_cpu(pf_eqe->rdma.rdma_op_len); pfault->rdma.rdma_va = be64_to_cpu(pf_eqe->rdma.rdma_va); mlx5_ib_dbg(eq->dev, "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n", pfault->type, pfault->token, pfault->rdma.r_key); mlx5_ib_dbg(eq->dev, "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n", pfault->rdma.rdma_op_len, pfault->rdma.rdma_va); break; case MLX5_PFAULT_SUBTYPE_WQE: /* WQE based event */ pfault->type = (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7; pfault->token = be32_to_cpu(pf_eqe->wqe.token); pfault->wqe.wq_num = be32_to_cpu(pf_eqe->wqe.pftype_wq) & MLX5_24BIT_MASK; pfault->wqe.wqe_index = be16_to_cpu(pf_eqe->wqe.wqe_index); pfault->wqe.packet_size = be16_to_cpu(pf_eqe->wqe.packet_length); mlx5_ib_dbg(eq->dev, "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n", pfault->type, pfault->token, pfault->wqe.wq_num, pfault->wqe.wqe_index); break; default: mlx5_ib_warn(eq->dev, "Unsupported page fault event sub-type: 0x%02hhx\n", eqe->sub_type); /* Unsupported page faults should still be * resolved by the page fault handler */ } pfault->eq = eq; INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action); queue_work(eq->wq, &pfault->work); cc = mlx5_eq_update_cc(eq->core, ++cc); } mlx5_eq_update_ci(eq->core, cc, 1); } static irqreturn_t mlx5_ib_eq_pf_int(int irq, void *eq_ptr) { struct mlx5_ib_pf_eq *eq = eq_ptr; unsigned long flags; if (spin_trylock_irqsave(&eq->lock, flags)) { mlx5_ib_eq_pf_process(eq); spin_unlock_irqrestore(&eq->lock, flags); } else { schedule_work(&eq->work); } return IRQ_HANDLED; } /* mempool_refill() was proposed but unfortunately wasn't accepted * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html * Cheap workaround. */ static void mempool_refill(mempool_t *pool) { while (pool->curr_nr < pool->min_nr) mempool_free(mempool_alloc(pool, GFP_KERNEL), pool); } static void mlx5_ib_eq_pf_action(struct work_struct *work) { struct mlx5_ib_pf_eq *eq = container_of(work, struct mlx5_ib_pf_eq, work); mempool_refill(eq->pool); spin_lock_irq(&eq->lock); mlx5_ib_eq_pf_process(eq); spin_unlock_irq(&eq->lock); } enum { MLX5_IB_NUM_PF_EQE = 0x1000, MLX5_IB_NUM_PF_DRAIN = 64, }; static int mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) { struct mlx5_eq_param param = {}; int err; INIT_WORK(&eq->work, mlx5_ib_eq_pf_action); spin_lock_init(&eq->lock); eq->dev = dev; eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN, sizeof(struct mlx5_pagefault)); if (!eq->pool) return -ENOMEM; eq->wq = alloc_workqueue("mlx5_ib_page_fault", WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, MLX5_NUM_CMD_EQE); if (!eq->wq) { err = -ENOMEM; goto err_mempool; } param = (struct mlx5_eq_param) { .index = MLX5_EQ_PFAULT_IDX, .mask = 1 << MLX5_EVENT_TYPE_PAGE_FAULT, .nent = MLX5_IB_NUM_PF_EQE, .context = eq, .handler = mlx5_ib_eq_pf_int }; eq->core = mlx5_eq_create_generic(dev->mdev, "mlx5_ib_page_fault_eq", &param); if (IS_ERR(eq->core)) { err = PTR_ERR(eq->core); goto err_wq; } return 0; err_wq: destroy_workqueue(eq->wq); err_mempool: mempool_destroy(eq->pool); return err; } static int mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) { int err; err = mlx5_eq_destroy_generic(dev->mdev, eq->core); cancel_work_sync(&eq->work); destroy_workqueue(eq->wq); mempool_destroy(eq->pool); return err; } void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) { if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) return; switch (ent->order - 2) { case MLX5_IMR_MTT_CACHE_ENTRY: ent->page = PAGE_SHIFT; ent->xlt = MLX5_IMR_MTT_ENTRIES * sizeof(struct mlx5_mtt) / MLX5_IB_UMR_OCTOWORD; ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; ent->limit = 0; break; case MLX5_IMR_KSM_CACHE_ENTRY: ent->page = MLX5_KSM_PAGE_SHIFT; ent->xlt = mlx5_imr_ksm_entries * sizeof(struct mlx5_klm) / MLX5_IB_UMR_OCTOWORD; ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM; ent->limit = 0; break; } } int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) { int ret = 0; if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) { ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey); if (ret) { mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret); return ret; } } if (!MLX5_CAP_GEN(dev->mdev, pg)) return ret; ret = mlx5_ib_create_pf_eq(dev, &dev->odp_pf_eq); return ret; } void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev) { if (!MLX5_CAP_GEN(dev->mdev, pg)) return; mlx5_ib_destroy_pf_eq(dev, &dev->odp_pf_eq); } int mlx5_ib_odp_init(void) { mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) - MLX5_IMR_MTT_BITS); return 0; }