litmus-rt.git - The LITMUS^RT kernel.

	Commit message (Expand)	Author	Age
*	[SCSI] libosd: SCSI/OSD Sense decoding support	Boaz Harrosh	2009-03-12
*	[SCSI] osd_uld: OSD scsi ULD	Boaz Harrosh	2009-03-12
*	[SCSI] libosd: OSDv1 preliminary implementation	Boaz Harrosh	2009-03-12

/* * GK20A Cycle stats snapshots support (subsystem for gr_gk20a). * * Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #include <linux/dma-mapping.h> #include <linux/dma-buf.h> #include <nvgpu/bitops.h> #include <nvgpu/kmem.h> #include <nvgpu/lock.h> #include <nvgpu/dma.h> #include <nvgpu/mm.h> #include <nvgpu/sizes.h> #include <nvgpu/barrier.h> #include <nvgpu/log.h> #include <nvgpu/bug.h> #include "gk20a.h" #include "css_gr_gk20a.h" #include <nvgpu/hw/gk20a/hw_perf_gk20a.h> #include <nvgpu/hw/gk20a/hw_mc_gk20a.h> /* check client for pointed perfmon ownership */ #define CONTAINS_PERFMON(cl, pm) \ ((cl)->perfmon_start <= (pm) && \ ((pm) - (cl)->perfmon_start) < (cl)->perfmon_count) /* address of fifo entry by offset */ #define CSS_FIFO_ENTRY(fifo, offs) \ ((struct gk20a_cs_snapshot_fifo_entry *)(((char *)(fifo)) + (offs))) /* calculate area capacity in number of fifo entries */ #define CSS_FIFO_ENTRY_CAPACITY(s) \ (((s) - sizeof(struct gk20a_cs_snapshot_fifo)) \ / sizeof(struct gk20a_cs_snapshot_fifo_entry)) /* reserved to indicate failures with data */ #define CSS_FIRST_PERFMON_ID 32 /* should correlate with size of gk20a_cs_snapshot_fifo_entry::perfmon_id */ #define CSS_MAX_PERFMON_IDS 256 /* reports whether the hw queue overflowed */ static inline bool css_hw_get_overflow_status(struct gk20a *g) { const u32 st = perf_pmasys_control_membuf_status_overflowed_f(); return st == (gk20a_readl(g, perf_pmasys_control_r()) & st); } /* returns how many pending snapshot entries are pending */ static inline u32 css_hw_get_pending_snapshots(struct gk20a *g) { return gk20a_readl(g, perf_pmasys_mem_bytes_r()) / sizeof(struct gk20a_cs_snapshot_fifo_entry); } /* informs hw how many snapshots have been processed (frees up fifo space) */ void css_hw_set_handled_snapshots(struct gk20a *g, u32 done) { if (done > 0) { gk20a_writel(g, perf_pmasys_mem_bump_r(), done * sizeof(struct gk20a_cs_snapshot_fifo_entry)); } } /* disable streaming to memory */ static void css_hw_reset_streaming(struct gk20a *g) { u32 engine_status; /* reset the perfmon */ g->ops.mc.reset(g, mc_enable_perfmon_enabled_f()); /* RBUFEMPTY must be set -- otherwise we'll pick up */ /* snapshot that have been queued up from earlier */ engine_status = gk20a_readl(g, perf_pmasys_enginestatus_r()); WARN_ON(0 == (engine_status & perf_pmasys_enginestatus_rbufempty_empty_f())); /* turn off writes */ gk20a_writel(g, perf_pmasys_control_r(), perf_pmasys_control_membuf_clear_status_doit_f()); /* pointing all pending snapshots as handled */ css_hw_set_handled_snapshots(g, css_hw_get_pending_snapshots(g)); } /* * WARNING: all css_gr_XXX functions are local and expected to be called * from locked context (protected by cs_lock) */ static int css_gr_create_shared_data(struct gr_gk20a *gr) { struct gk20a_cs_snapshot *data; if (gr->cs_data) return 0; data = nvgpu_kzalloc(gr->g, sizeof(*data)); if (!data) return -ENOMEM; nvgpu_init_list_node(&data->clients); gr->cs_data = data; return 0; } int css_hw_enable_snapshot(struct channel_gk20a *ch, struct gk20a_cs_snapshot_client *cs_client) { struct gk20a *g = ch->g; struct gr_gk20a *gr = &g->gr; struct gk20a_cs_snapshot *data = gr->cs_data; u32 snapshot_size = cs_client->snapshot_size; int ret; u32 virt_addr_lo; u32 virt_addr_hi; u32 inst_pa_page; if (data->hw_snapshot) return 0; if (snapshot_size < CSS_MIN_HW_SNAPSHOT_SIZE) snapshot_size = CSS_MIN_HW_SNAPSHOT_SIZE; ret = nvgpu_dma_alloc_map_sys(g->mm.pmu.vm, snapshot_size, &data->hw_memdesc); if (ret) return ret; /* perf output buffer may not cross a 4GB boundary - with a separate */ /* va smaller than that, it won't but check anyway */ if (!data->hw_memdesc.cpu_va || data->hw_memdesc.size < snapshot_size || data->hw_memdesc.gpu_va + u64_lo32(snapshot_size) > SZ_4G) { ret = -EFAULT; goto failed_allocation; } data->hw_snapshot = (struct gk20a_cs_snapshot_fifo_entry *)data->hw_memdesc.cpu_va; data->hw_end = data->hw_snapshot + snapshot_size / sizeof(struct gk20a_cs_snapshot_fifo_entry); data->hw_get = data->hw_snapshot; memset(data->hw_snapshot, 0xff, snapshot_size); /* address and size are aligned to 32 bytes, the lowest bits read back * as zeros */ virt_addr_lo = u64_lo32(data->hw_memdesc.gpu_va); virt_addr_hi = u64_hi32(data->hw_memdesc.gpu_va); css_hw_reset_streaming(g); gk20a_writel(g, perf_pmasys_outbase_r(), virt_addr_lo); gk20a_writel(g, perf_pmasys_outbaseupper_r(), perf_pmasys_outbaseupper_ptr_f(virt_addr_hi)); gk20a_writel(g, perf_pmasys_outsize_r(), snapshot_size); /* this field is aligned to 4K */ inst_pa_page = nvgpu_inst_block_addr(g, &g->mm.hwpm.inst_block) >> 12; /* A write to MEM_BLOCK triggers the block bind operation. MEM_BLOCK * should be written last */ gk20a_writel(g, perf_pmasys_mem_block_r(), perf_pmasys_mem_block_base_f(inst_pa_page) | perf_pmasys_mem_block_valid_true_f() | perf_pmasys_mem_block_target_lfb_f()); gk20a_dbg_info("cyclestats: buffer for hardware snapshots enabled\n"); return 0; failed_allocation: if (data->hw_memdesc.size) { nvgpu_dma_unmap_free(g->mm.pmu.vm, &data->hw_memdesc); memset(&data->hw_memdesc, 0, sizeof(data->hw_memdesc)); } data->hw_snapshot = NULL; return ret; } void css_hw_disable_snapshot(struct gr_gk20a *gr) { struct gk20a *g = gr->g; struct gk20a_cs_snapshot *data = gr->cs_data; if (!data->hw_snapshot) return; css_hw_reset_streaming(g); gk20a_writel(g, perf_pmasys_outbase_r(), 0); gk20a_writel(g, perf_pmasys_outbaseupper_r(), perf_pmasys_outbaseupper_ptr_f(0)); gk20a_writel(g, perf_pmasys_outsize_r(), 0); gk20a_writel(g, perf_pmasys_mem_block_r(), perf_pmasys_mem_block_base_f(0) | perf_pmasys_mem_block_valid_false_f() | perf_pmasys_mem_block_target_f(0)); nvgpu_dma_unmap_free(g->mm.pmu.vm, &data->hw_memdesc); memset(&data->hw_memdesc, 0, sizeof(data->hw_memdesc)); data->hw_snapshot = NULL; gk20a_dbg_info("cyclestats: buffer for hardware snapshots disabled\n"); } static void css_gr_free_shared_data(struct gr_gk20a *gr) { struct gk20a *g = gr->g; if (gr->cs_data) { /* the clients list is expected to be empty */ g->ops.css.disable_snapshot(gr); /* release the objects */ nvgpu_kfree(gr->g, gr->cs_data); gr->cs_data = NULL; } } static struct gk20a_cs_snapshot_client* css_gr_search_client(struct nvgpu_list_node *clients, u32 perfmon) { struct gk20a_cs_snapshot_client *client; nvgpu_list_for_each_entry(client, clients, gk20a_cs_snapshot_client, list) { if (CONTAINS_PERFMON(client, perfmon)) return client; } return NULL; } static int css_gr_flush_snapshots(struct channel_gk20a *ch) { struct gk20a *g = ch->g; struct gr_gk20a *gr = &g->gr; struct gk20a_cs_snapshot *css = gr->cs_data; struct gk20a_cs_snapshot_client *cur; u32 pending, completed; bool hw_overflow; int err; /* variables for iterating over HW entries */ u32 sid; struct gk20a_cs_snapshot_fifo_entry *src; /* due to data sharing with userspace we allowed update only */ /* overflows and put field in the fifo header */ struct gk20a_cs_snapshot_fifo *dst; struct gk20a_cs_snapshot_fifo_entry *dst_get; struct gk20a_cs_snapshot_fifo_entry *dst_put; struct gk20a_cs_snapshot_fifo_entry *dst_nxt; struct gk20a_cs_snapshot_fifo_entry *dst_head; struct gk20a_cs_snapshot_fifo_entry *dst_tail; if (!css) return -EINVAL; if (nvgpu_list_empty(&css->clients)) return -EBADF; /* check data available */ err = g->ops.css.check_data_available(ch, &pending, &hw_overflow); if (err) return err; if (!pending) return 0; if (hw_overflow) { nvgpu_list_for_each_entry(cur, &css->clients, gk20a_cs_snapshot_client, list) { cur->snapshot->hw_overflow_events_occured++; } nvgpu_warn(g, "cyclestats: hardware overflow detected"); } /* process all items in HW buffer */ sid = 0; completed = 0; cur = NULL; dst = NULL; dst_put = NULL; src = css->hw_get; /* proceed all completed records */ while (sid < pending && 0 == src->zero0) { /* we may have a new perfmon_id which required to */ /* switch to a new client -> let's forget current */ if (cur && !CONTAINS_PERFMON(cur, src->perfmon_id)) { dst->put = (char *)dst_put - (char *)dst; dst = NULL; cur = NULL; } /* now we have to select a new current client */ /* the client selection rate depends from experiment */ /* activity but on Android usually happened 1-2 times */ if (!cur) { cur = css_gr_search_client(&css->clients, src->perfmon_id); if (cur) { /* found - setup all required data */ dst = cur->snapshot; dst_get = CSS_FIFO_ENTRY(dst, dst->get); dst_put = CSS_FIFO_ENTRY(dst, dst->put); dst_head = CSS_FIFO_ENTRY(dst, dst->start); dst_tail = CSS_FIFO_ENTRY(dst, dst->end); dst_nxt = dst_put + 1; if (dst_nxt == dst_tail) dst_nxt = dst_head; } else { /* client not found - skipping this entry */ nvgpu_warn(g, "cyclestats: orphaned perfmon %u", src->perfmon_id); goto next_hw_fifo_entry; } } /* check for software overflows */ if (dst_nxt == dst_get) { /* no data copy, no pointer updates */ dst->sw_overflow_events_occured++; nvgpu_warn(g, "cyclestats: perfmon %u soft overflow", src->perfmon_id); } else { *dst_put = *src; completed++; dst_put = dst_nxt++; if (dst_nxt == dst_tail) dst_nxt = dst_head; } next_hw_fifo_entry: sid++; if (++src >= css->hw_end) src = css->hw_snapshot; } /* update client put pointer if necessary */ if (cur && dst) dst->put = (char *)dst_put - (char *)dst; /* re-set HW buffer after processing taking wrapping into account */ if (css->hw_get < src) { memset(css->hw_get, 0xff, (src - css->hw_get) * sizeof(*src)); } else { memset(css->hw_snapshot, 0xff, (src - css->hw_snapshot) * sizeof(*src)); memset(css->hw_get, 0xff, (css->hw_end - css->hw_get) * sizeof(*src)); } gr->cs_data->hw_get = src; if (g->ops.css.set_handled_snapshots) g->ops.css.set_handled_snapshots(g, sid); if (completed != sid) { /* not all entries proceed correctly. some of problems */ /* reported as overflows, some as orphaned perfmons, */ /* but it will be better notify with summary about it */ nvgpu_warn(g, "cyclestats: completed %u from %u entries", completed, pending); } return 0; } u32 css_gr_allocate_perfmon_ids(struct gk20a_cs_snapshot *data, u32 count) { unsigned long *pids = data->perfmon_ids; unsigned int f; f = bitmap_find_next_zero_area(pids, CSS_MAX_PERFMON_IDS, CSS_FIRST_PERFMON_ID, count, 0); if (f > CSS_MAX_PERFMON_IDS) f = 0; else bitmap_set(pids, f, count); return f; } u32 css_gr_release_perfmon_ids(struct gk20a_cs_snapshot *data, u32 start, u32 count) { unsigned long *pids = data->perfmon_ids; u32 end = start + count; u32 cnt = 0; if (start >= CSS_FIRST_PERFMON_ID && end <= CSS_MAX_PERFMON_IDS) { bitmap_clear(pids, start, count); cnt = count; } return cnt; } static int css_gr_free_client_data(struct gk20a *g, struct gk20a_cs_snapshot *data, struct gk20a_cs_snapshot_client *client) { int ret = 0; if (client->list.next && client->list.prev) nvgpu_list_del(&client->list); if (client->perfmon_start && client->perfmon_count && g->ops.css.release_perfmon_ids) { if (client->perfmon_count != g->ops.css.release_perfmon_ids(data, client->perfmon_start, client->perfmon_count)) ret = -EINVAL; } return ret; } static int css_gr_create_client_data(struct gk20a *g, struct gk20a_cs_snapshot *data, u32 perfmon_count, struct gk20a_cs_snapshot_client *cur) { memset(cur->snapshot, 0, sizeof(*cur->snapshot)); cur->snapshot->start = sizeof(*cur->snapshot); /* we should be ensure that can fit all fifo entries here */ cur->snapshot->end = CSS_FIFO_ENTRY_CAPACITY(cur->snapshot_size) * sizeof(struct gk20a_cs_snapshot_fifo_entry) + sizeof(struct gk20a_cs_snapshot_fifo); cur->snapshot->get = cur->snapshot->start; cur->snapshot->put = cur->snapshot->start; cur->perfmon_count = perfmon_count; /* In virtual case, perfmon ID allocation is handled by the server * at the time of the attach (allocate_perfmon_ids is NULL in this case) */ if (cur->perfmon_count && g->ops.css.allocate_perfmon_ids) { cur->perfmon_start = g->ops.css.allocate_perfmon_ids(data, cur->perfmon_count); if (!cur->perfmon_start) return -ENOENT; } nvgpu_list_add_tail(&cur->list, &data->clients); return 0; } int gr_gk20a_css_attach(struct channel_gk20a *ch, u32 perfmon_count, u32 *perfmon_start, struct gk20a_cs_snapshot_client *cs_client) { int ret = 0; struct gk20a *g = ch->g; struct gr_gk20a *gr; /* we must have a placeholder to store pointer to client structure */ if (!cs_client) return -EINVAL; if (!perfmon_count || perfmon_count > CSS_MAX_PERFMON_IDS - CSS_FIRST_PERFMON_ID) return -EINVAL; nvgpu_speculation_barrier(); gr = &g->gr; nvgpu_mutex_acquire(&gr->cs_lock); ret = css_gr_create_shared_data(gr); if (ret) goto failed; ret = css_gr_create_client_data(g, gr->cs_data, perfmon_count, cs_client); if (ret) goto failed; ret = g->ops.css.enable_snapshot(ch, cs_client); if (ret) goto failed; if (perfmon_start) *perfmon_start = cs_client->perfmon_start; nvgpu_mutex_release(&gr->cs_lock); return 0; failed: if (gr->cs_data) { if (cs_client) { css_gr_free_client_data(g, gr->cs_data, cs_client); cs_client = NULL; } if (nvgpu_list_empty(&gr->cs_data->clients)) css_gr_free_shared_data(gr); } nvgpu_mutex_release(&gr->cs_lock); if (perfmon_start) *perfmon_start = 0; return ret; } int gr_gk20a_css_detach(struct channel_gk20a *ch, struct gk20a_cs_snapshot_client *cs_client) { int ret = 0; struct gk20a *g = ch->g; struct gr_gk20a *gr; if (!cs_client) return -EINVAL; gr = &g->gr; nvgpu_mutex_acquire(&gr->cs_lock); if (gr->cs_data) { struct gk20a_cs_snapshot *data = gr->cs_data; if (g->ops.css.detach_snapshot) g->ops.css.detach_snapshot(ch, cs_client); ret = css_gr_free_client_data(g, data, cs_client); if (nvgpu_list_empty(&data->clients)) css_gr_free_shared_data(gr); } else { ret = -EBADF; } nvgpu_mutex_release(&gr->cs_lock); return ret; } int gr_gk20a_css_flush(struct channel_gk20a *ch, struct gk20a_cs_snapshot_client *cs_client) { int ret = 0; struct gk20a *g = ch->g; struct gr_gk20a *gr; if (!cs_client) return -EINVAL; gr = &g->gr; nvgpu_mutex_acquire(&gr->cs_lock); ret = css_gr_flush_snapshots(ch); nvgpu_mutex_release(&gr->cs_lock); return ret; } /* helper function with locking to cleanup snapshot code code in gr_gk20a.c */ void gr_gk20a_free_cyclestats_snapshot_data(struct gk20a *g) { struct gr_gk20a *gr = &g->gr; nvgpu_mutex_acquire(&gr->cs_lock); css_gr_free_shared_data(gr); nvgpu_mutex_release(&gr->cs_lock); nvgpu_mutex_destroy(&gr->cs_lock); } int css_hw_check_data_available(struct channel_gk20a *ch, u32 *pending, bool *hw_overflow) { struct gk20a *g = ch->g; struct gr_gk20a *gr = &g->gr; struct gk20a_cs_snapshot *css = gr->cs_data; if (!css->hw_snapshot) return -EINVAL; *pending = css_hw_get_pending_snapshots(g); if (!*pending) return 0; *hw_overflow = css_hw_get_overflow_status(g); return 0; }