606 lines
15 KiB
C
606 lines
15 KiB
C
/**************************************************************************
|
|
*
|
|
* Copyright (c) 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA
|
|
* All Rights Reserved.
|
|
* Copyright (c) 2009 VMware, Inc., Palo Alto, CA., USA
|
|
* All Rights Reserved.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
* under the terms and conditions of the GNU General Public License,
|
|
* version 2, as published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
* more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License along with
|
|
* this program; if not, write to the Free Software Foundation, Inc.,
|
|
* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
|
|
*
|
|
**************************************************************************/
|
|
/*
|
|
* Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
|
|
*/
|
|
|
|
#include "psb_ttm_fence_api.h"
|
|
#include "psb_ttm_fence_driver.h"
|
|
#include <linux/wait.h>
|
|
#include <linux/sched.h>
|
|
|
|
#include <drm/drmP.h>
|
|
|
|
/*
|
|
* Simple implementation for now.
|
|
*/
|
|
|
|
static void ttm_fence_lockup(struct ttm_fence_object *fence, uint32_t mask)
|
|
{
|
|
struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
|
|
|
|
printk(KERN_ERR "GPU lockup dectected on engine %u "
|
|
"fence type 0x%08x\n",
|
|
(unsigned int)fence->fence_class, (unsigned int)mask);
|
|
/*
|
|
* Give engines some time to idle?
|
|
*/
|
|
|
|
write_lock(&fc->lock);
|
|
ttm_fence_handler(fence->fdev, fence->fence_class,
|
|
fence->sequence, mask, -EBUSY);
|
|
write_unlock(&fc->lock);
|
|
}
|
|
|
|
/*
|
|
* Convenience function to be called by fence::wait methods that
|
|
* need polling.
|
|
*/
|
|
|
|
int ttm_fence_wait_polling(struct ttm_fence_object *fence, bool lazy,
|
|
bool interruptible, uint32_t mask)
|
|
{
|
|
struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
|
|
const struct ttm_fence_driver *driver = ttm_fence_driver(fence);
|
|
uint32_t count = 0;
|
|
int ret;
|
|
unsigned long end_jiffies = fence->timeout_jiffies;
|
|
|
|
DECLARE_WAITQUEUE(entry, current);
|
|
add_wait_queue(&fc->fence_queue, &entry);
|
|
|
|
ret = 0;
|
|
|
|
for (;;) {
|
|
__set_current_state((interruptible) ?
|
|
TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
|
|
if (ttm_fence_object_signaled(fence, mask))
|
|
break;
|
|
if (time_after_eq(jiffies, end_jiffies)) {
|
|
if (driver->lockup)
|
|
driver->lockup(fence, mask);
|
|
else
|
|
ttm_fence_lockup(fence, mask);
|
|
continue;
|
|
}
|
|
if (lazy)
|
|
schedule_timeout(1);
|
|
else if ((++count & 0x0F) == 0) {
|
|
__set_current_state(TASK_RUNNING);
|
|
schedule();
|
|
__set_current_state((interruptible) ?
|
|
TASK_INTERRUPTIBLE :
|
|
TASK_UNINTERRUPTIBLE);
|
|
}
|
|
if (interruptible && signal_pending(current)) {
|
|
ret = -ERESTART;
|
|
break;
|
|
}
|
|
}
|
|
__set_current_state(TASK_RUNNING);
|
|
remove_wait_queue(&fc->fence_queue, &entry);
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Typically called by the IRQ handler.
|
|
*/
|
|
|
|
void ttm_fence_handler(struct ttm_fence_device *fdev, uint32_t fence_class,
|
|
uint32_t sequence, uint32_t type, uint32_t error)
|
|
{
|
|
int wake = 0;
|
|
uint32_t diff;
|
|
uint32_t relevant_type;
|
|
uint32_t new_type;
|
|
struct ttm_fence_class_manager *fc = &fdev->fence_class[fence_class];
|
|
const struct ttm_fence_driver *driver = ttm_fence_driver_from_dev(fdev);
|
|
struct list_head *head;
|
|
struct ttm_fence_object *fence, *next;
|
|
bool found = false;
|
|
|
|
if (list_empty(&fc->ring))
|
|
return;
|
|
|
|
list_for_each_entry(fence, &fc->ring, ring) {
|
|
diff = (sequence - fence->sequence) & fc->sequence_mask;
|
|
if (diff > fc->wrap_diff) {
|
|
found = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
fc->waiting_types &= ~type;
|
|
head = (found) ? &fence->ring : &fc->ring;
|
|
|
|
list_for_each_entry_safe_reverse(fence, next, head, ring) {
|
|
if (&fence->ring == &fc->ring)
|
|
break;
|
|
|
|
DRM_DEBUG("Fence 0x%08lx, sequence 0x%08x, type 0x%08x\n",
|
|
(unsigned long)fence, fence->sequence,
|
|
fence->fence_type);
|
|
|
|
if (error) {
|
|
fence->info.error = error;
|
|
fence->info.signaled_types = fence->fence_type;
|
|
list_del_init(&fence->ring);
|
|
wake = 1;
|
|
break;
|
|
}
|
|
|
|
relevant_type = type & fence->fence_type;
|
|
new_type = (fence->info.signaled_types | relevant_type) ^
|
|
fence->info.signaled_types;
|
|
|
|
if (new_type) {
|
|
fence->info.signaled_types |= new_type;
|
|
DRM_DEBUG("Fence 0x%08lx signaled 0x%08x\n",
|
|
(unsigned long)fence,
|
|
fence->info.signaled_types);
|
|
|
|
if (unlikely(driver->signaled))
|
|
driver->signaled(fence);
|
|
|
|
if (driver->needed_flush)
|
|
fc->pending_flush |=
|
|
driver->needed_flush(fence);
|
|
|
|
if (new_type & fence->waiting_types)
|
|
wake = 1;
|
|
}
|
|
|
|
fc->waiting_types |=
|
|
fence->waiting_types & ~fence->info.signaled_types;
|
|
|
|
if (!(fence->fence_type & ~fence->info.signaled_types)) {
|
|
DRM_DEBUG("Fence completely signaled 0x%08lx\n",
|
|
(unsigned long)fence);
|
|
list_del_init(&fence->ring);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Reinstate lost waiting types.
|
|
*/
|
|
|
|
if ((fc->waiting_types & type) != type) {
|
|
head = head->prev;
|
|
list_for_each_entry(fence, head, ring) {
|
|
if (&fence->ring == &fc->ring)
|
|
break;
|
|
diff =
|
|
(fc->highest_waiting_sequence -
|
|
fence->sequence) & fc->sequence_mask;
|
|
if (diff > fc->wrap_diff)
|
|
break;
|
|
|
|
fc->waiting_types |=
|
|
fence->waiting_types & ~fence->info.signaled_types;
|
|
}
|
|
}
|
|
|
|
if (wake)
|
|
wake_up_all(&fc->fence_queue);
|
|
}
|
|
|
|
static void ttm_fence_unring(struct ttm_fence_object *fence)
|
|
{
|
|
struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
|
|
unsigned long irq_flags;
|
|
|
|
write_lock_irqsave(&fc->lock, irq_flags);
|
|
list_del_init(&fence->ring);
|
|
write_unlock_irqrestore(&fc->lock, irq_flags);
|
|
}
|
|
|
|
bool ttm_fence_object_signaled(struct ttm_fence_object *fence, uint32_t mask)
|
|
{
|
|
unsigned long flags;
|
|
bool signaled;
|
|
const struct ttm_fence_driver *driver = ttm_fence_driver(fence);
|
|
struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
|
|
|
|
mask &= fence->fence_type;
|
|
read_lock_irqsave(&fc->lock, flags);
|
|
signaled = (mask & fence->info.signaled_types) == mask;
|
|
read_unlock_irqrestore(&fc->lock, flags);
|
|
if (!signaled && driver->poll) {
|
|
write_lock_irqsave(&fc->lock, flags);
|
|
driver->poll(fence->fdev, fence->fence_class, mask);
|
|
signaled = (mask & fence->info.signaled_types) == mask;
|
|
write_unlock_irqrestore(&fc->lock, flags);
|
|
}
|
|
return signaled;
|
|
}
|
|
|
|
int ttm_fence_object_flush(struct ttm_fence_object *fence, uint32_t type)
|
|
{
|
|
const struct ttm_fence_driver *driver = ttm_fence_driver(fence);
|
|
struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
|
|
unsigned long irq_flags;
|
|
uint32_t saved_pending_flush;
|
|
uint32_t diff;
|
|
bool call_flush;
|
|
|
|
if (type & ~fence->fence_type) {
|
|
DRM_ERROR("Flush trying to extend fence type, "
|
|
"0x%x, 0x%x\n", type, fence->fence_type);
|
|
return -EINVAL;
|
|
}
|
|
|
|
write_lock_irqsave(&fc->lock, irq_flags);
|
|
fence->waiting_types |= type;
|
|
fc->waiting_types |= fence->waiting_types;
|
|
diff = (fence->sequence - fc->highest_waiting_sequence) &
|
|
fc->sequence_mask;
|
|
|
|
if (diff < fc->wrap_diff)
|
|
fc->highest_waiting_sequence = fence->sequence;
|
|
|
|
/*
|
|
* fence->waiting_types has changed. Determine whether
|
|
* we need to initiate some kind of flush as a result of this.
|
|
*/
|
|
|
|
saved_pending_flush = fc->pending_flush;
|
|
if (driver->needed_flush)
|
|
fc->pending_flush |= driver->needed_flush(fence);
|
|
|
|
if (driver->poll)
|
|
driver->poll(fence->fdev, fence->fence_class,
|
|
fence->waiting_types);
|
|
|
|
call_flush = (fc->pending_flush != 0);
|
|
write_unlock_irqrestore(&fc->lock, irq_flags);
|
|
|
|
if (call_flush && driver->flush)
|
|
driver->flush(fence->fdev, fence->fence_class);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Make sure old fence objects are signaled before their fence sequences are
|
|
* wrapped around and reused.
|
|
*/
|
|
|
|
void ttm_fence_flush_old(struct ttm_fence_device *fdev,
|
|
uint32_t fence_class, uint32_t sequence)
|
|
{
|
|
struct ttm_fence_class_manager *fc = &fdev->fence_class[fence_class];
|
|
struct ttm_fence_object *fence;
|
|
unsigned long irq_flags;
|
|
const struct ttm_fence_driver *driver = fdev->driver;
|
|
bool call_flush;
|
|
|
|
uint32_t diff;
|
|
|
|
write_lock_irqsave(&fc->lock, irq_flags);
|
|
|
|
list_for_each_entry_reverse(fence, &fc->ring, ring) {
|
|
diff = (sequence - fence->sequence) & fc->sequence_mask;
|
|
if (diff <= fc->flush_diff)
|
|
break;
|
|
|
|
fence->waiting_types = fence->fence_type;
|
|
fc->waiting_types |= fence->fence_type;
|
|
|
|
if (driver->needed_flush)
|
|
fc->pending_flush |= driver->needed_flush(fence);
|
|
}
|
|
|
|
if (driver->poll)
|
|
driver->poll(fdev, fence_class, fc->waiting_types);
|
|
|
|
call_flush = (fc->pending_flush != 0);
|
|
write_unlock_irqrestore(&fc->lock, irq_flags);
|
|
|
|
if (call_flush && driver->flush)
|
|
driver->flush(fdev, fence->fence_class);
|
|
|
|
/*
|
|
* FIXME: Shold we implement a wait here for really old fences?
|
|
*/
|
|
|
|
}
|
|
|
|
int ttm_fence_object_wait(struct ttm_fence_object *fence,
|
|
bool lazy, bool interruptible, uint32_t mask)
|
|
{
|
|
const struct ttm_fence_driver *driver = ttm_fence_driver(fence);
|
|
struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
|
|
int ret = 0;
|
|
unsigned long timeout;
|
|
unsigned long cur_jiffies;
|
|
unsigned long to_jiffies;
|
|
|
|
if (mask & ~fence->fence_type) {
|
|
DRM_ERROR("Wait trying to extend fence type"
|
|
" 0x%08x 0x%08x\n", mask, fence->fence_type);
|
|
BUG();
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (driver->wait)
|
|
return driver->wait(fence, lazy, interruptible, mask);
|
|
|
|
ttm_fence_object_flush(fence, mask);
|
|
retry:
|
|
if (!driver->has_irq ||
|
|
driver->has_irq(fence->fdev, fence->fence_class, mask)) {
|
|
|
|
cur_jiffies = jiffies;
|
|
to_jiffies = fence->timeout_jiffies;
|
|
|
|
timeout = (time_after(to_jiffies, cur_jiffies)) ?
|
|
to_jiffies - cur_jiffies : 1;
|
|
|
|
if (interruptible)
|
|
ret = wait_event_interruptible_timeout
|
|
(fc->fence_queue,
|
|
ttm_fence_object_signaled(fence, mask), timeout);
|
|
else
|
|
ret = wait_event_timeout
|
|
(fc->fence_queue,
|
|
ttm_fence_object_signaled(fence, mask), timeout);
|
|
|
|
if (unlikely(ret == -ERESTARTSYS))
|
|
return -ERESTART;
|
|
|
|
if (unlikely(ret == 0)) {
|
|
if (driver->lockup)
|
|
driver->lockup(fence, mask);
|
|
else
|
|
ttm_fence_lockup(fence, mask);
|
|
goto retry;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
return ttm_fence_wait_polling(fence, lazy, interruptible, mask);
|
|
}
|
|
|
|
int ttm_fence_object_emit(struct ttm_fence_object *fence, uint32_t fence_flags,
|
|
uint32_t fence_class, uint32_t type)
|
|
{
|
|
const struct ttm_fence_driver *driver = ttm_fence_driver(fence);
|
|
struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
|
|
unsigned long flags;
|
|
uint32_t sequence;
|
|
unsigned long timeout;
|
|
int ret;
|
|
|
|
ttm_fence_unring(fence);
|
|
ret = driver->emit(fence->fdev,
|
|
fence_class, fence_flags, &sequence, &timeout);
|
|
if (ret)
|
|
return ret;
|
|
|
|
write_lock_irqsave(&fc->lock, flags);
|
|
fence->fence_class = fence_class;
|
|
fence->fence_type = type;
|
|
fence->waiting_types = 0;
|
|
fence->info.signaled_types = 0;
|
|
fence->info.error = 0;
|
|
fence->sequence = sequence;
|
|
fence->timeout_jiffies = timeout;
|
|
if (list_empty(&fc->ring))
|
|
fc->highest_waiting_sequence = sequence - 1;
|
|
list_add_tail(&fence->ring, &fc->ring);
|
|
fc->latest_queued_sequence = sequence;
|
|
write_unlock_irqrestore(&fc->lock, flags);
|
|
return 0;
|
|
}
|
|
|
|
int ttm_fence_object_init(struct ttm_fence_device *fdev,
|
|
uint32_t fence_class,
|
|
uint32_t type,
|
|
uint32_t create_flags,
|
|
void (*destroy) (struct ttm_fence_object *),
|
|
struct ttm_fence_object *fence)
|
|
{
|
|
int ret = 0;
|
|
|
|
kref_init(&fence->kref);
|
|
fence->fence_class = fence_class;
|
|
fence->fence_type = type;
|
|
fence->info.signaled_types = 0;
|
|
fence->waiting_types = 0;
|
|
fence->sequence = 0;
|
|
fence->info.error = 0;
|
|
fence->fdev = fdev;
|
|
fence->destroy = destroy;
|
|
INIT_LIST_HEAD(&fence->ring);
|
|
atomic_inc(&fdev->count);
|
|
|
|
if (create_flags & TTM_FENCE_FLAG_EMIT) {
|
|
ret = ttm_fence_object_emit(fence, create_flags,
|
|
fence->fence_class, type);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
int ttm_fence_object_create(struct ttm_fence_device *fdev,
|
|
uint32_t fence_class,
|
|
uint32_t type,
|
|
uint32_t create_flags,
|
|
struct ttm_fence_object **c_fence)
|
|
{
|
|
struct ttm_fence_object *fence;
|
|
int ret;
|
|
|
|
ret = ttm_mem_global_alloc(fdev->mem_glob,
|
|
sizeof(*fence),
|
|
false,
|
|
false);
|
|
if (unlikely(ret != 0)) {
|
|
printk(KERN_ERR "Out of memory creating fence object\n");
|
|
return ret;
|
|
}
|
|
|
|
fence = kmalloc(sizeof(*fence), GFP_KERNEL);
|
|
if (!fence) {
|
|
printk(KERN_ERR "Out of memory creating fence object\n");
|
|
ttm_mem_global_free(fdev->mem_glob, sizeof(*fence));
|
|
return -ENOMEM;
|
|
}
|
|
|
|
ret = ttm_fence_object_init(fdev, fence_class, type,
|
|
create_flags, NULL, fence);
|
|
if (ret) {
|
|
ttm_fence_object_unref(&fence);
|
|
return ret;
|
|
}
|
|
*c_fence = fence;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void ttm_fence_object_destroy(struct kref *kref)
|
|
{
|
|
struct ttm_fence_object *fence =
|
|
container_of(kref, struct ttm_fence_object, kref);
|
|
struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
|
|
unsigned long irq_flags;
|
|
|
|
write_lock_irqsave(&fc->lock, irq_flags);
|
|
list_del_init(&fence->ring);
|
|
write_unlock_irqrestore(&fc->lock, irq_flags);
|
|
|
|
atomic_dec(&fence->fdev->count);
|
|
if (fence->destroy)
|
|
fence->destroy(fence);
|
|
else {
|
|
ttm_mem_global_free(fence->fdev->mem_glob,
|
|
sizeof(*fence));
|
|
kfree(fence);
|
|
}
|
|
}
|
|
|
|
void ttm_fence_device_release(struct ttm_fence_device *fdev)
|
|
{
|
|
kfree(fdev->fence_class);
|
|
}
|
|
|
|
int
|
|
ttm_fence_device_init(int num_classes,
|
|
struct ttm_mem_global *mem_glob,
|
|
struct ttm_fence_device *fdev,
|
|
const struct ttm_fence_class_init *init,
|
|
bool replicate_init,
|
|
const struct ttm_fence_driver *driver)
|
|
{
|
|
struct ttm_fence_class_manager *fc;
|
|
const struct ttm_fence_class_init *fci;
|
|
int i;
|
|
|
|
fdev->mem_glob = mem_glob;
|
|
fdev->fence_class = kzalloc(num_classes *
|
|
sizeof(*fdev->fence_class), GFP_KERNEL);
|
|
|
|
if (unlikely(!fdev->fence_class))
|
|
return -ENOMEM;
|
|
|
|
fdev->num_classes = num_classes;
|
|
atomic_set(&fdev->count, 0);
|
|
fdev->driver = driver;
|
|
|
|
for (i = 0; i < fdev->num_classes; ++i) {
|
|
fc = &fdev->fence_class[i];
|
|
fci = &init[(replicate_init) ? 0 : i];
|
|
|
|
fc->wrap_diff = fci->wrap_diff;
|
|
fc->flush_diff = fci->flush_diff;
|
|
fc->sequence_mask = fci->sequence_mask;
|
|
|
|
rwlock_init(&fc->lock);
|
|
INIT_LIST_HEAD(&fc->ring);
|
|
init_waitqueue_head(&fc->fence_queue);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
struct ttm_fence_info ttm_fence_get_info(struct ttm_fence_object *fence)
|
|
{
|
|
struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
|
|
struct ttm_fence_info tmp;
|
|
unsigned long irq_flags;
|
|
|
|
read_lock_irqsave(&fc->lock, irq_flags);
|
|
tmp = fence->info;
|
|
read_unlock_irqrestore(&fc->lock, irq_flags);
|
|
|
|
return tmp;
|
|
}
|
|
|
|
void ttm_fence_object_unref(struct ttm_fence_object **p_fence)
|
|
{
|
|
struct ttm_fence_object *fence = *p_fence;
|
|
|
|
*p_fence = NULL;
|
|
(void)kref_put(&fence->kref, &ttm_fence_object_destroy);
|
|
}
|
|
|
|
/*
|
|
* Placement / BO sync object glue.
|
|
*/
|
|
|
|
bool ttm_fence_sync_obj_signaled(void *sync_obj, void *sync_arg)
|
|
{
|
|
struct ttm_fence_object *fence = (struct ttm_fence_object *)sync_obj;
|
|
uint32_t fence_types = (uint32_t) (unsigned long)sync_arg;
|
|
|
|
return ttm_fence_object_signaled(fence, fence_types);
|
|
}
|
|
|
|
int ttm_fence_sync_obj_wait(void *sync_obj, void *sync_arg,
|
|
bool lazy, bool interruptible)
|
|
{
|
|
struct ttm_fence_object *fence = (struct ttm_fence_object *)sync_obj;
|
|
uint32_t fence_types = (uint32_t) (unsigned long)sync_arg;
|
|
|
|
return ttm_fence_object_wait(fence, lazy, interruptible, fence_types);
|
|
}
|
|
|
|
int ttm_fence_sync_obj_flush(void *sync_obj, void *sync_arg)
|
|
{
|
|
struct ttm_fence_object *fence = (struct ttm_fence_object *)sync_obj;
|
|
uint32_t fence_types = (uint32_t) (unsigned long)sync_arg;
|
|
|
|
return ttm_fence_object_flush(fence, fence_types);
|
|
}
|
|
|
|
void ttm_fence_sync_obj_unref(void **sync_obj)
|
|
{
|
|
ttm_fence_object_unref((struct ttm_fence_object **)sync_obj);
|
|
}
|
|
|
|
void *ttm_fence_sync_obj_ref(void *sync_obj)
|
|
{
|
|
return (void *)
|
|
ttm_fence_object_ref((struct ttm_fence_object *)sync_obj);
|
|
}
|