gcc/libgcc/config/spu/cachemgr.c

/* Copyright (C) 2008-2015 Free Software Foundation, Inc.

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3, or (at your option) any later
version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.

You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
<http://www.gnu.org/licenses/>.  */

#include <spu_mfcio.h>
#include <spu_internals.h>
#include <spu_intrinsics.h>
#include <spu_cache.h>

extern unsigned long long __ea_local_store;
extern char __cache_tag_array_size;

#define LINE_SIZE 128
#define TAG_MASK (LINE_SIZE - 1)

#define WAYS 4
#define SET_MASK ((int) &__cache_tag_array_size - LINE_SIZE)

#define CACHE_LINES ((int) &__cache_tag_array_size /		\
		     sizeof (struct __cache_tag_array) * WAYS)

struct __cache_tag_array
{
  unsigned int tag_lo[WAYS];
  unsigned int tag_hi[WAYS];
  void *base[WAYS];
  int reserved[WAYS];
  vector unsigned short dirty_bits[WAYS];
};

extern struct __cache_tag_array __cache_tag_array[];
extern char __cache[];

/* In order to make the code seem a little cleaner, and to avoid having
   64/32 bit ifdefs all over the place, we use macros.  */

#ifdef __EA64__
typedef unsigned long long addr;

#define CHECK_TAG(_entry, _way, _tag)			\
  ((_entry)->tag_lo[(_way)] == ((_tag) & 0xFFFFFFFF)	\
   && (_entry)->tag_hi[(_way)] == ((_tag) >> 32))

#define GET_TAG(_entry, _way) \
  ((unsigned long long)(_entry)->tag_hi[(_way)] << 32	\
   | (unsigned long long)(_entry)->tag_lo[(_way)])

#define SET_TAG(_entry, _way, _tag)			\
  (_entry)->tag_lo[(_way)] = (_tag) & 0xFFFFFFFF;	\
  (_entry)->tag_hi[(_way)] = (_tag) >> 32

#else /*__EA32__*/
typedef unsigned long addr;

#define CHECK_TAG(_entry, _way, _tag)			\
  ((_entry)->tag_lo[(_way)] == (_tag))

#define GET_TAG(_entry, _way)				\
  ((_entry)->tag_lo[(_way)])

#define SET_TAG(_entry, _way, _tag)			\
  (_entry)->tag_lo[(_way)] = (_tag)

#endif

/* In GET_ENTRY, we cast away the high 32 bits,
   as the tag is only in the low 32.  */

#define GET_ENTRY(_addr)						   \
  ((struct __cache_tag_array *)						   \
   si_to_uint (si_a (si_and (si_from_uint ((unsigned int) (addr) (_addr)), \
			     si_from_uint (SET_MASK)),			   \
	       si_from_uint ((unsigned int) __cache_tag_array))))

#define GET_CACHE_LINE(_addr, _way) \
  ((void *) (__cache + ((_addr) & SET_MASK) * WAYS) + ((_way) * LINE_SIZE));

#define CHECK_DIRTY(_vec) (si_to_uint (si_orx ((qword) (_vec))))
#define SET_EMPTY(_entry, _way) ((_entry)->tag_lo[(_way)] = 1)
#define CHECK_EMPTY(_entry, _way) ((_entry)->tag_lo[(_way)] == 1)

#define LS_FLAG 0x80000000
#define SET_IS_LS(_entry, _way) ((_entry)->reserved[(_way)] |= LS_FLAG)
#define CHECK_IS_LS(_entry, _way) ((_entry)->reserved[(_way)] & LS_FLAG)
#define GET_LRU(_entry, _way) ((_entry)->reserved[(_way)] & ~LS_FLAG)

static int dma_tag = 32;

static void
__cache_evict_entry (struct __cache_tag_array *entry, int way)
{
  addr tag = GET_TAG (entry, way);

  if (CHECK_DIRTY (entry->dirty_bits[way]) && !CHECK_IS_LS (entry, way))
    {
#ifdef NONATOMIC
      /* Non-atomic writes.  */
      unsigned int oldmask, mach_stat;
      char *line = ((void *) 0);

      /* Enter critical section.  */
      mach_stat = spu_readch (SPU_RdMachStat);
      spu_idisable ();

      /* Issue DMA request.  */
      line = GET_CACHE_LINE (entry->tag_lo[way], way);
      mfc_put (line, tag, LINE_SIZE, dma_tag, 0, 0);

      /* Wait for DMA completion.  */
      oldmask = mfc_read_tag_mask ();
      mfc_write_tag_mask (1 << dma_tag);
      mfc_read_tag_status_all ();
      mfc_write_tag_mask (oldmask);

      /* Leave critical section.  */
      if (__builtin_expect (mach_stat & 1, 0))
	spu_ienable ();
#else
      /* Allocate a buffer large enough that we know it has 128 bytes
         that are 128 byte aligned (for DMA). */

      char buffer[LINE_SIZE + 127];
      qword *buf_ptr = (qword *) (((unsigned int) (buffer) + 127) & ~127);
      qword *line = GET_CACHE_LINE (entry->tag_lo[way], way);
      qword bits;
      unsigned int mach_stat;

      /* Enter critical section.  */
      mach_stat = spu_readch (SPU_RdMachStat);
      spu_idisable ();

      do
	{
	  /* We atomically read the current memory into a buffer
	     modify the dirty bytes in the buffer, and write it
	     back. If writeback fails, loop and try again.  */

	  mfc_getllar (buf_ptr, tag, 0, 0);
	  mfc_read_atomic_status ();

	  /* The method we're using to write 16 dirty bytes into
	     the buffer at a time uses fsmb which in turn uses
	     the least significant 16 bits of word 0, so we
	     load the bits and rotate so that the first bit of
	     the bitmap is in the first bit that fsmb will use.  */

	  bits = (qword) entry->dirty_bits[way];
	  bits = si_rotqbyi (bits, -2);

	  /* Si_fsmb creates the mask of dirty bytes.
	     Use selb to nab the appropriate bits.  */
	  buf_ptr[0] = si_selb (buf_ptr[0], line[0], si_fsmb (bits));

	  /* Rotate to next 16 byte section of cache.  */
	  bits = si_rotqbyi (bits, 2);

	  buf_ptr[1] = si_selb (buf_ptr[1], line[1], si_fsmb (bits));
	  bits = si_rotqbyi (bits, 2);
	  buf_ptr[2] = si_selb (buf_ptr[2], line[2], si_fsmb (bits));
	  bits = si_rotqbyi (bits, 2);
	  buf_ptr[3] = si_selb (buf_ptr[3], line[3], si_fsmb (bits));
	  bits = si_rotqbyi (bits, 2);
	  buf_ptr[4] = si_selb (buf_ptr[4], line[4], si_fsmb (bits));
	  bits = si_rotqbyi (bits, 2);
	  buf_ptr[5] = si_selb (buf_ptr[5], line[5], si_fsmb (bits));
	  bits = si_rotqbyi (bits, 2);
	  buf_ptr[6] = si_selb (buf_ptr[6], line[6], si_fsmb (bits));
	  bits = si_rotqbyi (bits, 2);
	  buf_ptr[7] = si_selb (buf_ptr[7], line[7], si_fsmb (bits));
	  bits = si_rotqbyi (bits, 2);

	  mfc_putllc (buf_ptr, tag, 0, 0);
	}
      while (mfc_read_atomic_status ());

      /* Leave critical section.  */
      if (__builtin_expect (mach_stat & 1, 0))
	spu_ienable ();
#endif
    }

  /* In any case, marking the lo tag with 1 which denotes empty.  */
  SET_EMPTY (entry, way);
  entry->dirty_bits[way] = (vector unsigned short) si_from_uint (0);
}

void
__cache_evict (__ea void *ea)
{
  addr tag = (addr) ea & ~TAG_MASK;
  struct __cache_tag_array *entry = GET_ENTRY (ea);
  int i = 0;

  /* Cycles through all the possible ways an address could be at
     and evicts the way if found.  */

  for (i = 0; i < WAYS; i++)
    if (CHECK_TAG (entry, i, tag))
      __cache_evict_entry (entry, i);
}

static void *
__cache_fill (int way, addr tag)
{
  unsigned int oldmask, mach_stat;
  char *line = ((void *) 0);

  /* Reserve our DMA tag.  */
  if (dma_tag == 32)
    dma_tag = mfc_tag_reserve ();

  /* Enter critical section.  */
  mach_stat = spu_readch (SPU_RdMachStat);
  spu_idisable ();

  /* Issue DMA request.  */
  line = GET_CACHE_LINE (tag, way);
  mfc_get (line, tag, LINE_SIZE, dma_tag, 0, 0);

  /* Wait for DMA completion.  */
  oldmask = mfc_read_tag_mask ();
  mfc_write_tag_mask (1 << dma_tag);
  mfc_read_tag_status_all ();
  mfc_write_tag_mask (oldmask);

  /* Leave critical section.  */
  if (__builtin_expect (mach_stat & 1, 0))
    spu_ienable ();

  return (void *) line;
}

static void
__cache_miss (__ea void *ea, struct __cache_tag_array *entry, int way)
{

  addr tag = (addr) ea & ~TAG_MASK;
  unsigned int lru = 0;
  int i = 0;
  int idx = 0;

  /* If way > 4, then there are no empty slots, so we must evict
     the least recently used entry. */
  if (way >= 4)
    {
      for (i = 0; i < WAYS; i++)
	{
	  if (GET_LRU (entry, i) > lru)
	    {
	      lru = GET_LRU (entry, i);
	      idx = i;
	    }
	}
      __cache_evict_entry (entry, idx);
      way = idx;
    }

  /* Set the empty entry's tag and fill it's cache line. */

  SET_TAG (entry, way, tag);
  entry->reserved[way] = 0;

  /* Check if the address is just an effective address within the
     SPU's local store. */

  /* Because the LS is not 256k aligned, we can't do a nice and mask
     here to compare, so we must check the whole range.  */

  if ((addr) ea >= (addr) __ea_local_store
      && (addr) ea < (addr) (__ea_local_store + 0x40000))
    {
      SET_IS_LS (entry, way);
      entry->base[way] =
	(void *) ((unsigned int) ((addr) ea -
				  (addr) __ea_local_store) & ~0x7f);
    }
  else
    {
      entry->base[way] = __cache_fill (way, tag);
    }
}

void *
__cache_fetch_dirty (__ea void *ea, int n_bytes_dirty)
{
#ifdef __EA64__
  unsigned int tag_hi;
  qword etag_hi;
#endif
  unsigned int tag_lo;
  struct __cache_tag_array *entry;

  qword etag_lo;
  qword equal;
  qword bit_mask;
  qword way;

  /* This first chunk, we merely fill the pointer and tag.  */

  entry = GET_ENTRY (ea);

#ifndef __EA64__
  tag_lo =
    si_to_uint (si_andc
		(si_shufb
		 (si_from_uint ((addr) ea), si_from_uint (0),
		  si_from_uint (0x00010203)), si_from_uint (TAG_MASK)));
#else
  tag_lo =
    si_to_uint (si_andc
		(si_shufb
		 (si_from_ullong ((addr) ea), si_from_uint (0),
		  si_from_uint (0x04050607)), si_from_uint (TAG_MASK)));

  tag_hi =
    si_to_uint (si_shufb
		(si_from_ullong ((addr) ea), si_from_uint (0),
		 si_from_uint (0x00010203)));
#endif

  /* Increment LRU in reserved bytes.  */
  si_stqd (si_ai (si_lqd (si_from_ptr (entry), 48), 1),
	   si_from_ptr (entry), 48);

missreturn:
  /* Check if the entry's lo_tag is equal to the address' lo_tag.  */
  etag_lo = si_lqd (si_from_ptr (entry), 0);
  equal = si_ceq (etag_lo, si_from_uint (tag_lo));
#ifdef __EA64__
  /* And the high tag too.  */
  etag_hi = si_lqd (si_from_ptr (entry), 16);
  equal = si_and (equal, (si_ceq (etag_hi, si_from_uint (tag_hi))));
#endif

  if ((si_to_uint (si_orx (equal)) == 0))
    goto misshandler;

  if (n_bytes_dirty)
    {
      /* way = 0x40,0x50,0x60,0x70 for each way, which is also the
         offset of the appropriate dirty bits.  */
      way = si_shli (si_clz (si_gbb (equal)), 2);

      /* To create the bit_mask, we set it to all 1s (uint -1), then we
         shift it over (128 - n_bytes_dirty) times.  */

      bit_mask = si_from_uint (-1);

      bit_mask =
	si_shlqby (bit_mask, si_from_uint ((LINE_SIZE - n_bytes_dirty) / 8));

      bit_mask =
	si_shlqbi (bit_mask, si_from_uint ((LINE_SIZE - n_bytes_dirty) % 8));

      /* Rotate it around to the correct offset.  */
      bit_mask =
	si_rotqby (bit_mask,
		   si_from_uint (-1 * ((addr) ea & TAG_MASK) / 8));

      bit_mask =
	si_rotqbi (bit_mask,
		   si_from_uint (-1 * ((addr) ea & TAG_MASK) % 8));

      /* Update the dirty bits.  */
      si_stqx (si_or (si_lqx (si_from_ptr (entry), way), bit_mask),
	       si_from_ptr (entry), way);
    };

  /* We've definitely found the right entry, set LRU (reserved) to 0
     maintaining the LS flag (MSB).  */

  si_stqd (si_andc
	   (si_lqd (si_from_ptr (entry), 48),
	    si_and (equal, si_from_uint (~(LS_FLAG)))),
	   si_from_ptr (entry), 48);

  return (void *)
    si_to_uint (si_a
		(si_orx
		 (si_and (si_lqd (si_from_ptr (entry), 32), equal)),
		 si_from_uint (((unsigned int) (addr) ea) & TAG_MASK)));

misshandler:
  equal = si_ceqi (etag_lo, 1);
  __cache_miss (ea, entry, (si_to_uint (si_clz (si_gbb (equal))) - 16) >> 2);
  goto missreturn;
}

void *
__cache_fetch (__ea void *ea)
{
  return __cache_fetch_dirty (ea, 0);
}

void
__cache_touch (__ea void *ea __attribute__ ((unused)))
{
  /* NO-OP for now.  */
}

void __cache_flush (void) __attribute__ ((destructor));
void
__cache_flush (void)
{
  struct __cache_tag_array *entry = __cache_tag_array;
  unsigned int i;
  int j;

  /* Cycle through each cache entry and evict all used ways.  */

  for (i = 0; i < CACHE_LINES / WAYS; i++)
    {
      for (j = 0; j < WAYS; j++)
	if (!CHECK_EMPTY (entry, j))
	  __cache_evict_entry (entry, j);

      entry++;
    }
}