v2.4.8 -> v2.4.8.1
[opensuse:kernel.git] / drivers / char / drm / r128_cce.c
1 /* r128_cce.c -- ATI Rage 128 driver -*- linux-c -*-
2  * Created: Wed Apr  5 19:24:19 2000 by kevin@precisioninsight.com
3  *
4  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
5  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
6  * All Rights Reserved.
7  *
8  * Permission is hereby granted, free of charge, to any person obtaining a
9  * copy of this software and associated documentation files (the "Software"),
10  * to deal in the Software without restriction, including without limitation
11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12  * and/or sell copies of the Software, and to permit persons to whom the
13  * Software is furnished to do so, subject to the following conditions:
14  *
15  * The above copyright notice and this permission notice (including the next
16  * paragraph) shall be included in all copies or substantial portions of the
17  * Software.
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
22  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
23  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
24  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25  * DEALINGS IN THE SOFTWARE.
26  *
27  * Authors:
28  *    Gareth Hughes <gareth@valinux.com>
29  */
30
31 #define __NO_VERSION__
32 #include "r128.h"
33 #include "drmP.h"
34 #include "r128_drv.h"
35
36 #include <linux/interrupt.h>    /* For task queue support */
37 #include <linux/delay.h>
38
39 #define R128_FIFO_DEBUG         0
40
41
42 /* CCE microcode (from ATI) */
43 static u32 r128_cce_microcode[] = {
44         0, 276838400, 0, 268449792, 2, 142, 2, 145, 0, 1076765731, 0,
45         1617039951, 0, 774592877, 0, 1987540286, 0, 2307490946U, 0,
46         599558925, 0, 589505315, 0, 596487092, 0, 589505315, 1,
47         11544576, 1, 206848, 1, 311296, 1, 198656, 2, 912273422, 11,
48         262144, 0, 0, 1, 33559837, 1, 7438, 1, 14809, 1, 6615, 12, 28,
49         1, 6614, 12, 28, 2, 23, 11, 18874368, 0, 16790922, 1, 409600, 9,
50         30, 1, 147854772, 16, 420483072, 3, 8192, 0, 10240, 1, 198656,
51         1, 15630, 1, 51200, 10, 34858, 9, 42, 1, 33559823, 2, 10276, 1,
52         15717, 1, 15718, 2, 43, 1, 15936948, 1, 570480831, 1, 14715071,
53         12, 322123831, 1, 33953125, 12, 55, 1, 33559908, 1, 15718, 2,
54         46, 4, 2099258, 1, 526336, 1, 442623, 4, 4194365, 1, 509952, 1,
55         459007, 3, 0, 12, 92, 2, 46, 12, 176, 1, 15734, 1, 206848, 1,
56         18432, 1, 133120, 1, 100670734, 1, 149504, 1, 165888, 1,
57         15975928, 1, 1048576, 6, 3145806, 1, 15715, 16, 2150645232U, 2,
58         268449859, 2, 10307, 12, 176, 1, 15734, 1, 15735, 1, 15630, 1,
59         15631, 1, 5253120, 6, 3145810, 16, 2150645232U, 1, 15864, 2, 82,
60         1, 343310, 1, 1064207, 2, 3145813, 1, 15728, 1, 7817, 1, 15729,
61         3, 15730, 12, 92, 2, 98, 1, 16168, 1, 16167, 1, 16002, 1, 16008,
62         1, 15974, 1, 15975, 1, 15990, 1, 15976, 1, 15977, 1, 15980, 0,
63         15981, 1, 10240, 1, 5253120, 1, 15720, 1, 198656, 6, 110, 1,
64         180224, 1, 103824738, 2, 112, 2, 3145839, 0, 536885440, 1,
65         114880, 14, 125, 12, 206975, 1, 33559995, 12, 198784, 0,
66         33570236, 1, 15803, 0, 15804, 3, 294912, 1, 294912, 3, 442370,
67         1, 11544576, 0, 811612160, 1, 12593152, 1, 11536384, 1,
68         14024704, 7, 310382726, 0, 10240, 1, 14796, 1, 14797, 1, 14793,
69         1, 14794, 0, 14795, 1, 268679168, 1, 9437184, 1, 268449792, 1,
70         198656, 1, 9452827, 1, 1075854602, 1, 1075854603, 1, 557056, 1,
71         114880, 14, 159, 12, 198784, 1, 1109409213, 12, 198783, 1,
72         1107312059, 12, 198784, 1, 1109409212, 2, 162, 1, 1075854781, 1,
73         1073757627, 1, 1075854780, 1, 540672, 1, 10485760, 6, 3145894,
74         16, 274741248, 9, 168, 3, 4194304, 3, 4209949, 0, 0, 0, 256, 14,
75         174, 1, 114857, 1, 33560007, 12, 176, 0, 10240, 1, 114858, 1,
76         33560018, 1, 114857, 3, 33560007, 1, 16008, 1, 114874, 1,
77         33560360, 1, 114875, 1, 33560154, 0, 15963, 0, 256, 0, 4096, 1,
78         409611, 9, 188, 0, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
79         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
80         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
81         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
82         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
83         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
84         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
85 };
86
87
88 int R128_READ_PLL(drm_device_t *dev, int addr)
89 {
90         drm_r128_private_t *dev_priv = dev->dev_private;
91
92         R128_WRITE8(R128_CLOCK_CNTL_INDEX, addr & 0x1f);
93         return R128_READ(R128_CLOCK_CNTL_DATA);
94 }
95
96 #if R128_FIFO_DEBUG
97 static void r128_status( drm_r128_private_t *dev_priv )
98 {
99         printk( "GUI_STAT           = 0x%08x\n",
100                 (unsigned int)R128_READ( R128_GUI_STAT ) );
101         printk( "PM4_STAT           = 0x%08x\n",
102                 (unsigned int)R128_READ( R128_PM4_STAT ) );
103         printk( "PM4_BUFFER_DL_WPTR = 0x%08x\n",
104                 (unsigned int)R128_READ( R128_PM4_BUFFER_DL_WPTR ) );
105         printk( "PM4_BUFFER_DL_RPTR = 0x%08x\n",
106                 (unsigned int)R128_READ( R128_PM4_BUFFER_DL_RPTR ) );
107         printk( "PM4_MICRO_CNTL     = 0x%08x\n",
108                 (unsigned int)R128_READ( R128_PM4_MICRO_CNTL ) );
109         printk( "PM4_BUFFER_CNTL    = 0x%08x\n",
110                 (unsigned int)R128_READ( R128_PM4_BUFFER_CNTL ) );
111 }
112 #endif
113
114
115 /* ================================================================
116  * Engine, FIFO control
117  */
118
119 static int r128_do_pixcache_flush( drm_r128_private_t *dev_priv )
120 {
121         u32 tmp;
122         int i;
123
124         tmp = R128_READ( R128_PC_NGUI_CTLSTAT ) | R128_PC_FLUSH_ALL;
125         R128_WRITE( R128_PC_NGUI_CTLSTAT, tmp );
126
127         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
128                 if ( !(R128_READ( R128_PC_NGUI_CTLSTAT ) & R128_PC_BUSY) ) {
129                         return 0;
130                 }
131                 udelay( 1 );
132         }
133
134 #if R128_FIFO_DEBUG
135         DRM_ERROR( "%s failed!\n", __FUNCTION__ );
136 #endif
137         return -EBUSY;
138 }
139
140 static int r128_do_wait_for_fifo( drm_r128_private_t *dev_priv, int entries )
141 {
142         int i;
143
144         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
145                 int slots = R128_READ( R128_GUI_STAT ) & R128_GUI_FIFOCNT_MASK;
146                 if ( slots >= entries ) return 0;
147                 udelay( 1 );
148         }
149
150 #if R128_FIFO_DEBUG
151         DRM_ERROR( "%s failed!\n", __FUNCTION__ );
152 #endif
153         return -EBUSY;
154 }
155
156 int r128_do_wait_for_idle( drm_r128_private_t *dev_priv )
157 {
158         int i, ret;
159
160         ret = r128_do_wait_for_fifo( dev_priv, 64 );
161         if ( ret < 0 ) return ret;
162
163         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
164                 if ( !(R128_READ( R128_GUI_STAT ) & R128_GUI_ACTIVE) ) {
165                         r128_do_pixcache_flush( dev_priv );
166                         return 0;
167                 }
168                 udelay( 1 );
169         }
170
171 #if R128_FIFO_DEBUG
172         DRM_ERROR( "%s failed!\n", __FUNCTION__ );
173 #endif
174         return -EBUSY;
175 }
176
177
178 /* ================================================================
179  * CCE control, initialization
180  */
181
182 /* Load the microcode for the CCE */
183 static void r128_cce_load_microcode( drm_r128_private_t *dev_priv )
184 {
185         int i;
186
187         DRM_DEBUG( "%s\n", __FUNCTION__ );
188
189         r128_do_wait_for_idle( dev_priv );
190
191         R128_WRITE( R128_PM4_MICROCODE_ADDR, 0 );
192         for ( i = 0 ; i < 256 ; i++ ) {
193                 R128_WRITE( R128_PM4_MICROCODE_DATAH,
194                             r128_cce_microcode[i * 2] );
195                 R128_WRITE( R128_PM4_MICROCODE_DATAL,
196                             r128_cce_microcode[i * 2 + 1] );
197         }
198 }
199
200 /* Flush any pending commands to the CCE.  This should only be used just
201  * prior to a wait for idle, as it informs the engine that the command
202  * stream is ending.
203  */
204 static void r128_do_cce_flush( drm_r128_private_t *dev_priv )
205 {
206         u32 tmp;
207
208         tmp = R128_READ( R128_PM4_BUFFER_DL_WPTR ) | R128_PM4_BUFFER_DL_DONE;
209         R128_WRITE( R128_PM4_BUFFER_DL_WPTR, tmp );
210 }
211
212 /* Wait for the CCE to go idle.
213  */
214 int r128_do_cce_idle( drm_r128_private_t *dev_priv )
215 {
216         int i;
217
218         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
219                 if ( GET_RING_HEAD( &dev_priv->ring ) == dev_priv->ring.tail ) {
220                         int pm4stat = R128_READ( R128_PM4_STAT );
221                         if ( ( (pm4stat & R128_PM4_FIFOCNT_MASK) >=
222                                dev_priv->cce_fifo_size ) &&
223                              !(pm4stat & (R128_PM4_BUSY |
224                                           R128_PM4_GUI_ACTIVE)) ) {
225                                 return r128_do_pixcache_flush( dev_priv );
226                         }
227                 }
228                 udelay( 1 );
229         }
230
231 #if R128_FIFO_DEBUG
232         DRM_ERROR( "failed!\n" );
233         r128_status( dev_priv );
234 #endif
235         return -EBUSY;
236 }
237
238 /* Start the Concurrent Command Engine.
239  */
240 static void r128_do_cce_start( drm_r128_private_t *dev_priv )
241 {
242         r128_do_wait_for_idle( dev_priv );
243
244         R128_WRITE( R128_PM4_BUFFER_CNTL,
245                     dev_priv->cce_mode | dev_priv->ring.size_l2qw );
246         R128_READ( R128_PM4_BUFFER_ADDR ); /* as per the sample code */
247         R128_WRITE( R128_PM4_MICRO_CNTL, R128_PM4_MICRO_FREERUN );
248
249         dev_priv->cce_running = 1;
250 }
251
252 /* Reset the Concurrent Command Engine.  This will not flush any pending
253  * commands, so you must wait for the CCE command stream to complete
254  * before calling this routine.
255  */
256 static void r128_do_cce_reset( drm_r128_private_t *dev_priv )
257 {
258         R128_WRITE( R128_PM4_BUFFER_DL_WPTR, 0 );
259         R128_WRITE( R128_PM4_BUFFER_DL_RPTR, 0 );
260         SET_RING_HEAD( &dev_priv->ring, 0 );
261         dev_priv->ring.tail = 0;
262 }
263
264 /* Stop the Concurrent Command Engine.  This will not flush any pending
265  * commands, so you must flush the command stream and wait for the CCE
266  * to go idle before calling this routine.
267  */
268 static void r128_do_cce_stop( drm_r128_private_t *dev_priv )
269 {
270         R128_WRITE( R128_PM4_MICRO_CNTL, 0 );
271         R128_WRITE( R128_PM4_BUFFER_CNTL, R128_PM4_NONPM4 );
272
273         dev_priv->cce_running = 0;
274 }
275
276 /* Reset the engine.  This will stop the CCE if it is running.
277  */
278 static int r128_do_engine_reset( drm_device_t *dev )
279 {
280         drm_r128_private_t *dev_priv = dev->dev_private;
281         u32 clock_cntl_index, mclk_cntl, gen_reset_cntl;
282
283         r128_do_pixcache_flush( dev_priv );
284
285         clock_cntl_index = R128_READ( R128_CLOCK_CNTL_INDEX );
286         mclk_cntl = R128_READ_PLL( dev, R128_MCLK_CNTL );
287
288         R128_WRITE_PLL( R128_MCLK_CNTL,
289                         mclk_cntl | R128_FORCE_GCP | R128_FORCE_PIPE3D_CP );
290
291         gen_reset_cntl = R128_READ( R128_GEN_RESET_CNTL );
292
293         /* Taken from the sample code - do not change */
294         R128_WRITE( R128_GEN_RESET_CNTL,
295                     gen_reset_cntl | R128_SOFT_RESET_GUI );
296         R128_READ( R128_GEN_RESET_CNTL );
297         R128_WRITE( R128_GEN_RESET_CNTL,
298                     gen_reset_cntl & ~R128_SOFT_RESET_GUI );
299         R128_READ( R128_GEN_RESET_CNTL );
300
301         R128_WRITE_PLL( R128_MCLK_CNTL, mclk_cntl );
302         R128_WRITE( R128_CLOCK_CNTL_INDEX, clock_cntl_index );
303         R128_WRITE( R128_GEN_RESET_CNTL, gen_reset_cntl );
304
305         /* Reset the CCE ring */
306         r128_do_cce_reset( dev_priv );
307
308         /* The CCE is no longer running after an engine reset */
309         dev_priv->cce_running = 0;
310
311         /* Reset any pending vertex, indirect buffers */
312         r128_freelist_reset( dev );
313
314         return 0;
315 }
316
317 static void r128_cce_init_ring_buffer( drm_device_t *dev,
318                                        drm_r128_private_t *dev_priv )
319 {
320         u32 ring_start;
321         u32 tmp;
322
323         DRM_DEBUG( "%s\n", __FUNCTION__ );
324
325         /* The manual (p. 2) says this address is in "VM space".  This
326          * means it's an offset from the start of AGP space.
327          */
328 #if __REALLY_HAVE_AGP
329         if ( !dev_priv->is_pci )
330                 ring_start = dev_priv->cce_ring->offset - dev->agp->base;
331         else
332 #endif
333                 ring_start = dev_priv->cce_ring->offset - dev->sg->handle;
334
335         R128_WRITE( R128_PM4_BUFFER_OFFSET, ring_start | R128_AGP_OFFSET );
336
337         R128_WRITE( R128_PM4_BUFFER_DL_WPTR, 0 );
338         R128_WRITE( R128_PM4_BUFFER_DL_RPTR, 0 );
339
340         /* DL_RPTR_ADDR is a physical address in AGP space. */
341         SET_RING_HEAD( &dev_priv->ring, 0 );
342
343         if ( !dev_priv->is_pci ) {
344                 R128_WRITE( R128_PM4_BUFFER_DL_RPTR_ADDR,
345                             dev_priv->ring_rptr->offset );
346         } else {
347                 drm_sg_mem_t *entry = dev->sg;
348                 unsigned long tmp_ofs, page_ofs;
349
350                 tmp_ofs = dev_priv->ring_rptr->offset - dev->sg->handle;
351                 page_ofs = tmp_ofs >> PAGE_SHIFT;
352
353                 R128_WRITE( R128_PM4_BUFFER_DL_RPTR_ADDR,
354                             page_to_bus(entry->pagelist[page_ofs]));
355
356                 DRM_DEBUG( "ring rptr: offset=0x%08lx handle=0x%08lx\n",
357                            page_to_bus(entry->pagelist[page_ofs]),
358                            entry->handle + tmp_ofs );
359         }
360
361         /* Set watermark control */
362         R128_WRITE( R128_PM4_BUFFER_WM_CNTL,
363                     ((R128_WATERMARK_L/4) << R128_WMA_SHIFT)
364                     | ((R128_WATERMARK_M/4) << R128_WMB_SHIFT)
365                     | ((R128_WATERMARK_N/4) << R128_WMC_SHIFT)
366                     | ((R128_WATERMARK_K/64) << R128_WB_WM_SHIFT) );
367
368         /* Force read.  Why?  Because it's in the examples... */
369         R128_READ( R128_PM4_BUFFER_ADDR );
370
371         /* Turn on bus mastering */
372         tmp = R128_READ( R128_BUS_CNTL ) & ~R128_BUS_MASTER_DIS;
373         R128_WRITE( R128_BUS_CNTL, tmp );
374 }
375
376 static int r128_do_init_cce( drm_device_t *dev, drm_r128_init_t *init )
377 {
378         drm_r128_private_t *dev_priv;
379         struct list_head *list;
380
381         DRM_DEBUG( "%s\n", __FUNCTION__ );
382
383         dev_priv = DRM(alloc)( sizeof(drm_r128_private_t), DRM_MEM_DRIVER );
384         if ( dev_priv == NULL )
385                 return -ENOMEM;
386
387         memset( dev_priv, 0, sizeof(drm_r128_private_t) );
388
389         dev_priv->is_pci = init->is_pci;
390
391         if ( dev_priv->is_pci && !dev->sg ) {
392                 DRM_ERROR( "PCI GART memory not allocated!\n" );
393                 dev->dev_private = (void *)dev_priv;
394                 r128_do_cleanup_cce( dev );
395                 return -EINVAL;
396         }
397
398         dev_priv->usec_timeout = init->usec_timeout;
399         if ( dev_priv->usec_timeout < 1 ||
400              dev_priv->usec_timeout > R128_MAX_USEC_TIMEOUT ) {
401                 DRM_DEBUG( "TIMEOUT problem!\n" );
402                 dev->dev_private = (void *)dev_priv;
403                 r128_do_cleanup_cce( dev );
404                 return -EINVAL;
405         }
406
407         dev_priv->cce_mode = init->cce_mode;
408
409         /* GH: Simple idle check.
410          */
411         atomic_set( &dev_priv->idle_count, 0 );
412
413         /* We don't support anything other than bus-mastering ring mode,
414          * but the ring can be in either AGP or PCI space for the ring
415          * read pointer.
416          */
417         if ( ( init->cce_mode != R128_PM4_192BM ) &&
418              ( init->cce_mode != R128_PM4_128BM_64INDBM ) &&
419              ( init->cce_mode != R128_PM4_64BM_128INDBM ) &&
420              ( init->cce_mode != R128_PM4_64BM_64VCBM_64INDBM ) ) {
421                 DRM_DEBUG( "Bad cce_mode!\n" );
422                 dev->dev_private = (void *)dev_priv;
423                 r128_do_cleanup_cce( dev );
424                 return -EINVAL;
425         }
426
427         switch ( init->cce_mode ) {
428         case R128_PM4_NONPM4:
429                 dev_priv->cce_fifo_size = 0;
430                 break;
431         case R128_PM4_192PIO:
432         case R128_PM4_192BM:
433                 dev_priv->cce_fifo_size = 192;
434                 break;
435         case R128_PM4_128PIO_64INDBM:
436         case R128_PM4_128BM_64INDBM:
437                 dev_priv->cce_fifo_size = 128;
438                 break;
439         case R128_PM4_64PIO_128INDBM:
440         case R128_PM4_64BM_128INDBM:
441         case R128_PM4_64PIO_64VCBM_64INDBM:
442         case R128_PM4_64BM_64VCBM_64INDBM:
443         case R128_PM4_64PIO_64VCPIO_64INDPIO:
444                 dev_priv->cce_fifo_size = 64;
445                 break;
446         }
447
448         switch ( init->fb_bpp ) {
449         case 16:
450                 dev_priv->color_fmt = R128_DATATYPE_RGB565;
451                 break;
452         case 32:
453         default:
454                 dev_priv->color_fmt = R128_DATATYPE_ARGB8888;
455                 break;
456         }
457         dev_priv->front_offset  = init->front_offset;
458         dev_priv->front_pitch   = init->front_pitch;
459         dev_priv->back_offset   = init->back_offset;
460         dev_priv->back_pitch    = init->back_pitch;
461
462         switch ( init->depth_bpp ) {
463         case 16:
464                 dev_priv->depth_fmt = R128_DATATYPE_RGB565;
465                 break;
466         case 24:
467         case 32:
468         default:
469                 dev_priv->depth_fmt = R128_DATATYPE_ARGB8888;
470                 break;
471         }
472         dev_priv->depth_offset  = init->depth_offset;
473         dev_priv->depth_pitch   = init->depth_pitch;
474         dev_priv->span_offset   = init->span_offset;
475
476         dev_priv->front_pitch_offset_c = (((dev_priv->front_pitch/8) << 21) |
477                                           (dev_priv->front_offset >> 5));
478         dev_priv->back_pitch_offset_c = (((dev_priv->back_pitch/8) << 21) |
479                                          (dev_priv->back_offset >> 5));
480         dev_priv->depth_pitch_offset_c = (((dev_priv->depth_pitch/8) << 21) |
481                                           (dev_priv->depth_offset >> 5) |
482                                           R128_DST_TILE);
483         dev_priv->span_pitch_offset_c = (((dev_priv->depth_pitch/8) << 21) |
484                                          (dev_priv->span_offset >> 5));
485
486         list_for_each(list, &dev->maplist->head) {
487                 drm_map_list_t *r_list = (drm_map_list_t *)list;
488                 if( r_list->map &&
489                     r_list->map->type == _DRM_SHM &&
490                     r_list->map->flags & _DRM_CONTAINS_LOCK ) {
491                         dev_priv->sarea = r_list->map;
492                         break;
493                 }
494         }
495         if(!dev_priv->sarea) {
496                 DRM_ERROR("could not find sarea!\n");
497                 dev->dev_private = (void *)dev_priv;
498                 r128_do_cleanup_cce( dev );
499                 return -EINVAL;
500         }
501
502         DRM_FIND_MAP( dev_priv->fb, init->fb_offset );
503         if(!dev_priv->fb) {
504                 DRM_ERROR("could not find framebuffer!\n");
505                 dev->dev_private = (void *)dev_priv;
506                 r128_do_cleanup_cce( dev );
507                 return -EINVAL;
508         }
509         DRM_FIND_MAP( dev_priv->mmio, init->mmio_offset );
510         if(!dev_priv->mmio) {
511                 DRM_ERROR("could not find mmio region!\n");
512                 dev->dev_private = (void *)dev_priv;
513                 r128_do_cleanup_cce( dev );
514                 return -EINVAL;
515         }
516         DRM_FIND_MAP( dev_priv->cce_ring, init->ring_offset );
517         if(!dev_priv->cce_ring) {
518                 DRM_ERROR("could not find cce ring region!\n");
519                 dev->dev_private = (void *)dev_priv;
520                 r128_do_cleanup_cce( dev );
521                 return -EINVAL;
522         }
523         DRM_FIND_MAP( dev_priv->ring_rptr, init->ring_rptr_offset );
524         if(!dev_priv->ring_rptr) {
525                 DRM_ERROR("could not find ring read pointer!\n");
526                 dev->dev_private = (void *)dev_priv;
527                 r128_do_cleanup_cce( dev );
528                 return -EINVAL;
529         }
530         DRM_FIND_MAP( dev_priv->buffers, init->buffers_offset );
531         if(!dev_priv->buffers) {
532                 DRM_ERROR("could not find dma buffer region!\n");
533                 dev->dev_private = (void *)dev_priv;
534                 r128_do_cleanup_cce( dev );
535                 return -EINVAL;
536         }
537
538         if ( !dev_priv->is_pci ) {
539                 DRM_FIND_MAP( dev_priv->agp_textures,
540                               init->agp_textures_offset );
541                 if(!dev_priv->agp_textures) {
542                         DRM_ERROR("could not find agp texture region!\n");
543                         dev->dev_private = (void *)dev_priv;
544                         r128_do_cleanup_cce( dev );
545                         return -EINVAL;
546                 }
547         }
548
549         dev_priv->sarea_priv =
550                 (drm_r128_sarea_t *)((u8 *)dev_priv->sarea->handle +
551                                      init->sarea_priv_offset);
552
553         if ( !dev_priv->is_pci ) {
554                 DRM_IOREMAP( dev_priv->cce_ring );
555                 DRM_IOREMAP( dev_priv->ring_rptr );
556                 DRM_IOREMAP( dev_priv->buffers );
557                 if(!dev_priv->cce_ring->handle ||
558                    !dev_priv->ring_rptr->handle ||
559                    !dev_priv->buffers->handle) {
560                         DRM_ERROR("Could not ioremap agp regions!\n");
561                         dev->dev_private = (void *)dev_priv;
562                         r128_do_cleanup_cce( dev );
563                         return -ENOMEM;
564                 }
565         } else {
566                 dev_priv->cce_ring->handle =
567                         (void *)dev_priv->cce_ring->offset;
568                 dev_priv->ring_rptr->handle =
569                         (void *)dev_priv->ring_rptr->offset;
570                 dev_priv->buffers->handle = (void *)dev_priv->buffers->offset;
571         }
572
573 #if __REALLY_HAVE_AGP
574         if ( !dev_priv->is_pci )
575                 dev_priv->cce_buffers_offset = dev->agp->base;
576         else
577 #endif
578                 dev_priv->cce_buffers_offset = dev->sg->handle;
579
580         dev_priv->ring.head = ((__volatile__ u32 *)
581                                dev_priv->ring_rptr->handle);
582
583         dev_priv->ring.start = (u32 *)dev_priv->cce_ring->handle;
584         dev_priv->ring.end = ((u32 *)dev_priv->cce_ring->handle
585                               + init->ring_size / sizeof(u32));
586         dev_priv->ring.size = init->ring_size;
587         dev_priv->ring.size_l2qw = DRM(order)( init->ring_size / 8 );
588
589         dev_priv->ring.tail_mask =
590                 (dev_priv->ring.size / sizeof(u32)) - 1;
591
592         dev_priv->ring.high_mark = 128;
593
594         dev_priv->sarea_priv->last_frame = 0;
595         R128_WRITE( R128_LAST_FRAME_REG, dev_priv->sarea_priv->last_frame );
596
597         dev_priv->sarea_priv->last_dispatch = 0;
598         R128_WRITE( R128_LAST_DISPATCH_REG,
599                     dev_priv->sarea_priv->last_dispatch );
600
601         if ( dev_priv->is_pci ) {
602                 dev_priv->phys_pci_gart = DRM(ati_pcigart_init)( dev );
603                 if ( !dev_priv->phys_pci_gart ) {
604                         DRM_ERROR( "failed to init PCI GART!\n" );
605                         dev->dev_private = (void *)dev_priv;
606                         r128_do_cleanup_cce( dev );
607                         return -ENOMEM;
608                 }
609                 R128_WRITE( R128_PCI_GART_PAGE,
610                             virt_to_bus( (void *)dev_priv->phys_pci_gart ) );
611         }
612
613         r128_cce_init_ring_buffer( dev, dev_priv );
614         r128_cce_load_microcode( dev_priv );
615
616         dev->dev_private = (void *)dev_priv;
617
618         r128_do_engine_reset( dev );
619
620         return 0;
621 }
622
623 int r128_do_cleanup_cce( drm_device_t *dev )
624 {
625         if ( dev->dev_private ) {
626                 drm_r128_private_t *dev_priv = dev->dev_private;
627
628                 if ( !dev_priv->is_pci ) {
629                         DRM_IOREMAPFREE( dev_priv->cce_ring );
630                         DRM_IOREMAPFREE( dev_priv->ring_rptr );
631                         DRM_IOREMAPFREE( dev_priv->buffers );
632                 }
633
634                 DRM(free)( dev->dev_private, sizeof(drm_r128_private_t),
635                            DRM_MEM_DRIVER );
636                 dev->dev_private = NULL;
637         }
638
639         return 0;
640 }
641
642 int r128_cce_init( struct inode *inode, struct file *filp,
643                    unsigned int cmd, unsigned long arg )
644 {
645         drm_file_t *priv = filp->private_data;
646         drm_device_t *dev = priv->dev;
647         drm_r128_init_t init;
648
649         DRM_DEBUG( "%s\n", __FUNCTION__ );
650
651         if ( copy_from_user( &init, (drm_r128_init_t *)arg, sizeof(init) ) )
652                 return -EFAULT;
653
654         switch ( init.func ) {
655         case R128_INIT_CCE:
656                 return r128_do_init_cce( dev, &init );
657         case R128_CLEANUP_CCE:
658                 return r128_do_cleanup_cce( dev );
659         }
660
661         return -EINVAL;
662 }
663
664 int r128_cce_start( struct inode *inode, struct file *filp,
665                     unsigned int cmd, unsigned long arg )
666 {
667         drm_file_t *priv = filp->private_data;
668         drm_device_t *dev = priv->dev;
669         drm_r128_private_t *dev_priv = dev->dev_private;
670         DRM_DEBUG( "%s\n", __FUNCTION__ );
671
672         LOCK_TEST_WITH_RETURN( dev );
673
674         if ( dev_priv->cce_running || dev_priv->cce_mode == R128_PM4_NONPM4 ) {
675                 DRM_DEBUG( "%s while CCE running\n", __FUNCTION__ );
676                 return 0;
677         }
678
679         r128_do_cce_start( dev_priv );
680
681         return 0;
682 }
683
684 /* Stop the CCE.  The engine must have been idled before calling this
685  * routine.
686  */
687 int r128_cce_stop( struct inode *inode, struct file *filp,
688                    unsigned int cmd, unsigned long arg )
689 {
690         drm_file_t *priv = filp->private_data;
691         drm_device_t *dev = priv->dev;
692         drm_r128_private_t *dev_priv = dev->dev_private;
693         drm_r128_cce_stop_t stop;
694         int ret;
695         DRM_DEBUG( "%s\n", __FUNCTION__ );
696
697         LOCK_TEST_WITH_RETURN( dev );
698
699         if ( copy_from_user( &stop, (drm_r128_init_t *)arg, sizeof(stop) ) )
700                 return -EFAULT;
701
702         /* Flush any pending CCE commands.  This ensures any outstanding
703          * commands are exectuted by the engine before we turn it off.
704          */
705         if ( stop.flush ) {
706                 r128_do_cce_flush( dev_priv );
707         }
708
709         /* If we fail to make the engine go idle, we return an error
710          * code so that the DRM ioctl wrapper can try again.
711          */
712         if ( stop.idle ) {
713                 ret = r128_do_cce_idle( dev_priv );
714                 if ( ret < 0 ) return ret;
715         }
716
717         /* Finally, we can turn off the CCE.  If the engine isn't idle,
718          * we will get some dropped triangles as they won't be fully
719          * rendered before the CCE is shut down.
720          */
721         r128_do_cce_stop( dev_priv );
722
723         /* Reset the engine */
724         r128_do_engine_reset( dev );
725
726         return 0;
727 }
728
729 /* Just reset the CCE ring.  Called as part of an X Server engine reset.
730  */
731 int r128_cce_reset( struct inode *inode, struct file *filp,
732                     unsigned int cmd, unsigned long arg )
733 {
734         drm_file_t *priv = filp->private_data;
735         drm_device_t *dev = priv->dev;
736         drm_r128_private_t *dev_priv = dev->dev_private;
737         DRM_DEBUG( "%s\n", __FUNCTION__ );
738
739         LOCK_TEST_WITH_RETURN( dev );
740
741         if ( !dev_priv ) {
742                 DRM_DEBUG( "%s called before init done\n", __FUNCTION__ );
743                 return -EINVAL;
744         }
745
746         r128_do_cce_reset( dev_priv );
747
748         /* The CCE is no longer running after an engine reset */
749         dev_priv->cce_running = 0;
750
751         return 0;
752 }
753
754 int r128_cce_idle( struct inode *inode, struct file *filp,
755                    unsigned int cmd, unsigned long arg )
756 {
757         drm_file_t *priv = filp->private_data;
758         drm_device_t *dev = priv->dev;
759         drm_r128_private_t *dev_priv = dev->dev_private;
760         DRM_DEBUG( "%s\n", __FUNCTION__ );
761
762         LOCK_TEST_WITH_RETURN( dev );
763
764         if ( dev_priv->cce_running ) {
765                 r128_do_cce_flush( dev_priv );
766         }
767
768         return r128_do_cce_idle( dev_priv );
769 }
770
771 int r128_engine_reset( struct inode *inode, struct file *filp,
772                        unsigned int cmd, unsigned long arg )
773 {
774         drm_file_t *priv = filp->private_data;
775         drm_device_t *dev = priv->dev;
776         DRM_DEBUG( "%s\n", __FUNCTION__ );
777
778         LOCK_TEST_WITH_RETURN( dev );
779
780         return r128_do_engine_reset( dev );
781 }
782
783
784 /* ================================================================
785  * Fullscreen mode
786  */
787
788 static int r128_do_init_pageflip( drm_device_t *dev )
789 {
790         drm_r128_private_t *dev_priv = dev->dev_private;
791         DRM_DEBUG( "%s\n", __FUNCTION__ );
792
793         dev_priv->crtc_offset =      R128_READ( R128_CRTC_OFFSET );
794         dev_priv->crtc_offset_cntl = R128_READ( R128_CRTC_OFFSET_CNTL );
795
796         R128_WRITE( R128_CRTC_OFFSET, dev_priv->front_offset );
797         R128_WRITE( R128_CRTC_OFFSET_CNTL,
798                     dev_priv->crtc_offset_cntl | R128_CRTC_OFFSET_FLIP_CNTL );
799
800         dev_priv->page_flipping = 1;
801         dev_priv->current_page = 0;
802
803         return 0;
804 }
805
806 int r128_do_cleanup_pageflip( drm_device_t *dev )
807 {
808         drm_r128_private_t *dev_priv = dev->dev_private;
809         DRM_DEBUG( "%s\n", __FUNCTION__ );
810
811         R128_WRITE( R128_CRTC_OFFSET,      dev_priv->crtc_offset );
812         R128_WRITE( R128_CRTC_OFFSET_CNTL, dev_priv->crtc_offset_cntl );
813
814         dev_priv->page_flipping = 0;
815         dev_priv->current_page = 0;
816
817         return 0;
818 }
819
820 int r128_fullscreen( struct inode *inode, struct file *filp,
821                      unsigned int cmd, unsigned long arg )
822 {
823         drm_file_t *priv = filp->private_data;
824         drm_device_t *dev = priv->dev;
825         drm_r128_fullscreen_t fs;
826
827         LOCK_TEST_WITH_RETURN( dev );
828
829         if ( copy_from_user( &fs, (drm_r128_fullscreen_t *)arg, sizeof(fs) ) )
830                 return -EFAULT;
831
832         switch ( fs.func ) {
833         case R128_INIT_FULLSCREEN:
834                 return r128_do_init_pageflip( dev );
835         case R128_CLEANUP_FULLSCREEN:
836                 return r128_do_cleanup_pageflip( dev );
837         }
838
839         return -EINVAL;
840 }
841
842
843 /* ================================================================
844  * Freelist management
845  */
846 #define R128_BUFFER_USED        0xffffffff
847 #define R128_BUFFER_FREE        0
848
849 #if 0
850 static int r128_freelist_init( drm_device_t *dev )
851 {
852         drm_device_dma_t *dma = dev->dma;
853         drm_r128_private_t *dev_priv = dev->dev_private;
854         drm_buf_t *buf;
855         drm_r128_buf_priv_t *buf_priv;
856         drm_r128_freelist_t *entry;
857         int i;
858
859         dev_priv->head = DRM(alloc)( sizeof(drm_r128_freelist_t),
860                                      DRM_MEM_DRIVER );
861         if ( dev_priv->head == NULL )
862                 return -ENOMEM;
863
864         memset( dev_priv->head, 0, sizeof(drm_r128_freelist_t) );
865         dev_priv->head->age = R128_BUFFER_USED;
866
867         for ( i = 0 ; i < dma->buf_count ; i++ ) {
868                 buf = dma->buflist[i];
869                 buf_priv = buf->dev_private;
870
871                 entry = DRM(alloc)( sizeof(drm_r128_freelist_t),
872                                     DRM_MEM_DRIVER );
873                 if ( !entry ) return -ENOMEM;
874
875                 entry->age = R128_BUFFER_FREE;
876                 entry->buf = buf;
877                 entry->prev = dev_priv->head;
878                 entry->next = dev_priv->head->next;
879                 if ( !entry->next )
880                         dev_priv->tail = entry;
881
882                 buf_priv->discard = 0;
883                 buf_priv->dispatched = 0;
884                 buf_priv->list_entry = entry;
885
886                 dev_priv->head->next = entry;
887
888                 if ( dev_priv->head->next )
889                         dev_priv->head->next->prev = entry;
890         }
891
892         return 0;
893
894 }
895 #endif
896
897 drm_buf_t *r128_freelist_get( drm_device_t *dev )
898 {
899         drm_device_dma_t *dma = dev->dma;
900         drm_r128_private_t *dev_priv = dev->dev_private;
901         drm_r128_buf_priv_t *buf_priv;
902         drm_buf_t *buf;
903         int i, t;
904
905         /* FIXME: Optimize -- use freelist code */
906
907         for ( i = 0 ; i < dma->buf_count ; i++ ) {
908                 buf = dma->buflist[i];
909                 buf_priv = buf->dev_private;
910                 if ( buf->pid == 0 )
911                         return buf;
912         }
913
914         for ( t = 0 ; t < dev_priv->usec_timeout ; t++ ) {
915                 u32 done_age = R128_READ( R128_LAST_DISPATCH_REG );
916
917                 for ( i = 0 ; i < dma->buf_count ; i++ ) {
918                         buf = dma->buflist[i];
919                         buf_priv = buf->dev_private;
920                         if ( buf->pending && buf_priv->age <= done_age ) {
921                                 /* The buffer has been processed, so it
922                                  * can now be used.
923                                  */
924                                 buf->pending = 0;
925                                 return buf;
926                         }
927                 }
928                 udelay( 1 );
929         }
930
931         DRM_ERROR( "returning NULL!\n" );
932         return NULL;
933 }
934
935 void r128_freelist_reset( drm_device_t *dev )
936 {
937         drm_device_dma_t *dma = dev->dma;
938         int i;
939
940         for ( i = 0 ; i < dma->buf_count ; i++ ) {
941                 drm_buf_t *buf = dma->buflist[i];
942                 drm_r128_buf_priv_t *buf_priv = buf->dev_private;
943                 buf_priv->age = 0;
944         }
945 }
946
947
948 /* ================================================================
949  * CCE command submission
950  */
951
952 int r128_wait_ring( drm_r128_private_t *dev_priv, int n )
953 {
954         drm_r128_ring_buffer_t *ring = &dev_priv->ring;
955         int i;
956
957         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
958                 r128_update_ring_snapshot( ring );
959                 if ( ring->space >= n )
960                         return 0;
961                 udelay( 1 );
962         }
963
964         /* FIXME: This is being ignored... */
965         DRM_ERROR( "failed!\n" );
966         return -EBUSY;
967 }
968
969 static int r128_cce_get_buffers( drm_device_t *dev, drm_dma_t *d )
970 {
971         int i;
972         drm_buf_t *buf;
973
974         for ( i = d->granted_count ; i < d->request_count ; i++ ) {
975                 buf = r128_freelist_get( dev );
976                 if ( !buf ) return -EAGAIN;
977
978                 buf->pid = current->pid;
979
980                 if ( copy_to_user( &d->request_indices[i], &buf->idx,
981                                    sizeof(buf->idx) ) )
982                         return -EFAULT;
983                 if ( copy_to_user( &d->request_sizes[i], &buf->total,
984                                    sizeof(buf->total) ) )
985                         return -EFAULT;
986
987                 d->granted_count++;
988         }
989         return 0;
990 }
991
992 int r128_cce_buffers( struct inode *inode, struct file *filp,
993                       unsigned int cmd, unsigned long arg )
994 {
995         drm_file_t *priv = filp->private_data;
996         drm_device_t *dev = priv->dev;
997         drm_device_dma_t *dma = dev->dma;
998         int ret = 0;
999         drm_dma_t d;
1000
1001         LOCK_TEST_WITH_RETURN( dev );
1002
1003         if ( copy_from_user( &d, (drm_dma_t *) arg, sizeof(d) ) )
1004                 return -EFAULT;
1005
1006         /* Please don't send us buffers.
1007          */
1008         if ( d.send_count != 0 ) {
1009                 DRM_ERROR( "Process %d trying to send %d buffers via drmDMA\n",
1010                            current->pid, d.send_count );
1011                 return -EINVAL;
1012         }
1013
1014         /* We'll send you buffers.
1015          */
1016         if ( d.request_count < 0 || d.request_count > dma->buf_count ) {
1017                 DRM_ERROR( "Process %d trying to get %d buffers (of %d max)\n",
1018                            current->pid, d.request_count, dma->buf_count );
1019                 return -EINVAL;
1020         }
1021
1022         d.granted_count = 0;
1023
1024         if ( d.request_count ) {
1025                 ret = r128_cce_get_buffers( dev, &d );
1026         }
1027
1028         if ( copy_to_user( (drm_dma_t *) arg, &d, sizeof(d) ) )
1029                 return -EFAULT;
1030
1031         return ret;
1032 }