2 * Acess2 VirtIO Common Code
3 * - By John Hodge (thePowersGang)
12 #include <semaphore.h>
13 #include "include/virtio.h"
14 #include "include/virtio_hw.h"
17 typedef struct sVirtIO_Queue tVirtIO_Queue;
32 tVirtIO_QueueCallback Callback;
35 volatile struct sVirtIO_RingDesc *Entries;
36 tShortSpinlock lAvailQueue;
37 volatile struct sVirtIO_AvailRing *Avail;
40 volatile struct sVirtIO_UsedRing *Used;
42 tSemaphore FreeDescsSem;
43 tShortSpinlock lFreeList;
46 tVirtIO_Buf Buffers[];
58 struct sVirtIO_Queue *Queues[];
62 int VirtIO_Install(char **Arguments);
63 int VirtIO_Cleanup(void);
64 void VirtIO_IRQHandler(int IRQ, void *Ptr);
67 MODULE_DEFINE(0, VERSION, VirtIOCommon, VirtIO_Install, VirtIO_Cleanup, NULL);
70 int VirtIO_Install(char **Arguments)
75 int VirtIO_Cleanup(void)
77 Log_Warning("VirtIO", "TODO: Cleanup");
82 // - Device management
83 tVirtIO_Dev *VirtIO_InitDev(Uint16 IOBase, Uint IRQ, Uint32 Features, int MaxQueues, size_t DataSize)
87 // Reset and init device
88 outb(IOBase + VIRTIO_REG_DEVSTS, 0);
89 outb(IOBase + VIRTIO_REG_DEVSTS, VIRTIO_DEVSTS_ACKNOWLEDGE);
90 outb(IOBase + VIRTIO_REG_DEVSTS, VIRTIO_DEVSTS_DRIVER);
93 Uint32 support_feat = ind(IOBase + VIRTIO_REG_DEVFEAT);
94 outd(IOBase + VIRTIO_REG_GUESTFEAT, Features & support_feat);
95 LOG("Features: (Dev 0x%08x, Driver 0x%08x)", support_feat, Features);
98 ret = malloc( offsetof(tVirtIO_Dev, Queues[MaxQueues]) + DataSize );
100 ret->IOBase = IOBase;
101 ret->nQueues = MaxQueues;
102 ret->DataPtr = &ret->Queues[MaxQueues];
104 // TODO: MSI-X makes this move
105 ret->DevCfgBase = IOBase + VIRTIO_REG_DEVSPEC_0;
107 // Discover virtqueues
108 for( int i = 0; i < MaxQueues; i ++ )
110 outw(IOBase + VIRTIO_REG_QUEUESELECT, i);
111 size_t qsz = inw(IOBase + VIRTIO_REG_QUEUESIZE);
112 LOG("Queue #%i: QSZ = %i", i, qsz);
114 ret->Queues[i] = NULL;
117 // TODO: Assert that qsz is a power of 2
119 tVirtIO_Queue *queue = calloc( offsetof(tVirtIO_Queue, Buffers[qsz]), 1 );
121 queue->FirstUnused = 0;
123 Semaphore_Init(&queue->FreeDescsSem, qsz, qsz, "VirtIO", "FreeDescs");
125 // Allocate virtqueue spaces
126 size_t sz1 = qsz*16 + offsetof(struct sVirtIO_AvailRing, Ring[qsz])+2;
127 size_t sz2 = offsetof(struct sVirtIO_UsedRing, Ring[qsz]) + 2;
128 sz1 = (sz1 + PAGE_SIZE-1) & ~(PAGE_SIZE-1);
129 LOG(" sz{1,2} = 0x%x,0x%x", sz1, sz2);
130 queue->Entries = MM_AllocDMA( (sz1+sz2+0xFFF)>>12, 32+12, NULL );
131 queue->Avail = (void*)(queue->Entries + qsz);
132 queue->Used = (void*)((char*)queue->Entries + sz1);
134 // Clear and prepare unused list
135 memset((void*)queue->Entries, 0, sz1 + sz2);
136 for( int j = 0; j < qsz; j ++ )
138 queue->Entries[j].Flags = 1;
139 queue->Entries[j].Next = j+1;
141 queue->Buffers[j].Idx = j;
142 queue->Buffers[j].Queue = i;
143 queue->Buffers[j].Dev = ret;
145 queue->Entries[qsz-1].Flags = 0;
147 ret->Queues[i] = queue;
149 Uint32 queueaddr = MM_GetPhysAddr(queue->Entries) / 4096;
150 LOG(" Phys %P", MM_GetPhysAddr(queue->Entries));
151 outd(IOBase + VIRTIO_REG_QUEUEADDR, queueaddr);
152 ASSERTC(queueaddr, ==, ind(IOBase + VIRTIO_REG_QUEUEADDR));
155 // Register IRQ Handler
156 IRQ_AddHandler(IRQ, VirtIO_IRQHandler, ret);
157 Uint8 isr = inb(IOBase + VIRTIO_REG_ISRSTS);
158 LOG("isr = %x", isr);
161 outb(IOBase + VIRTIO_REG_DEVSTS, VIRTIO_DEVSTS_DRIVER_OK);
166 Uint32 VirtIO_GetFeatures(tVirtIO_Dev *Dev)
168 return ind(Dev->IOBase + VIRTIO_REG_GUESTFEAT);
170 Uint32 VirtIO_GetDevConfig(tVirtIO_Dev *Dev, int Size, Uint8 Offset)
175 return inb(Dev->DevCfgBase + Offset);
177 return inw(Dev->DevCfgBase + Offset);
179 return ind(Dev->DevCfgBase + Offset);
183 void *VirtIO_GetDataPtr(tVirtIO_Dev *Dev)
187 void VirtIO_RemoveDev(tVirtIO_Dev *Dev)
193 * \brief Sets the Queue Callback
195 * The queue callback is called when
196 * a) a read-only queue entry is retired (device writes it to the Available ring)
197 * b) a write-only queue is handed to the guest (devices writes it to the used ring)
199 int VirtIO_SetQueueCallback(tVirtIO_Dev *Dev, int QueueID, tVirtIO_QueueCallback Callback, int NoAutoRel)
201 ASSERTCR(QueueID, <, Dev->nQueues, -1);
203 Dev->Queues[QueueID]->Callback = Callback;
204 Dev->Queues[QueueID]->NoAutoRel = NoAutoRel;
206 if( !Callback && NoAutoRel ) {
207 Log_Warning("VirtIO", "%p:%i has had callback==NULL with auto release disabled",
214 int VirtIO_int_AllocQueueEntry(tVirtIO_Queue *Queue)
216 if( Semaphore_Wait(&Queue->FreeDescsSem, 1) != 1 ) {
220 SHORTLOCK(&Queue->lFreeList);
221 int idx = Queue->FirstUnused;
222 ASSERT( Queue->Entries[idx].Flags & VRING_DESC_F_NEXT );
223 Queue->FirstUnused = Queue->Entries[idx].Next;
224 SHORTREL(&Queue->lFreeList);
229 tVirtIO_Buf *VirtIO_int_AllocBuf(tVirtIO_Queue *Queue, const void *Ptr, size_t Size, Uint Flags, Uint16 Next)
231 int idx = VirtIO_int_AllocQueueEntry(Queue);
232 tVirtIO_Buf *buf = &Queue->Buffers[idx];
233 ASSERTC(idx, ==, buf->Idx);
235 LOG("%p:%i[%i] = {%P+0x%x}",
236 buf->Dev, buf->Queue, buf->Idx,
237 MM_GetPhysAddr(Ptr), Size);
239 Queue->Entries[idx].Addr = MM_GetPhysAddr(Ptr);
240 Queue->Entries[idx].Len = Size;
241 Queue->Entries[idx].Flags = Flags;
242 Queue->Entries[idx].Next = Next;
250 tVirtIO_Buf *VirtIO_int_AllocBufV(tVirtIO_Queue *Queue, const char *Ptr, size_t Size, Uint Flags, Uint16 Next)
252 if( ((tVAddr)Ptr & (PAGE_SIZE-1)) + Size > PAGE_SIZE*2 )
254 Log_Error("VirtIO", ">2 page buffers are not supported");
260 tPAddr phys = MM_GetPhysAddr(Ptr);
261 if( phys + Size-1 != MM_GetPhysAddr( Ptr + Size-1 ) )
263 size_t fp_size = PAGE_SIZE-(phys%PAGE_SIZE);
264 tVirtIO_Buf *last = VirtIO_int_AllocBuf(Queue, Ptr+fp_size, Size-fp_size, Flags, Next);
265 ret = VirtIO_int_AllocBuf(Queue, Ptr, fp_size, Flags|VRING_DESC_F_NEXT, last->Idx);
269 ret = VirtIO_int_AllocBuf(Queue, Ptr, Size, Flags, Next);
275 * Append a ring descriptor to the available ring
277 void VirtIO_int_AddAvailBuf(tVirtIO_Queue *Queue, tVirtIO_Buf *Buf)
279 __sync_synchronize();
280 SHORTLOCK(&Queue->lAvailQueue);
281 Queue->Avail->Ring[ Queue->Avail->Idx & (Queue->Size-1) ] = Buf->Idx;
282 Queue->Avail->Idx ++;
283 SHORTREL(&Queue->lAvailQueue);
286 __sync_synchronize();
287 // TODO: Delay notifications
288 tVirtIO_Dev *dev = Buf->Dev;
289 outw(dev->IOBase + VIRTIO_REG_QUEUENOTIFY, Buf->Queue);
290 LOG("Notifying %p:%i", Buf->Dev, Buf->Queue);
293 // Send a set of RO buffers
294 tVirtIO_Buf *VirtIO_SendBuffers(tVirtIO_Dev *Dev, int QueueID, int nBufs, size_t Sizes[], const void *Ptrs[], void *Handle)
296 tVirtIO_Queue *queue = Dev->Queues[QueueID];
297 tVirtIO_Buf *prev = NULL;
299 // Allocate buffers for each non-contiguious region
300 // - these come from the queue's unallocated pool
301 size_t totalsize = 0;
302 for( int i = nBufs; i --; )
305 prev = VirtIO_int_AllocBufV(queue, Ptrs[i], Sizes[i], VRING_DESC_F_NEXT, prev->Idx);
307 prev = VirtIO_int_AllocBufV(queue, Ptrs[i], Sizes[i], 0, 0);
308 totalsize += Sizes[i];
310 LOG("Total size 0x%x", totalsize);
312 // Final buffer has the handle set to the passed handle
313 // - all others get NULL
314 prev->Handle = Handle;
316 // Add first to avaliable ring
317 VirtIO_int_AddAvailBuf(queue, prev);
322 // Supply a single WO buffer for the device
323 tVirtIO_Buf *VirtIO_ReceiveBuffer(tVirtIO_Dev *Dev, int QueueID, size_t Size, void *Ptr, void *Handle)
325 LOG("%p:%i - Add %p+0x%x for RX", Dev, QueueID, Ptr, Size);
326 tVirtIO_Queue *queue = Dev->Queues[QueueID];
327 tVirtIO_Buf *ret = VirtIO_int_AllocBufV(queue, Ptr, Size, VRING_DESC_F_WRITE, 0);
328 ret->Handle = Handle;
330 VirtIO_int_AddAvailBuf(queue, ret);
334 tVirtIO_Buf *VirtIO_PopBuffer(tVirtIO_Dev *Dev, int QueueID, size_t *Size, const void **Ptr)
336 ASSERTCR(QueueID, <, Dev->nQueues, NULL);
337 tVirtIO_Queue *queue = Dev->Queues[QueueID];
340 if( queue->NextUsedPop == queue->Used->Idx )
342 int qidx = queue->NextUsedPop;
343 queue->NextUsedPop ++;
345 int idx = queue->Used->Ring[qidx].ID;
347 *Size = queue->Used->Ring[qidx].Len;
349 *Ptr = queue->Buffers[idx].BufPtr;
350 ASSERTC(MM_GetPhysAddr(*Ptr), ==, queue->Entries[idx].Addr);
352 return &queue->Buffers[idx];
355 const void *VirtIO_GetBufferPtr(tVirtIO_Buf *Buf, size_t *Size)
357 tVirtIO_Queue *queue = Buf->Dev->Queues[Buf->Queue];
359 *Size = queue->Entries[Buf->Idx].Len;
362 tVirtIO_Dev *VirtIO_GetBufferDev(tVirtIO_Buf *Buf)
367 void VirtIO_int_ReleaseQDesc(tVirtIO_Queue *Queue, Uint16 Index)
369 LOG("Release QDesc %p:%i into free pool",
371 SHORTLOCK(&Queue->lFreeList);
372 Queue->Entries[Index].Next = Queue->FirstUnused;
373 Queue->Entries[Index].Flags = VRING_DESC_F_NEXT;
374 Queue->FirstUnused = Index;
375 SHORTREL(&Queue->lFreeList);
376 Semaphore_Signal(&Queue->FreeDescsSem, 1);
380 * \brief Releases all qdescs in the buffer to the free list
382 void VirtIO_ReleaseBuffer(tVirtIO_Buf *Buffer)
384 int idx = Buffer->Idx;
385 tVirtIO_Queue *queue = Buffer->Dev->Queues[Buffer->Queue];
387 LOG("Releasing chain at %p:%i/%i",
388 Buffer->Dev, Buffer->Queue, Buffer->Idx);
392 has_next = !!(queue->Entries[idx].Flags & VRING_DESC_F_NEXT);
393 int next_idx = queue->Entries[idx].Next;
394 ASSERTC(!has_next || next_idx,!=,idx);
396 VirtIO_int_ReleaseQDesc(queue, idx);
402 void VirtIO_int_ProcessUsedList(tVirtIO_Dev *Dev, tVirtIO_Queue *Queue, int UsedIdx)
404 Uint16 qent = Queue->Used->Ring[UsedIdx].ID;
405 size_t len = Queue->Used->Ring[UsedIdx].Len;
406 LOG("QEnt %i (0x%x bytes) callback w/ Handle=%p",
407 qent, len, Queue->Buffers[qent].Handle);
408 if( Queue->Callback )
409 Queue->Callback(Dev, qent, len, Queue->Buffers[qent].Handle);
411 if( !Queue->NoAutoRel )
413 // Return the buffer to the avaliable pool
414 VirtIO_ReleaseBuffer(&Queue->Buffers[qent]);
415 if(Queue->NextUsedPop == UsedIdx)
416 Queue->NextUsedPop ++;
420 void VirtIO_IRQHandler(int IRQ, void *Ptr)
422 tVirtIO_Dev *Dev = Ptr;
423 Uint8 isr = inb(Dev->IOBase + VIRTIO_REG_ISRSTS);
424 LOG("IRQ for %p - ISR = 0x%x", Dev, isr);
426 // ISR == 0: Interrupt was not from this card
431 for( int i = 0; i < Dev->nQueues; i ++ )
433 tVirtIO_Queue *queue = Dev->Queues[i];
435 LOG("Queue %i Used: %i ?!= %i (Avail: %i)",
436 i, queue->LastSeenUsed, queue->Used->Idx, queue->Avail->Idx);
437 while( queue->LastSeenUsed != queue->Used->Idx )
439 int idx = queue->LastSeenUsed;
440 queue->LastSeenUsed ++;
441 VirtIO_int_ProcessUsedList(Dev, queue, idx);