Playing video in a Golang game

#gamedev #go

Please proceed here for a new version of the article. The method described below doesn't work anymore (and it's not really good either).

In videogames, we often need to show the player beautiful cinematics, either for ending or in the middle of the playthrough. To do it, we need to somehow load video frames and render them on the screen. Here's how to do that using Pixel game library and goav, Golang bindings for FFmpeg.

Initial setup

First we need to create a GLFW window for rendering OpenGL stuff:

package main

import (
    "fmt"

    "github.com/faiface/pixel"
    "github.com/faiface/pixel/pixelgl"
    colors "golang.org/x/image/colornames"
)

const (
    // WindowWidth is the width of the window.
    WindowWidth = 1280
    // WindowHeight is the height of the window.
    WindowHeight = 720
)

func run() {
    // Create a new window.
    cfg := pixelgl.WindowConfig{
        Title:  "Pixel Rocks!",
        Bounds: pixel.R(0, 0,
            float64(WindowWidth), float64(WindowHeight)),
        VSync:  false,
    }
    win, err := pixelgl.NewWindow(cfg)
    handleError(err)

    fps := 0
    perSecond := time.Tick(time.Second)

    for !win.Closed() {
        win.Clear(colors.White)

        win.Update()

        // Show FPS in the window title.
        fps++

        select {
        case <-perSecond:
            win.SetTitle(fmt.Sprintf("%s | FPS: %d",
                cfg.Title, fps))
            fps = 0

        default:
        }
    }
}

func main() {
    pixelgl.Run(run)
}

func handleError(err error) {
    if err != nil {
        panic(err)
    }
}

Obtaining video frames

Now we need to obtain frames from the video stream of the file. goav provides an example code on how to do that. To make it work with Pixel game library, we need to set the frame decoding format to avcodec.AV_PIX_FMT_RGBA which is used by Pixel.

We also need a way to send the decoded frames to the renderer. For this task we will use a thread-safe frame buffer channel. First we should specify its size:

const (
    FrameBufferSize = 1024
)

The greater the buffer size, the faster the frame transfer from the decoder to the renderer. Now to the channel creation:

frameBuffer := make(chan *pixel.PictureData, FrameBufferSize)

When the frame transfer is complete, we need to close the channel, but we also have to make sure the renderer got all the frames from the buffer. So here's the code for closing the frame buffer:

go func() {
    for {
        if len(frameBuffer) <= 0 {
            close(frameBuffer)
            break
        }
    }
}()

The complete code for reading video frames is presented below:

func readVideoFrames(videoPath string) <-chan *pixel.PictureData {
    // Create a frame buffer.
    frameBuffer := make(chan *pixel.PictureData, FrameBufferSize)

    go func() {
        // Open a video file.
        pFormatContext := avformat.AvformatAllocContext()

        if avformat.AvformatOpenInput(&pFormatContext, videoPath, nil, nil) != 0 {
            fmt.Printf("Unable to open file %s\n", videoPath)
            os.Exit(1)
        }

        // Retrieve the stream information.
        if pFormatContext.AvformatFindStreamInfo(nil) < 0 {
            fmt.Println("Couldn't find stream information")
            os.Exit(1)
        }

        // Dump information about the video to stderr.
        pFormatContext.AvDumpFormat(0, videoPath, 0)

        // Find the first video stream
        for i := 0; i < int(pFormatContext.NbStreams()); i++ {
            switch pFormatContext.Streams()[i].
                CodecParameters().AvCodecGetType() {
            case avformat.AVMEDIA_TYPE_VIDEO:

                // Get a pointer to the codec context for the video stream
                pCodecCtxOrig := pFormatContext.Streams()[i].Codec()
                // Find the decoder for the video stream
                pCodec := avcodec.AvcodecFindDecoder(avcodec.
                    CodecId(pCodecCtxOrig.GetCodecId()))

                if pCodec == nil {
                    fmt.Println("Unsupported codec!")
                    os.Exit(1)
                }

                // Copy context
                pCodecCtx := pCodec.AvcodecAllocContext3()

                if pCodecCtx.AvcodecCopyContext((*avcodec.
                    Context)(unsafe.Pointer(pCodecCtxOrig))) != 0 {
                    fmt.Println("Couldn't copy codec context")
                    os.Exit(1)
                }

                // Open codec
                if pCodecCtx.AvcodecOpen2(pCodec, nil) < 0 {
                    fmt.Println("Could not open codec")
                    os.Exit(1)
                }

                // Allocate video frame
                pFrame := avutil.AvFrameAlloc()

                // Allocate an AVFrame structure
                pFrameRGB := avutil.AvFrameAlloc()

                if pFrameRGB == nil {
                    fmt.Println("Unable to allocate RGB Frame")
                    os.Exit(1)
                }

                // Determine required buffer size and allocate buffer
                numBytes := uintptr(avcodec.AvpictureGetSize(
                    avcodec.AV_PIX_FMT_RGBA, pCodecCtx.Width(),
                    pCodecCtx.Height()))
                buffer := avutil.AvMalloc(numBytes)

                // Assign appropriate parts of buffer to image planes in pFrameRGB
                // Note that pFrameRGB is an AVFrame, but AVFrame is a superset
                // of AVPicture
                avp := (*avcodec.Picture)(unsafe.Pointer(pFrameRGB))
                avp.AvpictureFill((*uint8)(buffer),
                    avcodec.AV_PIX_FMT_RGBA, pCodecCtx.Width(), pCodecCtx.Height())

                // initialize SWS context for software scaling
                swsCtx := swscale.SwsGetcontext(
                    pCodecCtx.Width(),
                    pCodecCtx.Height(),
                    (swscale.PixelFormat)(pCodecCtx.PixFmt()),
                    pCodecCtx.Width(),
                    pCodecCtx.Height(),
                    avcodec.AV_PIX_FMT_RGBA,
                    avcodec.SWS_BILINEAR,
                    nil,
                    nil,
                    nil,
                )

                // Read frames and save first five frames to disk
                packet := avcodec.AvPacketAlloc()

                for pFormatContext.AvReadFrame(packet) >= 0 {
                    // Is this a packet from the video stream?
                    if packet.StreamIndex() == i {
                        // Decode video frame
                        response := pCodecCtx.AvcodecSendPacket(packet)

                        if response < 0 {
                            fmt.Printf("Error while sending a packet to the decoder: %s\n",
                                avutil.ErrorFromCode(response))
                        }

                        for response >= 0 {
                            response = pCodecCtx.AvcodecReceiveFrame(
                                (*avcodec.Frame)(unsafe.Pointer(pFrame)))

                            if response == avutil.AvErrorEAGAIN ||
                                response == avutil.AvErrorEOF {
                                break
                            } else if response < 0 {
                                //fmt.Printf("Error while receiving a frame from the decoder: %s\n",
                                //avutil.ErrorFromCode(response))

                                //return
                            }

                            // Convert the image from its native format to RGB
                            swscale.SwsScale2(swsCtx, avutil.Data(pFrame),
                                avutil.Linesize(pFrame), 0, pCodecCtx.Height(),
                                avutil.Data(pFrameRGB), avutil.Linesize(pFrameRGB))

                            // Save the frame to the frame buffer.
                            frame := getFrameRGBA(pFrameRGB,
                                pCodecCtx.Width(), pCodecCtx.Height())
                            frameBuffer <- frame
                        }
                    }

                    // Free the packet that was allocated by av_read_frame
                    packet.AvFreePacket()
                }

                go func() {
                    for {
                        if len(frameBuffer) <= 0 {
                            close(frameBuffer)
                            break
                        }
                    }
                }()

                // Free the RGB image
                avutil.AvFree(buffer)
                avutil.AvFrameFree(pFrameRGB)

                // Free the YUV frame
                avutil.AvFrameFree(pFrame)

                // Close the codecs
                pCodecCtx.AvcodecClose()
                (*avcodec.Context)(unsafe.Pointer(pCodecCtxOrig)).AvcodecClose()

                // Close the video file
                pFormatContext.AvformatCloseInput()

                // Stop after saving frames of first video straem
                break

            default:
                fmt.Println("Didn't find a video stream")
                os.Exit(1)
            }
        }
    }()

    return frameBuffer
}

We allocate a separate goroutine for the frame decoder so it can do its work in its own pace and just put all the results in the frame buffer.

Converting video frames

Now we need to bring the extracted video frame to the form appropriate for Pixel. First we should extract raw RGBA bytes:

func getFrameRGBA(frame *avutil.Frame, width, height int) *pixel.PictureData {
    pix := []byte{}

    for y := 0; y < height; y++ {
        data0 := avutil.Data(frame)[0]
        buf := make([]byte, width*4)
        startPos := uintptr(unsafe.Pointer(data0)) +
            uintptr(y)*uintptr(avutil.Linesize(frame)[0])

        for i := 0; i < width*4; i++ {
            element := *(*uint8)(unsafe.Pointer(startPos + uintptr(i)))
            buf[i] = element
        }

        pix = append(pix, buf...)
    }

    return pixToPictureData(pix, width, height)
}

To create *pixel.PictureData out of these bytes, we'll use this simple code:

func pixToPictureData(pixels []byte, width, height int) *pixel.PictureData {
    picData := pixel.MakePictureData(pixel.
        R(0, 0, float64(width), float64(height)))

    for y := height - 1; y >= 0; y-- {
        for x := 0; x < width; x++ {
            picData.Pix[(height-y-1)*width+x].R = pixels[y*width*4+x*4+0]
            picData.Pix[(height-y-1)*width+x].G = pixels[y*width*4+x*4+1]
            picData.Pix[(height-y-1)*width+x].B = pixels[y*width*4+x*4+2]
            picData.Pix[(height-y-1)*width+x].A = pixels[y*width*4+x*4+3]
        }
    }

    return picData
}

We need to fill the Pix array vice versa because this is how Pixel treats picture data.

Rendering video frames

Now let's create a new animated sprite to output the video frames:

videoSprite := pixel.NewSprite(nil, pixel.Rect{})
videoTransform := pixel.IM.Moved(pixel.V(
    float64(WindowWidth)/2, float64(WindowHeight)/2))
frameBuffer := readVideoFrames(os.Args[1])

The path to the video file is specified as a command line argument.

Then let's start rendering the video frames:

select {
    case frame, ok: = <-frameBuffer:
        if !ok {
            os.Exit(0)
        }

        if frame != nil {
            videoSprite.Set(frame, frame.Rect)
        }

    default:
}

videoSprite.Draw(win, videoTransform)