Kinectからの画像をXNAで表示する

公開されたばかりのKinectSDKのドキュメントを読みながらDirectXもWPFも使いたくない！
と思ったのでXNAでKinectから取り込んだ画像を表示できるようにしてみました。
ピクセルフォーマットとか直すところでつまづいたけど、VideoStreamとDepthStreamから得られる画像はきちんと表示できるようになりました。

土日のうちにボーンの描画までして何か作りたいですね。

view plainprint?

1. using System;  
2. using System.Collections.Generic;  
3. using System.Linq;  
4. using System.Windows;  
5. using Microsoft.Xna.Framework;  
6. using Microsoft.Xna.Framework.Audio;  
7. using Microsoft.Xna.Framework.Content;  
8. using Microsoft.Xna.Framework.GamerServices;  
9. using Microsoft.Xna.Framework.Graphics;  
10. using Microsoft.Xna.Framework.Input;  
11. using Microsoft.Xna.Framework.Media;  
12. using Microsoft.Research.Kinect.Nui;  
13.   
14. namespace XNAFirst  
15. {  
16.     /// <summary>  
17.     /// This is the main type for your game  
18.     /// </summary>  
19.     public class Game1 : Microsoft.Xna.Framework.Game  
20.     {  
21.         GraphicsDeviceManager graphics;  
22.         SpriteBatch spriteBatch;  
23.   
24.         //Kinectの変数  
25.         Runtime nui;  
26.         int totalFrames = 0;  
27.         int lastFrames = 0;  
28.         DateTime lastTime = DateTime.MaxValue;  
29.   
30.         //テクスチャ  
31.         private Texture2D texDepth = null;  
32.         private Texture2D texBorn = null;  
33.         private Texture2D texImage = null;  
34.   
35.         const int ALPHA_IDX = 3;  
36.         const int RED_IDX = 0;  
37.         const int GREEN_IDX = 1;  
38.         const int BLUE_IDX = 2;  
39.   
40.         byte[] depthFrame32 = new byte[320 * 240 * 4];  
41.         byte[] colorFrame32 = new byte[640 * 480 * 4];  
42.   
43.         public Game1()  
44.         {  
45.             graphics = new GraphicsDeviceManager(this);  
46.             Content.RootDirectory = "Content";  
47.             graphics.PreferredBackBufferWidth = 640;  
48.             graphics.PreferredBackBufferHeight = 480;  
49.         }  
50.   
51.         protected override void Initialize()  
52.         {  
53.             //kinectの初期化  
54.             nui = new Runtime();  
55.             try {  
56.                 nui.Initialize(RuntimeOptions.UseDepthAndPlayerIndex | RuntimeOptions.UseSkeletalTracking | RuntimeOptions.UseColor);  
57.             } catch (InvalidOperationException) {  
58.                 System.Windows.MessageBox.Show("Runtime initialization failed. Please make sure Kinect device is plugged in.");  
59.                 return;  
60.             }  
61.             //video stream と depth stream のOpenをする  
62.             try {  
63.                 nui.VideoStream.Open(ImageStreamType.Video , 2 , ImageResolution.Resolution640x480 , ImageType.Color);  
64.                 nui.DepthStream.Open(ImageStreamType.Depth , 2 , ImageResolution.Resolution320x240 , ImageType.DepthAndPlayerIndex);  
65.             } catch (InvalidOperationException) {  
66.                 System.Windows.MessageBox.Show("Failed to open stream. Please make sure to specify a supported image type and resolution.");  
67.                 return;  
68.             }  
69.             //FPS制御のための時間取得  
70.             lastTime = DateTime.Now;  
71.   
72.             //イベント駆動以外でframeを取得したいのでここはちょっと保留  
73.             nui.DepthFrameReady += new EventHandler<ImageFrameReadyEventArgs>(nui_DepthFrameReady);  
74.             nui.SkeletonFrameReady += new EventHandler<SkeletonFrameReadyEventArgs>(nui_SkeletonFrameReady);  
75.             nui.VideoFrameReady += new EventHandler<ImageFrameReadyEventArgs>(nui_VideoFrameReady);  
76.             base.Initialize();  
77.         }  
78.   
79.         void nui_VideoFrameReady(object sender, ImageFrameReadyEventArgs e)  
80.         {  
81.             //32bit per pixel RGBA image  
82.             PlanarImage Image = e.ImageFrame.Image;  
83.             texImage = new Texture2D(this.GraphicsDevice, Image.Width, Image.Height , false , SurfaceFormat.Color);  
84.             byte[] converted = convertColorFrame(Image.Bits);  
85.             texImage.SetData(converted);  
86.         }  
87.   
88.         void nui_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)  
89.         {  
90.             //とりあえず保留  
91.         }  
92.   
93.         void nui_DepthFrameReady(object sender, ImageFrameReadyEventArgs e)  
94.         {  
95.             PlanarImage Image = e.ImageFrame.Image;  
96.             texDepth = new Texture2D(this.GraphicsDevice, Image.Width, Image.Height , false , SurfaceFormat.Color);  
97.             //byteデータで取得  
98.             //depthバッファを見やすくする  
99.             byte[] convertedDepthFrame = convertDepthFrame(Image.Bits);  
100.               
101.             //byteデータから画像orスプライトを作成する  
102.             texDepth.SetData(convertedDepthFrame);  
103.   
104.             totalFrames++;  
105.   
106.             //fps測定  
107.             DateTime cur = DateTime.Now;  
108.             if (cur.Subtract(lastTime) > TimeSpan.FromSeconds(1)) {  
109.                 int frameDiff = totalFrames - lastFrames;  
110.                 lastFrames = totalFrames;  
111.                 lastTime = cur;  
112.             }  
113.         }  
114.   
115.         protected override void LoadContent()  
116.         {  
117.             // Create a new SpriteBatch, which can be used to draw textures.  
118.             spriteBatch = new SpriteBatch(GraphicsDevice);  
119.   
120.             // TODO: use this.Content to load your game content here  
121.         }  
122.   
123.         protected override void UnloadContent()  
124.         {  
125.             nui.Uninitialize();  
126.         }  
127.   
128.         protected override void Update(GameTime gameTime)  
129.         {  
130.             // Allows the game to exit  
131.             if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed)  
132.                 this.Exit();  
133.   
134.             // TODO: Add your update logic here  
135.   
136.             base.Update(gameTime);  
137.         }  
138.   
139.         protected override void Draw(GameTime gameTime)  
140.         {  
141.             GraphicsDevice.Clear(Color.White);  
142.   
143.             this.spriteBatch.Begin();  
144.   
145.             this.spriteBatch.Draw(texImage, new Rectangle(0, 0, graphics.PreferredBackBufferWidth, graphics.PreferredBackBufferHeight), Color.White);  
146.             //this.spriteBatch.Draw(texDepth, new Rectangle(0, 0, graphics.PreferredBackBufferWidth, graphics.PreferredBackBufferHeight), Color.White);  
147.   
148.             this.spriteBatch.End();  
149.   
150.             base.Draw(gameTime);  
151.         }  
152.   
153.         //kinectから取り込んだColorFrameをXNAのColorフォーマットに直す  
154.         byte[] convertColorFrame(byte[] colorFrame)  
155.         {  
156.             for (int i = 0 ; i < colorFrame32.Length ; i += 4) {  
157.                 colorFrame32[i + ALPHA_IDX] = 255;  
158.                 colorFrame32[i + RED_IDX] = colorFrame[i + 2];  
159.                 colorFrame32[i + GREEN_IDX] = colorFrame[i + 1];  
160.                 colorFrame32[i + BLUE_IDX] = colorFrame[i];  
161.             }  
162.             return colorFrame32;  
163.         }  
164.   
165.         // Converts a 16-bit grayscale depth frame which includes player indexes into a 32-bit frame  
166.         // that displays different players in different colors  
167.         //XNAのテクスチャーではColorフォーマットを使うことを想定している  
168.         byte[] convertDepthFrame(byte[] depthFrame16)  
169.         {  
170.             for (int i16 = 0, i32 = 0; i16 < depthFrame16.Length && i32 < depthFrame32.Length; i16 += 2, i32 += 4)  
171.             {  
172.                 int player = depthFrame16[i16] & 0x07;  
173.                 int realDepth = (depthFrame16[i16 + 1] << 5) | (depthFrame16[i16] >> 3);  
174.                 // transform 13-bit depth information into an 8-bit intensity appropriate  
175.                 // for display (we disregard information in most significant bit)  
176.                 byte intensity = (byte)(255 - (255 * realDepth / 0x0fff));  
177.   
178.                 depthFrame32[i32 + ALPHA_IDX] = 255;  
179.                 depthFrame32[i32 + RED_IDX] = 0;  
180.                 depthFrame32[i32 + GREEN_IDX] = 0;  
181.                 depthFrame32[i32 + BLUE_IDX] = 0;  
182.   
183.                 // choose different display colors based on player  
184.                 switch (player)  
185.                 {  
186.                     case 0:  
187.                         depthFrame32[i32 + RED_IDX] = (byte)(intensity / 2);  
188.                         depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 2);  
189.                         depthFrame32[i32 + BLUE_IDX] = (byte)(intensity / 2);  
190.                         break;  
191.                     case 1:  
192.                         depthFrame32[i32 + RED_IDX] = intensity;  
193.                         break;  
194.                     case 2:  
195.                         depthFrame32[i32 + GREEN_IDX] = intensity;  
196.                         break;  
197.                     case 3:  
198.                         depthFrame32[i32 + RED_IDX] = (byte)(intensity / 4);  
199.                         depthFrame32[i32 + GREEN_IDX] = (byte)(intensity);  
200.                         depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);  
201.                         break;  
202.                     case 4:  
203.                         depthFrame32[i32 + RED_IDX] = (byte)(intensity);  
204.                         depthFrame32[i32 + GREEN_IDX] = (byte)(intensity);  
205.                         depthFrame32[i32 + BLUE_IDX] = (byte)(intensity / 4);  
206.                         break;  
207.                     case 5:  
208.                         depthFrame32[i32 + RED_IDX] = (byte)(intensity);  
209.                         depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 4);  
210.                         depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);  
211.                         break;  
212.                     case 6:  
213.                         depthFrame32[i32 + RED_IDX] = (byte)(intensity / 2);  
214.                         depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 2);  
215.                         depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);  
216.                         break;  
217.                     case 7:  
218.                         depthFrame32[i32 + RED_IDX] = (byte)(255 - intensity);  
219.                         depthFrame32[i32 + GREEN_IDX] = (byte)(255 - intensity);  
220.                         depthFrame32[i32 + BLUE_IDX] = (byte)(255 - intensity);  
221.                         break;  
222.                 }  
223.             }  
224.             return depthFrame32;  
225.         }  
226.     }  
227. }  

using System;
using System.Collections.Generic;
using System.Linq;
using System.Windows;
using Microsoft.Xna.Framework;
using Microsoft.Xna.Framework.Audio;
using Microsoft.Xna.Framework.Content;
using Microsoft.Xna.Framework.GamerServices;
using Microsoft.Xna.Framework.Graphics;
using Microsoft.Xna.Framework.Input;
using Microsoft.Xna.Framework.Media;
using Microsoft.Research.Kinect.Nui;

namespace XNAFirst
{
    /// <summary>
    /// This is the main type for your game
    /// </summary>
    public class Game1 : Microsoft.Xna.Framework.Game
    {
        GraphicsDeviceManager graphics;
        SpriteBatch spriteBatch;

        //Kinectの変数
        Runtime nui;
        int totalFrames = 0;
        int lastFrames = 0;
        DateTime lastTime = DateTime.MaxValue;

        //テクスチャ
        private Texture2D texDepth = null;
        private Texture2D texBorn = null;
        private Texture2D texImage = null;

        const int ALPHA_IDX = 3;
        const int RED_IDX = 0;
        const int GREEN_IDX = 1;
        const int BLUE_IDX = 2;

        byte[] depthFrame32 = new byte[320 * 240 * 4];
        byte[] colorFrame32 = new byte[640 * 480 * 4];

        public Game1()
        {
            graphics = new GraphicsDeviceManager(this);
            Content.RootDirectory = "Content";
            graphics.PreferredBackBufferWidth = 640;
            graphics.PreferredBackBufferHeight = 480;
        }

        protected override void Initialize()
        {
            //kinectの初期化
            nui = new Runtime();
            try {
                nui.Initialize(RuntimeOptions.UseDepthAndPlayerIndex | RuntimeOptions.UseSkeletalTracking | RuntimeOptions.UseColor);
            } catch (InvalidOperationException) {
                System.Windows.MessageBox.Show("Runtime initialization failed. Please make sure Kinect device is plugged in.");
                return;
            }
            //video stream と depth stream のOpenをする
            try {
                nui.VideoStream.Open(ImageStreamType.Video , 2 , ImageResolution.Resolution640x480 , ImageType.Color);
                nui.DepthStream.Open(ImageStreamType.Depth , 2 , ImageResolution.Resolution320x240 , ImageType.DepthAndPlayerIndex);
            } catch (InvalidOperationException) {
                System.Windows.MessageBox.Show("Failed to open stream. Please make sure to specify a supported image type and resolution.");
                return;
            }
            //FPS制御のための時間取得
            lastTime = DateTime.Now;

            //イベント駆動以外でframeを取得したいのでここはちょっと保留
            nui.DepthFrameReady += new EventHandler<ImageFrameReadyEventArgs>(nui_DepthFrameReady);
            nui.SkeletonFrameReady += new EventHandler<SkeletonFrameReadyEventArgs>(nui_SkeletonFrameReady);
            nui.VideoFrameReady += new EventHandler<ImageFrameReadyEventArgs>(nui_VideoFrameReady);
            base.Initialize();
        }

        void nui_VideoFrameReady(object sender, ImageFrameReadyEventArgs e)
        {
            //32bit per pixel RGBA image
            PlanarImage Image = e.ImageFrame.Image;
            texImage = new Texture2D(this.GraphicsDevice, Image.Width, Image.Height , false , SurfaceFormat.Color);
            byte[] converted = convertColorFrame(Image.Bits);
            texImage.SetData(converted);
        }

        void nui_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
        {
            //とりあえず保留
        }

        void nui_DepthFrameReady(object sender, ImageFrameReadyEventArgs e)
        {
            PlanarImage Image = e.ImageFrame.Image;
            texDepth = new Texture2D(this.GraphicsDevice, Image.Width, Image.Height , false , SurfaceFormat.Color);
            //byteデータで取得
            //depthバッファを見やすくする
            byte[] convertedDepthFrame = convertDepthFrame(Image.Bits);
            
            //byteデータから画像orスプライトを作成する
            texDepth.SetData(convertedDepthFrame);

            totalFrames++;

            //fps測定
            DateTime cur = DateTime.Now;
            if (cur.Subtract(lastTime) > TimeSpan.FromSeconds(1)) {
                int frameDiff = totalFrames - lastFrames;
                lastFrames = totalFrames;
                lastTime = cur;
            }
        }

        protected override void LoadContent()
        {
            // Create a new SpriteBatch, which can be used to draw textures.
            spriteBatch = new SpriteBatch(GraphicsDevice);

            // TODO: use this.Content to load your game content here
        }

        protected override void UnloadContent()
        {
            nui.Uninitialize();
        }

        protected override void Update(GameTime gameTime)
        {
            // Allows the game to exit
            if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed)
                this.Exit();

            // TODO: Add your update logic here

            base.Update(gameTime);
        }

        protected override void Draw(GameTime gameTime)
        {
            GraphicsDevice.Clear(Color.White);

            this.spriteBatch.Begin();

            this.spriteBatch.Draw(texImage, new Rectangle(0, 0, graphics.PreferredBackBufferWidth, graphics.PreferredBackBufferHeight), Color.White);
            //this.spriteBatch.Draw(texDepth, new Rectangle(0, 0, graphics.PreferredBackBufferWidth, graphics.PreferredBackBufferHeight), Color.White);

            this.spriteBatch.End();

            base.Draw(gameTime);
        }

        //kinectから取り込んだColorFrameをXNAのColorフォーマットに直す
        byte[] convertColorFrame(byte[] colorFrame)
        {
            for (int i = 0 ; i < colorFrame32.Length ; i += 4) {
                colorFrame32[i + ALPHA_IDX] = 255;
                colorFrame32[i + RED_IDX] = colorFrame[i + 2];
                colorFrame32[i + GREEN_IDX] = colorFrame[i + 1];
                colorFrame32[i + BLUE_IDX] = colorFrame[i];
            }
            return colorFrame32;
        }

        // Converts a 16-bit grayscale depth frame which includes player indexes into a 32-bit frame
        // that displays different players in different colors
        //XNAのテクスチャーではColorフォーマットを使うことを想定している
        byte[] convertDepthFrame(byte[] depthFrame16)
        {
            for (int i16 = 0, i32 = 0; i16 < depthFrame16.Length && i32 < depthFrame32.Length; i16 += 2, i32 += 4)
            {
                int player = depthFrame16[i16] & 0x07;
                int realDepth = (depthFrame16[i16 + 1] << 5) | (depthFrame16[i16] >> 3);
                // transform 13-bit depth information into an 8-bit intensity appropriate
                // for display (we disregard information in most significant bit)
                byte intensity = (byte)(255 - (255 * realDepth / 0x0fff));

                depthFrame32[i32 + ALPHA_IDX] = 255;
                depthFrame32[i32 + RED_IDX] = 0;
                depthFrame32[i32 + GREEN_IDX] = 0;
                depthFrame32[i32 + BLUE_IDX] = 0;

                // choose different display colors based on player
                switch (player)
                {
                    case 0:
                        depthFrame32[i32 + RED_IDX] = (byte)(intensity / 2);
                        depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 2);
                        depthFrame32[i32 + BLUE_IDX] = (byte)(intensity / 2);
                        break;
                    case 1:
                        depthFrame32[i32 + RED_IDX] = intensity;
                        break;
                    case 2:
                        depthFrame32[i32 + GREEN_IDX] = intensity;
                        break;
                    case 3:
                        depthFrame32[i32 + RED_IDX] = (byte)(intensity / 4);
                        depthFrame32[i32 + GREEN_IDX] = (byte)(intensity);
                        depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);
                        break;
                    case 4:
                        depthFrame32[i32 + RED_IDX] = (byte)(intensity);
                        depthFrame32[i32 + GREEN_IDX] = (byte)(intensity);
                        depthFrame32[i32 + BLUE_IDX] = (byte)(intensity / 4);
                        break;
                    case 5:
                        depthFrame32[i32 + RED_IDX] = (byte)(intensity);
                        depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 4);
                        depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);
                        break;
                    case 6:
                        depthFrame32[i32 + RED_IDX] = (byte)(intensity / 2);
                        depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 2);
                        depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);
                        break;
                    case 7:
                        depthFrame32[i32 + RED_IDX] = (byte)(255 - intensity);
                        depthFrame32[i32 + GREEN_IDX] = (byte)(255 - intensity);
                        depthFrame32[i32 + BLUE_IDX] = (byte)(255 - intensity);
                        break;
                }
            }
            return depthFrame32;
        }
    }
}