Unity-Technologies / Unity-Technologies/com.unity.webrtc

[BUG]: Video Stream does not show on the client in version 2.4.0-exp.4

Open
#623 10 comments 0 reactions 1 assignee View on GitHub

@karasusan is already working on this.

Since Feb 3, 2022.

bug issued
Dominant language
Assembly
Stars
852
Forks
238
PR merge metrics
No merged PRs in 30d

Description

Package version

2.4.0-exp.4

Environment
* OS: Windows 10
* Unity version: 2020.3.27f1
Steps To Reproduce

Code (snippets) below works on all the webRTC versions up to 2.4.0.-exp.3. In version 2.4.0-exp.4 the video stream seems to be going to the client (it shows in webRTC internals) but there is no video. I have looked at the changes between exp.3 and exp.4 and I cannot figure out what might be wrong (codec?). Maybe you could reproduce it using below code snippets and see what change/update is needed/

// Initialize WebRTC
public void Awake()
    {

        DontDestroyOnLoad(this.gameObject);
        Instance = this;
        WebRTC.Initialize(EncoderType.Hardware);
        EnhancedTouchSupport.Enable();

    }

   public void Update()
    {
        StartCoroutine(CaptureScreen());
    }

   // Create video stream track if not created. 
  // Get screen pixels
  // Call 'Update' delegate on the VideoStreamTrack
 //
    private IEnumerator CaptureScreen()
    {


        // Are we shutting down?
        if (Instance == null) yield return null;

        if (m_screen == null || (Screen.width != m_screen.width || Screen.height != m_screen.height))
        {
            CreateOrUpdateTrack();
        }
        

        yield return new WaitForEndOfFrame();
        m_screen.ReadPixels(new Rect(0, 0, Screen.width, Screen.height), 0, 0);
        m_screen.Apply();

        foreach (var update in m_listVideoStreamTrackUpdaters.Values)
        {
            try
            {
                update();
            }
            catch (Exception e)
            {
                Debug.LogErrorFormat("Failure during track update {0}", e);
            }
        }

    }

private void CreateOrUpdateTrack()
    {
        if (m_screen == null)
        {
            Screen.SetResolution(m_width, m_height, false, 60);
            m_screen = new Texture2D(Screen.width, Screen.height, TextureFormat.BGRA32, false);
        }
        else
        {
            m_screen.Resize(Screen.width, Screen.height);
        }

        var videoTrack = new VideoStreamTrack("screen", m_screen);
        // WebRTC.Update begins an infinite loop and does not allow us to perform our own capture before calling Update on the individual tracks.
        // Additionally, Track.Update is internal. This should be addressed.
        var trackUpdateMethod = videoTrack.GetType().GetMethod("Update", BindingFlags.NonPublic | BindingFlags.Instance);
        var update = (Action)trackUpdateMethod.CreateDelegate(typeof(Action), videoTrack);
        m_listVideoStreamTrackUpdaters.Add(videoTrack, update);
    }

////////// THE FOLLOWING PART IS TO SHOW WEBRTC NEGOTIATION, CONFIGURATION AND HOW AND WHEN THE VIDEO TRACK IS ADDED TO THE RTCPeerConnection.

    void BeginNegotiation(Signaling signaling, PlatformSessionDescription e)
    {
        RTCSessionDescription offer;
        offer.type = RTCSdpType.Offer;
        offer.sdp = e.sdp;
        var connectionId = e.playerId;
        if (m_mapConnectionIdAndPeer.TryGetValue(connectionId, out RTCPeerConnection target))
        {
            Debug.Log("Removing existing player: " + connectionId);
            target.Close();
            m_mapConnectionIdAndPeer.Remove(connectionId);
        }

        var pc = new RTCPeerConnection();
        if (!m_mapConnectionIdAndPeer.ContainsKey(e.playerId))
        {
            m_mapConnectionIdAndPeer.Add(e.playerId, pc);
        }

        pc.OnDataChannel = new DelegateOnDataChannel(channel => { OnDataChannel(pc, e.playerId, channel); });
        pc.OnIceCandidate = new DelegateOnIceCandidate(candidate =>
        {
            signaling.SendCandidate(e.playerId, candidate);
        });
        pc.OnIceConnectionChange = new DelegateOnIceConnectionChange(state =>
        {
            Debug.Log("PeerConnection ICE Connection State:" + state);
            if (state == RTCIceConnectionState.Disconnected)
            {
                pc.Close();
                m_mapConnectionIdAndPeer.Remove(e.playerId);
                if (e.playerId == 1)
                {
                    signaling.Close();
                }
            }
        });

        var conf = new RTCConfiguration
        {
            iceServers = m_clientConfiguration.peerConnectionOptions.iceServers,
            iceCandidatePoolSize = m_clientConfiguration.peerConnectionOptions.iceCandidatePoolSize == 0 ? 4 : m_clientConfiguration.peerConnectionOptions.iceCandidatePoolSize,
            iceTransportPolicy = m_clientConfiguration.peerConnectionOptions.iceTransportPolicy == "relay" ? RTCIceTransportPolicy.Relay : RTCIceTransportPolicy.All
        };

        Debug.Log("SetConfiguration: " + JsonUtility.ToJson(conf));

        var configError = pc.SetConfiguration(ref conf);
        Debug.Log("Config error: " + configError);
        if (configError != RTCErrorType.None)
        {
            throw new Exception("Unable to set WebRTC configuration");
        }

        // PLAT-1009
        // The WebRTC package supports receiving as well as sending video streams, but with different codecs (H264 for sending, VP8 for receiving).
        // The Hardware Encoder only supports H264 (which is what we want anyways). 
        // Forcing this into receive (from the browser pov)  forces the correct codecs.
        offer.sdp = offer.sdp.Replace("a=sendrecv", "a=recvonly");
        var setRemoteDescriptionOp = pc.SetRemoteDescription(ref offer);
        WaitForAsyncOperation(setRemoteDescriptionOp);

        foreach (var pair in m_listVideoStreamTrackUpdaters)
        {
            var sender = pc.AddTrack(pair.Key);
            m_rtpVideoSenders[pc] = sender;
        }

        if (m_audioStream != null)
        {
            foreach (var track in m_audioStream.GetTracks())
            {
                pc.AddTrack(track);
            }
        }

        RTCAnswerOptions options = default;
        var createAnswserOp = pc.CreateAnswer(ref options);
        WaitForAsyncOperation(createAnswserOp);


        var desc = createAnswserOp.Desc;
        signaling.SendAnswer(connectionId, desc);

        var setLocationDescriptionOp = pc.SetLocalDescription(ref desc);
        WaitForAsyncOperation(setLocationDescriptionOp);
    }
Current Behavior

Stream does not show on the browser client. WebRTC-internals show that the frames are going through fine but nothing on the client shows up.

Expected Behavior

No response

Anything else?

Please let me know what else I can provide to help.

Thank you,

Jacek

Contributor guide

Open the contributing guide

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.