Unity-Technologies / Unity-Technologies/com.unity.webrtc
[BUG]: Video Stream does not show on the client in version 2.4.0-exp.4
@karasusan is already working on this.
Since Feb 3, 2022.
- Dominant language
- Assembly
- Stars
- 852
- Forks
- 238
- PR merge metrics
- No merged PRs in 30d
Description
Package version
2.4.0-exp.4
Environment
* OS: Windows 10
* Unity version: 2020.3.27f1
Steps To Reproduce
Code (snippets) below works on all the webRTC versions up to 2.4.0.-exp.3. In version 2.4.0-exp.4 the video stream seems to be going to the client (it shows in webRTC internals) but there is no video. I have looked at the changes between exp.3 and exp.4 and I cannot figure out what might be wrong (codec?). Maybe you could reproduce it using below code snippets and see what change/update is needed/
// Initialize WebRTC
public void Awake()
{
DontDestroyOnLoad(this.gameObject);
Instance = this;
WebRTC.Initialize(EncoderType.Hardware);
EnhancedTouchSupport.Enable();
}
public void Update()
{
StartCoroutine(CaptureScreen());
}
// Create video stream track if not created.
// Get screen pixels
// Call 'Update' delegate on the VideoStreamTrack
//
private IEnumerator CaptureScreen()
{
// Are we shutting down?
if (Instance == null) yield return null;
if (m_screen == null || (Screen.width != m_screen.width || Screen.height != m_screen.height))
{
CreateOrUpdateTrack();
}
yield return new WaitForEndOfFrame();
m_screen.ReadPixels(new Rect(0, 0, Screen.width, Screen.height), 0, 0);
m_screen.Apply();
foreach (var update in m_listVideoStreamTrackUpdaters.Values)
{
try
{
update();
}
catch (Exception e)
{
Debug.LogErrorFormat("Failure during track update {0}", e);
}
}
}
private void CreateOrUpdateTrack()
{
if (m_screen == null)
{
Screen.SetResolution(m_width, m_height, false, 60);
m_screen = new Texture2D(Screen.width, Screen.height, TextureFormat.BGRA32, false);
}
else
{
m_screen.Resize(Screen.width, Screen.height);
}
var videoTrack = new VideoStreamTrack("screen", m_screen);
// WebRTC.Update begins an infinite loop and does not allow us to perform our own capture before calling Update on the individual tracks.
// Additionally, Track.Update is internal. This should be addressed.
var trackUpdateMethod = videoTrack.GetType().GetMethod("Update", BindingFlags.NonPublic | BindingFlags.Instance);
var update = (Action)trackUpdateMethod.CreateDelegate(typeof(Action), videoTrack);
m_listVideoStreamTrackUpdaters.Add(videoTrack, update);
}
////////// THE FOLLOWING PART IS TO SHOW WEBRTC NEGOTIATION, CONFIGURATION AND HOW AND WHEN THE VIDEO TRACK IS ADDED TO THE RTCPeerConnection.
void BeginNegotiation(Signaling signaling, PlatformSessionDescription e)
{
RTCSessionDescription offer;
offer.type = RTCSdpType.Offer;
offer.sdp = e.sdp;
var connectionId = e.playerId;
if (m_mapConnectionIdAndPeer.TryGetValue(connectionId, out RTCPeerConnection target))
{
Debug.Log("Removing existing player: " + connectionId);
target.Close();
m_mapConnectionIdAndPeer.Remove(connectionId);
}
var pc = new RTCPeerConnection();
if (!m_mapConnectionIdAndPeer.ContainsKey(e.playerId))
{
m_mapConnectionIdAndPeer.Add(e.playerId, pc);
}
pc.OnDataChannel = new DelegateOnDataChannel(channel => { OnDataChannel(pc, e.playerId, channel); });
pc.OnIceCandidate = new DelegateOnIceCandidate(candidate =>
{
signaling.SendCandidate(e.playerId, candidate);
});
pc.OnIceConnectionChange = new DelegateOnIceConnectionChange(state =>
{
Debug.Log("PeerConnection ICE Connection State:" + state);
if (state == RTCIceConnectionState.Disconnected)
{
pc.Close();
m_mapConnectionIdAndPeer.Remove(e.playerId);
if (e.playerId == 1)
{
signaling.Close();
}
}
});
var conf = new RTCConfiguration
{
iceServers = m_clientConfiguration.peerConnectionOptions.iceServers,
iceCandidatePoolSize = m_clientConfiguration.peerConnectionOptions.iceCandidatePoolSize == 0 ? 4 : m_clientConfiguration.peerConnectionOptions.iceCandidatePoolSize,
iceTransportPolicy = m_clientConfiguration.peerConnectionOptions.iceTransportPolicy == "relay" ? RTCIceTransportPolicy.Relay : RTCIceTransportPolicy.All
};
Debug.Log("SetConfiguration: " + JsonUtility.ToJson(conf));
var configError = pc.SetConfiguration(ref conf);
Debug.Log("Config error: " + configError);
if (configError != RTCErrorType.None)
{
throw new Exception("Unable to set WebRTC configuration");
}
// PLAT-1009
// The WebRTC package supports receiving as well as sending video streams, but with different codecs (H264 for sending, VP8 for receiving).
// The Hardware Encoder only supports H264 (which is what we want anyways).
// Forcing this into receive (from the browser pov) forces the correct codecs.
offer.sdp = offer.sdp.Replace("a=sendrecv", "a=recvonly");
var setRemoteDescriptionOp = pc.SetRemoteDescription(ref offer);
WaitForAsyncOperation(setRemoteDescriptionOp);
foreach (var pair in m_listVideoStreamTrackUpdaters)
{
var sender = pc.AddTrack(pair.Key);
m_rtpVideoSenders[pc] = sender;
}
if (m_audioStream != null)
{
foreach (var track in m_audioStream.GetTracks())
{
pc.AddTrack(track);
}
}
RTCAnswerOptions options = default;
var createAnswserOp = pc.CreateAnswer(ref options);
WaitForAsyncOperation(createAnswserOp);
var desc = createAnswserOp.Desc;
signaling.SendAnswer(connectionId, desc);
var setLocationDescriptionOp = pc.SetLocalDescription(ref desc);
WaitForAsyncOperation(setLocationDescriptionOp);
}
Current Behavior
Stream does not show on the browser client. WebRTC-internals show that the frames are going through fine but nothing on the client shows up.
Expected Behavior
No response
Anything else?
Please let me know what else I can provide to help.
Thank you,
Jacek
Contributor guide
First steps
- Read the whole issue, then the project's contributing guide.
- Comment on the issue to say you are picking it up — it saves two people doing the same work.
- Fork the repository and make your change on a branch.
- Open a pull request that references the issue number.
Assessment
This issue has not been assessed yet.