diff --git a/app/assets/styles/blink/_ConferenceMatrixParticipant.scss b/app/assets/styles/blink/_ConferenceMatrixParticipant.scss index add1e17..762791a 100644 --- a/app/assets/styles/blink/_ConferenceMatrixParticipant.scss +++ b/app/assets/styles/blink/_ConferenceMatrixParticipant.scss @@ -1,72 +1,79 @@ .container { flex: 1; width: 100%; } .portraitTabletContainer { flex-basis: 50%; width: 50%; } .landscapeTabletContainer { flex-basis: 50%; height: 50%; } .soloContainer { position: absolute; top: 0; bottom: 0; left: 0; right: 0; } .videoContainer { height: 100%; width: 100%; } .video { height: 100%; width: 100%; } .controlsTop { position: absolute; top: 0; right: 0; left: 0; z-index: 1; display: flex; align-items: center; flex-direction: row; max-height: 50px; min-height: 50px; padding-left: 20px; } .badge { background-color: #5cb85c; margin-bottom: 10px; font-size: 14px; font-weight: 500; } .controls { position: absolute; right: 0; bottom: 0; left: 0; z-index: 1; display: flex; align-items: flex-end; flex-direction: row; max-height: 114px; min-height: 114px; padding-left: 20px; } .lead { color: #fff; margin-bottom: 10px; } + +.status { + color: #fff; + font-size: 8px; + margin-bottom: 16px; + margin-left: 5px; +} diff --git a/app/components/AudioCallBox.js b/app/components/AudioCallBox.js index 6b24375..5a4d7b4 100644 --- a/app/components/AudioCallBox.js +++ b/app/components/AudioCallBox.js @@ -1,334 +1,334 @@ import React, { Component } from 'react'; import { View, Platform } from 'react-native'; import { IconButton, Dialog, Text, ActivityIndicator, Colors } from 'react-native-paper'; import PropTypes from 'prop-types'; import autoBind from 'auto-bind'; import EscalateConferenceModal from './EscalateConferenceModal'; import CallOverlay from './CallOverlay'; import DTMFModal from './DTMFModal'; import UserIcon from './UserIcon'; import styles from '../assets/styles/blink/_AudioCallBox.scss'; import utils from '../utils'; import TrafficStats from './BarChart'; function toTitleCase(str) { return str.replace( /\w\S*/g, function(txt) { return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase(); } ); } class AudioCallBox extends Component { constructor(props) { super(props); autoBind(this); this.state = { remoteUri : this.props.remoteUri, remoteDisplayName : this.props.remoteDisplayName, photo : this.props.photo, active : false, audioMuted : this.props.muted, showDtmfModal : false, showEscalateConferenceModal : false, call : this.props.call, reconnectingCall : this.props.reconnectingCall, - bandwidth : this.props.bandwidth, + info : this.props.info, packetLossQueue : [], audioBandwidthQueue : [], latencyQueue : [] }; this.remoteAudio = React.createRef(); this.userHangup = false; } componentDidMount() { // This component is used both for as 'local media' and as the in-call component. // Thus, if the call is not null it means we are beyond the 'local media' phase // so don't call the mediaPlaying prop. if (this.state.call != null) { switch (this.state.call.state) { case 'established': this.attachStream(this.state.call); break; case 'incoming': this.props.mediaPlaying(); // fall through default: this.state.call.on('stateChanged', this.callStateChanged); break; } } else { this.props.mediaPlaying(); } } componentWillUnmount() { if (this.state.call != null) { this.state.call.removeListener('stateChanged', this.callStateChanged); } } //getDerivedStateFromProps(nextProps, state) { UNSAFE_componentWillReceiveProps(nextProps) { if (nextProps.call !== null && nextProps.call !== this.state.call) { if (nextProps.call.state === 'established') { this.attachStream(nextProps.call); this.setState({reconnectingCall: false}); } else if (nextProps.call.state === 'incoming') { this.props.mediaPlaying(); } nextProps.call.on('stateChanged', this.callStateChanged); if (this.state.call !== null) { this.state.call.removeListener('stateChanged', this.callStateChanged); } this.setState({call: nextProps.call}); } if (nextProps.reconnectingCall != this.state.reconnectingCall) { this.setState({reconnectingCall: nextProps.reconnectingCall}); } if (nextProps.hasOwnProperty('muted')) { this.setState({audioMuted: nextProps.muted}); } - if (nextProps.hasOwnProperty('bandwidth')) { - this.setState({bandwidth: nextProps.bandwidth}); + if (nextProps.hasOwnProperty('info')) { + this.setState({info: nextProps.info}); } if (nextProps.hasOwnProperty('packetLossQueue')) { this.setState({packetLossQueue: nextProps.packetLossQueue}); } if (nextProps.hasOwnProperty('audioBandwidthQueue')) { this.setState({audioBandwidthQueue: nextProps.audioBandwidthQueue}); } if (nextProps.hasOwnProperty('latencyQueue')) { this.setState({latencyQueue: nextProps.latencyQueue}); } this.setState({remoteUri: nextProps.remoteUri, remoteDisplayName: nextProps.remoteDisplayName, photo: nextProps.photo }); } componentWillUnmount() { if (this.state.call != null) { this.state.call.removeListener('stateChanged', this.callStateChanged); } clearTimeout(this.callTimer); } callStateChanged(oldState, newState, data) { if (newState === 'established') { this.attachStream(this.state.call); this.setState({reconnectingCall: false}); } } attachStream(call) { this.setState({stream: call.getRemoteStreams()[0]}); //we dont use it anywhere though as audio gets automatically piped } escalateToConference(participants) { this.props.escalateToConference(participants); } hangupCall(event) { event.preventDefault(); this.props.hangupCall('user_hangup_call'); this.userHangup = true; } cancelCall(event) { event.preventDefault(); this.props.hangupCall('user_cancelled_call'); } muteAudio(event) { event.preventDefault(); this.props.toggleMute(this.props.call.id, !this.state.audioMuted); } showDtmfModal() { this.setState({showDtmfModal: true}); } hideDtmfModal() { this.setState({showDtmfModal: false}); } toggleEscalateConferenceModal() { this.setState({ showEscalateConferenceModal: !this.state.showEscalateConferenceModal }); } render() { let buttonContainerClass; let userIconContainerClass; let remoteIdentity = {uri: this.state.remoteUri || '', displayName: this.state.remoteDisplayName || '', photo: this.state.photo }; let displayName = this.state.remoteUri ? toTitleCase(this.state.remoteUri.split('@')[0]) : ''; if (this.state.remoteDisplayName && this.state.remoteUri !== this.state.remoteDisplayName) { displayName = this.state.remoteDisplayName; } if (this.props.isTablet) { buttonContainerClass = this.props.orientation === 'landscape' ? styles.tabletLandscapeButtonContainer : styles.tabletPortraitButtonContainer; userIconContainerClass = styles.tabletUserIconContainer; } else { buttonContainerClass = this.props.orientation === 'landscape' ? styles.landscapeButtonContainer : styles.portraitButtonContainer; userIconContainerClass = styles.userIconContainer; } const buttonSize = this.props.isTablet ? 40 : 34; const buttonClass = (Platform.OS === 'ios') ? styles.iosButton : styles.androidButton; return ( {displayName} {this.state.remoteUri} {this.props.orientation !== 'landscape' && this.state.reconnectingCall ? : null } {this.state.call && ((this.state.call.state === 'accepted' || this.state.call.state === 'established') && !this.state.reconnectingCall) ? : } ); } } AudioCallBox.propTypes = { remoteUri : PropTypes.string, remoteDisplayName : PropTypes.string, photo : PropTypes.string, call : PropTypes.object, connection : PropTypes.object, accountId : PropTypes.string, escalateToConference : PropTypes.func, - bandwidth : PropTypes.string, + info : PropTypes.string, hangupCall : PropTypes.func, mediaPlaying : PropTypes.func, callKeepSendDtmf : PropTypes.func, toggleMute : PropTypes.func, toggleSpeakerPhone : PropTypes.func, speakerPhoneEnabled : PropTypes.bool, orientation : PropTypes.string, isTablet : PropTypes.bool, reconnectingCall : PropTypes.bool, muted : PropTypes.bool, packetLossQueue : PropTypes.array, videoBandwidthQueue : PropTypes.array, audioBandwidthQueue : PropTypes.array, latencyQueue : PropTypes.array }; export default AudioCallBox; diff --git a/app/components/Call.js b/app/components/Call.js index be5448b..b9ac478 100644 --- a/app/components/Call.js +++ b/app/components/Call.js @@ -1,917 +1,925 @@ import React, { Component } from 'react'; import { View } from 'react-native'; import PropTypes from 'prop-types'; import assert from 'assert'; import debug from 'react-native-debug'; import autoBind from 'auto-bind'; import uuid from 'react-native-uuid'; import AudioCallBox from './AudioCallBox'; import LocalMedia from './LocalMedia'; import VideoBox from './VideoBox'; import config from '../config'; import utils from '../utils'; function randomIntFromInterval(min,max) { return Math.floor(Math.random()*(max-min+1)+min); } function FixedQueue( size, initialValues ){ // If there are no initial arguments, default it to // an empty value so we can call the constructor in // a uniform way. initialValues = (initialValues || []); // Create the fixed queue array value. var queue = Array.apply( null, initialValues ); // Store the fixed size in the queue. queue.fixedSize = size; // Add the class methods to the queue. Some of these have // to override the native Array methods in order to make // sure the queue lenght is maintained. queue.push = FixedQueue.push; queue.splice = FixedQueue.splice; queue.unshift = FixedQueue.unshift; // Trim any initial excess from the queue. FixedQueue.trimTail.call( queue ); // Return the new queue. return( queue ); } // I trim the queue down to the appropriate size, removing // items from the beginning of the internal array. FixedQueue.trimHead = function(){ // Check to see if any trimming needs to be performed. if (this.length <= this.fixedSize){ // No trimming, return out. return; } // Trim whatever is beyond the fixed size. Array.prototype.splice.call( this, 0, (this.length - this.fixedSize) ); }; // I trim the queue down to the appropriate size, removing // items from the end of the internal array. FixedQueue.trimTail = function(){ // Check to see if any trimming needs to be performed. if (this.length <= this.fixedSize){ // No trimming, return out. return; } // Trim whatever is beyond the fixed size. Array.prototype.splice.call( this, this.fixedSize, (this.length - this.fixedSize) ); }; // I synthesize wrapper methods that call the native Array // methods followed by a trimming method. FixedQueue.wrapMethod = function( methodName, trimMethod ){ // Create a wrapper that calls the given method. var wrapper = function(){ // Get the native Array method. var method = Array.prototype[ methodName ]; // Call the native method first. var result = method.apply( this, arguments ); // Trim the queue now that it's been augmented. trimMethod.call( this ); // Return the original value. return( result ); }; // Return the wrapper method. return( wrapper ); }; // Wrap the native methods. FixedQueue.push = FixedQueue.wrapMethod( "push", FixedQueue.trimHead ); FixedQueue.splice = FixedQueue.wrapMethod( "splice", FixedQueue.trimTail ); FixedQueue.unshift = FixedQueue.wrapMethod( "unshift", FixedQueue.trimTail ); class Call extends Component { constructor(props) { super(props); autoBind(this); this.samples = 30; this.sampleInterval = 3; this.defaultWaitInterval = 60; // until we can connect or reconnect this.waitCounter = 0; this.waitInterval = this.defaultWaitInterval; this.videoBytesSent = 0; this.audioBytesSent = 0; this.videoBytesReceived = 0; this.audioBytesReceived = 0; this.packetLoss = 0; this.packetLossQueue = FixedQueue(this.samples); this.latencyQueue = FixedQueue(this.samples); this.audioBandwidthQueue = FixedQueue(this.samples); this.videoBandwidthQueue = FixedQueue(this.samples); this.mediaLost = false; let callUUID; let remoteUri = ''; let remoteDisplayName = ''; let callState = null; let direction = null; let callEnded = false; this.mediaIsPlaying = false; this.ended = false; this.answering = false; if (this.props.call) { // If current call is available on mount we must have incoming this.props.call.on('stateChanged', this.callStateChanged); remoteUri = this.props.call.remoteIdentity.uri; remoteDisplayName = this.props.call.remoteIdentity.displayName || this.props.call.remoteIdentity.uri; direction = this.props.call.direction; callUUID = this.props.call.id; } else { remoteUri = this.props.targetUri; remoteDisplayName = this.props.targetUri; callUUID = this.props.callUUID; direction = callUUID ? 'outgoing' : 'incoming'; } if (this.props.connection) { this.props.connection.on('stateChanged', this.connectionStateChanged); } let audioOnly = false; if (this.props.localMedia && this.props.localMedia.getVideoTracks().length === 0) { audioOnly = true; } this.state = { call: this.props.call, targetUri: this.props.targetUri, audioOnly: audioOnly, boo: false, remoteUri: remoteUri, remoteDisplayName: remoteDisplayName, localMedia: this.props.localMedia, connection: this.props.connection, accountId: this.props.account ? this.props.account.id : null, callState: callState, direction: direction, callUUID: callUUID, reconnectingCall: this.props.reconnectingCall, - bandwidth: '', + info: '', packetLossQueue: [], audioBandwidthQueue: [], videoBandwidthQueue: [], latencyQueue: [] } this.statisticsTimer = setInterval(() => { this.getConnectionStats(); }, this.sampleInterval * 1000); this.resetStats(); } resetStats() { this.setState({ bandwidth: '', packetLossQueue: [], audioBandwidthQueue: [], videoBandwidthQueue: [], latencyQueue: [] }); } //getDerivedStateFromProps(nextProps, state) { UNSAFE_componentWillReceiveProps(nextProps) { // Needed for switching to incoming call while in a call if (this.ended) { return; } //console.log('Call got props...'); this.setState({connection: nextProps.connection, accountId: nextProps.account ? nextProps.account.id : null}); if (nextProps.call !== null) { if (this.state.call !== nextProps.call) { nextProps.call.on('stateChanged', this.callStateChanged); this.setState({ call: nextProps.call, remoteUri: nextProps.call.remoteIdentity.uri, direction: nextProps.call.direction, callUUID: nextProps.call.id, remoteDisplayName: nextProps.call.remoteIdentity.displayName }); if (nextProps.call.direction === 'incoming') { this.mediaPlaying(); } this.lookupContact(); } } else { if (nextProps.callUUID !== null && this.state.callUUID !== nextProps.callUUID) { this.setState({'callUUID': nextProps.callUUID, 'direction': 'outgoing', 'call': null }); this.startCallWhenReady(nextProps.callUUID); } } if (nextProps.reconnectingCall !== this.state.reconnectingCall) { this.setState({reconnectingCall: nextProps.reconnectingCall}); } if (nextProps.targetUri !== this.state.targetUri && this.state.direction === 'outgoing') { this.setState({targetUri: nextProps.targetUri}); } this.setState({registrationState: nextProps.registrationState}); if (nextProps.localMedia !== null && nextProps.localMedia !== this.state.localMedia) { let audioOnly = false; if (nextProps.localMedia.getVideoTracks().length === 0) { audioOnly = true; } this.setState({localMedia: nextProps.localMedia, audioOnly: audioOnly}); this.mediaPlaying(nextProps.localMedia); } } getConnectionStats() { //console.log('getConnectionStats'); let speed = 0; let diff = 0; let delay = 0; let audioPackets = 0; let videoPackets = 0; let audioPacketsLost = 0; let videoPacketsLost = 0; let audioPacketLoss = 0; let videoPacketLoss = 0; let bandwidthUpload = 0; let bandwidthDownload = 0; let mediaType; let foundVideo = false; if (!this.state.call || !this.state.call._pc) { this.resetStats(); return; } this.state.call._pc.getStats(null).then(stats => { stats.forEach(report => { if (report.type === "ssrc") { report.values.forEach(object => { if (object.mediaType) { mediaType = object.mediaType; } }); report.values.forEach(object => { if (object.bytesReceived) { const bytesReceived = Math.floor(object.bytesReceived); if (mediaType === 'audio') { if (this.audioBytesReceived > 0 && this.audioBytesReceived < bytesReceived) { diff = bytesReceived - this.audioBytesReceived; diff = bytesReceived - this.audioBytesReceived; speed = Math.floor(diff / this.sampleInterval * 8 / 1000); //console.log('Audio bandwidth received', speed, 'kbit/s'); bandwidthDownload = bandwidthDownload + speed; if (this.audioBandwidthQueue.length < this.samples) { var n = this.samples; while (n > 0) { this.audioBandwidthQueue.push(0); n = n - 1; } } this.audioBandwidthQueue.push(speed); } this.audioBytesReceived = bytesReceived; } else if (mediaType === 'video') { foundVideo = true; if (this.videoBytesReceived > 0 && this.videoBytesReceived < bytesReceived) { diff = bytesReceived - this.videoBytesReceived; speed = Math.floor(diff / this.sampleInterval * 8 / 1000); //console.log('Video bandwidth received', speed, 'kbit/s'); bandwidthDownload = bandwidthDownload + speed; if (this.videoBandwidthQueue.length < this.samples) { var n = this.samples; while (n > 0) { this.videoBandwidthQueue.push(0); n = n - 1; } } this.videoBandwidthQueue.push(speed) } this.videoBytesReceived = bytesReceived; } } else if (object.bytesSent) { const bytesSent = Math.floor(object.bytesSent); if (mediaType === 'audio') { if (this.audioBytesSent > 0 && bytesSent > this.audioBytesSent) { const diff = bytesSent - this.audioBytesSent; const speed = Math.floor(diff / this.sampleInterval * 8 / 1000); bandwidthUpload = bandwidthUpload + speed; //console.log('Audio bandwidth sent', speed, 'kbit/s'); } this.audioBytesSent = bytesSent; } else if (mediaType === 'video') { foundVideo = true; if (this.videoBytesSent > 0 && bytesSent > this.videoBytesSent) { const diff = bytesSent - this.videoBytesSent; const speed = Math.floor(diff / this.sampleInterval * 8 / 1000); bandwidthUpload = bandwidthUpload + speed; //console.log('Video bandwidth sent', speed, 'kbit/s'); } this.videoBytesSent = bytesSent; } } else if (object.packetsLost) { if (mediaType === 'audio') { audioPackets = audioPackets + Math.floor(object.packetsLost); audioPacketsLost = audioPacketsLost + Math.floor(object.packetsLost); } else if (mediaType === 'video') { videoPackets = videoPackets + Math.floor(object.packetsLost); videoPacketsLost = videoPacketsLost + Math.floor(object.packetsLost); } } else if (object.packetsReceived) { if (mediaType === 'audio') { audioPackets = audioPackets + Math.floor(object.packetsReceived); } else if (mediaType === 'video') { videoPackets = videoPackets + Math.floor(object.packetsReceived); } } else if (object.googCurrentDelayMs) { delay = object.googCurrentDelayMs; } //console.log(object); }); }}); // packet loss videoPacketLoss = 0; if (videoPackets > 0) { videoPacketLoss = Math.floor(videoPacketsLost / videoPackets * 100); if (videoPacketLoss > 0) { console.log('Video packet loss', videoPacketLoss, '%'); } } audioPacketLoss = 0; if (audioPackets > 0) { audioPacketLoss = Math.floor(audioPacketsLost / audioPackets * 100); if (audioPacketLoss > 0) { console.log('Audio packet loss', videoPacketLoss, '%'); } } this.packetLoss = videoPacketLoss > audioPacketLoss ? videoPacketLoss : audioPacketLoss; //this.packetLoss = randomIntFromInterval(2, 10); + if (this.packetLoss < 3) { + this.packetLoss = 0; + } + if (this.packetLossQueue.length < this.samples) { var n = this.samples; while (n > 0) { this.packetLossQueue.push(0); n = n - 1; } } if (this.latencyQueue.length < this.samples) { var n = this.samples; while (n > 0) { this.latencyQueue.push(0); n = n - 1; } } this.latencyQueue.push(Math.ceil(delay)); this.packetLossQueue.push(this.packetLoss); this.audioPacketLoss = audioPacketLoss; this.videoPacketLoss = videoPacketLoss; - let bandwidth; + let info; let suffix = ' kbit/s'; if (foundVideo) { suffix = ' Mbit/s'; bandwidthUpload = Math.ceil(bandwidthUpload / 1000 * 100) / 100; bandwidthDownload = Math.ceil(bandwidthDownload / 1000 * 100) / 100; } if (bandwidthDownload > 0 && bandwidthUpload > 0) { - bandwidth = '⇣' + bandwidthDownload + ' ⇡' + bandwidthUpload; + info = '⇣' + bandwidthDownload + ' ⇡' + bandwidthUpload; } else if (bandwidthDownload > 0) { - bandwidth = '⇣' + bandwidthDownload; + info = '⇣' + bandwidthDownload; } else if (bandwidthUpload > 0) { - bandwidth = '⇡' + this.bandwidthUpload; + info = '⇡' + this.bandwidthUpload; } - if (bandwidth) { - bandwidth = bandwidth + suffix; + if (info) { + info = info + suffix; } if (this.packetLoss > 2) { - bandwidth = bandwidth + ' - ' + Math.ceil(this.packetLoss) + '% loss'; + info = info + ' - ' + Math.ceil(this.packetLoss) + '% loss'; + } + + if (delay > 150) { + info = info + ' - ' + Math.ceil(this.delay) + ' ms'; } this.setState({packetLossQueue: this.packetLossQueue, latencyQueue: this.latencyQueue, videoBandwidthQueue: this.videoBandwidthQueue, audioBandwidthQueue: this.audioBandwidthQueue, - bandwidth: bandwidth + info: info }); }); }; mediaPlaying(localMedia) { if (this.state.direction === 'incoming') { const media = localMedia ? localMedia : this.state.localMedia; this.answerCall(media); } else { this.mediaIsPlaying = true; } } answerCall(localMedia) { const media = localMedia ? localMedia : this.state.localMedia; if (this.state.call && this.state.call.state === 'incoming' && media) { let options = {pcConfig: {iceServers: config.iceServers}}; options.localStream = media; if (!this.answering) { this.answering = true; const connectionState = this.state.connection.state ? this.state.connection.state : null; utils.timestampedLog('Call: answering call in connection state', connectionState); this.state.call.answer(options); } else { utils.timestampedLog('Call: answering call in progress...'); } } else { if (!media) { utils.timestampedLog('Call: waiting for local media'); } if (!this.state.call) { utils.timestampedLog('Call: waiting for incoming call data'); } } } componentDidMount() { this.lookupContact(); if (this.state.direction === 'outgoing' && this.state.callUUID) { this.startCallWhenReady(this.state.callUUID); } } componentWillUnmount() { this.ended = true; this.answering = false; if (this.state.call) { this.state.call.removeListener('stateChanged', this.callStateChanged); } if (this.state.connection) { this.state.connection.removeListener('stateChanged', this.connectionStateChanged); } } lookupContact() { let photo = null; let remoteUri = this.state.remoteUri || ''; let remoteDisplayName = this.state.remoteDisplayName || ''; if (!remoteUri) { return; } if (remoteUri.indexOf('3333@') > -1) { remoteDisplayName = 'Video Test'; } else if (remoteUri.indexOf('4444@') > -1) { remoteDisplayName = 'Echo Test'; } else if (this.props.contacts) { let username = remoteUri.split('@')[0]; let isPhoneNumber = username.match(/^(\+|0)(\d+)$/); if (isPhoneNumber) { var contact_obj = this.findObjectByKey(this.props.contacts, 'remoteParty', username); } else { var contact_obj = this.findObjectByKey(this.props.contacts, 'remoteParty', remoteUri); } if (contact_obj) { remoteDisplayName = contact_obj.displayName; photo = contact_obj.photo; if (isPhoneNumber) { remoteUri = username; } } else { if (isPhoneNumber) { remoteUri = username; remoteDisplayName = username; } } } this.setState({remoteDisplayName: remoteDisplayName, remoteUri: remoteUri, photo: photo }); } callStateChanged(oldState, newState, data) { //console.log('Call: callStateChanged', oldState, '->', newState); if (this.ended) { return; } let remoteHasNoVideoTracks; let remoteIsRecvOnly; let remoteIsInactive; let remoteStreams; this.answering = false; if (newState === 'established') { this.setState({reconnectingCall: false}); const currentCall = this.state.call; if (currentCall) { remoteStreams = currentCall.getRemoteStreams(); if (remoteStreams) { if (remoteStreams.length > 0) { const remotestream = remoteStreams[0]; remoteHasNoVideoTracks = remotestream.getVideoTracks().length === 0; remoteIsRecvOnly = currentCall.remoteMediaDirections.video[0] === 'recvonly'; remoteIsInactive = currentCall.remoteMediaDirections.video[0] === 'inactive'; } } } if (remoteStreams && (remoteHasNoVideoTracks || remoteIsRecvOnly || remoteIsInactive) && !this.state.audioOnly) { //console.log('Media type changed to audio'); // Stop local video if (this.state.localMedia.getVideoTracks().length !== 0) { currentCall.getLocalStreams()[0].getVideoTracks()[0].stop(); } this.setState({audioOnly: true}); } else { this.forceUpdate(); } } else if (newState === 'accepted') { // Switch if we have audioOnly and local videotracks. This means // the call object switched and we are transitioning to an // incoming call. if (this.state.audioOnly && this.state.localMedia && this.state.localMedia.getVideoTracks().length !== 0) { //console.log('Media type changed to video on accepted'); this.setState({audioOnly: false}); } } this.forceUpdate(); } connectionStateChanged(oldState, newState) { switch (newState) { case 'closed': break; case 'ready': break; case 'disconnected': if (oldState === 'ready' && this.state.direction === 'outgoing') { utils.timestampedLog('Call: reconnecting the call...'); this.waitInterval = this.defaultWaitInterval; } break; default: break; } } findObjectByKey(array, key, value) { for (var i = 0; i < array.length; i++) { if (array[i][key] === value) { return array[i]; } } return null; } canConnect() { if (!this.state.connection) { console.log('Call: no connection yet'); return false; } if (this.state.connection.state !== 'ready') { console.log('Call: connection is not ready'); return false; } if (this.props.registrationState !== 'registered') { console.log('Call: account not ready yet'); return false; } if (!this.mediaIsPlaying) { console.log('Call: media is not playing'); return false; } return true; } async startCallWhenReady(callUUID) { utils.timestampedLog('Call: start call', callUUID, 'when ready to', this.state.targetUri); this.waitCounter = 0; let diff = 0; while (this.waitCounter < this.waitInterval) { if (this.waitCounter === 1) { utils.timestampedLog('Call: waiting for establishing call', this.waitInterval, 'seconds'); } if (this.userHangup) { this.hangupCall('user_cancelled'); return; } if (this.ended) { return; } if (this.waitCounter >= this.waitInterval - 1) { this.hangupCall('timeout'); } if (!this.canConnect()) { //utils.timestampedLog('Call: waiting for connection', this.waitInterval - this.waitCounter, 'seconds'); if (this.state.call && this.state.call.id === callUUID && this.state.call.state !== 'terminated') { return; } console.log('Wait', this.waitCounter); await this._sleep(1000); } else { this.waitCounter = 0; this.start(); return; } this.waitCounter++; } } _sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } start() { utils.timestampedLog('Call: starting call', this.state.callUUID); if (this.state.localMedia === null) { console.log('Call: cannot create new call without local media'); return; } let options = {pcConfig: {iceServers: config.iceServers}, id: this.state.callUUID}; options.localStream = this.state.localMedia; let call = this.props.account.call(this.state.targetUri, options); if (call) { call.on('stateChanged', this.callStateChanged); } } hangupCall(reason) { let callUUID = this.state.call ? this.state.call.id : this.state.callUUID; this.waitInterval = this.defaultWaitInterval; if (this.state.call) { this.state.call.removeListener('stateChanged', this.callStateChanged); } if (this.props.connection) { this.props.connection.removeListener('stateChanged', this.connectionStateChanged); } if (this.waitCounter > 0) { this.waitCounter = this.waitInterval; } this.props.hangupCall(callUUID, reason); } render() { let box = null; if (this.state.localMedia !== null) { if (this.state.audioOnly) { box = ( ); } else { if (this.state.call !== null && (this.state.call.state === 'established' || (this.state.call.state === 'terminated' && this.state.reconnectingCall))) { box = ( ); } else { if (this.state.call && this.state.call.state === 'terminated' && this.state.reconnectingCall) { //console.log('Skip render local media because we will reconnect'); } else { box = ( ); } } } } else { box = ( ); } return box; } } Call.propTypes = { targetUri : PropTypes.string, account : PropTypes.object, hangupCall : PropTypes.func, connection : PropTypes.object, registrationState : PropTypes.string, call : PropTypes.object, localMedia : PropTypes.object, shareScreen : PropTypes.func, escalateToConference : PropTypes.func, generatedVideoTrack : PropTypes.bool, callKeepSendDtmf : PropTypes.func, toggleMute : PropTypes.func, toggleSpeakerPhone : PropTypes.func, speakerPhoneEnabled : PropTypes.bool, callUUID : PropTypes.string, contacts : PropTypes.array, intercomDtmfTone : PropTypes.string, orientation : PropTypes.string, isTablet : PropTypes.bool, reconnectingCall : PropTypes.bool, muted : PropTypes.bool }; export default Call; diff --git a/app/components/CallOverlay.js b/app/components/CallOverlay.js index 77bba35..1511094 100644 --- a/app/components/CallOverlay.js +++ b/app/components/CallOverlay.js @@ -1,213 +1,218 @@ import React from 'react'; import { View, Text } from 'react-native'; import PropTypes from 'prop-types'; import classNames from 'classnames'; import moment from 'moment'; import momentFormat from 'moment-duration-format'; import autoBind from 'auto-bind'; import { Appbar } from 'react-native-paper'; import Icon from 'react-native-vector-icons/MaterialCommunityIcons'; import { Colors } from 'react-native-paper'; import styles from '../assets/styles/blink/_AudioCallBox.scss'; function toTitleCase(str) { return str.replace( /\w\S*/g, function(txt) { return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase(); } ); } class CallOverlay extends React.Component { constructor(props) { super(props); autoBind(this); this.state = { call: this.props.call, callState: this.props.call ? this.props.call.state : null, direction: this.props.call ? this.props.call.direction: null, remoteUri: this.props.remoteUri, remoteDisplayName: this.props.remoteDisplayName, reconnectingCall: this.props.reconnectingCall } this.duration = null; this.finalDuration = null; this.timer = null; this._isMounted = true; } componentDidMount() { if (this.state.call) { if (this.state.call.state === 'established') { this.startTimer(); } this.state.call.on('stateChanged', this.callStateChanged); this.setState({callState: this.state.call.state}); } } //getDerivedStateFromProps(nextProps, state) { UNSAFE_componentWillReceiveProps(nextProps) { if (!this._isMounted) { return; } if (nextProps.reconnectingCall != this.state.reconnectingCall) { this.setState({reconnectingCall: nextProps.reconnectingCall}); } if (nextProps.call !== null && nextProps.call !== this.state.call) { nextProps.call.on('stateChanged', this.callStateChanged); if (this.state.call !== null) { this.state.call.removeListener('stateChanged', this.callStateChanged); } this.setState({call: nextProps.call}); } this.setState({remoteDisplayName: nextProps.remoteDisplayName, remoteUri: nextProps.remoteUri}); } componentWillUnmount() { this._isMounted = false; if (this.state.call) { this.state.call.removeListener('stateChanged', this.callStateChanged); } clearTimeout(this.timer); } callStateChanged(oldState, newState, data) { if (newState === 'established' && this._isMounted && !this.props.terminated) { this.startTimer(); } if (newState === 'terminated') { if (this.state.call) { this.state.call.removeListener('stateChanged', this.callStateChanged); } clearTimeout(this.timer); this.finalDuration = this.duration; this.duration = null; this.timer = null; } if (!this._isMounted) { return; } this.setState({callState: newState}); } startTimer() { if (this.timer !== null) { // already armed return; } // TODO: consider using window.requestAnimationFrame const startTime = new Date(); - this.duration = moment.duration(new Date() - startTime).format('hh:mm:ss', {trim: false}); - this.timer = setInterval(() => { - this.duration = moment.duration(new Date() - startTime).format('hh:mm:ss', {trim: false}); + const duration = moment.duration(new Date() - startTime); + + if (this.duration > 3600) { + this.duration = duration.format('hh:mm:ss', {trim: false}); + } else { + this.duration = duration.format('mm:ss', {trim: false}); + } + if (this.props.show) { this.forceUpdate(); } }, 1000); } render() { let header = null; if (this.props.terminated) { clearTimeout(this.timer); this.duration = null; this.timer = null; } let displayName = this.state.remoteUri; if (this.state.remoteDisplayName && this.state.remoteDisplayName !== this.state.remoteUri) { displayName = this.state.remoteDisplayName; } if (this.props.show) { let callDetail; if (this.duration) { callDetail = {this.duration}; callDetail = 'Duration: ' + this.duration; } else { if (this.state.reconnectingCall) { callDetail = 'Reconnecting the call...'; } else if (this.props.terminated) { callDetail = 'Call ended'; } else if (this.state.callState === 'terminated') { callDetail = this.finalDuration ? 'Call ended after ' + this.finalDuration : 'Call ended'; } else { if (this.state.callState) { if (this.state.callState === 'incoming') { callDetail = 'Waiting for incoming call...'; } else { callDetail = toTitleCase(this.state.callState); } } else if (this.state.direction) { callDetail = 'Connecting', this.state.direction, 'call...'; } else { callDetail = 'Connecting...'; } } } - if (this.props.bandwidth) { - callDetail = callDetail + ' - ' + this.props.bandwidth; + if (this.props.info) { + callDetail = callDetail + ' - ' + this.props.info; } if (this.state.remoteUri && this.state.remoteUri.search('videoconference') > -1) { displayName = this.state.remoteUri.split('@')[0]; header = ( ); } else { header = ( ); } } return header } } CallOverlay.propTypes = { show: PropTypes.bool.isRequired, remoteUri: PropTypes.string, remoteDisplayName: PropTypes.string, call: PropTypes.object, connection: PropTypes.object, reconnectingCall: PropTypes.bool, terminated : PropTypes.bool, - bandwidth: PropTypes.string + info: PropTypes.string }; export default CallOverlay; diff --git a/app/components/ConferenceBox.js b/app/components/ConferenceBox.js index 2e6e912..a55ef7d 100644 --- a/app/components/ConferenceBox.js +++ b/app/components/ConferenceBox.js @@ -1,1422 +1,1472 @@ 'use strict'; import React, {Component, Fragment} from 'react'; import { View, Platform, TouchableWithoutFeedback, Dimensions, SafeAreaView, ScrollView, FlatList } from 'react-native'; import PropTypes from 'prop-types'; import * as sylkrtc from 'react-native-sylkrtc'; import classNames from 'classnames'; import debug from 'react-native-debug'; import superagent from 'superagent'; import autoBind from 'auto-bind'; import { RTCView } from 'react-native-webrtc'; import { IconButton, Appbar, Portal, Modal, Surface, Paragraph } from 'react-native-paper'; import config from '../config'; import utils from '../utils'; //import AudioPlayer from './AudioPlayer'; import ConferenceDrawer from './ConferenceDrawer'; import ConferenceDrawerLog from './ConferenceDrawerLog'; // import ConferenceDrawerFiles from './ConferenceDrawerFiles'; import ConferenceDrawerParticipant from './ConferenceDrawerParticipant'; import ConferenceDrawerParticipantList from './ConferenceDrawerParticipantList'; import ConferenceDrawerSpeakerSelection from './ConferenceDrawerSpeakerSelection'; import ConferenceDrawerSpeakerSelectionWrapper from './ConferenceDrawerSpeakerSelectionWrapper'; import ConferenceHeader from './ConferenceHeader'; import ConferenceCarousel from './ConferenceCarousel'; import ConferenceParticipant from './ConferenceParticipant'; import ConferenceMatrixParticipant from './ConferenceMatrixParticipant'; import ConferenceParticipantSelf from './ConferenceParticipantSelf'; import InviteParticipantsModal from './InviteParticipantsModal'; import ConferenceAudioParticipantList from './ConferenceAudioParticipantList'; import ConferenceAudioParticipant from './ConferenceAudioParticipant'; import styles from '../assets/styles/blink/_ConferenceBox.scss'; const DEBUG = debug('blinkrtc:ConferenceBox'); debug.enable('*'); function toTitleCase(str) { return str.replace( /\w\S*/g, function(txt) { return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase(); } ); } class ConferenceBox extends Component { constructor(props) { super(props); autoBind(this); this.audioBytesReceived = new Map(); this.audioBandwidth = new Map(); this.bandwidthDownload = 0; this.bandwidthUpload = 0; this.videoBytesReceived = new Map(); this.videoBandwidth = new Map(); this.audioPacketLoss = new Map(); this.videoPacketLoss = new Map(); this.packetLoss = new Map(); + this.latency = new Map(); + this.mediaLost = new Map(); + this.sampleInterval = 5; + this.state = { callOverlayVisible: true, ended: false, audioMuted: this.props.muted, videoMuted: !this.props.inFocus, videoMutedbyUser: false, participants: props.call.participants.slice(), showInviteModal: false, showDrawer: false, showFiles: false, shareOverlayVisible: false, showSpeakerSelection: false, activeSpeakers: props.call.activeParticipants.slice(), selfDisplayedLarge: false, eventLog: [], sharedFiles: props.call.sharedFiles.slice(), largeVideoStream: null, previousParticipants: this.props.previousParticipants, inFocus: this.props.inFocus, reconnectingCall: this.props.reconnectingCall, terminated: this.props.terminated }; const friendlyName = this.props.remoteUri.split('@')[0]; //if (window.location.origin.startsWith('file://')) { this.callUrl = `${config.publicUrl}/conference/${friendlyName}`; //} else { // this.callUrl = `${window.location.origin}/conference/${friendlyName}`; //} const emailMessage = `You can join me in the conference using a Web browser at ${this.callUrl} ` + 'or by using the freely available Sylk WebRTC client app at http://sylkserver.com'; const subject = 'Join me, maybe?'; this.emailLink = `mailto:?subject=${encodeURI(subject)}&body=${encodeURI(emailMessage)}`; this.overlayTimer = null; this.logEvent = {}; this.haveVideo = false; this.uploads = []; this.selectSpeaker = 1; this.foundContacts = new Map(); if (this.props.call) { this.lookupContact(this.props.call.localIdentity._uri, this.props.call.localIdentity._displayName); } [ 'error', 'warning', 'info', 'debug' ].forEach((level) => { this.logEvent[level] = ( (action, messages, originator) => { const log = this.state.eventLog.slice(); log.unshift({originator, originator, level: level, action: action, messages: messages}); this.setState({eventLog: log}); } ); }); this.invitedParticipants = new Map(); props.initialParticipants.forEach((uri) => { this.invitedParticipants.set(uri, {timestamp: Date.now(), status: 'Invited'}) this.lookupContact(uri); }); this.participantsTimer = setInterval(() => { this.updateParticipantsStatus(); - }, 5000); + }, this.sampleInterval * 1000); } - speed() { - let speed; + getInfo() { + let info; if (this.bandwidthDownload > 0 && this.bandwidthUpload > 0) { - speed = '⇣' + this.bandwidthDownload + ' ⇡' + this.bandwidthUpload; + info = '⇣' + this.bandwidthDownload + ' ⇡' + this.bandwidthUpload; } else if (this.bandwidthDownload > 0) { - speed = '⇣' + this.bandwidthDownload ; + info = '⇣' + this.bandwidthDownload ; } else if (this.bandwidthUpload > 0) { - speed = '⇡' + this.bandwidthUpload; + info = '⇡' + this.bandwidthUpload; } - if (speed) { - return speed + ' Mbit/s'; + if (info) { + return info + ' Mbit/s'; } - return speed; + return info; } updateParticipantsStatus() { let participants_uris = []; this.state.participants.forEach((p) => { participants_uris.push(p.identity._uri); }); this.getConnectionStats(); const invitedParties = Array.from(this.invitedParticipants.keys()); //console.log('Invited participants', invitedParties); //console.log('Current participants', participants_uris); let p; let interval; invitedParties.forEach((_uri) => { if (participants_uris.indexOf(_uri) > 0) { this.invitedParticipants.delete(_uri); } p = this.invitedParticipants.get(_uri); if (!p) { return; } interval = Math.floor((Date.now() - p.timestamp) / 1000); //console.log(_uri, 'was invited', interval, 'seconds ago'); if (interval >= 60) { this.invitedParticipants.delete(_uri); this.forceUpdate(); } if (p.status.indexOf('Invited') > -1 && interval > 5) { p.status = '.'; } if (p.status.indexOf('.') > -1) { if (interval > 45) { p.status = 'No answer'; } else { p.status = p.status + '.'; } } }); this.forceUpdate(); } componentDidMount() { for (let p of this.state.participants) { p.on('stateChanged', this.onParticipantStateChanged); p.attach(); } this.props.call.on('participantJoined', this.onParticipantJoined); this.props.call.on('participantLeft', this.onParticipantLeft); this.props.call.on('roomConfigured', this.onConfigureRoom); this.props.call.on('fileSharing', this.onFileSharing); if (this.state.participants.length > 1) { this.armOverlayTimer(); } // attach to ourselves first if there are no other participants if (this.state.participants.length === 0) { setTimeout(() => { const item = { stream: this.props.call.getLocalStreams()[0], identity: this.props.call.localIdentity }; this.selectVideo(item); }); } else { this.state.participants.forEach((p) => { if (p.identity._uri.search('guest.') === -1 && p.identity._uri !== this.props.call.localIdentity._uri) { // used for history item this.props.saveParticipant(this.props.call.id, this.props.remoteUri.split('@')[0], p.identity._uri); this.lookupContact(p.identity._uri, p.identity._displayName); } }); // this.changeResolution(); } if (this.props.call.getLocalStreams()[0].getVideoTracks().length !== 0) { this.haveVideo = true; } if (this.state.videoMuted) { this._muteVideo(); } } componentWillUnmount() { clearTimeout(this.overlayTimer); clearTimeout(this.participantsTimer); this.uploads.forEach((upload) => { this.props.notificationCenter().removeNotification(upload[1]); upload[0].abort(); }) } //getDerivedStateFromProps(nextProps, state) { UNSAFE_componentWillReceiveProps(nextProps) { if (nextProps.hasOwnProperty('muted')) { this.setState({audioMuted: nextProps.muted}); } if (nextProps.inFocus !== this.state.inFocus) { if (nextProps.inFocus) { if (!this.state.videoMutedbyUser) { this._resumeVideo(); } } else { this._muteVideo(); } this.setState({inFocus: nextProps.inFocus}); } if (nextProps.reconnectingCall !== this.state.reconnectingCall) { this.setState({reconnectingCall: nextProps.reconnectingCall}); } this.setState({terminated: nextProps.terminated}); } findObjectByKey(array, key, value) { for (var i = 0; i < array.length; i++) { if (array[i][key] === value) { return array[i]; } } return null; } lookupContact(uri, displayName) { let photo; let username = uri.split('@')[0]; if (this.props.contacts) { let username = uri.split('@')[0]; let isPhoneNumber = username.match(/^(\+|0)(\d+)$/); if (isPhoneNumber) { var contact_obj = this.findObjectByKey(this.props.contacts, 'remoteParty', username); } else { var contact_obj = this.findObjectByKey(this.props.contacts, 'remoteParty', uri); } if (contact_obj) { displayName = contact_obj.displayName; photo = contact_obj.photo; if (isPhoneNumber) { uri = username; } } else { if (isPhoneNumber) { uri = username; displayName = toTitleCase(username); } } } const c = {photo: photo, displayName: displayName || toTitleCase(username)}; this.foundContacts.set(uri, c) } getConnectionStats() { let audioPackets = 0; let videoPackets = 0; + let delay = 0; let audioPacketsLost = 0; let videoPacketsLost = 0; let audioPacketLoss = 0; let videoPacketLoss = 0; let totalPackets = 0; let totalPacketsLost = 0; let totalPacketLoss = 0; let totalAudioBandwidth = 0; let totalVideoBandwidth = 0; let totalSpeed = 0; let bandwidthUpload = 0; let mediaType; if (this.state.participants.length === 0) { this.bandwidthDownload = 0; this.videoBandwidth.set('total', 0); this.audioBandwidth.set('total', 0); } let participants = this.state.participants.concat(this.props.call); participants.forEach((p) => { if (!p._pc) { return; } let identity; if (p.identity) { identity = p.identity.uri; } else { identity = 'myself'; } p._pc.getStats(null).then(stats => { audioPackets = 0; videoPackets = 0; audioPacketsLost = 0; videoPacketsLost = 0; audioPacketLoss = 0; videoPacketLoss = 0; stats.forEach(report => { if (report.type === "ssrc") { report.values.forEach(object => { if (object.mediaType) { mediaType = object.mediaType; } }); report.values.forEach(object => { if (object.bytesReceived && identity !== 'myself') { const bytesReceived = Math.floor(object.bytesReceived); if (mediaType === 'audio') { if (this.audioBytesReceived.has(p.id)) { const lastBytes = this.audioBytesReceived.get(p.id); const diff = bytesReceived - lastBytes; - const speed = Math.floor(diff / 5 * 8 / 1000); + const speed = Math.floor(diff / this.sampleInterval * 8 / 1000); totalAudioBandwidth = totalAudioBandwidth + speed; totalSpeed = totalSpeed + speed; - //console.log('Audio bandwidth', speed, 'kbit/s from', identity); + //console.log(identity, 'audio bandwidth', speed, 'kbit/s from', identity); this.audioBandwidth.set(p.id, speed); } this.audioBytesReceived.set(p.id, bytesReceived); } else if (mediaType === 'video') { if (this.videoBytesReceived.has(p.id)) { const lastBytes = this.videoBytesReceived.get(p.id); const diff = bytesReceived - lastBytes; - const speed = Math.floor(diff / 5 * 8 / 1000); + const speed = Math.floor(diff / this.sampleInterval * 8 / 1000); totalVideoBandwidth = totalVideoBandwidth + speed; totalSpeed = totalSpeed + speed; - //console.log('Video bandwidth', speed, 'kbit/s from', identity); + //console.log(identity, 'video bandwidth', speed, 'kbit/s from', identity); this.videoBandwidth.set(p.id, speed); } this.videoBytesReceived.set(p.id, bytesReceived); } } else if (object.bytesSent && identity === 'myself') { - const bytesReceived = Math.floor(object.bytesSent); + const bytesSent = Math.floor(object.bytesSent); if (mediaType === 'audio') { if (this.audioBytesReceived.has(p.id)) { const lastBytes = this.audioBytesReceived.get(p.id); - const diff = bytesReceived - lastBytes; - const speed = Math.floor(diff / 5 * 8 / 1000); + const diff = bytesSent - lastBytes; + const speed = Math.floor(diff / this.sampleInterval * 8 / 1000); bandwidthUpload = bandwidthUpload + speed; - //console.log('Audio bandwidth', speed, 'kbit/s from', identity); + //console.log(identity, 'audio bandwidth', speed, 'kbit/s from', identity); this.audioBandwidth.set(p.id, speed); } - this.audioBytesReceived.set(p.id, bytesReceived); + this.audioBytesReceived.set(p.id, bytesSent); } else if (mediaType === 'video') { if (this.videoBytesReceived.has(p.id)) { const lastBytes = this.videoBytesReceived.get(p.id); - const diff = bytesReceived - lastBytes; - const speed = Math.floor(diff / 5 * 8 / 1000); + const diff = bytesSent - lastBytes; + const speed = Math.floor(diff / this.sampleInterval * 8 / 1000); bandwidthUpload = bandwidthUpload + speed; - //console.log('Video bandwidth', speed, 'kbit/s from', identity); + //console.log(identity, 'video bandwidth', speed, 'kbit/s from', identity); this.videoBandwidth.set(p.id, speed); } - this.videoBytesReceived.set(p.id, bytesReceived); + this.videoBytesReceived.set(p.id, bytesSent); } } else if (object.totalAudioEnergy) { //console.log('Total audio energy', object.totalAudioEnergy, 'from', identity); } else if (object.audioOutputLevel) { //console.log('Output level', object.audioOutputLevel, 'from', identity); this.mediaLost.set(p.id, Math.floor(object.audioOutputLevel) < 5 ? true : false); } else if (object.audioInputLevel) { //console.log('Input level', object.audioInputLevel, 'from', identity); this.mediaLost.set(p.id, Math.floor(object.audioInputLevel) < 5 ? true : false); } else if (object.packetsLost) { totalPackets = totalPackets + Math.floor(object.packetsLost); totalPacketsLost = totalPacketsLost + Math.floor(object.packetsLost); if (mediaType === 'audio') { audioPackets = audioPackets + Math.floor(object.packetsLost); audioPacketsLost = audioPacketsLost + Math.floor(object.packetsLost); } else if (mediaType === 'video') { videoPackets = videoPackets + Math.floor(object.packetsLost); - videoPacketsLost = videoPacketsLost + Math.floor(object.packetsLost); + videoPacketsLost = videoPacketsLost + Math.floor(object.packetsLost); + } + if (object.packetsLost > 0) { + //console.log(identity, mediaType, 'packetsLost', object.packetsLost); } - } else if (object.packetsReceived) { + } else if (object.packetsReceived && identity !== 'myself') { totalPackets = totalPackets + Math.floor(object.packetsReceived); if (mediaType === 'audio') { audioPackets = audioPackets + Math.floor(object.packetsReceived); } else if (mediaType === 'video') { videoPackets = videoPackets + Math.floor(object.packetsReceived); } + //console.log(identity, mediaType, 'packetsReceived', object.packetsReceived); + } else if (object.packetsSent && identity === 'myself') { + totalPackets = totalPackets + Math.floor(object.packetsSent); + + if (mediaType === 'audio') { + audioPackets = audioPackets + Math.floor(object.packetsSent); + } else if (mediaType === 'video') { + videoPackets = videoPackets + Math.floor(object.packetsSent); + } + //console.log(identity, mediaType, 'packetsSent', object.packetsSent); + } else if (object.googCurrentDelayMs && identity !== 'myself') { + delay = object.googCurrentDelayMs; + //console.log('mediaType', mediaType, 'identity', identity, 'delay', delay); + this.latency.set(p.id, Math.ceil(delay)); + //console.log(object); } - if (identity !== 'myself') { + if (identity === 'myself') { //console.log(object); } }); if (videoPackets > 0) { videoPacketLoss = Math.floor(videoPacketsLost / videoPackets * 100); } if (audioPackets > 0) { audioPacketLoss = Math.floor(audioPacketsLost / audioPackets * 100); } if (totalPackets > 0) { totalPacketLoss = Math.floor(totalPacketsLost / totalPackets * 100); } this.audioPacketLoss.set(p.id, audioPacketLoss); this.videoPacketLoss.set(p.id, videoPacketLoss); this.packetLoss.set(p.id, totalPacketLoss); - }}); + console.log(identity, 'audio loss', audioPacketLoss, '%, video loss', videoPacketLoss, '%, total loss', totalPacketLoss, '%'); + const bandwidthDownload = totalVideoBandwidth + totalAudioBandwidth; - this.bandwidthDownload = Math.ceil(bandwidthDownload / 1024 * 100) / 100; + this.bandwidthDownload = Math.ceil(bandwidthDownload / 1000 * 100) / 100; - this.bandwidthUpload = Math.ceil(bandwidthUpload / 1024 * 100) / 100; + this.bandwidthUpload = Math.ceil(bandwidthUpload / 1000 * 100) / 100; this.videoBandwidth.set('total', totalVideoBandwidth); this.audioBandwidth.set('total', totalAudioBandwidth); //console.log('audio bandwidth', totalAudioBandwidth); //console.log('video bandwidth', totalVideoBandwidth); //console.log('total bandwidth', this.bandwidthDownload); + //console.log('this.latency', this.latency); }); }); }; onParticipantJoined(p) { //console.log(p.identity.uri, 'joined the conference'); if (p.identity._uri.search('guest.') === -1 && p.identity._uri !== this.props.call.localIdentity._uri) { // used for history item this.props.saveParticipant(this.props.call.id, this.props.remoteUri.split('@')[0], p.identity._uri); } this.lookupContact(p.identity._uri, p.identity._displayName); if (this.invitedParticipants.has(p.identity._uri)) { this.invitedParticipants.delete(p.identity._uri); } // this.refs.audioPlayerParticipantJoined.play(); p.on('stateChanged', this.onParticipantStateChanged); p.attach(); this.setState({ participants: this.state.participants.concat([p]) }); // this.changeResolution(); if (this.state.participants.length > 1) { this.armOverlayTimer(); } else { this.setState({callOverlayVisible: true}); } } onParticipantLeft(p) { //console.log(p.identity.uri, 'left the conference'); const participants = this.state.participants.slice(); this.audioBandwidth.delete(p.id); this.videoBandwidth.delete(p.id); + this.latency.delete(p.id); + this.audioBytesReceived.delete(p.id); this.videoBytesReceived.delete(p.id); this.audioPacketLoss.delete(p.id); this.videoPacketLoss.delete(p.id); this.packetLoss.delete(p.id); this.mediaLost.delete(p.id); const idx = participants.indexOf(p); if (idx !== -1) { participants.splice(idx, 1); this.setState({ participants: participants }); } p.detach(true); // this.changeResolution(); if (this.state.participants.length > 1) { this.armOverlayTimer(); } else { this.setState({callOverlayVisible: true}); } } onParticipantStateChanged(oldState, newState) { if (newState === 'established' || newState === null) { this.maybeSwitchLargeVideo(); } } onConfigureRoom(config) { const newState = {}; newState.activeSpeakers = config.activeParticipants; this.setState(newState); if (config.activeParticipants.length === 0) { this.logEvent.info('set speakers to', ['Nobody'], config.originator); } else { const speakers = config.activeParticipants.map((p) => {return p.identity.displayName || p.identity.uri}); this.logEvent.info('set speakers to', speakers, config.originator); } this.maybeSwitchLargeVideo(); } onFileSharing(files) { let stateFiles = this.state.sharedFiles; stateFiles = stateFiles.concat(files); this.setState({sharedFiles: stateFiles}); files.forEach((file)=>{ if (file.session !== this.props.call.id) { this.props.notificationCenter().postFileShared(file, this.showFiles); } }) } onVideoSelected(item) { const participants = this.state.participants.slice(); const idx = participants.indexOf(item); participants.splice(idx, 1); participants.unshift(item); if (item.videoPaused) { item.resumeVideo(); } this.setState({ participants: participants }); } changeResolution() { let stream = this.props.call.getLocalStreams()[0]; if (this.state.participants.length < 2) { this.props.call.scaleLocalTrack(stream, 1.5); } else if (this.state.participants.length < 5) { this.props.call.scaleLocalTrack(stream, 2); } else { this.props.call.scaleLocalTrack(stream, 1); } } selectVideo(item) { DEBUG('Switching video to: %o', item); if (item.stream) { this.setState({selfDisplayedLarge: true, largeVideoStream: item.stream}); } } maybeSwitchLargeVideo() { // Switch the large video to another source, maybe. if (this.state.participants.length === 0 && !this.state.selfDisplayedLarge) { // none of the participants are eligible, show ourselves const item = { stream: this.props.call.getLocalStreams()[0], identity: this.props.call.localIdentity }; this.selectVideo(item); } else if (this.state.selfDisplayedLarge) { this.setState({selfDisplayedLarge: false}); } } handleClipboardButton() { utils.copyToClipboard(this.callUrl); this.props.notificationCenter().postSystemNotification('Join me, maybe?', {body: 'Link copied to the clipboard'}); this.setState({shareOverlayVisible: false}); } handleEmailButton(event) { // if (navigator.userAgent.indexOf('Chrome') > 0) { // let emailWindow = window.open(this.emailLink, '_blank'); // setTimeout(() => { // emailWindow.close(); // }, 500); // } else { // window.open(this.emailLink, '_self'); // } this.setState({shareOverlayVisible: false}); } handleShareOverlayEntered() { this.setState({shareOverlayVisible: true}); } handleShareOverlayExited() { this.setState({shareOverlayVisible: false}); } handleActiveSpeakerSelected(participant, secondVideo=false) { // eslint-disable-line space-infix-ops let newActiveSpeakers = this.state.activeSpeakers.slice(); if (secondVideo) { if (participant.id !== 'none') { if (newActiveSpeakers.length >= 1) { newActiveSpeakers[1] = participant; } else { newActiveSpeakers[0] = participant; } } else { newActiveSpeakers.splice(1,1); } } else { if (participant.id !== 'none') { newActiveSpeakers[0] = participant; } else { newActiveSpeakers.shift(); } } this.props.call.configureRoom(newActiveSpeakers.map((element) => element.publisherId), (error) => { if (error) { // This causes a state update, hence the drawer lists update this.logEvent.error('set speakers failed', [], this.localIdentity); } }); } handleDrop(files) { DEBUG('Dropped file %o', files); this.uploadFiles(files); }; handleFiles(e) { DEBUG('Selected files %o', e.target.files); this.uploadFiles(e.target.files); event.target.value = ''; } toggleSpeakerSelection() { this.setState({showSpeakerSelection: !this.state.showSpeakerSelection}); } startSpeakerSelection(number) { this.selectSpeaker = number; this.toggleSpeakerSelection(); } uploadFiles(files) { for (var key in files) { // is the item a File? if (files.hasOwnProperty(key) && files[key] instanceof File) { let uploadRequest; let complete = false; const filename = files[key].name let progressNotification = this.props.notificationCenter().postFileUploadProgress( filename, (notification) => { if (!complete) { uploadRequest.abort(); this.uploads.splice(this.uploads.indexOf(uploadRequest), 1); } } ); uploadRequest = superagent .post(`${config.fileSharingUrl}/${this.props.remoteUri}/${this.props.call.id}/${filename}`) .send(files[key]) .on('progress', (e) => { this.props.notificationCenter().editFileUploadNotification(e.percent, progressNotification); }) .end((err, response) => { complete = true; this.props.notificationCenter().removeFileUploadNotification(progressNotification); if (err) { this.props.notificationCenter().postFileUploadFailed(filename); } this.uploads.splice(this.uploads.indexOf(uploadRequest), 1); }); this.uploads.push([uploadRequest, progressNotification]); } } } downloadFile(filename) { // const a = document.createElement('a'); // a.href = `${config.fileSharingUrl}/${this.props.remoteUri}/${this.props.call.id}/${filename}`; // a.target = '_blank'; // a.download = filename; // const clickEvent = document.createEvent('MouseEvent'); // clickEvent.initMouseEvent('click', true, true, window, 0, // clickEvent.screenX, clickEvent.screenY, clickEvent.clientX, clickEvent.clientY, // clickEvent.ctrlKey, clickEvent.altKey, clickEvent.shiftKey, clickEvent.metaKey, // 0, null); // a.dispatchEvent(clickEvent); } preventOverlay(event) { // Stop the overlay when we are the thumbnail bar event.stopPropagation(); } muteAudio(event) { event.preventDefault(); this.props.toggleMute(this.props.call.id, !this.state.audioMuted); } toggleCamera(event) { event.preventDefault(); const localStream = this.props.call.getLocalStreams()[0]; if (localStream.getVideoTracks().length > 0) { const track = localStream.getVideoTracks()[0]; track._switchCamera(); } } muteVideo(event) { event.preventDefault(); if (this.state.videoMuted) { this._resumeVideo(); this.setState({videoMutedbyUser: false}); } else { this.setState({videoMutedbyUser: true}); this._muteVideo(); } } _muteVideo() { const localStream = this.props.call.getLocalStreams()[0]; if (localStream && localStream.getVideoTracks().length > 0) { const track = localStream.getVideoTracks()[0]; if (!this.state.videoMuted) { console.log('Mute camera'); track.enabled = false; this.setState({videoMuted: true}); } } } _resumeVideo() { const localStream = this.props.call.getLocalStreams()[0]; if (localStream && localStream.getVideoTracks().length > 0) { const track = localStream.getVideoTracks()[0]; if (this.state.videoMuted) { console.log('Resume camera'); track.enabled = true; this.setState({videoMuted: false}); } } } hangup(event) { event.preventDefault(); for (let participant of this.state.participants) { participant.detach(); } this.props.hangup('user_hangup_conference'); } armOverlayTimer() { if (this.props.audioOnly) { return; } clearTimeout(this.overlayTimer); this.overlayTimer = setTimeout(() => { this.setState({callOverlayVisible: false}); }, 4000); } showOverlay() { if (this.props.audioOnly) { return; } // if (!this.state.shareOverlayVisible && !this.state.showDrawer && !this.state.showFiles) { // if (!this.state.callOverlayVisible) { this.setState({callOverlayVisible: !this.state.callOverlayVisible}); // } // this.armOverlayTimer(); // } } toggleInviteModal() { this.setState({showInviteModal: !this.state.showInviteModal}); } toggleDrawer() { this.setState({callOverlayVisible: true, showDrawer: !this.state.showDrawer, showFiles: false, showSpeakerSelection: false}); clearTimeout(this.overlayTimer); } toggleFiles() { this.setState({callOverlayVisible: true, showFiles: !this.state.showFiles, showDrawer: false}); clearTimeout(this.overlayTimer); } showFiles() { this.setState({callOverlayVisible: true, showFiles: true, showDrawer: false}); clearTimeout(this.overlayTimer); } inviteParticipants(uris) { this.props.call.inviteParticipants(uris); uris.forEach((uri) => { uri = uri.replace(/ /g, ''); if (this.props.call.localIdentity._uri === uri) { return; } this.invitedParticipants.set(uri, {timestamp: Date.now(), status: 'Invited'}) this.props.saveParticipant(this.props.call.id, this.props.remoteUri.split('@')[0], uri); this.lookupContact(uri); }); this.forceUpdate() } render() { if (this.props.call === null) { return (); } let watermark; const largeVideoClasses = classNames({ 'animated' : true, 'fadeIn' : true, 'large' : true, 'mirror' : !this.props.call.sharingScreen && !this.props.generatedVideoTrack, 'fit' : this.props.call.sharingScreen }); let matrixClasses = classNames({ 'matrix' : true }); const containerClasses = classNames({ 'video-container': true, 'conference': true, 'drawer-visible': this.state.showDrawer || this.state.showFiles }); const remoteUri = this.props.remoteUri.split('@')[0]; // const shareOverlay = ( // // // // // Invite other online users of this service, share this link with others or email, so they can easily join this conference. // // // // // // // // // // // // ); const buttons = {}; // const commonButtonTopClasses = classNames({ // 'btn' : true, // 'btn-link' : true // }); // const fullScreenButtonIcons = classNames({ // 'fa' : true, // 'fa-2x' : true, // 'fa-expand' : !this.isFullScreen(), // 'fa-compress' : this.isFullScreen() // }); const topButtons = []; // if (!this.state.showFiles) { // if (this.state.sharedFiles.length !== 0) { // topButtons.push( // // // // ); // } // } if (!this.state.showDrawer) { topButtons.push(); } buttons.top = {right: topButtons}; const muteButtonIcons = this.state.audioMuted ? 'microphone-off' : 'microphone'; const muteVideoButtonIcons = this.state.videoMuted ? 'video-off' : 'video'; const buttonClass = (Platform.OS === 'ios') ? styles.iosButton : styles.androidButton; const bottomButtons = []; if (!this.state.reconnectingCall) { bottomButtons.push( ); } if (this.haveVideo) { bottomButtons.push( ); } bottomButtons.push( ); if (this.haveVideo) { bottomButtons.push( ); } if (!this.state.reconnectingCall) { bottomButtons.push( ) // bottomButtons.push( // // // // ); } bottomButtons.push( ); buttons.bottom = bottomButtons; const audioParticipants = []; let _contact; let _identity; let participants_uris = []; if (this.props.audioOnly) { _contact = this.foundContacts.get(this.props.call.localIdentity._uri); _identity = {uri: this.props.call.localIdentity._uri, displayName: _contact.displayName, photo: _contact.photo }; participants_uris.push(this.props.call.localIdentity._uri); audioParticipants.push( ); this.state.participants.forEach((p) => { _contact = this.foundContacts.get(p.identity._uri); _identity = {uri: p.identity._uri.indexOf('@guest') > -1 ? 'From the web': p.identity._uri, displayName: (_contact && _contact.displayName != p.identity._displayName) ? _contact.displayName : p.identity._displayName, photo: _contact ? _contact.photo: null }; participants_uris.push(p.identity._uri); let status; if (this.mediaLost.has(p.id) && this.mediaLost.get(p.id)) { status = 'Muted'; - } else if (this.packetLoss.has(p.id) && this.packetLoss.get(p.id) > 2) { + } else if (this.packetLoss.has(p.id) && this.packetLoss.get(p.id) > 3) { status = this.packetLoss.get(p.id) + '% loss'; + } else if (this.latency.has(p.id) && this.latency.get(p.id) > 200) { + status = this.latency.get(p.id) + ' ms delay'; //} else if (this.audioBandwidth.has(p.id)) { // status = this.audioBandwidth.get(p.id) + ' kbit/s'; } + audioParticipants.push( ); }); const invitedParties = Array.from(this.invitedParticipants.keys()); let p; invitedParties.forEach((_uri) => { if (participants_uris.indexOf(_uri) > 0) { return; } p = this.invitedParticipants.get(_uri); _contact = this.foundContacts.get(_uri); _identity = {uri: _uri, displayName: (_contact && _contact.displayName ) ? _contact.displayName : _uri, photo: _contact ? _contact.photo: null }; audioParticipants.push( ); }); const alreadyInvitedParticipants = this.invitedParticipants ? Array.from(this.invitedParticipants.keys()) : []; return ( {audioParticipants} {return p.identity.uri})} close={this.toggleInviteModal} room={this.props.remoteUri.split('@')[0]} defaultDomain = {this.props.defaultDomain} accountId = {this.props.call.localIdentity._uri} notificationCenter = {this.props.notificationCenter} /> {drawerParticipants} ); } const participants = []; const drawerParticipants = []; if (this.state.participants.length > 0) { if (this.state.activeSpeakers.findIndex((element) => {return element.id === this.props.call.id}) === -1) { participants.push( ); } } drawerParticipants.push( ); let videos = []; - let status; + let status = ''; if (this.state.participants.length === 0) { videos.push( ); } else { const activeSpeakers = this.state.activeSpeakers; const activeSpeakersCount = activeSpeakers.length; if (activeSpeakersCount > 0) { activeSpeakers.forEach((p) => { + status = ''; + if (this.mediaLost.has(p.id) && this.mediaLost.get(p.id)) { + status = 'Muted'; + } else if (this.packetLoss.has(p.id) && this.packetLoss.get(p.id) > 3) { + status = this.packetLoss.get(p.id) + '% loss'; + } else if (this.latency.has(p.id) && this.latency.get(p.id) > 100) { + status = this.latency.get(p.id) + ' ms delay'; + } + if (this.mediaLost.has(p.id) && this.mediaLost.get(p.id)) { status = 'Muted'; } else if (this.packetLoss.has(p.id) && this.packetLoss.get(p.id) > 2) { status = this.packetLoss.get(p.id) + '% loss'; //} else if (this.audioBandwidth.has(p.id)) { // status = this.audioBandwidth.get(p.id) + ' kbit/s'; } videos.push( ); }); this.state.participants.forEach((p) => { + status = ''; + if (this.mediaLost.has(p.id) && this.mediaLost.get(p.id)) { + status = 'Muted'; + } else if (this.packetLoss.has(p.id) && this.packetLoss.get(p.id) > 3) { + status = this.packetLoss.get(p.id) + '% loss'; + } else if (this.latency.has(p.id) && this.latency.get(p.id) > 100) { + status = this.latency.get(p.id) + ' ms delay'; + } + if (this.state.activeSpeakers.indexOf(p) === -1) { participants.push( {}} pauseVideo={true} display={false} status={status} /> ); } drawerParticipants.push( ); }); } else { this.state.participants.forEach((p, idx) => { + status = ''; + console.log(this.packetLoss); if (this.mediaLost.has(p.id) && this.mediaLost.get(p.id)) { status = 'Muted'; - } else if (this.packetLoss.has(p.id) && this.packetLoss.get(p.id) > 2) { + } else if (this.packetLoss.has(p.id) && this.packetLoss.get(p.id) > 3) { status = this.packetLoss.get(p.id) + '% loss'; - //} else if (this.audioBandwidth.has(p.id)) { - // status = this.audioBandwidth.get(p.id) + ' kbit/s'; + } else if (this.latency.has(p.id) && this.latency.get(p.id) > 100) { + status = this.latency.get(p.id) + ' ms'; } videos.push( = 4) || (idx >= 2 && this.props.isTablet === false)} isLandscape={this.props.isLandscape} isTablet={this.props.isTablet} useTwoRows={this.state.participants.length > 2} status={status} /> ); if (idx >= 4 || idx >= 2 && this.props.isTablet === false) { participants.push( ); } drawerParticipants.push( ); }); } } // let filesDrawerContent = ( // // ); const currentParticipants = this.state.participants.map((p) => {return p.identity.uri}) const alreadyInvitedParticipants = this.invitedParticipants ? Array.from(this.invitedParticipants.keys()) : []; return ( {videos} {participants} {drawerParticipants} ); } } ConferenceBox.propTypes = { notificationCenter : PropTypes.func.isRequired, call : PropTypes.object, connection : PropTypes.object, hangup : PropTypes.func, saveParticipant : PropTypes.func, previousParticipants: PropTypes.array, remoteUri : PropTypes.string, generatedVideoTrack : PropTypes.bool, toggleMute : PropTypes.func, toggleSpeakerPhone : PropTypes.func, speakerPhoneEnabled : PropTypes.bool, isLandscape : PropTypes.bool, isTablet : PropTypes.bool, muted : PropTypes.bool, defaultDomain : PropTypes.string, inFocus : PropTypes.bool, reconnectingCall : PropTypes.bool, audioOnly : PropTypes.bool, contacts : PropTypes.array, initialParticipants : PropTypes.array, terminated : PropTypes.bool }; export default ConferenceBox; diff --git a/app/components/ConferenceHeader.js b/app/components/ConferenceHeader.js index bc26378..d5b90d1 100644 --- a/app/components/ConferenceHeader.js +++ b/app/components/ConferenceHeader.js @@ -1,104 +1,110 @@ import React, { useState, useEffect, useRef, Fragment } from 'react'; import { View } from 'react-native'; import PropTypes from 'prop-types'; import moment from 'moment'; import momentFormat from 'moment-duration-format'; import { Text, Appbar } from 'react-native-paper'; import Icon from 'react-native-vector-icons/MaterialCommunityIcons'; import styles from '../assets/styles/blink/_ConferenceHeader.scss'; const useInterval = (callback, delay) => { const savedCallback = useRef(); // Remember the latest callback. useEffect(() => { savedCallback.current = callback; }, [callback]); // Set up the interval. useEffect(() => { function tick() { savedCallback.current(); } if (delay !== null) { let id = setInterval(tick, delay); return () => clearInterval(id); } }, [delay]); } const ConferenceHeader = (props) => { let [seconds, setSeconds] = useState(0); useInterval(() => { setSeconds(seconds + 1); }, 1000); - const duration = moment.duration(seconds, 'seconds').format('hh:mm:ss', {trim: false}); + let duration = moment.duration(seconds, 'seconds'); + + if (duration > 3600) { + duration = duration.format('hh:mm:ss', {trim: false}); + } else { + duration = duration.format('mm:ss', {trim: false}); + } let videoHeader; let callButtons; if (props.show) { const participantCount = props.participants.length + 1; // const callDetail = ( // // {duration} - {participantCount} participant{participantCount > 1 ? 's' : ''} // // ); const room = props.remoteUri.split('@')[0]; let callDetail; if (props.reconnectingCall) { callDetail = 'Reconnecting call...'; } else if (props.terminated) { callDetail = 'Call ended'; } else { callDetail = `${duration} - ${participantCount} participant${participantCount > 1 ? 's' : ''}`; } - if (props.bandwidth) { - callDetail = callDetail + ' - ' + props.bandwidth; + if (props.info) { + callDetail = callDetail + ' - ' + props.info; } videoHeader = ( {props.audioOnly ? null : props.buttons.top.right} ); callButtons = ( {props.buttons.bottom} ); } return ( {videoHeader} {callButtons} ); } ConferenceHeader.propTypes = { show: PropTypes.bool.isRequired, remoteUri: PropTypes.string.isRequired, participants: PropTypes.array.isRequired, buttons: PropTypes.object.isRequired, reconnectingCall: PropTypes.bool, audioOnly: PropTypes.bool, terminated: PropTypes.bool, - bandwidth: PropTypes.string + info: PropTypes.string }; export default ConferenceHeader; diff --git a/app/components/ConferenceMatrixParticipant.js b/app/components/ConferenceMatrixParticipant.js index 347182f..abe3f19 100644 --- a/app/components/ConferenceMatrixParticipant.js +++ b/app/components/ConferenceMatrixParticipant.js @@ -1,152 +1,162 @@ import React, { Component } from 'react'; import PropTypes from 'prop-types'; // const hark = require('hark'); import classNames from 'classnames'; import autoBind from 'auto-bind'; -import { Title, Badge } from 'react-native-paper'; +import { Title, Badge, Text } from 'react-native-paper'; import LinearGradient from 'react-native-linear-gradient'; import { RTCView } from 'react-native-webrtc'; import { View } from 'react-native'; import styles from '../assets/styles/blink/_ConferenceMatrixParticipant.scss'; class ConferenceMatrixParticipant extends Component { constructor(props) { super(props); autoBind(this); this.state = { active: false, hasVideo: false, sharesScreen: false, audioMuted: false, - stream: null + stream: null, + status: this.props.status } this.speechEvents = null; this.videoElement = React.createRef(); if (!props.isLocal) { props.participant.on('stateChanged', this.onParticipantStateChanged); } } + UNSAFE_componentWillReceiveProps(nextProps) { + if (nextProps.hasOwnProperty('status')) { + this.setState({status: nextProps.status}); + } + } + componentDidMount() { this.maybeAttachStream(); if (!this.props.pauseVideo && this.props.participant.videoPaused) { this.props.participant.resumeVideo(); } // this.videoElement.current.oncontextmenu = (e) => { // // disable right click for video elements // e.preventDefault(); // }; // this.videoElement.current.onresize = (event) => { // this.handleResize(event); // }; } componentWillUnmount() { if (!this.props.isLocal) { this.props.participant.removeListener('stateChanged', this.onParticipantStateChanged); } // if (this.speechEvents !== null) { // this.speechEvents.stop(); // this.speechEvents = null; // } } onParticipantStateChanged(oldState, newState) { if (newState === 'established') { this.maybeAttachStream(); } } handleResize(event) { // console.log(event.srcElement.videoWidth); const resolutions = ['1280x720', '960x540', '640x480', '640x360', '480x270', '320x180']; if (this.state.hasVideo) { const videoResolution = event.target.videoWidth + 'x' + event.target.videoHeight; if (resolutions.indexOf(videoResolution) === -1) { this.setState({sharesScreen: true}); } else { this.setState({sharesScreen: false}); } } } maybeAttachStream() { const streams = this.props.participant.streams; if (streams.length > 0) { this.setState({stream: streams[0], hasVideo: streams[0].getVideoTracks().length > 0}); // const options = { // interval: 150, // play: false // }; // this.speechEvents = hark(streams[0], options); // this.speechEvents.on('speaking', () => { // this.setState({active: true}); // }); // this.speechEvents.on('stopped_speaking', () => { // this.setState({active: false}); // }); } } render() { // const classes = classNames({ // 'poster' : !this.state.hasVideo, // 'fit' : this.state.sharesScreen // }); // const remoteVideoClasses = classNames({ // 'remote-video' : true, // 'large' : this.props.large, // 'conference-active' : this.state.active // }); + //console.log('Participant', this.props.participant.identity.uri, 'status', this.state.status); + const participantInfo = ( - {this.props.participant.identity.displayName || this.props.participant.identity.uri} {this.props.status} + {this.props.participant.identity.displayName || this.props.participant.identity.uri} + {this.state.status} ); let activeIcon; if (this.props.isLocal) { activeIcon = ( Speaker ); } let style = null; if (this.props.isTablet === true && this.props.useTwoRows) { style = styles.portraitTabletContainer; if (this.props.isLandscape) { style = styles.landscapeTabletContainer; } } return ( {activeIcon} {participantInfo} ); } } ConferenceMatrixParticipant.propTypes = { participant: PropTypes.object.isRequired, large: PropTypes.bool, isLocal: PropTypes.bool, isTablet: PropTypes.bool, isLandscape: PropTypes.bool, status: PropTypes.string }; export default ConferenceMatrixParticipant; diff --git a/app/components/VideoBox.js b/app/components/VideoBox.js index 3af209f..9ea10a6 100644 --- a/app/components/VideoBox.js +++ b/app/components/VideoBox.js @@ -1,389 +1,405 @@ import React, { Component } from 'react'; import PropTypes from 'prop-types'; import classNames from 'classnames'; import dtmf from 'react-native-dtmf'; import debug from 'react-native-debug'; import autoBind from 'auto-bind'; import { IconButton, ActivityIndicator, Colors } from 'react-native-paper'; import { View, Dimensions, TouchableWithoutFeedback, Platform } from 'react-native'; import { RTCView } from 'react-native-webrtc'; import CallOverlay from './CallOverlay'; import EscalateConferenceModal from './EscalateConferenceModal'; import DTMFModal from './DTMFModal'; import config from '../config'; import styles from '../assets/styles/blink/_VideoBox.scss'; +//import TrafficStats from './BarChart'; const DEBUG = debug('blinkrtc:Video'); debug.enable('*'); class VideoBox extends Component { constructor(props) { super(props); autoBind(this); this.state = { call: this.props.call, reconnectingCall: this.props.reconnectingCall, audioMuted: this.props.muted, mirror: true, callOverlayVisible: true, videoMuted: false, localVideoShow: true, remoteVideoShow: true, remoteSharesScreen: false, showEscalateConferenceModal: false, localStream: this.props.call.getLocalStreams()[0], remoteStream: this.props.call.getRemoteStreams()[0], - bandwidth: this.props.bandwidth, + info: this.props.info, showDtmfModal: false, - doorOpened: false + doorOpened: false, + packetLossQueue : [], + audioBandwidthQueue : [], + latencyQueue : [] }; this.overlayTimer = null; this.localVideo = React.createRef(); this.remoteVideo = React.createRef(); this.userHangup = false; } //getDerivedStateFromProps(nextProps, state) { UNSAFE_componentWillReceiveProps(nextProps) { if (nextProps.hasOwnProperty('muted')) { this.setState({audioMuted: nextProps.muted}); } - if (nextProps.hasOwnProperty('bandwidth')) { - this.setState({bandwidth: nextProps.bandwidth}); + if (nextProps.hasOwnProperty('info')) { + this.setState({info: nextProps.info}); + } + + if (nextProps.hasOwnProperty('packetLossQueue')) { + this.setState({packetLossQueue: nextProps.packetLossQueue}); + } + + if (nextProps.hasOwnProperty('audioBandwidthQueue')) { + this.setState({audioBandwidthQueue: nextProps.audioBandwidthQueue}); + } + + if (nextProps.hasOwnProperty('latencyQueue')) { + this.setState({latencyQueue: nextProps.latencyQueue}); } if (nextProps.call && nextProps.call !== this.state.call) { nextProps.call.on('stateChanged', this.callStateChanged); if (this.state.call !== null) { this.state.call.removeListener('stateChanged', this.callStateChanged); } this.setState({call: nextProps.call, localStream: nextProps.call.getLocalStreams()[0], remoteStream: nextProps.call.getRemoteStreams()[0] }); } if (nextProps.reconnectingCall != this.state.reconnectingCall) { this.setState({reconnectingCall: nextProps.reconnectingCall}); } } callStateChanged(oldState, newState, data) { this.forceUpdate(); } openDoor() { const tone = this.props.intercomDtmfTone; DEBUG('DTMF tone sent to intercom: ' + tone); this.setState({doorOpened: true}); this.forceUpdate(); dtmf.stopTone(); //don't play a tone at the same time as another dtmf.playTone(dtmf['DTMF_' + tone], 1000); if (this.state.call !== null && this.state.call.state === 'established') { this.state.call.sendDtmf(tone); } } componentDidMount() { if (this.state.call) { this.state.call.on('stateChanged', this.callStateChanged); } this.armOverlayTimer(); } componentWillUnmount() { if (this.state.call != null) { this.state.call.removeListener('stateChanged', this.callStateChanged); } } showDtmfModal() { this.setState({showDtmfModal: true}); } hideDtmfModal() { this.setState({showDtmfModal: false}); } handleFullscreen(event) { event.preventDefault(); // this.toggleFullscreen(); } handleRemoteVideoPlaying() { this.setState({remoteVideoShow: true}); } handleRemoteResize(event, target) { const resolutions = [ '1280x720', '960x540', '640x480', '640x360', '480x270','320x180']; const videoResolution = event.target.videoWidth + 'x' + event.target.videoHeight; if (resolutions.indexOf(videoResolution) === -1) { this.setState({remoteSharesScreen: true}); } else { this.setState({remoteSharesScreen: false}); } } muteAudio(event) { event.preventDefault(); this.props.toggleMute(this.state.call.id, !this.state.audioMuted); } muteVideo(event) { event.preventDefault(); const localStream = this.state.localStream; if (localStream.getVideoTracks().length > 0) { const track = localStream.getVideoTracks()[0]; if(this.state.videoMuted) { DEBUG('Unmute camera'); track.enabled = true; this.setState({videoMuted: false}); } else { DEBUG('Mute camera'); track.enabled = false; this.setState({videoMuted: true}); } } } toggleCamera(event) { event.preventDefault(); const localStream = this.state.localStream; if (localStream.getVideoTracks().length > 0) { const track = localStream.getVideoTracks()[0]; track._switchCamera(); this.setState({mirror: !this.state.mirror}); } } hangupCall(event) { event.preventDefault(); this.props.hangupCall('user_hangup_call'); this.userHangup = true; } cancelCall(event) { event.preventDefault(); this.props.hangupCall('user_cancelled_call'); } escalateToConference(participants) { this.props.escalateToConference(participants); } armOverlayTimer() { clearTimeout(this.overlayTimer); this.overlayTimer = setTimeout(() => { this.setState({callOverlayVisible: false}); }, 4000); } toggleCallOverlay() { this.setState({callOverlayVisible: !this.state.callOverlayVisible}); } toggleEscalateConferenceModal() { this.setState({ callOverlayVisible : false, showEscalateConferenceModal : !this.state.showEscalateConferenceModal }); } render() { if (this.state.call === null) { return null; } // 'mirror' : !this.state.call.sharingScreen && !this.props.generatedVideoTrack, // we do not want mirrored local video once the call has started, just in preview const localVideoClasses = classNames({ 'video-thumbnail' : true, 'hidden' : !this.state.localVideoShow, 'animated' : true, 'fadeIn' : this.state.localVideoShow || this.state.videoMuted, 'fadeOut' : this.state.videoMuted, 'fit' : this.state.call.sharingScreen }); const remoteVideoClasses = classNames({ 'poster' : !this.state.remoteVideoShow, 'animated' : true, 'fadeIn' : this.state.remoteVideoShow, 'large' : true, 'fit' : this.state.remoteSharesScreen }); let buttonContainerClass; let buttons; const muteButtonIcons = this.state.audioMuted ? 'microphone-off' : 'microphone'; const muteVideoButtonIcons = this.state.videoMuted ? 'video-off' : 'video'; const buttonClass = (Platform.OS === 'ios') ? styles.iosButton : styles.androidButton; const buttonSize = this.props.isTablet ? 40 : 34; if (this.props.isTablet) { buttonContainerClass = this.props.orientation === 'landscape' ? styles.tabletLandscapeButtonContainer : styles.tabletPortraitButtonContainer; userIconContainerClass = styles.tabletUserIconContainer; } else { buttonContainerClass = this.props.orientation === 'landscape' ? styles.landscapeButtonContainer : styles.portraitButtonContainer; } if (this.state.callOverlayVisible) { let content = ( ); if (this.props.intercomDtmfTone) { content = ( ); } buttons = ({content}); } const remoteStreamUrl = this.state.remoteStream ? this.state.remoteStream.toURL() : null const show = this.state.callOverlayVisible || this.state.reconnectingCall; return ( {this.state.remoteVideoShow && !this.state.reconnectingCall ? : null } { this.state.localVideoShow ? : null } {this.state.reconnectingCall ? : null } {buttons} ); } } VideoBox.propTypes = { call : PropTypes.object, connection : PropTypes.object, photo : PropTypes.string, accountId : PropTypes.string, remoteUri : PropTypes.string, remoteDisplayName : PropTypes.string, localMedia : PropTypes.object, hangupCall : PropTypes.func, - bandwidth : PropTypes.string, + info : PropTypes.string, shareScreen : PropTypes.func, escalateToConference : PropTypes.func, generatedVideoTrack : PropTypes.bool, callKeepSendDtmf : PropTypes.func, toggleMute : PropTypes.func, toggleSpeakerPhone : PropTypes.func, speakerPhoneEnabled : PropTypes.bool, intercomDtmfTone : PropTypes.string, orientation : PropTypes.string, isTablet : PropTypes.bool, reconnectingCall : PropTypes.bool, muted : PropTypes.bool }; export default VideoBox;