enable audioswitch, use a payloader to transfer buffer timestamps through the shm-bridge and keep a/v sync intact

6 files changed, 151 insertions, 59 deletions

diff --git a/voctocore/lib/audiomix.py b/voctocore/lib/audiomix.py
new file mode 100644
index 0000000..326e7a5
--- /dev/null
+++ b/voctocore/lib/audiomix.py
@@ -0,0 +1,36 @@
+#!/usr/bin/python3
+import time, logging
+from gi.repository import GLib, Gst
+
+from lib.config import Config
+
+class AudioMix(Gst.Bin):
+	log = logging.getLogger('AudioMix')
+	mixerpads = []
+
+	def __init__(self):
+		super().__init__()
+
+		self.switch = Gst.ElementFactory.make('input-selector', 'switch')
+
+		self.add(self.switch)
+		self.switch.set_property('sync-streams', True)
+		self.switch.set_property('sync-mode', 1) #GST_INPUT_SELECTOR_SYNC_MODE_CLOCK
+		self.switch.set_property('cache-buffers', True)
+
+		self.add_pad(
+			Gst.GhostPad.new('src', self.switch.get_static_pad('src'))
+		)
+
+	def request_mixer_pad(self):
+		mixerpad = self.switch.get_request_pad('sink_%u')
+		self.mixerpads.append(mixerpad)
+
+		self.log.info('requested mixerpad %u (named %s)', len(self.mixerpads) - 1, mixerpad.get_name())
+		ghostpad = Gst.GhostPad.new(mixerpad.get_name(), mixerpad)
+		self.add_pad(ghostpad)
+		return ghostpad
+
+	def set_active(self, target):
+		self.log.info('switching to audiosource %u', target)
+		self.switch.set_property('active-pad', self.mixerpads[target])
diff --git a/voctocore/lib/failaudiosrc.py b/voctocore/lib/failaudiosrc.py
new file mode 100644
index 0000000..0ce885e
--- /dev/null
+++ b/voctocore/lib/failaudiosrc.py
@@ -0,0 +1,25 @@
+#!/usr/bin/python3
+import time, logging
+from gi.repository import GLib, Gst
+
+from lib.config import Config
+
+class FailAudioSrc(Gst.Bin):
+	log = logging.getLogger('FailAudioSrc')
+
+	def __init__(self, idx, name):
+		super().__init__()
+
+		# Create elements
+		self.failsrc = Gst.ElementFactory.make('audiotestsrc', None)
+
+		# Add elements to Bin
+		self.add(self.failsrc)
+
+		# Set properties
+		self.failsrc.set_property('freq', 400+idx*50)
+
+		# Add Ghost Pads
+		self.add_pad(
+			Gst.GhostPad.new('src', self.failsrc.get_static_pad('src'))
+		)
diff --git a/voctocore/lib/pipeline.py b/voctocore/lib/pipeline.py
index 529aeca..6267e9e 100644
--- a/voctocore/lib/pipeline.py
+++ b/voctocore/lib/pipeline.py
@@ -9,10 +9,11 @@ from lib.controlserver import controlServerEntrypoint
 from lib.config import Config
 from lib.quadmix import QuadMix
 from lib.videomix import VideoMix
-# from lib.audiomix import AudioMix
+from lib.audiomix import AudioMix
 from lib.distributor import TimesTwoDistributor
 from lib.shmsrc import FailsafeShmSrc
 from lib.failvideosrc import FailVideoSrc
+from lib.failaudiosrc import FailAudioSrc
 
 class Pipeline(Gst.Pipeline):
 	"""mixing, streaming and encoding pipeline constuction and control"""
@@ -31,8 +32,8 @@ class Pipeline(Gst.Pipeline):
 		self.videomixer = VideoMix()
 		self.add(self.videomixer)
 
-		# self.audiomixer = AudioMix()
-		# self.add(self.audiomixer)
+		self.audiomixer = AudioMix()
+		self.add(self.audiomixer)
 
 		# read the path where the shm-control-sockets are located and ensure it exists
 		socketpath = Config.get('sources', 'socketpath')
@@ -44,13 +45,16 @@ class Pipeline(Gst.Pipeline):
 				raise
 
 		self.videonames = Config.getlist('sources', 'video')
-		self.audionames = Config.getlist('sources', 'video')
+		self.audionames = Config.getlist('sources', 'audio')
+
+		caps = Gst.Caps.from_string(Config.get('sources', 'videocaps'))
+		self.log.debug('parsing videocaps from config: %s', caps.to_string())
 
 		for idx, name in enumerate(self.videonames):
 			socket = os.path.join(socketpath, 'v-'+name)
 
 			self.log.info('Creating video-source "%s" at socket-path %s', name, socket)
-			sourcebin = FailsafeShmSrc(socket, FailVideoSrc(idx, name))
+			sourcebin = FailsafeShmSrc(socket, caps, FailVideoSrc(idx, name))
 			self.add(sourcebin)
 
 			distributor = TimesTwoDistributor()
@@ -63,32 +67,63 @@ class Pipeline(Gst.Pipeline):
 			mixerpad = self.videomixer.request_mixer_pad()
 			distributor.get_static_pad('src_b').link(mixerpad)
 
-		# for audiosource in Config.getlist('sources', 'audio'):
-		# 	sourcebin = FailsafeShmSrc(os.path.join(socketpath, audiosource))
 
-		# 	self.add(sourcebin)
-		# 	sourcebin.link(self.audiomixer)
+		caps = Gst.Caps.from_string(Config.get('sources', 'audiocaps'))
+		self.log.debug('parsing videocaps from config: %s', caps.to_string())
+
+		for idx, name in enumerate(self.audionames):
+			socket = os.path.join(socketpath, 'a-'+name)
+
+			self.log.info('Creating audio-source "%s" at socket-path %s', name, socket)
+			sourcebin = FailsafeShmSrc(socket, caps, FailAudioSrc(idx, name))
+			self.add(sourcebin)
+
+			mixerpad = self.audiomixer.request_mixer_pad()
+			sourcebin.get_static_pad('src').link(mixerpad)
 
 		# tell the quadmix that this were all sources and no more sources will come after this
 		self.quadmixer.finalize()
 
 		self.quadmixer.set_active(0)
 		self.videomixer.set_active(0)
+		self.audiomixer.set_active(0)
+
+		self.audioconv = Gst.ElementFactory.make('audioconvert', 'audioconv')
+		self.audioenc = Gst.ElementFactory.make('avenc_mp2', 'audioenc')
+
+		self.videoconv = Gst.ElementFactory.make('videoconvert', 'videoconv')
+		self.videoenc = Gst.ElementFactory.make('avenc_mpeg2video', 'videoenc')
+
+		self.mux = Gst.ElementFactory.make('mpegtsmux', 'mux')
+		self.sink = Gst.ElementFactory.make('filesink', 'sink')
+
+		self.add(self.audioconv)
+		self.add(self.audioenc)
+		self.add(self.videoconv)
+		self.add(self.videoenc)
+
+		self.add(self.mux)
+		self.add(self.sink)
+
+		self.videomixer.link(self.videoconv)
+		self.videoconv.link(self.videoenc)
+
+		self.audiomixer.link(self.audioconv)
+		self.audioconv.link(self.audioenc)
+
+		self.videoenc.link(self.mux)
+		self.audioenc.link(self.mux)
+
+		self.mux.link(self.sink)
+
+		self.sink.set_property('location', '/home/peter/test.ts')
+
 
 		self.quadmixsink = Gst.ElementFactory.make('autovideosink', 'quadmixsink')
 		self.quadmixsink.set_property('sync', False)
 		self.add(self.quadmixsink)
 		self.quadmixer.link(self.quadmixsink)
 
-		self.videosink = Gst.ElementFactory.make('autovideosink', 'videosink')
-		self.videosink.set_property('sync', False)
-		self.add(self.videosink)
-		self.videomixer.link(self.videosink)
-
-		# self.audiosink = Gst.ElementFactory.make('autoaudiosink', 'audiosink')
-		# self.add(self.audiosink)
-		# self.audiomixer.link(self.audiosink)
-
 	def run(self):
 		self.set_state(Gst.State.PAUSED)
 		time.sleep(0.5)
@@ -327,7 +362,13 @@ class Pipeline(Gst.Pipeline):
 	@controlServerEntrypoint
 	def switchAudio(self, audiosource):
 		"""switch audio to the selected audio"""
-		raise NotImplementedError("audio is not implemented yet")
+		idx = int(audiosource)
+		if idx >= len(self.audionames):
+			return 'unknown audio-source: %s' % (audiosource)
+
+		self.log.info("switching mixer to audio-source %u", idx)
+		self.audiomixer.set_active(idx)
+
 
 
 	@controlServerEntrypoint
@@ -339,23 +380,12 @@ class Pipeline(Gst.Pipeline):
 
 	@controlServerEntrypoint
 	def switchVideo(self, videosource):
-		"""switch audio to the selected video"""
-		if videosource.isnumeric():
-			idx = int(videosource)
-			self.log.info("interpreted input as videosource-index %u", idx)
-			if idx >= len(self.videonames):
-				idx = None
-		else:
-			try:
-				idx = self.videonames.index(videosource)
-				self.log.info("interpreted input as videosource-name, lookup to %u", idx)
-			except IndexError:
-				idx = None
-
-		if idx == None:
+		"""switch video to the selected video"""
+		idx = int(videosource)
+		if idx >= len(self.videonames):
 			return 'unknown video-source: %s' % (videosource)
 
-		self.log.info("switching quadmix to video-source %u", idx)
+		self.log.info("switching mixer to video-source %u", idx)
 		self.quadmixer.set_active(idx)
 		self.videomixer.set_active(idx)
 
diff --git a/voctocore/lib/shmsrc.py b/voctocore/lib/shmsrc.py
index 1158887..4e3c402 100644
--- a/voctocore/lib/shmsrc.py
+++ b/voctocore/lib/shmsrc.py
@@ -10,24 +10,24 @@ class FailsafeShmSrc(Gst.Bin):
 	last_restart_retry = 0
 	is_in_failstate = True
 
-	def __init__(self, socket, failsrc):
+	def __init__(self, socket, caps, failsrc):
 		super().__init__()
 
-		caps = Gst.Caps.from_string(Config.get('sources', 'videocaps'))
-		self.log.debug('parsing videocaps from config: %s', caps.to_string())
-
 		# Create elements
 		self.shmsrc = Gst.ElementFactory.make('shmsrc', None)
+		self.depay = Gst.ElementFactory.make('gdpdepay', None)
 		self.capsfilter = Gst.ElementFactory.make('capsfilter', None)
 		self.failsrcsyncer = Gst.ElementFactory.make('identity', None)
 		self.switch = Gst.ElementFactory.make('input-selector', None)
-		self.failsrc = failsrc;
+		self.failsrc = failsrc
+		self.capsstr = caps.to_string()
 
 		if not self.shmsrc or not self.capsfilter or not self.failsrcsyncer or not self.switch or not self.failsrc:
 			self.log.error('could not create elements')
 
 		# Add elements to Bin
 		self.add(self.shmsrc)
+		self.add(self.depay)
 		self.add(self.capsfilter)
 		self.add(self.failsrcsyncer)
 		self.add(self.switch)
@@ -39,14 +39,13 @@ class FailsafeShmSrc(Gst.Bin):
 
 		# Set properties
 		self.shmsrc.set_property('socket-path', socket)
-		self.shmsrc.set_property('is-live', True)
-		self.shmsrc.set_property('do-timestamp', True)
+		self.shmsrc.link(self.depay)
 		self.switch.set_property('active-pad', self.failpad)
 		self.failsrcsyncer.set_property('sync', True)
 		self.capsfilter.set_property('caps', caps)
 
 		# Link elements
-		self.shmsrc.link(self.capsfilter)
+		self.depay.link(self.capsfilter)
 		self.capsfilter.get_static_pad('src').link(self.goodpad)
 
 		self.failsrc.link_filtered(self.failsrcsyncer, caps)
@@ -57,7 +56,12 @@ class FailsafeShmSrc(Gst.Bin):
 		self.shmsrc.get_static_pad('src').add_probe(Gst.PadProbeType.BLOCK | Gst.PadProbeType.BUFFER, self.data_probe, None)
 
 		# Install Watchdog
-		GLib.timeout_add(250, self.watchdog)
+		if self.capsstr.startswith('audio'):
+			timeoutms = 1000
+		else:
+			timeoutms = 250
+
+		GLib.timeout_add(timeoutms, self.watchdog)
 
 		# Add Ghost Pads
 		self.add_pad(
@@ -81,7 +85,6 @@ class FailsafeShmSrc(Gst.Bin):
 		
 		return Gst.PadProbeReturn.PASS
 
-
 	def data_probe(self, pad, info, ud):
 		self.last_buffer_arrived = time.time()
 		self.switch_to_goodstate()
diff --git a/voctocore/scripts/videosrc1.sh b/voctocore/scripts/videosrc1.sh
index 39e5212..fe34b16 100755
--- a/voctocore/scripts/videosrc1.sh
+++ b/voctocore/scripts/videosrc1.sh
@@ -1,8 +1,9 @@
 #!/bin/sh
+mkdir -p /tmp/voctomix-sockets/
 rm -f /tmp/voctomix-sockets/v-cam1 /tmp/voctomix-sockets/a-cam1
 gst-launch-1.0 -v \
 	uridecodebin \
-		uri=file:///home/peter/ED_1280.avi \
+		uri=file:///home/peter/avsync.mp4 \
 		name=src \
 	\
 	src. !\
@@ -12,20 +13,18 @@ gst-launch-1.0 -v \
 	videorate !\
 	videoscale !\
 	video/x-raw,format=RGBx,width=1280,height=720,framerate=25/1 !\
+	gdppay !\
 	shmsink \
-		sync=true \
 		socket-path=/tmp/voctomix-sockets/v-cam1 \
-		wait-for-connection=false \
 		shm-size=100000000 \
 	\
 	src. !\
 	queue !\
+	audioresample !\
 	audioconvert !\
-	audiorate !\
-	audio/x-raw,format=F32LE,layout=interleaved,channels=2,rate=48000 !\
+	audio/x-raw,format=S16LE,layout=interleaved,rate=48000,channels=2 !\
+	gdppay !\
 	shmsink \
-		sync=true \
 		socket-path=/tmp/voctomix-sockets/a-cam1 \
-		wait-for-connection=false \
 		shm-size=10000000
 
diff --git a/voctocore/scripts/videosrc2.sh b/voctocore/scripts/videosrc2.sh
index e136495..1c5bc56 100755
--- a/voctocore/scripts/videosrc2.sh
+++ b/voctocore/scripts/videosrc2.sh
@@ -1,8 +1,9 @@
 #!/bin/sh
+mkdir -p /tmp/voctomix-sockets/
 rm -f /tmp/voctomix-sockets/v-cam2 /tmp/voctomix-sockets/a-cam2
 gst-launch-1.0 -v \
 	uridecodebin \
-		uri=file:///home/peter/Sintel.2010.720p.mkv \
+		uri=file:///home/peter/ED_1280.avi \
 		name=src \
 	\
 	src. !\
@@ -11,20 +12,18 @@ gst-launch-1.0 -v \
 	videoconvert !\
 	videorate !\
 	videoscale !\
-	video/x-raw,format=RGBx,width=1280,height=720,framerate=25/1 !\
+	video/x-raw,format=RGBx,width=1280,height=720,framerate=25/1,pixel-aspect-ratio=1/1 !\
+	gdppay !\
 	shmsink \
-		sync=true \
 		socket-path=/tmp/voctomix-sockets/v-cam2 \
-		wait-for-connection=false \
 		shm-size=100000000 \
 	\
 	src. !\
 	queue !\
+	audioresample !\
 	audioconvert !\
-	audiorate !\
-	audio/x-raw,format=F32LE,layout=interleaved,channels=2,rate=48000 !\
+	audio/x-raw,format=S16LE,layout=interleaved,rate=48000,channels=2 !\
+	gdppay !\
 	shmsink \
-		sync=true \
 		socket-path=/tmp/voctomix-sockets/a-cam2 \
-		wait-for-connection=false \
 		shm-size=10000000