Fix VP9 out of order packets forwarding

worker/include/RTC/Codecs/PayloadDescriptorHandler.hpp

@@ -2,6 +2,8 @@
 #define MS_RTC_CODECS_PAYLOAD_DESCRIPTOR_HANDLER_HPP
 
 #include "common.hpp"
+#include "RTC/SeqManager.hpp"
+#include <deque>
 
 namespace RTC
 {
@@ -14,6 +16,65 @@ namespace RTC
 			virtual void Dump() const    = 0;
 		};
 
+		class PictureIdList
+		{
+			static constexpr uint16_t MaxCurrentLayerPictureIdNum{ 1000u };
+
+		public:
+			explicit PictureIdList()
+			{
+			}
+
+			~PictureIdList()
+			{
+				this->list.clear();
+			}
+
+			void Push(uint16_t pictureId, int16_t layer)
+			{
+				for (const auto& it : this->list)
+				{
+					// Layers can be changed only with ordered pictureId values.
+					// If pictureId is lower than the previous one, then it has rolled over the max value.
+					uint16_t diff = pictureId > it.first
+					                  ? pictureId - it.first
+					                  : pictureId + RTC::SeqManager<uint16_t, 15>::MaxValue - it.first;
+
+					if (diff > MaxCurrentLayerPictureIdNum)
+					{
+						this->list.pop_front();
+					}
+					else
+					{
+						break;
+					}
+				}
+
+				this->list.push_back({ pictureId, layer });
+			}
+
+			int16_t GetLayer(uint16_t pictureId) const
+			{
+				if (this->list.size() <= 1)
+				{
+					return -1;
+				}
+
+				for (auto it = std::next(this->list.begin()); it != this->list.end(); ++it)
+				{
+					if (RTC::SeqManager<uint16_t, 15>::IsSeqHigherThan(it->first, pictureId))
+					{
+						return std::prev(it)->second;
+					}
+				}
+
+				return -1;
+			}
+
+		private:
+			std::deque<std::pair<uint16_t, int16_t>> list;
+		};
+
 		// Encoding context used by PayloadDescriptorHandler to properly rewrite the
 		// PayloadDescriptor.
 		class EncodingContext
@@ -87,13 +148,60 @@ namespace RTC
 			}
 			virtual void SyncRequired() = 0;
 
+			void SetCurrentSpatialLayer(int16_t spatialLayer, uint16_t pictureId)
+			{
+				if (this->currentSpatialLayer == spatialLayer)
+				{
+					return;
+				}
+
+				this->spatialLayerPictureIdList.Push(pictureId, spatialLayer);
+
+				this->currentSpatialLayer = spatialLayer;
+			}
+			void SetCurrentTemporalLayer(int16_t temporalLayer, uint16_t pictureId)
+			{
+				if (this->currentTemporalLayer == temporalLayer)
+				{
+					return;
+				}
+
+				this->temporalLayerPictureIdList.Push(pictureId, temporalLayer);
+
+				this->currentTemporalLayer = temporalLayer;
+			}
+			int16_t GetSpatialLayerForPictureId(uint16_t pictureId) const
+			{
+				int16_t layer = this->spatialLayerPictureIdList.GetLayer(pictureId);
+				if (layer > -1)
+				{
+					return layer;
+				}
+
+				return this->currentSpatialLayer;
+			}
+			int16_t GetTemporalLayerForPictureId(uint16_t pictureId) const
+			{
+				int16_t layer = this->temporalLayerPictureIdList.GetLayer(pictureId);
+				if (layer > -1)
+				{
+					return layer;
+				}
+
+				return this->currentTemporalLayer;
+			}
+
 		private:
 			Params params;
 			int16_t targetSpatialLayer{ -1 };
 			int16_t targetTemporalLayer{ -1 };
 			int16_t currentSpatialLayer{ -1 };
 			int16_t currentTemporalLayer{ -1 };
 			bool ignoreDtx{ false };
+
+		private:
+			PictureIdList spatialLayerPictureIdList;
+			PictureIdList temporalLayerPictureIdList;
 		};
 
 		class PayloadDescriptorHandler

worker/src/RTC/Codecs/VP9.cpp

@@ -241,35 +241,16 @@ namespace RTC
 			);
 			// clang-format on
 
-			// Upgrade current spatial layer if needed.
-			if (context->GetTargetSpatialLayer() > context->GetCurrentSpatialLayer())
-			{
-				if (this->payloadDescriptor->isKeyFrame)
-				{
-					MS_DEBUG_DEV(
-					  "upgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8 ":%" PRIu8
-					  ")",
-					  context->GetCurrentSpatialLayer(),
-					  context->GetTargetSpatialLayer(),
-					  packetSpatialLayer,
-					  packetTemporalLayer);
-
-					tmpSpatialLayer  = context->GetTargetSpatialLayer();
-					tmpTemporalLayer = 0; // Just in case.
-				}
-			}
-			// Downgrade current spatial layer if needed.
-			else if (context->GetTargetSpatialLayer() < context->GetCurrentSpatialLayer())
+			if (!isOldPacket)
 			{
-				// In K-SVC we must wait for a keyframe.
-				if (context->IsKSvc())
+				// Upgrade current spatial layer if needed.
+				if (context->GetTargetSpatialLayer() > context->GetCurrentSpatialLayer())
 				{
 					if (this->payloadDescriptor->isKeyFrame)
-					// clang-format on
 					{
 						MS_DEBUG_DEV(
-						  "downgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8
-						  ":%" PRIu8 ") after keyframe (K-SVC)",
+						  "upgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8 ":%" PRIu8
+						  ")",
 						  context->GetCurrentSpatialLayer(),
 						  context->GetTargetSpatialLayer(),
 						  packetSpatialLayer,
@@ -279,41 +260,63 @@ namespace RTC
 						tmpTemporalLayer = 0; // Just in case.
 					}
 				}
-				// In full SVC we do not need a keyframe.
-				else
+				// Downgrade current spatial layer if needed.
+				else if (context->GetTargetSpatialLayer() < context->GetCurrentSpatialLayer())
 				{
-					// clang-format off
-					if (
-						packetSpatialLayer == context->GetTargetSpatialLayer() &&
-						this->payloadDescriptor->e
-					)
-					// clang-format on
+					// In K-SVC we must wait for a keyframe.
+					if (context->IsKSvc())
 					{
-						MS_DEBUG_DEV(
-						  "downgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8
-						  ":%" PRIu8 ") without keyframe (full SVC)",
-						  context->GetCurrentSpatialLayer(),
-						  context->GetTargetSpatialLayer(),
-						  packetSpatialLayer,
-						  packetTemporalLayer);
-
-						tmpSpatialLayer  = context->GetTargetSpatialLayer();
-						tmpTemporalLayer = 0; // Just in case.
+						if (this->payloadDescriptor->isKeyFrame)
+						// clang-format on
+						{
+							MS_DEBUG_DEV(
+							  "downgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8
+							  ":%" PRIu8 ") after keyframe (K-SVC)",
+							  context->GetCurrentSpatialLayer(),
+							  context->GetTargetSpatialLayer(),
+							  packetSpatialLayer,
+							  packetTemporalLayer);
+
+							tmpSpatialLayer  = context->GetTargetSpatialLayer();
+							tmpTemporalLayer = 0; // Just in case.
+						}
+					}
+					// In full SVC we do not need a keyframe.
+					else
+					{
+						// clang-format off
+						if (
+							packetSpatialLayer == context->GetTargetSpatialLayer() &&
+							this->payloadDescriptor->e
+						)
+						// clang-format on
+						{
+							MS_DEBUG_DEV(
+							  "downgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8
+							  ":%" PRIu8 ") without keyframe (full SVC)",
+							  context->GetCurrentSpatialLayer(),
+							  context->GetTargetSpatialLayer(),
+							  packetSpatialLayer,
+							  packetTemporalLayer);
+
+							tmpSpatialLayer  = context->GetTargetSpatialLayer();
+							tmpTemporalLayer = 0; // Just in case.
+						}
 					}
 				}
 			}
 
-			// Unless old packet filter spatial layers that are either
+			// Filter spatial layers that are either
 			// * higher than current one
 			// * different than the current one when KSVC is enabled and this is not a keyframe
 			// (interframe p bit = 1)
+			uint16_t spatialLayerForPictureId =
+			  isOldPacket ? context->GetSpatialLayerForPictureId(this->payloadDescriptor->pictureId)
+			              : tmpSpatialLayer;
 			// clang-format off
 			if (
-			  !isOldPacket &&
-			  (
-			  	packetSpatialLayer > tmpSpatialLayer ||
-			  	(context->IsKSvc() && this->payloadDescriptor->p && packetSpatialLayer != tmpSpatialLayer)
-			  )
+				packetSpatialLayer > spatialLayerForPictureId ||
+				(context->IsKSvc() && this->payloadDescriptor->p && packetSpatialLayer != spatialLayerForPictureId)
 			)
 			// clang-format on
 			{
@@ -369,12 +372,15 @@ namespace RTC
 						tmpTemporalLayer = context->GetTargetTemporalLayer();
 					}
 				}
+			}
 
-				// Filter temporal layers higher than current one.
-				if (packetTemporalLayer > tmpTemporalLayer)
-				{
-					return false;
-				}
+			// Filter temporal layers higher than current one.
+			uint16_t temporalLayerForPictureId =
+			  isOldPacket ? context->GetTemporalLayerForPictureId(this->payloadDescriptor->pictureId)
+			              : tmpTemporalLayer;
+			if (packetTemporalLayer > temporalLayerForPictureId)
+			{
+				return false;
 			}
 
 			// Set marker bit if needed.
@@ -394,13 +400,13 @@ namespace RTC
 			// Update current spatial layer if needed.
 			if (tmpSpatialLayer != context->GetCurrentSpatialLayer())
 			{
-				context->SetCurrentSpatialLayer(tmpSpatialLayer);
+				context->SetCurrentSpatialLayer(tmpSpatialLayer, this->payloadDescriptor->pictureId);
 			}
 
 			// Update current temporal layer if needed.
 			if (tmpTemporalLayer != context->GetCurrentTemporalLayer())
 			{
-				context->SetCurrentTemporalLayer(tmpTemporalLayer);
+				context->SetCurrentTemporalLayer(tmpTemporalLayer, this->payloadDescriptor->pictureId);
 			}
 
 			return true;