From cd6aaa1393cbc623d8040c9f5fdd0604ec40f028 Mon Sep 17 00:00:00 2001 From: Valentin Cocaud Date: Wed, 19 May 2021 14:39:54 +0200 Subject: [PATCH] Force key frame on PLI Also make the ReadCloser an Controllable allows to uncouple the controller implementation from the encoder. This is not needed for the 2 codec controller already implemented (openh264 and vpx) but is more future proof in case it required for other codecs. --- .gitignore | 2 + examples/go.sum | 59 ++++++++++++++++++----------- go.mod | 2 + ioreader.go | 24 +++++++++--- pkg/codec/codec.go | 28 ++++++++++++-- pkg/codec/mmal/mmal.go | 8 +--- pkg/codec/mmal/mmal_test.go | 19 ++++++++++ pkg/codec/openh264/openh264.go | 8 ++-- pkg/codec/openh264/openh264_test.go | 17 +++++++++ pkg/codec/opus/opus.go | 4 +- pkg/codec/opus/opus_test.go | 17 +++++++++ pkg/codec/vaapi/vp8.go | 8 +--- pkg/codec/vaapi/vp8_test.go | 27 +++++++++++++ pkg/codec/vaapi/vp9.go | 8 +--- pkg/codec/vaapi/vp9_test.go | 27 +++++++++++++ pkg/codec/vpx/vpx.go | 8 ++-- pkg/codec/vpx/vpx_test.go | 18 ++++++++- pkg/codec/x264/x264.go | 9 +++-- pkg/codec/x264/x264_test.go | 19 ++++++++++ rtpreader.go | 15 ++++++-- track.go | 44 ++++++++++++++++++++- 21 files changed, 304 insertions(+), 67 deletions(-) create mode 100644 pkg/codec/vaapi/vp8_test.go create mode 100644 pkg/codec/vaapi/vp9_test.go diff --git a/.gitignore b/.gitignore index e4588f24..64b73a94 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ scripts/cross coverage.txt + +.idea diff --git a/examples/go.sum b/examples/go.sum index 5eab9fce..7cb3cbe4 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -13,6 +13,7 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/gen2brain/malgo v0.10.29 h1:bTYiUTUKJsEomNby+W0hgyLrOttUXIk4lTEnKA54iqM= github.com/gen2brain/malgo v0.10.29/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs= github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -21,6 +22,7 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -36,18 +38,19 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg= github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXmf0= github.com/pion/datachannel v1.4.21/go.mod h1:oiNyP4gHx2DIwRzX/MFyH0Rz/Gz05OgBlayAI2hAWjg= -github.com/pion/dtls/v2 v2.0.8 h1:reGe8rNIMfO/UAeFLqO61tl64t154Qfkr4U3Gzu1tsg= -github.com/pion/dtls/v2 v2.0.8/go.mod h1:QuDII+8FVvk9Dp5t5vYIMTo7hh7uBkra+8QIm7QGm10= -github.com/pion/ice/v2 v2.0.16 h1:K6bzD8ef9vMKbGMTHaUweHXEyuNGnvr2zdqKoLKZPn0= -github.com/pion/ice/v2 v2.0.16/go.mod h1:SJNJzC27gDZoOW0UoxIoC8Hf2PDxG28hQyNdSexDu38= +github.com/pion/dtls/v2 v2.0.9 h1:7Ow+V++YSZQMYzggI0P9vLJz/hUFcffsfGMfT/Qy+u8= +github.com/pion/dtls/v2 v2.0.9/go.mod h1:O0Wr7si/Zj5/EBFlDzDd6UtVxx25CE1r7XM7BQKYQho= +github.com/pion/ice/v2 v2.1.7 h1:FjgDfUNrVYTxQabJrkBX6ld12tvYbgzHenqPh3PJF6E= +github.com/pion/ice/v2 v2.1.7/go.mod h1:kV4EODVD5ux2z8XncbLHIOtcXKtYXVgLVCeVqnpoeP0= github.com/pion/interceptor v0.0.12 h1:eC1iVneBIAQJEfaNAfDqAncJWhMDAnaXPRCJsltdokE= github.com/pion/interceptor v0.0.12/go.mod h1:qzeuWuD/ZXvPqOnxNcnhWfkCZ2e1kwwslicyyPnhoK4= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -60,6 +63,8 @@ github.com/pion/rtcp v1.2.6 h1:1zvwBbyd0TeEuuWftrd/4d++m+/kZSeiguxU61LFWpo= github.com/pion/rtcp v1.2.6/go.mod h1:52rMNPWFsjr39z9B9MhnkqhPLoeHTv1aN63o/42bWE0= github.com/pion/rtp v1.6.2 h1:iGBerLX6JiDjB9NXuaPzHyxHFG9JsIEdgwTC0lp5n/U= github.com/pion/rtp v1.6.2/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= +github.com/pion/rtp v1.6.5 h1:o2cZf8OascA5HF/b0PAbTxRKvOWxTQxWYt7SlToxFGI= +github.com/pion/rtp v1.6.5/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0= github.com/pion/sctp v1.7.12 h1:GsatLufywVruXbZZT1CKg+Jr8ZTkwiPnmUC/oO9+uuY= github.com/pion/sctp v1.7.12/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s= @@ -69,17 +74,14 @@ github.com/pion/srtp/v2 v2.0.2 h1:664iGzVmaY7KYS5M0gleY0DscRo9ReDfTxQrq4UgGoU= github.com/pion/srtp/v2 v2.0.2/go.mod h1:VEyLv4CuxrwGY8cxM+Ng3bmVy8ckz/1t6A0q/msKOw0= github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg= github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA= -github.com/pion/transport v0.10.0/go.mod h1:BnHnUipd0rZQyTVB2SBGojFHT9CBt5C5TcsJSQGkvSE= github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A= github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q= github.com/pion/transport v0.12.3 h1:vdBfvfU/0Wq8kd2yhUMSDB/x+O4Z9MYVl2fJ5BT4JZw= github.com/pion/transport v0.12.3/go.mod h1:OViWW9SP2peE/HbwBvARicmAVnesphkNkCVZIWJ6q9A= github.com/pion/turn/v2 v2.0.5 h1:iwMHqDfPEDEOFzwWKT56eFmh6DYC6o/+xnLAEzgISbA= github.com/pion/turn/v2 v2.0.5/go.mod h1:APg43CFyt/14Uy7heYUOGWdkem/Wu4PhCO/bjyrTqMw= -github.com/pion/udp v0.1.0 h1:uGxQsNyrqG3GLINv36Ff60covYmfrLoxzwnCsIYspXI= -github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths= -github.com/pion/webrtc/v3 v3.0.20 h1:Jj0sk45MqQdkR24E1wbFRmOzb1Lv258ot9zd2fYB/Pw= -github.com/pion/webrtc/v3 v3.0.20/go.mod h1:0eJnCpQrUMpRnvyonw4ZiWClToerpixrZ2KcoTxvX9M= +github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o= +github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -90,24 +92,32 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb h1:fqpd0EBDzlHRCjiphRR5Zo/RSWWQlWv34418dnEixWk= golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 h1:b0LrWgu8+q7z4J+0Y3Umo5q1dL7NXBkKBWkaVkAq17E= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210420210106-798c2154c571 h1:Q6Bg8xzKzpFPU4Oi1sBnBTHBwlMsLeEXpu4hYBY8rAg= +golang.org/x/net v0.0.0-20210420210106-798c2154c571/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -116,19 +126,23 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005 h1:pDMpM2zh2MT0kHy037cKlSby2nEhD50SYqwQk76Nm40= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20191110171634-ad39bd3f0407/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -147,5 +161,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.mod b/go.mod index 233ffd3d..e0308e58 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,9 @@ require ( github.com/gen2brain/malgo v0.10.35 github.com/google/uuid v1.3.0 github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329 + github.com/pion/interceptor v0.1.10 github.com/pion/logging v0.2.2 + github.com/pion/rtcp v1.2.9 github.com/pion/rtp v1.7.13 github.com/pion/webrtc/v3 v3.1.34 golang.org/x/image v0.0.0-20220617043117-41969df76e82 diff --git a/ioreader.go b/ioreader.go index 5cc71ae7..92ab9e94 100644 --- a/ioreader.go +++ b/ioreader.go @@ -1,5 +1,7 @@ package mediadevices +import "github.com/pion/mediadevices/pkg/codec" + type EncodedBuffer struct { Data []byte Samples uint32 @@ -8,11 +10,13 @@ type EncodedBuffer struct { type EncodedReadCloser interface { Read() (EncodedBuffer, func(), error) Close() error + codec.Controllable } type encodedReadCloserImpl struct { - readFn func() (EncodedBuffer, func(), error) - closeFn func() error + readFn func() (EncodedBuffer, func(), error) + closeFn func() error + controllerFn func() codec.EncoderController } func (r *encodedReadCloserImpl) Read() (EncodedBuffer, func(), error) { @@ -23,9 +27,14 @@ func (r *encodedReadCloserImpl) Close() error { return r.closeFn() } +func (r *encodedReadCloserImpl) Controller() codec.EncoderController { + return r.controllerFn() +} + type encodedIOReadCloserImpl struct { - readFn func([]byte) (int, error) - closeFn func() error + readFn func([]byte) (int, error) + closeFn func() error + controller func() codec.EncoderController } func newEncodedIOReadCloserImpl(reader EncodedReadCloser) *encodedIOReadCloserImpl { @@ -48,7 +57,8 @@ func newEncodedIOReadCloserImpl(reader EncodedReadCloser) *encodedIOReadCloserIm encoded.Data = encoded.Data[n:] return n, nil }, - closeFn: reader.Close, + closeFn: reader.Close, + controller: reader.Controller, } } @@ -59,3 +69,7 @@ func (r *encodedIOReadCloserImpl) Read(b []byte) (int, error) { func (r *encodedIOReadCloserImpl) Close() error { return r.closeFn() } + +func (r *encodedIOReadCloserImpl) Controller() codec.EncoderController { + return r.controller() +} diff --git a/pkg/codec/codec.go b/pkg/codec/codec.go index 8b9367f4..2208c392 100644 --- a/pkg/codec/codec.go +++ b/pkg/codec/codec.go @@ -112,15 +112,37 @@ type VideoEncoderBuilder interface { BuildVideoEncoder(r video.Reader, p prop.Media) (ReadCloser, error) } -// ReadCloser is an io.ReadCloser with methods for rate limiting: SetBitRate and ForceKeyFrame +// ReadCloser is an io.ReadCloser with a controller type ReadCloser interface { Read() (b []byte, release func(), err error) Close() error + Controllable +} + +// EncoderController is the interface allowing to control the encoder behaviour after it's initialisation. +// It will possibly have common control method in the future. +// A controller can have optional methods represented by *Controller interfaces +type EncoderController interface{} + +// Controllable is a interface representing a encoder which can be controlled +// after it's initialisation with an EncoderController +type Controllable interface { + Controller() EncoderController +} + +// KeyFrameController is a interface representing an encoder that can be forced to produce key frame on demand +type KeyFrameController interface { + EncoderController + // ForceKeyFrame forces the next frame to be a keyframe, aka intra-frame. + ForceKeyFrame() error +} + +// BitRateController is a interface representing an encoder which can have a variable bit rate +type BitRateController interface { + EncoderController // SetBitRate sets current target bitrate, lower bitrate means smaller data will be transmitted // but this also means that the quality will also be lower. SetBitRate(int) error - // ForceKeyFrame forces the next frame to be a keyframe, aka intra-frame. - ForceKeyFrame() error } // BaseParams represents an codec's encoding properties diff --git a/pkg/codec/mmal/mmal.go b/pkg/codec/mmal/mmal.go index b6fd4a93..f17c8bb4 100644 --- a/pkg/codec/mmal/mmal.go +++ b/pkg/codec/mmal/mmal.go @@ -91,12 +91,8 @@ func (e *encoder) Read() ([]byte, func(), error) { return encoded, func() {}, err } -func (e *encoder) SetBitRate(b int) error { - panic("SetBitRate is not implemented") -} - -func (e *encoder) ForceKeyFrame() error { - panic("ForceKeyFrame is not implemented") +func (e *encoder) Controller() codec.EncoderController { + return e } func (e *encoder) Close() error { diff --git a/pkg/codec/mmal/mmal_test.go b/pkg/codec/mmal/mmal_test.go index ba21f9be..ea3e00af 100644 --- a/pkg/codec/mmal/mmal_test.go +++ b/pkg/codec/mmal/mmal_test.go @@ -4,6 +4,7 @@ import ( "image" "testing" + "github.com/pion/mediadevices/pkg/codec" "github.com/pion/mediadevices/pkg/codec/internal/codectest" "github.com/pion/mediadevices/pkg/frame" "github.com/pion/mediadevices/pkg/prop" @@ -44,3 +45,21 @@ func TestEncoder(t *testing.T) { }) }) } + +func TestShouldImplementBitRateControl(t *testing.T) { + t.SkipNow() // TODO: Implement bit rate control + + e := &encoder{} + if _, ok := e.Controller().(codec.BitRateController); !ok { + t.Error() + } +} + +func TestShouldImplementKeyFrameControl(t *testing.T) { + t.SkipNow() // TODO: Implement key frame control + + e := &encoder{} + if _, ok := e.Controller().(codec.KeyFrameController); !ok { + t.Error() + } +} diff --git a/pkg/codec/openh264/openh264.go b/pkg/codec/openh264/openh264.go index c83e6f2f..8c8d2ab2 100644 --- a/pkg/codec/openh264/openh264.go +++ b/pkg/codec/openh264/openh264.go @@ -90,15 +90,15 @@ func (e *encoder) Read() ([]byte, func(), error) { return encoded, func() {}, nil } -func (e *encoder) SetBitRate(b int) error { - panic("SetBitRate is not implemented") -} - func (e *encoder) ForceKeyFrame() error { e.engine.force_key_frame = C.int(1) return nil } +func (e *encoder) Controller() codec.EncoderController { + return e +} + func (e *encoder) Close() error { e.mu.Lock() defer e.mu.Unlock() diff --git a/pkg/codec/openh264/openh264_test.go b/pkg/codec/openh264/openh264_test.go index 6842bdb3..04806ba1 100644 --- a/pkg/codec/openh264/openh264_test.go +++ b/pkg/codec/openh264/openh264_test.go @@ -4,11 +4,28 @@ import ( "image" "testing" + "github.com/pion/mediadevices/pkg/codec" "github.com/pion/mediadevices/pkg/codec/internal/codectest" "github.com/pion/mediadevices/pkg/frame" "github.com/pion/mediadevices/pkg/prop" ) +func TestShouldImplementBitRateControl(t *testing.T) { + t.SkipNow() // TODO: Implement bit rate control + + e := &encoder{} + if _, ok := e.Controller().(codec.BitRateController); !ok { + t.Error() + } +} + +func TestShouldImplementKeyFrameControl(t *testing.T) { + e := &encoder{} + if _, ok := e.Controller().(codec.KeyFrameController); !ok { + t.Error() + } +} + func TestEncoder(t *testing.T) { t.Run("SimpleRead", func(t *testing.T) { p, err := NewParams() diff --git a/pkg/codec/opus/opus.go b/pkg/codec/opus/opus.go index 5f0b0b0d..2d120f10 100644 --- a/pkg/codec/opus/opus.go +++ b/pkg/codec/opus/opus.go @@ -121,8 +121,8 @@ func (e *encoder) SetBitRate(bitRate int) error { return nil } -func (e *encoder) ForceKeyFrame() error { - panic("ForceKeyFrame is not implemented") +func (e *encoder) Controller() codec.EncoderController { + return e } func (e *encoder) Close() error { diff --git a/pkg/codec/opus/opus_test.go b/pkg/codec/opus/opus_test.go index 20c5d478..ea89d1c7 100644 --- a/pkg/codec/opus/opus_test.go +++ b/pkg/codec/opus/opus_test.go @@ -3,11 +3,28 @@ package opus import ( "testing" + "github.com/pion/mediadevices/pkg/codec" "github.com/pion/mediadevices/pkg/codec/internal/codectest" "github.com/pion/mediadevices/pkg/prop" "github.com/pion/mediadevices/pkg/wave" ) +func TestShouldImplementBitRateControl(t *testing.T) { + e := &encoder{} + if _, ok := e.Controller().(codec.BitRateController); !ok { + t.Error() + } +} + +func TestShouldImplementKeyFrameControl(t *testing.T) { + t.SkipNow() // TODO: Implement key frame control + + e := &encoder{} + if _, ok := e.Controller().(codec.KeyFrameController); !ok { + t.Error() + } +} + func TestEncoder(t *testing.T) { t.Run("SimpleRead", func(t *testing.T) { p, err := NewParams() diff --git a/pkg/codec/vaapi/vp8.go b/pkg/codec/vaapi/vp8.go index 6abaf41a..abf47e86 100644 --- a/pkg/codec/vaapi/vp8.go +++ b/pkg/codec/vaapi/vp8.go @@ -541,12 +541,8 @@ func (e *encoderVP8) Read() ([]byte, func(), error) { return encoded, func() {}, err } -func (e *encoderVP8) SetBitRate(b int) error { - panic("SetBitRate is not implemented") -} - -func (e *encoderVP8) ForceKeyFrame() error { - panic("ForceKeyFrame is not implemented") +func (e *encoderVP8) Controller() codec.EncoderController { + return e } func (e *encoderVP8) Close() error { diff --git a/pkg/codec/vaapi/vp8_test.go b/pkg/codec/vaapi/vp8_test.go new file mode 100644 index 00000000..5c966445 --- /dev/null +++ b/pkg/codec/vaapi/vp8_test.go @@ -0,0 +1,27 @@ +//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris +// +build dragonfly freebsd linux netbsd openbsd solaris + +package vaapi + +import ( + "github.com/pion/mediadevices/pkg/codec" + "testing" +) + +func TestVP8ShouldImplementBitRateControl(t *testing.T) { + t.SkipNow() // TODO: Implement bit rate control + + e := &encoderVP8{} + if _, ok := e.Controller().(codec.BitRateController); !ok { + t.Error() + } +} + +func TestVP8ShouldImplementKeyFrameControl(t *testing.T) { + t.SkipNow() // TODO: Implement key frame control + + e := &encoderVP8{} + if _, ok := e.Controller().(codec.KeyFrameController); !ok { + t.Error() + } +} diff --git a/pkg/codec/vaapi/vp9.go b/pkg/codec/vaapi/vp9.go index edd992d0..88f00396 100644 --- a/pkg/codec/vaapi/vp9.go +++ b/pkg/codec/vaapi/vp9.go @@ -476,12 +476,8 @@ func (e *encoderVP9) Read() ([]byte, func(), error) { return encoded, func() {}, err } -func (e *encoderVP9) SetBitRate(b int) error { - panic("SetBitRate is not implemented") -} - -func (e *encoderVP9) ForceKeyFrame() error { - panic("ForceKeyFrame is not implemented") +func (e *encoderVP9) Controller() codec.EncoderController { + return e } func (e *encoderVP9) Close() error { diff --git a/pkg/codec/vaapi/vp9_test.go b/pkg/codec/vaapi/vp9_test.go new file mode 100644 index 00000000..62dda076 --- /dev/null +++ b/pkg/codec/vaapi/vp9_test.go @@ -0,0 +1,27 @@ +//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris +// +build dragonfly freebsd linux netbsd openbsd solaris + +package vaapi + +import ( + "github.com/pion/mediadevices/pkg/codec" + "testing" +) + +func TestVP9ShouldImplementBitRateControl(t *testing.T) { + t.SkipNow() // TODO: Implement bit rate control + + e := &encoderVP9{} + if _, ok := e.Controller().(codec.BitRateController); !ok { + t.Error() + } +} + +func TestVP9ShouldImplementKeyFrameControl(t *testing.T) { + t.SkipNow() // TODO: Implement key frame control + + e := &encoderVP9{} + if _, ok := e.Controller().(codec.KeyFrameController); !ok { + t.Error() + } +} diff --git a/pkg/codec/vpx/vpx.go b/pkg/codec/vpx/vpx.go index f2d90271..c925387c 100644 --- a/pkg/codec/vpx/vpx.go +++ b/pkg/codec/vpx/vpx.go @@ -295,10 +295,6 @@ func (e *encoder) Read() ([]byte, func(), error) { return encoded, func() {}, err } -func (e *encoder) SetBitRate(b int) error { - panic("SetBitRate is not implemented") -} - func (e *encoder) ForceKeyFrame() error { e.mu.Lock() defer e.mu.Unlock() @@ -306,6 +302,10 @@ func (e *encoder) ForceKeyFrame() error { return nil } +func (e *encoder) Controller() codec.EncoderController { + return e +} + func (e *encoder) Close() error { e.mu.Lock() defer e.mu.Unlock() diff --git a/pkg/codec/vpx/vpx_test.go b/pkg/codec/vpx/vpx_test.go index d1ee9959..01fb5402 100644 --- a/pkg/codec/vpx/vpx_test.go +++ b/pkg/codec/vpx/vpx_test.go @@ -193,7 +193,7 @@ func TestRequestKeyFrame(t *testing.T) { t.Fatal(err) } rel() - r.ForceKeyFrame() + r.Controller().(codec.KeyFrameController).ForceKeyFrame() _, rel, err = r.Read() if err != nil { t.Fatal(err) @@ -210,3 +210,19 @@ func TestRequestKeyFrame(t *testing.T) { } } + +func TestShouldImplementBitRateControl(t *testing.T) { + t.SkipNow() // TODO: Implement bit rate control + + e := &encoder{} + if _, ok := e.Controller().(codec.BitRateController); !ok { + t.Error() + } +} + +func TestShouldImplementKeyFrameControl(t *testing.T) { + e := &encoder{} + if _, ok := e.Controller().(codec.KeyFrameController); !ok { + t.Error() + } +} diff --git a/pkg/codec/x264/x264.go b/pkg/codec/x264/x264.go index ee5e05f9..aae7581f 100644 --- a/pkg/codec/x264/x264.go +++ b/pkg/codec/x264/x264.go @@ -124,15 +124,18 @@ func (e *encoder) Read() ([]byte, func(), error) { return encoded, func() {}, err } -func (e *encoder) SetBitRate(b int) error { - panic("SetBitRate is not implemented") -} +// TODO: Implement bit rate control +//var _ codec.BitRateController = (*encoder)(nil) func (e *encoder) ForceKeyFrame() error { e.engine.force_key_frame = C.int(1) return nil } +func (e *encoder) Controller() codec.EncoderController { + return e +} + func (e *encoder) Close() error { e.mu.Lock() defer e.mu.Unlock() diff --git a/pkg/codec/x264/x264_test.go b/pkg/codec/x264/x264_test.go index ccd31ddb..a3e1eed2 100644 --- a/pkg/codec/x264/x264_test.go +++ b/pkg/codec/x264/x264_test.go @@ -4,6 +4,7 @@ import ( "image" "testing" + "github.com/pion/mediadevices/pkg/codec" "github.com/pion/mediadevices/pkg/codec/internal/codectest" "github.com/pion/mediadevices/pkg/frame" "github.com/pion/mediadevices/pkg/prop" @@ -46,3 +47,21 @@ func TestEncoder(t *testing.T) { }) }) } + +func TestShouldImplementKeyFrameControl(t *testing.T) { + t.SkipNow() // TODO: Implement key frame control + + e := &encoder{} + if _, ok := e.Controller().(codec.KeyFrameController); !ok { + t.Error() + } +} + +func TestShouldImplementBitRateControl(t *testing.T) { + t.SkipNow() // TODO: Implement bit rate control + + e := &encoder{} + if _, ok := e.Controller().(codec.BitRateController); !ok { + t.Error() + } +} diff --git a/rtpreader.go b/rtpreader.go index 99e47d2c..947646d1 100644 --- a/rtpreader.go +++ b/rtpreader.go @@ -1,15 +1,20 @@ package mediadevices -import "github.com/pion/rtp" +import ( + "github.com/pion/mediadevices/pkg/codec" + "github.com/pion/rtp" +) type RTPReadCloser interface { Read() (pkts []*rtp.Packet, release func(), err error) Close() error + codec.Controllable } type rtpReadCloserImpl struct { - readFn func() ([]*rtp.Packet, func(), error) - closeFn func() error + readFn func() ([]*rtp.Packet, func(), error) + closeFn func() error + controllerFn func() codec.EncoderController } func (r *rtpReadCloserImpl) Read() ([]*rtp.Packet, func(), error) { @@ -19,3 +24,7 @@ func (r *rtpReadCloserImpl) Read() ([]*rtp.Packet, func(), error) { func (r *rtpReadCloserImpl) Close() error { return r.closeFn() } + +func (r *rtpReadCloserImpl) Controller() codec.EncoderController { + return r.controllerFn() +} diff --git a/track.go b/track.go index ca57395e..4b5aaa01 100644 --- a/track.go +++ b/track.go @@ -3,6 +3,8 @@ package mediadevices import ( "errors" "fmt" + "github.com/pion/interceptor" + "github.com/pion/rtcp" "image" "io" "strings" @@ -157,6 +159,7 @@ func (track *baseTrack) bind(ctx webrtc.TrackLocalContext, specializedTrack Trac defer track.mu.Unlock() signalCh := make(chan chan<- struct{}) + var stopRead chan struct{} track.activePeerConnections[ctx.ID()] = signalCh var encodedReader RTPReadCloser @@ -182,6 +185,7 @@ func (track *baseTrack) bind(ctx webrtc.TrackLocalContext, specializedTrack Trac var doneCh chan<- struct{} writer := ctx.WriteStream() defer func() { + close(stopRead) encodedReader.Close() // When there's another call to unbind, it won't block since we mark the signalCh to be closed @@ -214,6 +218,40 @@ func (track *baseTrack) bind(ctx webrtc.TrackLocalContext, specializedTrack Trac } }() + keyFrameController, ok := encodedReader.Controller().(codec.KeyFrameController) + if ok { + stopRead = make(chan struct{}) + go func() { + reader := ctx.RTCPReader() + for { + select { + case <-stopRead: + return + default: + } + + var readerBuffer []byte + _, _, err := reader.Read(readerBuffer, interceptor.Attributes{}) + if err != nil { + track.onError(err) + return + } + + pkts, err := rtcp.Unmarshal(readerBuffer) + + for _, pkt := range pkts { + switch pkt.(type) { + case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest: + if err := keyFrameController.ForceKeyFrame(); err != nil { + track.onError(err) + return + } + } + } + } + }() + } + return selectedCodec, nil } @@ -334,7 +372,8 @@ func (track *VideoTrack) newEncodedReader(codecNames ...string) (EncodedReadClos } return buffer, release, err }, - closeFn: encodedReader.Close, + closeFn: encodedReader.Close, + controllerFn: encodedReader.Controller, }, selectedCodec, nil } @@ -372,7 +411,8 @@ func (track *VideoTrack) NewRTPReader(codecName string, ssrc uint32, mtu int) (R pkts := packetizer.Packetize(encoded.Data, encoded.Samples) return pkts, release, err }, - closeFn: encodedReader.Close, + closeFn: encodedReader.Close, + controllerFn: encodedReader.Controller, }, nil }