Try forward client microphone without encoding

Author: Cabbache

app/meson.build

@@ -25,6 +25,7 @@ src = [
     'src/keyboard_sdk.c',
     'src/mouse_capture.c',
     'src/mouse_sdk.c',
+    'src/microphone.c',
     'src/opengl.c',
     'src/options.c',
     'src/packet_merger.c',

app/src/main.c

@@ -16,6 +16,7 @@
 #include "util/net.h"
 #include "util/thread.h"
 #include "version.h"
+#include "microphone.h"
 
 #ifdef _WIN32
 #include <windows.h>
@@ -73,11 +74,16 @@ main_scrcpy(int argc, char *argv[]) {
     av_register_all();
 #endif
 
+/*
 #ifdef HAVE_V4L2
     if (args.opts.v4l2_device) {
         avdevice_register_all();
     }
 #endif
+*/
+
+		//NEED TO DO THIS for microphone
+		avdevice_register_all();
 
     if (!net_init()) {
         ret = SCRCPY_EXIT_FAILURE;

app/src/microphone.c

@@ -0,0 +1,74 @@
+#include "microphone.h"
+#include <libavcodec/avcodec.h>
+#include <libavdevice/avdevice.h>
+#include <libavformat/avformat.h>
+#include <stdio.h>
+#include "util/log.h"
+#include "util/net.h"
+
+int read_mic(void *data) {
+    sc_socket mic_socket = *(sc_socket*)data;
+    const char *input_format_name = "alsa";
+    const char *device_name = "default";
+
+    AVInputFormat *input_format = av_find_input_format(input_format_name);
+    if (!input_format) {
+        fprintf(stderr, "Could not find ALSA input format\n");
+        return 1;
+    }
+
+    AVFormatContext *fmt_ctx = NULL;
+    if (avformat_open_input(&fmt_ctx, device_name, input_format, NULL) < 0) {
+        fprintf(stderr, "Could not open ALSA device '%s'\n", device_name);
+        return 1;
+    }
+
+    if (avformat_find_stream_info(fmt_ctx, NULL) < 0) {
+        fprintf(stderr, "Could not read stream info\n");
+        avformat_close_input(&fmt_ctx);
+        return 1;
+    }
+
+    // ALSA has only one audio stream
+    int audio_stream_index = 0;
+    AVCodecParameters *codecpar = fmt_ctx->streams[audio_stream_index]->codecpar;
+    AVCodec *codec = avcodec_find_decoder(codecpar->codec_id);
+    if (!codec) {
+        fprintf(stderr, "Codec not found\n");
+        avformat_close_input(&fmt_ctx);
+        return 1;
+    }
+
+    AVCodecContext *codec_ctx = avcodec_alloc_context3(codec);
+    avcodec_parameters_to_context(codec_ctx, codecpar);
+    if (avcodec_open2(codec_ctx, codec, NULL) < 0) {
+        fprintf(stderr, "Could not open codec\n");
+        avcodec_free_context(&codec_ctx);
+        avformat_close_input(&fmt_ctx);
+        return 1;
+    }
+		LOGD("sample_fmt = %d\n", codec_ctx->sample_fmt);
+		LOGD("sample rate = %d\n", codec_ctx->sample_rate);
+
+    AVPacket pkt;
+    AVFrame *frame = av_frame_alloc();
+    printf("Recording audio (Ctrl+C to stop)...\n");
+
+    while (av_read_frame(fmt_ctx, &pkt) >= 0) {
+        // ALSA always gives audio packets, so skip the stream_index check
+        if (avcodec_send_packet(codec_ctx, &pkt) >= 0) {
+            while (avcodec_receive_frame(codec_ctx, frame) >= 0) {
+                int bytes_per_sample = av_get_bytes_per_sample(codec_ctx->sample_fmt);
+                int data_size = frame->nb_samples * frame->ch_layout.nb_channels * bytes_per_sample;
+
+                net_send_all(mic_socket, frame->data[0], data_size);
+            }
+        }
+        av_packet_unref(&pkt);
+    }
+
+    av_frame_free(&frame);
+    avcodec_free_context(&codec_ctx);
+    avformat_close_input(&fmt_ctx);
+    return 0;
+}

app/src/microphone.h

@@ -0,0 +1,2 @@
+#include "util/net.h"
+int read_mic(void *);

app/src/options.c

@@ -101,6 +101,7 @@ const struct scrcpy_options scrcpy_options_default = {
     .power_on = true,
     .video = true,
     .audio = true,
+    .microphone = true,
     .require_audio = false,
     .kill_adb_on_close = false,
     .camera_high_speed = false,

app/src/options.h

@@ -311,6 +311,7 @@ struct scrcpy_options {
     bool power_on;
     bool video;
     bool audio;
+    bool microphone;
     bool require_audio;
     bool kill_adb_on_close;
     bool camera_high_speed;

app/src/scrcpy.c

@@ -26,6 +26,7 @@
 #include "recorder.h"
 #include "screen.h"
 #include "server.h"
+#include "microphone.h"
 #include "uhid/gamepad_uhid.h"
 #include "uhid/keyboard_uhid.h"
 #include "uhid/mouse_uhid.h"
@@ -450,6 +451,7 @@ scrcpy(struct scrcpy_options *options) {
         .display_ime_policy = options->display_ime_policy,
         .video = options->video,
         .audio = options->audio,
+        .microphone = options->microphone,
         .audio_dup = options->audio_dup,
         .show_touches = options->show_touches,
         .stay_awake = options->stay_awake,
@@ -887,6 +889,14 @@ scrcpy(struct scrcpy_options *options) {
         audio_demuxer_started = true;
     }
 
+		if (options->microphone) {
+			sc_thread mic_thread;
+			bool ok = sc_thread_create(&mic_thread, read_mic, "scrcpy-mic", &s->server.mic_socket);
+			if (!ok) {
+				goto end;
+			}
+		}
+
     // If the device screen is to be turned off, send the control message after
     // everything is set up
     if (options->control && options->turn_screen_off) {

app/src/server.c

@@ -11,6 +11,7 @@
 #include "util/env.h"
 #include "util/file.h"
 #include "util/log.h"
+#include "util/net.h"
 #include "util/net_intr.h"
 #include "util/process.h"
 #include "util/str.h"
@@ -555,6 +556,7 @@ sc_server_init(struct sc_server *server, const struct sc_server_params *params,
     server->video_socket = SC_SOCKET_NONE;
     server->audio_socket = SC_SOCKET_NONE;
     server->control_socket = SC_SOCKET_NONE;
+    server->mic_socket = SC_SOCKET_NONE;
 
     sc_adb_tunnel_init(&server->tunnel);
 
@@ -597,10 +599,12 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) {
     bool video = server->params.video;
     bool audio = server->params.audio;
     bool control = server->params.control;
+    bool microphone = server->params.microphone;
 
     sc_socket video_socket = SC_SOCKET_NONE;
     sc_socket audio_socket = SC_SOCKET_NONE;
     sc_socket control_socket = SC_SOCKET_NONE;
+    sc_socket mic_socket = SC_SOCKET_NONE;
     if (!tunnel->forward) {
         if (video) {
             video_socket =
@@ -625,6 +629,14 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) {
                 goto fail;
             }
         }
+
+        if (microphone) {
+            mic_socket =
+                net_accept_intr(&server->intr, tunnel->server_socket);
+            if (mic_socket == SC_SOCKET_NONE) {
+                goto fail;
+            }
+        }
     } else {
         uint32_t tunnel_host = server->params.tunnel_host;
         if (!tunnel_host) {
@@ -681,14 +693,19 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) {
         }
     }
 
+    // Disable Nagle's algorithm for the control socket
+    // (it only impacts the sending side, so it is useless to set it
+    // for the other sockets)
     if (control_socket != SC_SOCKET_NONE) {
-        // Disable Nagle's algorithm for the control socket
-        // (it only impacts the sending side, so it is useless to set it
-        // for the other sockets)
         bool ok = net_set_tcp_nodelay(control_socket, true);
         (void) ok; // error already logged
     }
 
+    if (mic_socket != SC_SOCKET_NONE) {
+        bool ok = net_set_tcp_nodelay(mic_socket, true);
+        (void) ok; // error already logged
+    }
+
     // we don't need the adb tunnel anymore
     sc_adb_tunnel_close(tunnel, &server->intr, serial,
                         server->device_socket_name);
@@ -706,10 +723,12 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) {
     assert(!video || video_socket != SC_SOCKET_NONE);
     assert(!audio || audio_socket != SC_SOCKET_NONE);
     assert(!control || control_socket != SC_SOCKET_NONE);
+    assert(!microphone || mic_socket != SC_SOCKET_NONE);
 
     server->video_socket = video_socket;
     server->audio_socket = audio_socket;
     server->control_socket = control_socket;
+    server->mic_socket = mic_socket;
 
     return true;
 
@@ -732,6 +751,12 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) {
         }
     }
 
+    if (mic_socket != SC_SOCKET_NONE) {
+        if (!net_close(mic_socket)) {
+            LOGW("Could not close microphone socket");
+        }
+    }
+
     if (tunnel->enabled) {
         // Always leave this function with tunnel disabled
         sc_adb_tunnel_close(tunnel, &server->intr, serial,
@@ -1121,6 +1146,11 @@ run_server(void *data) {
         net_interrupt(server->control_socket);
     }
 
+    if (server->mic_socket != SC_SOCKET_NONE) {
+        // There is no control_socket if --no-microphone is set
+        net_interrupt(server->mic_socket);
+    }
+
     // Give some delay for the server to terminate properly
 #define WATCHDOG_DELAY SC_TICK_FROM_SEC(1)
     sc_tick deadline = sc_tick_now() + WATCHDOG_DELAY;
@@ -1189,6 +1219,9 @@ sc_server_destroy(struct sc_server *server) {
     if (server->control_socket != SC_SOCKET_NONE) {
         net_close(server->control_socket);
     }
+    if (server->mic_socket != SC_SOCKET_NONE) {
+        net_close(server->mic_socket);
+    }
 
     free(server->serial);
     free(server->device_socket_name);

app/src/server.h

@@ -53,6 +53,7 @@ struct sc_server_params {
     enum sc_display_ime_policy display_ime_policy;
     bool video;
     bool audio;
+    bool microphone;
     bool audio_dup;
     bool show_touches;
     bool stay_awake;
@@ -91,6 +92,7 @@ struct sc_server {
 
     sc_socket video_socket;
     sc_socket audio_socket;
+    sc_socket mic_socket;
     sc_socket control_socket;
 
     const struct sc_server_callbacks *cbs;

server/src/main/java/com/genymobile/scrcpy/Options.java

@@ -26,6 +26,7 @@ public class Options {
     private int scid = -1; // 31-bit non-negative value, or -1
     private boolean video = true;
     private boolean audio = true;
+    private boolean microphone = true;
     private int maxSize;
     private VideoCodec videoCodec = VideoCodec.H264;
     private AudioCodec audioCodec = AudioCodec.OPUS;
@@ -96,6 +97,10 @@ public boolean getAudio() {
         return audio;
     }
 
+    public boolean getMicrophone() {
+        return microphone;
+    }
+
     public int getMaxSize() {
         return maxSize;
     }
@@ -327,6 +332,9 @@ public static Options parse(String... args) {
                 case "audio":
                     options.audio = Boolean.parseBoolean(value);
                     break;
+                case "microphone":
+                    options.microphone = Boolean.parseBoolean(value);
+                    break;
                 case "video_codec":
                     VideoCodec videoCodec = VideoCodec.findByName(value);
                     if (videoCodec == null) {