Android Screen Sharing

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.20.3 (May 18, 2022)
+
+- [Fix]: Updates from DependatBot
+
 # 0.20.2 (May 16, 2022)
 
 - [Update]: Readme file update with Bintray instructions

README.md

@@ -109,11 +109,21 @@ If you try to archive the app and it fails, please do the following:
         }
 ```
 
-5. Sync Gradle
+5. Add the following to the `AndroidManifest.xml` file (needed for Screen Sharing to work):
 
-6. Make sure the following in your app's gradle `compileSdkVersion`, `buildToolsVersion`, `minSdkVersion`, and `targetSdkVersion` are greater than or equal to versions specified in the OpenTok React Native library.
+```xml
+    <service
+      android:name="com.opentokreactnative.OTForegroundService"
+      android:foregroundServiceType="mediaProjection"
+      android:enabled="true"
+      android:exported="false" />
+```
+
+6. Sync Gradle
+
+7. Make sure the following in your app's gradle `compileSdkVersion`, `buildToolsVersion`, `minSdkVersion`, and `targetSdkVersion` are greater than or equal to versions specified in the OpenTok React Native library.
 
-7. As for the older Android devices, ensure you add camera and audio permissions to your `AndroidManifest.xml` file:
+8. As for the older Android devices, ensure you add camera and audio permissions to your `AndroidManifest.xml` file:
 
 ```xml
     <uses-permission android:name="android.permission.CAMERA" />

android/src/main/java/com/opentokreactnative/OTForegroundService.java

@@ -0,0 +1,55 @@
+package com.opentokreactnative;
+
+import android.app.Notification;
+import android.app.NotificationChannel;
+import android.app.NotificationManager;
+import android.app.Service;
+import android.content.Context;
+import android.content.Intent;
+import android.os.Build;
+import android.os.IBinder;
+
+import androidx.annotation.Nullable;
+import androidx.annotation.RequiresApi;
+import androidx.core.app.NotificationCompat;
+
+public class OTForegroundService extends Service {
+    private static final int ID_SERVICE = 10101;
+
+    @Override
+    public int onStartCommand(Intent intent, int flags, int startId) {
+        return START_STICKY;
+    }
+
+    @Nullable
+    @Override
+    public IBinder onBind(Intent intent) {
+        return null;
+    }
+
+    @Override
+    public void onCreate() {
+        super.onCreate();
+
+        NotificationManager notificationManager = (NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE);
+        String channelId = Build.VERSION.SDK_INT >= Build.VERSION_CODES.O ? createNotificationChannel(notificationManager) : "";
+
+        NotificationCompat.Builder notificationBuilder = new NotificationCompat.Builder(this, channelId);
+        Notification notification = notificationBuilder.setOngoing(true)
+              .setPriority(Notification.PRIORITY_MIN)
+              .setCategory(NotificationCompat.CATEGORY_SERVICE)
+              .build();
+
+        startForeground(ID_SERVICE, notification);
+    }
+
+    @RequiresApi(Build.VERSION_CODES.O)
+    private String createNotificationChannel(NotificationManager notificationManager) {
+        String channelId = "ot_service_channelid";
+        String channelName = "OT Foreground Service";
+        NotificationChannel channel = new NotificationChannel(channelId, channelName, NotificationManager.IMPORTANCE_NONE);
+        channel.setLockscreenVisibility(Notification.VISIBILITY_PRIVATE);
+        notificationManager.createNotificationChannel(channel);
+        return channelId;
+    }
+}

android/src/main/java/com/opentokreactnative/OTNativeScreenRecorder.java

@@ -0,0 +1,121 @@
+package com.opentokreactnative;
+import android.app.Activity;
+import android.content.Context;
+import android.content.Intent;
+import android.graphics.PixelFormat;
+import android.hardware.display.DisplayManager;
+import android.hardware.display.VirtualDisplay;
+import android.media.ImageReader;
+import android.media.projection.MediaProjection;
+import android.media.projection.MediaProjectionManager;
+import android.util.DisplayMetrics;
+import androidx.annotation.NonNull;
+import com.facebook.react.bridge.*;
+
+public class OTNativeScreenRecorder extends ReactContextBaseJavaModule {
+    private static final String TAG = "OTNativeScreenRecorder";
+
+    private MediaProjectionManager projectManager;
+    private MediaProjection mediaProjection;
+    private VirtualDisplay virtualDisplay;
+
+    private final int REQUEST_CODE = 1010;
+    public int screenDensity;
+    public int screenWidth;
+    public int screenHeight;
+
+    private ImageReader imageReader;
+    ImageReader.OnImageAvailableListener listener;
+
+    class MediaProjectionCallback extends MediaProjection.Callback {
+        @Override
+        public void onStop() {
+            super.onStop();
+        }
+    }
+
+    private ActivityEventListener activityEventListener = new BaseActivityEventListener() {
+        @Override
+        public void onActivityResult(Activity activity, int requestCode, int resultCode, Intent data) {
+            if (requestCode == REQUEST_CODE && resultCode == Activity.RESULT_OK) {
+                try {
+                    mediaProjection = projectManager.getMediaProjection(resultCode, data);
+                    mediaProjection.registerCallback(new MediaProjectionCallback(), null);
+                    virtualDisplay = createVirtualDisplay();
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    };
+
+    public OTNativeScreenRecorder(ReactApplicationContext reactContext, ImageReader.OnImageAvailableListener listener) {
+        super(reactContext);
+        reactContext.addActivityEventListener(activityEventListener);
+        this.listener = listener;
+
+        DisplayMetrics metrics = new DisplayMetrics();
+        reactContext.getCurrentActivity().getWindowManager().getDefaultDisplay().getMetrics(metrics);
+        screenDensity = metrics.densityDpi;
+        screenWidth = metrics.widthPixels;
+        screenHeight = metrics.heightPixels;
+    }
+
+    private VirtualDisplay createVirtualDisplay() {
+        if (mediaProjection == null) {
+            return null;
+        }
+        return mediaProjection.createVirtualDisplay(
+                TAG,
+                screenWidth,
+                screenHeight,
+                screenDensity,
+                DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
+                imageReader.getSurface(),
+                null,
+                null
+        );
+    }
+
+    public void startRecording() {
+        try {
+            Intent serviceIntent = new Intent(getCurrentActivity(), OTForegroundService.class);
+            getReactApplicationContext().startService(serviceIntent);
+
+            if (projectManager == null) {
+                projectManager = (MediaProjectionManager) this.getReactApplicationContext().getSystemService(Context.MEDIA_PROJECTION_SERVICE);
+            }
+
+            imageReader = ImageReader.newInstance(screenWidth, screenHeight, PixelFormat.RGBA_8888, 1);
+            imageReader.setOnImageAvailableListener(listener, null);
+
+            Intent captureIntent = projectManager.createScreenCaptureIntent();
+            this.getCurrentActivity().startActivityForResult(captureIntent, REQUEST_CODE);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void stopRecording() {
+        try {
+            if (imageReader != null) {
+                imageReader.setOnImageAvailableListener(null, null);
+                mediaProjection.stop();
+                virtualDisplay = null;
+                imageReader = null;
+                mediaProjection = null;
+            }
+
+            Intent intent = new Intent(getCurrentActivity(), OTForegroundService.class);
+            getReactApplicationContext().stopService(intent);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    @NonNull
+    @Override
+    public String getName() {
+        return TAG;
+    }
+}

android/src/main/java/com/opentokreactnative/OTScreenCapturer.java

@@ -1,89 +1,40 @@
 package com.opentokreactnative;
 
-import android.content.Context;
 import android.graphics.Bitmap;
-import android.graphics.Canvas;
-import android.os.Handler;
+import android.media.Image;
+import android.media.ImageReader;
 import android.view.View;
 
+import com.facebook.react.bridge.ReactApplicationContext;
 import com.opentok.android.BaseVideoCapturer;
 
-public class OTScreenCapturer extends BaseVideoCapturer {
+import java.nio.ByteBuffer;
 
+public class OTScreenCapturer extends BaseVideoCapturer implements ImageReader.OnImageAvailableListener {
     private boolean capturing = false;
-    private View contentView;
+    private ReactApplicationContext reactContext;
 
-    private int fps = 15;
-    private int width = 20;
-    private int height = 20;
-    private int[] frame;
+    private OTNativeScreenRecorder nativeScreenRecorder;
 
-    private Bitmap bmp;
-    private Canvas canvas;
-
-    private Handler mHandler = new Handler();
-
-    private Runnable newFrame = new Runnable() {
-        @Override
-        public void run() {
-            if (capturing) {
-                int width = contentView.getWidth();
-                int height = contentView.getHeight();
-
-                if (frame == null ||
-                        OTScreenCapturer.this.width != width ||
-                        OTScreenCapturer.this.height != height) {
-
-                    OTScreenCapturer.this.width = width;
-                    OTScreenCapturer.this.height = height;
-
-                    if (bmp != null) {
-                        bmp.recycle();
-                        bmp = null;
-                    }
-                    bmp = Bitmap.createBitmap(width,
-                            height, Bitmap.Config.ARGB_8888);
-
-                    canvas = new Canvas(bmp);
-                    frame = new int[width * height];
-                }
-                canvas.save();
-                canvas.translate(-contentView.getScrollX(), - contentView.getScrollY());
-                contentView.draw(canvas);
-
-                bmp.getPixels(frame, 0, width, 0, 0, width, height);
-
-                provideIntArrayFrame(frame, ARGB, width, height, 0, false);
-
-                canvas.restore();
-
-                mHandler.postDelayed(newFrame, 1000 / fps);
-
-            }
-        }
-    };
-
-    public OTScreenCapturer(View view) {
-        this.contentView = view;
+    public OTScreenCapturer(ReactApplicationContext reactContext) {
+        this.reactContext = reactContext;
+        this.nativeScreenRecorder = new OTNativeScreenRecorder(reactContext, this);
     }
 
     @Override
-    public void init() {
-
-    }
+    public void init() {}
 
     @Override
     public int startCapture() {
         capturing = true;
-
-        mHandler.postDelayed(newFrame, 1000 / fps);
+        nativeScreenRecorder.startRecording();
         return 0;
     }
 
     @Override
     public int stopCapture() {
         capturing = false;
-        mHandler.removeCallbacks(newFrame);
+        nativeScreenRecorder.stopRecording();
         return 0;
     }
 
@@ -96,9 +47,9 @@ public boolean isCaptureStarted() {
     public CaptureSettings getCaptureSettings() {
 
         CaptureSettings settings = new CaptureSettings();
-        settings.fps = fps;
-        settings.width = width;
-        settings.height = height;
+        settings.fps = 15;
+        settings.width = nativeScreenRecorder.screenWidth;
+        settings.height = nativeScreenRecorder.screenHeight;
         settings.format = ARGB;
         return settings;
     }
@@ -118,4 +69,42 @@ public void onResume() {
 
     }
 
+    @Override
+    public void onImageAvailable(final ImageReader reader) {
+        Image image = null;
+        Bitmap bitmap = null;
+        try {
+            image = reader.acquireNextImage();
+
+            if (image != null) {
+                Image.Plane[] planes = image.getPlanes();
+                ByteBuffer buffer = planes[0].getBuffer();
+                buffer.rewind();
+
+                int pixelStride = planes[0].getPixelStride();
+                int rowStride = planes[0].getRowStride();
+                int rowPadding = rowStride - pixelStride * reader.getWidth();
+                int bitmapWidth = reader.getWidth() + rowPadding / pixelStride;
+                bitmap = Bitmap.createBitmap(bitmapWidth, reader.getHeight(), Bitmap.Config.ARGB_8888);
+                bitmap.copyPixelsFromBuffer(buffer);
+
+                int width = nativeScreenRecorder.screenWidth;
+                int height = nativeScreenRecorder.screenHeight;
+                int[] frame = new int[width * height];
+                bitmap.getPixels(frame, 0, width, 0, 0, width, height);
+
+                provideIntArrayFrame(frame, ARGB, width, height, 0, false);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            if (bitmap != null) {
+                bitmap.recycle();
+            }
+
+            if (image != null) {
+                image.close();
+            }
+        }
+    }
 }

android/src/main/java/com/opentokreactnative/OTSessionManager.java

@@ -6,9 +6,8 @@
 
 import android.util.Log;
 import android.widget.FrameLayout;
-import android.view.View;
-
 import androidx.annotation.Nullable;
+import android.view.View;
 
 import com.facebook.react.bridge.Arguments;
 import com.facebook.react.bridge.Callback;
@@ -156,8 +155,7 @@ public void initPublisher(String publisherId, ReadableMap properties, Callback c
         String videoSource = properties.getString("videoSource");
         Publisher mPublisher = null;
         if (videoSource.equals("screen")) {
-            View view = getCurrentActivity().getWindow().getDecorView().getRootView();
-            OTScreenCapturer capturer = new OTScreenCapturer(view);
+            OTScreenCapturer capturer = new OTScreenCapturer(this.getReactApplicationContext());
             mPublisher = new Publisher.Builder(this.getReactApplicationContext())
                     .audioTrack(audioTrack)
                     .videoTrack(videoTrack)