feat(🍏, 🤖): Realtime Skia effects on native views

apps/example/ios/Podfile.lock

@@ -1327,7 +1327,7 @@ PODS:
     - ReactCommon/turbomodule/bridging
     - ReactCommon/turbomodule/core
     - Yoga
-  - react-native-slider (4.5.6):
+  - react-native-slider (4.5.7):
     - DoubleConversion
     - glog
     - hermes-engine
@@ -1340,7 +1340,7 @@ PODS:
     - React-featureflags
     - React-graphics
     - React-ImageManager
-    - react-native-slider/common (= 4.5.6)
+    - react-native-slider/common (= 4.5.7)
     - React-NativeModulesApple
     - React-RCTFabric
     - React-rendererdebug
@@ -1349,7 +1349,7 @@ PODS:
     - ReactCommon/turbomodule/bridging
     - ReactCommon/turbomodule/core
     - Yoga
-  - react-native-slider/common (4.5.6):
+  - react-native-slider/common (4.5.7):
     - DoubleConversion
     - glog
     - hermes-engine
@@ -2194,7 +2194,7 @@ SPEC CHECKSUMS:
   React-microtasksnativemodule: 843f352b32aacbe13a9c750190d34df44c3e6c2c
   react-native-safe-area-context: 0f14bce545abcdfbff79ce2e3c78c109f0be283e
   react-native-skia: d696b43cb1a6560ec3902856cdc8f0838cc25b64
-  react-native-slider: bb7eb4732940fab78217e1c096bb647d8b0d1cf3
+  react-native-slider: 310d3f89edd6ca8344a974bfe83a29a3fbb60e5a
   React-NativeModulesApple: 88433b6946778bea9c153e27b671de15411bf225
   React-perflogger: 9e8d3c0dc0194eb932162812a168aa5dc662f418
   React-performancetimeline: 5a2d6efef52bdcefac079c7baa30934978acd023

apps/example/src/App.tsx

@@ -107,7 +107,7 @@ const App = () => {
             screenOptions={{
               headerLeft: HeaderLeft,
             }}
-            initialRouteName={CI ? "Tests" : "Home"}
+            initialRouteName={CI ? "Tests" : "LiquidGlass"}
           >
             <Stack.Screen
               name="Home"

apps/example/src/Examples/API/Snapshot.tsx

@@ -48,13 +48,16 @@ export const Snapshot = () => {
   const image = useSharedValue<SkImage | null>(null);
   const takeSnapshot = useCallback(async () => {
     if (viewRef.current != null) {
+      const start = performance.now();
       image.value = await makeImageFromView(viewRef as RefObject<View>);
+      const end = performance.now();
+      console.log("Performance: " + Math.round(end - start) + " ms");
     }
   }, [image]);
 
   return (
     <View style={{ flex: 1 }}>
-      <View ref={viewRef} style={styles.view}>
+      <View ref={viewRef} style={styles.view} collapsable={false}>
         <Component />
       </View>
       <Button title="Take snapshot" onPress={takeSnapshot} />

apps/example/src/Examples/LiquidGlass/List.tsx

@@ -26,6 +26,10 @@ export const examples = [
     screen: "Shader2",
     title: "🎨 Shader 2",
   },
+  {
+    screen: "NativeView",
+    title: "📱 Native View",
+  },
 ] as const;
 
 const styles = StyleSheet.create({

apps/example/src/Examples/LiquidGlass/NativeView.tsx

@@ -0,0 +1,109 @@
+import {
+  Canvas,
+  Skia,
+  useClock,
+  Image as SkImage,
+  Fill,
+  ColorMatrix,
+  notifyChange,
+} from "@shopify/react-native-skia";
+import React, { useLayoutEffect, useRef } from "react";
+import {
+  View,
+  PixelRatio,
+  StyleSheet,
+  findNodeHandle,
+  Platform,
+} from "react-native";
+import Animated, {
+  useAnimatedReaction,
+  useAnimatedStyle,
+  useDerivedValue,
+  useSharedValue,
+} from "react-native-reanimated";
+
+const style = {
+  flex: 1,
+  overflow: "hidden" as const,
+};
+
+export const NativeView = () => {
+  const canvasSize = useSharedValue({ width: 0, height: 0 });
+  const viewTag = useRef(-1);
+  const image = useSharedValue(Skia.Image.MakeNull());
+  const ref = useRef<View>(null);
+  const clock = useClock();
+  const imageHeight = 1920 / PixelRatio.get();
+
+  const translateY = useDerivedValue(() => clock.value / 10);
+
+  useLayoutEffect(() => {
+    viewTag.current = findNodeHandle(ref.current)!;
+    if (Platform.OS === "android") {
+      console.log("SetRenderEffect!");
+      Skia.Image.setRenderEffectAndroid(viewTag.current, "");
+    }
+  }, []);
+
+  const animatedStyle = useAnimatedStyle(() => {
+    return {
+      transform: [{ translateY: -translateY.value % imageHeight }],
+    };
+  });
+
+  useAnimatedReaction(
+    () => clock.value,
+    () => {
+      if (Platform.OS === "ios") {
+        Skia.Image.MakeImageFromViewTagSync(viewTag.current, image.value);
+        notifyChange(image);
+      }
+    }
+  );
+
+  const rect = useDerivedValue(() => ({
+    x: 0,
+    y: 0,
+    width: canvasSize.value.width,
+    height: canvasSize.value.height,
+  }));
+
+  return (
+    <View style={{ flex: 1 }}>
+      <View style={style} collapsable={false} ref={ref}>
+        <Animated.Image
+          style={[
+            {
+              resizeMode: "repeat",
+              width: "100%",
+              height: imageHeight * 2,
+            },
+            animatedStyle,
+          ]}
+          source={require("./assets/flowers.jpg")}
+        />
+        <Animated.Image
+          style={[
+            {
+              resizeMode: "repeat",
+              width: "100%",
+              height: imageHeight * 2,
+            },
+            animatedStyle,
+          ]}
+          source={require("./assets/flowers.jpg")}
+        />
+      </View>
+      <Canvas style={StyleSheet.absoluteFillObject} onSize={canvasSize}>
+        <SkImage image={image} rect={rect}>
+          <ColorMatrix
+            matrix={[
+              -0.578, 0.99, 0.588, 0, 0, 0.469, 0.535, -0.003, 0, 0, 0.015,
+              1.69, -0.703, 0, 0, 0, 0, 0, 1, 0,
+            ]}
+          />
+        </SkImage>
+      </Canvas>
+    </View>
+  );
+};

apps/example/src/Examples/LiquidGlass/Routes.ts

@@ -5,4 +5,5 @@ export type Routes = {
   DisplacementMap2: undefined;
   Shader1: undefined;
   Shader2: undefined;
+  NativeView: undefined;
 };

apps/example/src/Examples/LiquidGlass/index.tsx

@@ -8,11 +8,12 @@ import { DisplacementMap1 } from "./DisplacementMap1";
 import { DisplacementMap2 } from "./DisplacementMap2";
 import { Shader1 } from "./Shader1";
 import { Shader2 } from "./Shader2";
+import { NativeView } from "./NativeView";
 
 const Stack = createNativeStackNavigator<Routes>();
 export const LiquidGlass = () => {
   return (
-    <Stack.Navigator>
+    <Stack.Navigator initialRouteName="NativeView">
       <Stack.Screen
         name="List"
         component={List}
@@ -56,6 +57,13 @@ export const LiquidGlass = () => {
           title: "🎨 Shader 2",
         }}
       />
+      <Stack.Screen
+        name="NativeView"
+        component={NativeView}
+        options={{
+          title: "📱 Native View",
+        }}
+      />
     </Stack.Navigator>
   );
 };

packages/skia/android/cpp/jni/JniPlatformContext.cpp

@@ -130,6 +130,17 @@ sk_sp<SkImage> JniPlatformContext::takeScreenshotFromViewTag(size_t tag) {
   return skImage;
 }
 
+void JniPlatformContext::setRenderEffectAndroid(int viewTag, const std::string &shaderString) {
+  auto env = jni::Environment::current();
+  static auto method = javaPart_->getClass()->getMethod<void(int, jstring)>("setRenderEffectAndroid");
+
+  jstring jShaderString = env->NewStringUTF(shaderString.c_str());
+
+  method(javaPart_.get(), viewTag, jShaderString);
+
+  env->DeleteLocalRef(jShaderString);
+}
+
 void JniPlatformContext::performStreamOperation(
     const std::string &sourceUri,
     const std::function<void(std::unique_ptr<SkStreamAsset>)> &op) {

packages/skia/android/cpp/jni/include/JniPlatformContext.h

@@ -42,6 +42,8 @@ class JniPlatformContext : public jni::HybridClass<JniPlatformContext> {
 
   sk_sp<SkImage> takeScreenshotFromViewTag(size_t tag);
 
+  void setRenderEffectAndroid(int viewTag, const std::string &shaderString);
+
   jni::global_ref<jobject> createVideo(const std::string &url);
 
 private:

packages/skia/android/cpp/rnskia-android/RNSkAndroidPlatformContext.h

@@ -226,6 +226,10 @@ class RNSkAndroidPlatformContext : public RNSkPlatformContext {
     return _jniPlatformContext->takeScreenshotFromViewTag(tag);
   }
 
+  void setRenderEffectAndroid(int viewTag, const std::string &shaderString) override {
+    _jniPlatformContext->setRenderEffectAndroid(viewTag, shaderString);
+  }
+
 private:
   JniPlatformContext *_jniPlatformContext;
 };

packages/skia/android/src/main/java/com/shopify/reactnative/skia/PlatformContext.java

@@ -1,11 +1,18 @@
 package com.shopify.reactnative.skia;
 
+import android.os.Build;
 import android.os.Handler;
 import android.os.Looper;
+import android.graphics.RenderEffect;
+import android.graphics.Shader;
+import android.view.View;
 
 import com.facebook.jni.HybridData;
 import com.facebook.proguard.annotations.DoNotStrip;
 import com.facebook.react.bridge.ReactContext;
+import com.facebook.react.bridge.UIManager;
+import com.facebook.react.uimanager.UIManagerHelper;
+import com.facebook.react.uimanager.UIManagerModule;
 
 import java.io.BufferedInputStream;
 import java.io.ByteArrayOutputStream;
@@ -115,6 +122,26 @@ public void run() {
         });
     }
 
+    @DoNotStrip
+    public void setRenderEffectAndroid(final int tag, final String shaderString) {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
+            UIManager uiManager = UIManagerHelper.getUIManagerForReactTag(mContext, tag);
+            View view = null;
+            try {
+                view = uiManager.resolveView(tag);
+            } catch (RuntimeException e) {
+                mContext.handleException(e);
+            }
+            if (view != null) {
+                    // For now, ignore shaderString and use a fixed blur effect
+                    RenderEffect blurEffect = null;
+                    blurEffect = RenderEffect.createBlurEffect(
+                            10, 10, null, Shader.TileMode.CLAMP);
+                    view.setRenderEffect(blurEffect);
+            }
+        }
+    }
+
     // Private c++ native methods
     private native HybridData initHybrid(float pixelDensity);
 }

packages/skia/apple/ViewScreenshotService.mm

@@ -44,8 +44,11 @@ - (instancetype)initWithUiManager:(RCTUIManager *)uiManager {
   // up in the profiler!
   UIImage *image = [renderer
       imageWithActions:^(UIGraphicsImageRendererContext *_Nonnull context) {
+        //CFTimeInterval startTime = CACurrentMediaTime();
         [view drawViewHierarchyInRect:(CGRect){CGPointZero, size}
-                   afterScreenUpdates:YES];
+                   afterScreenUpdates:NO];
+        //CFTimeInterval endTime = CACurrentMediaTime();
+        //NSLog(@"drawViewHierarchyInRect took %.2f ms", (endTime - startTime) * 1000.0);
       }];
 
   // Convert from UIImage -> CGImage -> SkImage

packages/skia/cpp/api/JsiSkImageFactory.h

@@ -83,6 +83,29 @@ class JsiSkImageFactory : public JsiSkHostObject {
         });
   }
 
+  JSI_HOST_FUNCTION(MakeImageFromViewTagSync) {
+    // TODO: add safety checks on args[0] and args[1]
+    auto viewTag = arguments[0].asNumber();
+    auto jsiImage = arguments[1].asObject(runtime).asHostObject<JsiSkImage>(runtime);
+    auto context = getContext();
+    // TODO: check we are on the main thread
+    if (viewTag != -1) {
+      auto result = context->makeViewScreenshotSync(viewTag);
+      jsiImage->setObject(result);
+      return jsi::Object::createFromHostObject(
+            runtime, std::make_shared<JsiSkImage>(getContext(), std::move(result)));
+    }
+    return jsi::Value::undefined();
+  }
+
+  JSI_HOST_FUNCTION(setRenderEffectAndroid) {
+    auto viewTag = static_cast<int>(arguments[0].asNumber());
+    auto shaderString = arguments[1].asString(runtime).utf8(runtime);
+    auto context = getContext();
+    context->setRenderEffectAndroid(viewTag, shaderString);
+    return jsi::Value::undefined();
+  }
+
   JSI_HOST_FUNCTION(MakeImageFromNativeTextureUnstable) {
     auto texInfo = JsiTextureInfo::fromValue(runtime, arguments[0]);
     auto image = getContext()->makeImageFromNativeTexture(
@@ -104,6 +127,8 @@ class JsiSkImageFactory : public JsiSkHostObject {
 
   JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiSkImageFactory, MakeImageFromEncoded),
                        JSI_EXPORT_FUNC(JsiSkImageFactory, MakeImageFromViewTag),
+                       JSI_EXPORT_FUNC(JsiSkImageFactory, MakeImageFromViewTagSync),
+                       JSI_EXPORT_FUNC(JsiSkImageFactory, setRenderEffectAndroid),
                        JSI_EXPORT_FUNC(JsiSkImageFactory,
                                        MakeImageFromNativeBuffer),
                        JSI_EXPORT_FUNC(JsiSkImageFactory,

packages/skia/cpp/api/recorder/Drawings.h

@@ -496,22 +496,24 @@ class ImageCmd : public Command {
     auto [x, y, width, height, rect, fit, image, sampling] = props;
     if (image.has_value()) {
       auto img = image.value();
-      auto hasRect =
-          rect.has_value() || (width.has_value() && height.has_value());
-      if (hasRect) {
-        auto src = SkRect::MakeXYWH(0, 0, img->width(), img->height());
-        auto dst = rect.has_value()
-                       ? rect.value()
-                       : SkRect::MakeXYWH(x, y, width.value(), height.value());
-        auto rects = RNSkiaImage::fitRects(fit, src, dst);
-        ctx->canvas->drawImageRect(
-            img, rects.src, rects.dst,
-            sampling.value_or(SkSamplingOptions(SkFilterMode::kLinear)),
-            &(ctx->getPaint()), SkCanvas::kStrict_SrcRectConstraint);
-      } else {
-        throw std::runtime_error(
-            "Image node could not resolve image dimension props.");
-      }
+        if (img != nullptr) {
+            auto hasRect =
+            rect.has_value() || (width.has_value() && height.has_value());
+            if (hasRect) {
+                auto src = SkRect::MakeXYWH(0, 0, img->width(), img->height());
+                auto dst = rect.has_value()
+                ? rect.value()
+                : SkRect::MakeXYWH(x, y, width.value(), height.value());
+                auto rects = RNSkiaImage::fitRects(fit, src, dst);
+                ctx->canvas->drawImageRect(
+                                           img, rects.src, rects.dst,
+                                           sampling.value_or(SkSamplingOptions(SkFilterMode::kLinear)),
+                                           &(ctx->getPaint()), SkCanvas::kStrict_SrcRectConstraint);
+            } else {
+                throw std::runtime_error(
+                                         "Image node could not resolve image dimension props.");
+            }
+        }
     }
   }
 };