Implement the data encryption plugin for RocksDB (#1)

Most of the code are migrated from https://github.com/pegasus-kv/rocksdb/commits/v8.5.3-pegasus-encrypt, there is no functional changes, but just refactor to use the plugin architecture [1] of RocksDB. 1. https://github.com/facebook/rocksdb/tree/main/plugin#readme
pegasus-kv · Nov 8, 2023 · 58cf2b6 · 58cf2b6
1 parent f58c3d8
commit 58cf2b6
Show file tree

Hide file tree

Showing 29 changed files with 2,761 additions and 1 deletion.
diff --git a/.github/actions/build-folly/action.yml b/.github/actions/build-folly/action.yml
@@ -0,0 +1,8 @@
+name: build-folly
+runs:
+  using: composite
+  steps:
+  - name: Build folly and dependencies
+    working-directory: /__w/encfs/encfs/rocksdb
+    run: make build_folly -j4
+    shell: bash
diff --git a/.github/actions/increase-max-open-files-on-macos/action.yml b/.github/actions/increase-max-open-files-on-macos/action.yml
@@ -0,0 +1,10 @@
+name: increase-max-open-files-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Increase max open files
+    run: |-
+      sudo sysctl -w kern.maxfiles=1048576
+      sudo sysctl -w kern.maxfilesperproc=1048576
+      sudo launchctl limit maxfiles 1048576
+    shell: bash
diff --git a/.github/actions/install-cmake-on-macos/action.yml b/.github/actions/install-cmake-on-macos/action.yml
@@ -0,0 +1,7 @@
+name: install-cmake-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Install cmake on macos
+    run: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
+    shell: bash
diff --git a/.github/actions/install-gflags-on-macos/action.yml b/.github/actions/install-gflags-on-macos/action.yml
@@ -0,0 +1,7 @@
+name: install-gflags-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Install gflags on macos
+    run: HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
+    shell: bash
diff --git a/.github/actions/install-gflags/action.yml b/.github/actions/install-gflags/action.yml
@@ -0,0 +1,7 @@
+name: install-gflags
+runs:
+  using: composite
+  steps:
+  - name: Install gflags
+    run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+    shell: bash
diff --git a/.github/actions/install-jdk8-on-macos/action.yml b/.github/actions/install-jdk8-on-macos/action.yml
@@ -0,0 +1,7 @@
+name: install-jdk8-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Install JDK 8 on macos
+    run: brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
+    shell: bash
diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml
@@ -0,0 +1,25 @@
+name: post-steps
+runs:
+  using: composite
+  steps:
+  - uses: actions/[email protected]
+    with:
+      path: "/tmp/test-results"
+  - uses: actions/[email protected]
+    with:
+      path: LOG
+  - name: Compress Test Logs
+    run: tar -cvzf t.tar.gz t
+    if: failure()
+    shell: bash
+  - uses: actions/[email protected]
+    with:
+      path: t.tar.gz
+  - run: |-
+      mkdir -p /tmp/core_dumps
+      cp core.* /tmp/core_dumps
+    if: failure()
+    shell: bash
+  - uses: actions/[email protected]
+    with:
+      path: "/tmp/core_dumps"
diff --git a/.github/actions/pre-steps-macos/action.yml b/.github/actions/pre-steps-macos/action.yml
@@ -0,0 +1,28 @@
+name: pre-steps-macos
+runs:
+  using: composite
+  steps:
+  - name: Clone rocksdb
+    run: |
+      git clone --depth 1 --branch v8.5.3 https://github.com/facebook/rocksdb.git
+    shell: bash
+  - uses: actions/checkout@v4
+    with:
+      # Relative path under $GITHUB_WORKSPACE to place the repository
+      path: rocksdb/plugin/encfs
+  - name: Setup Environment Variables
+    run: |-
+      echo "ROCKSDB_PLUGINS=encfs" >> $GITHUB_ENV
+      echo "CIRCLECI=1" >> $GITHUB_ENV
+      echo "GTEST_THROW_ON_FAILURE=0" >> $GITHUB_ENV
+      echo "GTEST_OUTPUT=xml:/tmp/test-results/" >> $GITHUB_ENV
+      echo "SKIP_FORMAT_BUCK_CHECKS=1" >> $GITHUB_ENV
+      echo "GTEST_COLOR=1" >> $GITHUB_ENV
+      echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV
+      echo "CTEST_TEST_TIMEOUT=300" >> $GITHUB_ENV
+      echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> $GITHUB_ENV
+      echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> $GITHUB_ENV
+      echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> $GITHUB_ENV
+      echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $GITHUB_ENV
+      echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $GITHUB_ENV
+    shell: bash
diff --git a/.github/actions/pre-steps/action.yml b/.github/actions/pre-steps/action.yml
@@ -0,0 +1,35 @@
+name: pre-steps
+runs:
+  using: composite
+  steps:
+  - name: Free Disk Space
+    run: |-
+      rm -rf /usr/local/lib/android
+      rm -rf /usr/share/dotnet
+      rm -rf /opt/ghc
+      apt-get remove -y 'php.*'
+    shell: bash
+  - name: Clone rocksdb
+    run: |
+      git clone --depth 1 --branch v8.5.3 https://github.com/facebook/rocksdb.git
+    shell: bash
+  - uses: actions/checkout@v4
+    with:
+      # Relative path under $GITHUB_WORKSPACE to place the repository
+      path: rocksdb/plugin/encfs
+  - name: Setup Environment Variables
+    run: |-
+      echo "ROCKSDB_PLUGINS=encfs" >> $GITHUB_ENV
+      echo "CIRCLECI=1" >> $GITHUB_ENV
+      echo "GTEST_THROW_ON_FAILURE=0" >> $GITHUB_ENV
+      echo "GTEST_OUTPUT=xml:/tmp/test-results/" >> $GITHUB_ENV
+      echo "SKIP_FORMAT_BUCK_CHECKS=1" >> $GITHUB_ENV
+      echo "GTEST_COLOR=1" >> $GITHUB_ENV
+      echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV
+      echo "CTEST_TEST_TIMEOUT=300" >> $GITHUB_ENV
+      echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> $GITHUB_ENV
+      echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> $GITHUB_ENV
+      echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> $GITHUB_ENV
+      echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $GITHUB_ENV
+      echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $GITHUB_ENV
+    shell: bash
diff --git a/.github/actions/setup-folly/action.yml b/.github/actions/setup-folly/action.yml
@@ -0,0 +1,8 @@
+name: setup-folly
+runs:
+  using: composite
+  steps:
+  - name: Checkout folly sources
+    working-directory: /__w/encfs/encfs/rocksdb
+    run: make checkout_folly -j4
+    shell: bash
diff --git a/.github/workflows/build-fuzzers.yml b/.github/workflows/build-fuzzers.yml
@@ -0,0 +1,21 @@
+name: build-fuzzers
+on:
+  pull_request:
+    types: [ synchronize, opened, reopened ]
+    branches:
+      - main
+jobs:
+  build-fuzzers:
+    runs-on: ubuntu-latest
+    container:
+      image: zjay437/rocksdb:0.6
+    steps:
+    - uses: actions/checkout@v4
+    - uses: "./.github/actions/pre-steps"
+    - name: Build rocksdb
+      working-directory: /__w/encfs/encfs/rocksdb
+      run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
+    - name: Build fuzzers
+      working-directory: /__w/encfs/encfs/rocksdb
+      run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
+    - uses: "./.github/actions/post-steps"
diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml
@@ -0,0 +1,161 @@
+name: jobs-java
+on:
+  pull_request:
+    types: [ synchronize, opened, reopened ]
+    branches:
+      - main
+jobs:
+  build-linux-java:
+    runs-on: ubuntu-latest
+    container:
+      image: zjay437/rocksdb:0.6
+    steps:
+    - uses: actions/checkout@v4
+    - uses: "./.github/actions/pre-steps"
+    - name: Set Java Environment
+      run: |-
+        echo "JAVA_HOME=${JAVA_HOME}"
+        echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_ENV
+        which java && java -version
+        which javac && javac -version
+    - name: Test RocksDBJava
+      run: |-
+        cd /__w/encfs/encfs/rocksdb
+        make V=1 J=4 -j4 jtest
+    - uses: "./.github/actions/post-steps"
+  build-linux-java-static:
+    runs-on: ubuntu-latest
+    container:
+      image: zjay437/rocksdb:0.6
+    steps:
+    - uses: actions/checkout@v4
+    - uses: "./.github/actions/pre-steps"
+    - name: Set Java Environment
+      run: |-
+        echo "JAVA_HOME=${JAVA_HOME}"
+        echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_ENV
+        which java && java -version
+        which javac && javac -version
+    - name: Build RocksDBJava Static Library
+      run: |-
+        cd /__w/encfs/encfs/rocksdb
+        make V=1 J=4 -j4 rocksdbjavastatic
+    - uses: "./.github/actions/post-steps"
+  build-macos-java:
+    runs-on: macos-11
+    env:
+      JAVA_HOME: "/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home"
+      ROCKSDB_DISABLE_JEMALLOC: 1
+    steps:
+    - uses: actions/checkout@v4
+    - uses: maxim-lobanov/[email protected]
+      with:
+        xcode-version: 12.5.1
+    - uses: "./.github/actions/increase-max-open-files-on-macos"
+    - uses: "./.github/actions/install-gflags-on-macos"
+    - uses: "./.github/actions/install-jdk8-on-macos"
+    - uses: "./.github/actions/pre-steps-macos"
+    - name: Set Java Environment
+      run: |-
+        echo "JAVA_HOME=${JAVA_HOME}"
+        echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_ENV
+        which java && java -version
+        which javac && javac -version
+    - name: Test RocksDBJava
+      run: echo "skip this job"
+#      Undefined symbols for architecture x86_64:
+#          "encfs_reg", referenced from:
+#                         -u command line option
+#                         (maybe you meant: _encfs_reg)
+#      ld: symbol(s) not found for architecture x86_64
+#      clang:
+#        error: linker command failed with exit code 1 (use -v to see invocation)
+#      make: *** [ rocksdbjava ] Error 1
+#      run: |-
+#        cd /Users/runner/work/encfs/encfs/rocksdb
+#        make V=1 J=4 -j4 jtest
+    - uses: "./.github/actions/post-steps"
+  build-macos-java-static:
+    runs-on: macos-11
+    env:
+      JAVA_HOME: "/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home"
+    steps:
+    - uses: actions/checkout@v4
+    - uses: maxim-lobanov/[email protected]
+      with:
+        xcode-version: 12.5.1
+    - uses: "./.github/actions/increase-max-open-files-on-macos"
+    - uses: "./.github/actions/install-gflags-on-macos"
+    - uses: "./.github/actions/install-cmake-on-macos"
+    - uses: "./.github/actions/install-jdk8-on-macos"
+    - uses: "./.github/actions/pre-steps-macos"
+    - name: Set Java Environment
+      run: |-
+        echo "JAVA_HOME=${JAVA_HOME}"
+        echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_ENV
+        which java && java -version
+        which javac && javac -version
+    - name: Build RocksDBJava x86 and ARM Static Libraries
+      run: echo "skip this job"
+      # TODO(yingchun): resolve it
+      # build error:
+      #   Undefined symbols for architecture x86_64:
+      #       "_AES_decrypt", referenced from:
+      #                         rocksdb::encryption::AESBlockCipher::Decrypt(char*) in encryption.o
+      #         "_AES_encrypt", referenced from:
+      #                           rocksdb::encryption::AESCTRCipherStream::EncryptBlock(unsigned long long, char*, char*) in encryption.o
+      #                           rocksdb::encryption::AESBlockCipher::Encrypt(char*) in encryption.o
+      #         "_AES_set_decrypt_key", referenced from:
+      #                                   rocksdb::encryption::AESBlockCipher::InitKey(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) in encryption.o
+      #         "_AES_set_encrypt_key", referenced from:
+      #                                   rocksdb::encryption::AESBlockCipher::InitKey(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) in encryption.o
+      #   ld: symbol(s) not found for architecture x86_64
+      #   clang: error: linker command failed with exit code 1 (use -v to see invocation)
+      #   make[2]: *** [rocksdbjavastatic_javalib] Error 1
+      #   make[1]: *** [rocksdbjavastaticosx_arch_x86_64] Error 2
+      #   make: *** [rocksdbjavastaticosx_archs] Error 2
+      #   Error: Process completed with exit code 2.
+      # run: make V=1 J=4 -j4 rocksdbjavastaticosx
+    - uses: "./.github/actions/post-steps"
+  build-macos-java-static-universal:
+    runs-on: macos-11
+    env:
+      JAVA_HOME: "/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home"
+    steps:
+    - uses: actions/checkout@v4
+    - uses: maxim-lobanov/[email protected]
+      with:
+        xcode-version: 12.5.1
+    - uses: "./.github/actions/increase-max-open-files-on-macos"
+    - uses: "./.github/actions/install-gflags-on-macos"
+    - uses: "./.github/actions/install-cmake-on-macos"
+    - uses: "./.github/actions/install-jdk8-on-macos"
+    - uses: "./.github/actions/pre-steps-macos"
+    - name: Set Java Environment
+      run: |-
+        echo "JAVA_HOME=${JAVA_HOME}"
+        echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_ENV
+        which java && java -version
+        which javac && javac -version
+    - name: Build RocksDBJava Universal Binary Static Library
+      run: echo "skip this job"
+      # TODO(yingchun): resolve it
+      # build error:
+      #   Undefined symbols for architecture x86_64:
+      #       "_AES_decrypt", referenced from:
+      #                         rocksdb::encryption::AESBlockCipher::Decrypt(char*) in encryption.o
+      #         "_AES_encrypt", referenced from:
+      #                           rocksdb::encryption::AESCTRCipherStream::EncryptBlock(unsigned long long, char*, char*) in encryption.o
+      #                           rocksdb::encryption::AESBlockCipher::Encrypt(char*) in encryption.o
+      #         "_AES_set_decrypt_key", referenced from:
+      #                                   rocksdb::encryption::AESBlockCipher::InitKey(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) in encryption.o
+      #         "_AES_set_encrypt_key", referenced from:
+      #                                   rocksdb::encryption::AESBlockCipher::InitKey(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) in encryption.o
+      #   ld: symbol(s) not found for architecture x86_64
+      #   clang: error: linker command failed with exit code 1 (use -v to see invocation)
+      #   make[2]: *** [rocksdbjavastatic_javalib] Error 1
+      #   make[1]: *** [rocksdbjavastaticosx_arch_x86_64] Error 2
+      #   make: *** [rocksdbjavastaticosx_archs] Error 2
+      #   Error: Process completed with exit code 2.
+      # run: make V=1 J=4 -j4 rocksdbjavastaticosx_ub
+    - uses: "./.github/actions/post-steps"
diff --git a/.github/workflows/jobs-linux-arm.yml b/.github/workflows/jobs-linux-arm.yml
@@ -0,0 +1,18 @@
+name: jobs-linux-arm
+on:
+  pull_request:
+    types: [ synchronize, opened, reopened ]
+    branches:
+      - main
+jobs:
+  build-linux-arm:
+    runs-on: ubuntu-20.04
+    container:
+      image: zjay437/rocksdb:0.6
+    steps:
+    - run: echo "skip this job, Github Actions free plan does not support arm64 yet"
+#    - uses: actions/checkout@v4
+#    - uses: "./.github/actions/pre-steps"
+#    - uses: "./.github/actions/install-gflags"
+#    - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some encfs_test
+#    - uses: "./.github/actions/post-steps"