Merge branch 'main' into dont-swallow-errors

.circleci/config.yml

@@ -16,7 +16,8 @@ commands:
       - run:
           name: Install JDK 8 on macos
           command: |
-            brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
+            HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica
+            HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8
 
   increase-max-open-files-on-macos:
     steps:
@@ -77,6 +78,11 @@ commands:
           shell: powershell.exe
           command: |
             build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
+      - run:
+          name: "Test RocksJava"
+          command: |
+            cd build\java
+            & $Env:CTEST_BIN -C Debug -j 16
   pre-steps-macos:
       steps:
         - pre-steps
@@ -104,6 +110,15 @@ commands:
           path: /tmp/core_dumps
           when: on_fail
 
+  post-pmd-steps:
+    steps:
+      - store_artifacts:
+          path: /home/circleci/project/java/target/pmd.xml
+          when: on_fail
+      - store_artifacts:
+          path: /home/circleci/project/java/target/site
+          when: on_fail
+
   upgrade-cmake:
     steps:
       - run:
@@ -126,6 +141,13 @@ commands:
           command: |
             HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
 
+  install-maven:
+    steps:
+      - run:
+          name: Install maven
+          command: |
+            sudo apt-get update -y && sudo apt-get install -y maven
+
   setup-folly:
     steps:
       - run:
@@ -209,21 +231,21 @@ executors:
 jobs:
   build-macos:
     macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
     environment:
       ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc cause env_test hang, disable it for now
     steps:
       - increase-max-open-files-on-macos
       - install-gflags-on-macos
       - pre-steps-macos
-      - run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=32 -j32 all
+      - run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=16 -j16 all
       - post-steps
 
   build-macos-cmake:
     macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
     parameters:
       run_even_tests:
         description: run even or odd tests, used to split tests to 2 groups
@@ -239,20 +261,20 @@ jobs:
           command: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
       - run:
           name: "Build tests"
-          command: cd build && make V=1 -j32
+          command: cd build && make V=1 -j16
       - when:
           condition: << parameters.run_even_tests >>
           steps:
             - run:
                 name: "Run even tests"
-                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 0,,2
+                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2
       - when:
           condition:
             not: << parameters.run_even_tests >>
           steps:
             - run:
                 name: "Run odd tests"
-                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 1,,2
+                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2
       - post-steps
 
   build-linux:
@@ -549,6 +571,7 @@ jobs:
       THIRDPARTY_HOME: C:/Users/circleci/thirdparty
       CMAKE_HOME: C:/Program Files/CMake
       CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
       SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
       SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
       SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
@@ -564,6 +587,7 @@ jobs:
       THIRDPARTY_HOME: C:/Users/circleci/thirdparty
       CMAKE_HOME: C:/Program Files/CMake
       CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
       SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
       SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
       SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
@@ -588,6 +612,27 @@ jobs:
           command: make V=1 J=8 -j8 jtest
       - post-steps
 
+  build-linux-java-pmd:
+    machine: 
+      image: ubuntu-2004:202111-02
+    resource_class: large
+    environment:
+      JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+    steps:
+      - install-maven
+      - pre-steps
+      - run:
+          name: "Set Java Environment"
+          command: |
+            echo "JAVA_HOME=${JAVA_HOME}"
+            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
+            which java && java -version
+            which javac && javac -version
+      - run:
+          name: "PMD RocksDBJava"
+          command: make V=1 J=8 -j8 jpmd
+      - post-pmd-steps
+
   build-linux-java-static:
     executor: linux-docker
     resource_class: large
@@ -607,10 +652,10 @@ jobs:
 
   build-macos-java:
     macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
     environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
       ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc causes java 8 crash
     steps:
       - increase-max-open-files-on-macos
@@ -632,10 +677,10 @@ jobs:
 
   build-macos-java-static:
     macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
     environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
     steps:
       - increase-max-open-files-on-macos
       - install-gflags-on-macos
@@ -657,10 +702,10 @@ jobs:
 
   build-macos-java-static-universal:
     macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
     environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
     steps:
       - increase-max-open-files-on-macos
       - install-gflags-on-macos
@@ -852,6 +897,7 @@ workflows:
       - build-macos-java
       - build-macos-java-static
       - build-macos-java-static-universal
+      - build-linux-java-pmd
   jobs-macos:
     jobs:
       - build-macos

.github/workflows/sanity_check.yml

@@ -33,9 +33,7 @@ jobs:
       run: pip install argparse
 
     - name: Download clang-format-diff.py
-      uses: wei/wget@v1
-      with:
-        args: https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py
+      run: wget https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py
 
     - name: Check format
       run: VERBOSE_CHECK=1 make check-format

.gitignore

@@ -85,6 +85,7 @@ fbcode/
 fbcode
 buckifier/*.pyc
 buckifier/__pycache__
+.arcconfig
 
 compile_commands.json
 clang-format-diff.py
@@ -95,4 +96,5 @@ fuzz/crash-*
 
 cmake-build-*
 third-party/folly/
-.cache
+.cache
+*.sublime-*

CMakeLists.txt

@@ -254,7 +254,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")
 endif(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")
 
 set(PORTABLE 0 CACHE STRING "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU")
-if(PORTABLE STREQUAL 1)
+if(PORTABLE MATCHES "1|ON|YES|TRUE|Y")
   # Usually nothing to do; compiler default is typically the most general
   if(NOT MSVC)
     if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x")
@@ -264,14 +264,7 @@ if(PORTABLE STREQUAL 1)
       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=loongarch64")
     endif()
   endif()
-elseif(PORTABLE MATCHES [^0]+)
-  # Name of a CPU arch spec or feature set to require
-  if(MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:${PORTABLE}")
-  else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${PORTABLE}")
-  endif()
-else()
+elseif(PORTABLE MATCHES "0|OFF|NO|FALSE|N")
   if(MSVC)
     # NOTE: No auto-detection of current CPU, but instead assume some useful
     # level of optimization is supported
@@ -285,6 +278,13 @@ else()
       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
     endif()
   endif()
+else()
+  # Name of a CPU arch spec or feature set to require
+  if(MSVC)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:${PORTABLE}")
+  else()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${PORTABLE}")
+  endif()
 endif()
 
 include(CheckCXXSourceCompiles)
@@ -632,6 +632,7 @@ set(SOURCES
         cache/secondary_cache.cc
         cache/secondary_cache_adapter.cc
         cache/sharded_cache.cc
+        cache/tiered_secondary_cache.cc
         db/arena_wrapped_db_iter.cc
         db/blob/blob_contents.cc
         db/blob/blob_fetcher.cc
@@ -714,6 +715,7 @@ set(SOURCES
         db/wal_manager.cc
         db/wide/wide_column_serialization.cc
         db/wide/wide_columns.cc
+        db/wide/wide_columns_helper.cc
         db/write_batch.cc
         db/write_batch_base.cc
         db/write_controller.cc
@@ -1262,6 +1264,7 @@ if(WITH_TESTS)
         cache/cache_test.cc
         cache/compressed_secondary_cache_test.cc
         cache/lru_cache_test.cc
+        cache/tiered_secondary_cache_test.cc
         db/blob/blob_counting_iterator_test.cc
         db/blob/blob_file_addition_test.cc
         db/blob/blob_file_builder_test.cc
@@ -1357,6 +1360,7 @@ if(WITH_TESTS)
         db/wal_edit_test.cc
         db/wide/db_wide_basic_test.cc
         db/wide/wide_column_serialization_test.cc
+        db/wide/wide_columns_helper_test.cc
         db/write_batch_test.cc
         db/write_callback_test.cc
         db/write_controller_test.cc
@@ -1592,3 +1596,6 @@ option(WITH_BENCHMARK "build benchmark tests" OFF)
 if(WITH_BENCHMARK)
   add_subdirectory(${PROJECT_SOURCE_DIR}/microbench/)
 endif()
+
+target_include_directories(${PROJECT_NAME} PUBLIC
+  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>)