Initial commit

mlc-ai · Aug 27, 2024 · a1bb7a5 · a1bb7a5
commit a1bb7a5
Show file tree

Hide file tree

Showing 96 changed files with 10,743 additions and 0 deletions.
diff --git a/.clang-format b/.clang-format
@@ -0,0 +1,2 @@
+BasedOnStyle: LLVM
+ColumnLimit: 120
diff --git a/.clangd b/.clangd
@@ -0,0 +1,2 @@
+CompileFlags:
+  CompilationDatabase: build-vscode/
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,82 @@
+name: CI
+
+on: [push, pull_request]
+env:
+  CIBW_BUILD_VERBOSITY: 3
+  CIBW_TEST_REQUIRES: "pytest"
+  CIBW_TEST_COMMAND: "pytest -svv {project}/tests/python/"
+  MLC_CIBW_VERSION: "2.20.0"
+  MLC_PYTHON_VERSION: "3.9"
+  MLC_CIBW_WIN_BUILD: "cp39-win_amd64"
+  MLC_CIBW_MAC_BUILD: "cp39-macosx_arm64"
+  MLC_CIBW_LINUX_BUILD: "cp313-manylinux_x86_64"
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.MLC_PYTHON_VERSION }}
+      - uses: pre-commit/[email protected]
+  windows:
+    name: Windows
+    runs-on: windows-latest
+    needs: pre-commit
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.MLC_PYTHON_VERSION }}
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel=="${{ env.MLC_CIBW_VERSION }}"
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BEFORE_ALL: ".\\scripts\\cpp_tests.bat"
+          CIBW_BUILD: ${{ env.MLC_CIBW_WIN_BUILD }}
+      - name: Show package contents
+        run: python scripts/show_wheel_content.py wheelhouse
+  macos:
+    name: MacOS
+    runs-on: macos-latest
+    needs: pre-commit
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.MLC_PYTHON_VERSION }}
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==${{ env.MLC_CIBW_VERSION }}
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BEFORE_ALL: "./scripts/cpp_tests.sh"
+          CIBW_BUILD: ${{ env.MLC_CIBW_MAC_BUILD }}
+      - name: Show package contents
+        run: python scripts/show_wheel_content.py wheelhouse
+  linux:
+    name: Linux
+    runs-on: ubuntu-latest
+    needs: pre-commit
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.MLC_PYTHON_VERSION }}
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==${{ env.MLC_CIBW_VERSION }}
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BEFORE_ALL: "./scripts/setup_manylinux2014.sh && ./scripts/cpp_tests.sh"
+          CIBW_BUILD: ${{ env.MLC_CIBW_LINUX_BUILD }}
+      - name: Show package contents
+        run: python scripts/show_wheel_content.py wheelhouse
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,124 @@
+name: Release
+
+on:
+  release:
+    types: [published]
+
+env:
+  CIBW_BUILD_VERBOSITY: 3
+  CIBW_TEST_REQUIRES: "pytest"
+  CIBW_TEST_COMMAND: "pytest -svv {project}/tests/python/"
+  CIBW_SKIP: "cp313-win_amd64"  # Python 3.13 is not quite ready yet
+  MLC_CIBW_VERSION: "2.20.0"
+  MLC_PYTHON_VERSION: "3.9"
+  MLC_CIBW_WIN_BUILD: "cp3*-win_amd64"
+  MLC_CIBW_MAC_BUILD: "cp3*-macosx_arm64"
+  MLC_CIBW_LINUX_BUILD: "cp3*-manylinux_x86_64"
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.MLC_PYTHON_VERSION }}
+      - uses: pre-commit/[email protected]
+  windows:
+    name: Windows
+    runs-on: windows-latest
+    needs: pre-commit
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.MLC_PYTHON_VERSION }}
+      - name: Install cibuildwheel
+        run: python -m pip install twine cibuildwheel=="${{ env.MLC_CIBW_VERSION }}"
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BUILD: ${{ env.MLC_CIBW_WIN_BUILD }}
+      - name: Show package contents
+        run: python scripts/show_wheel_content.py wheelhouse
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels-windows
+          path: ./wheelhouse/*.whl
+  macos:
+    name: MacOS
+    runs-on: macos-latest
+    needs: pre-commit
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.MLC_PYTHON_VERSION }}
+      - name: Install cibuildwheel
+        run: python -m pip install twine cibuildwheel==${{ env.MLC_CIBW_VERSION }}
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BUILD: ${{ env.MLC_CIBW_MAC_BUILD }}
+      - name: Show package contents
+        run: python scripts/show_wheel_content.py wheelhouse
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels-macos
+          path: ./wheelhouse/*.whl
+  linux:
+    name: Linux
+    runs-on: ubuntu-latest
+    needs: pre-commit
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.MLC_PYTHON_VERSION }}
+      - name: Install cibuildwheel
+        run: python -m pip install twine cibuildwheel==${{ env.MLC_CIBW_VERSION }}
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BUILD: ${{ env.MLC_CIBW_LINUX_BUILD }}
+      - name: Show package contents
+        run: python scripts/show_wheel_content.py wheelhouse
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels-linux
+          path: ./wheelhouse/*.whl
+  publish:
+    name: Publish
+    runs-on: ubuntu-latest
+    needs: [windows, macos, linux]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/mlc-python
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.MLC_PYTHON_VERSION }}
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: ./wheelhouse
+      - name: Prepare distribution files
+        run: |
+          mkdir -p dist
+          mv wheelhouse/wheels-macos/*.whl dist/
+          mv wheelhouse/wheels-linux/*.whl dist/
+          mv wheelhouse/wheels-windows/*.whl dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/
+          password: ${{ secrets.MLC_PYPI_API_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,4 @@
+.vscode
+*.dSYM
+build
+build-cpp-tests
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,9 @@
+[submodule "3rdparty/dlpack"]
+	path = 3rdparty/dlpack
+	url = https://github.com/dmlc/dlpack.git
+[submodule "3rdparty/libbacktrace"]
+	path = 3rdparty/libbacktrace
+	url = https://github.com/ianlancetaylor/libbacktrace
+[submodule "3rdparty/googletest"]
+	path = 3rdparty/googletest
+	url = https://github.com/google/googletest.git
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,48 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: trailing-whitespace
+      - id: mixed-line-ending
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.7
+    hooks:
+      - id: ruff
+        types_or: [python, pyi, jupyter]
+        args: [--fix]
+      - id: ruff-format
+        types_or: [python, pyi, jupyter]
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: "v1.11.1"
+    hooks:
+      - id: mypy
+        additional_dependencies: ['numpy >= 1.22', "ml-dtypes >= 0.1", "pytest"]
+        args: [--show-error-codes]
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: "v18.1.5"
+    hooks:
+      - id: clang-format
+  - repo: https://github.com/MarcoGorelli/cython-lint
+    rev: v0.16.2
+    hooks:
+      - id: cython-lint
+      - id: double-quote-cython-strings
+  - repo: https://github.com/scop/pre-commit-shfmt
+    rev: v3.8.0-1
+    hooks:
+      - id: shfmt
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.10.0.1
+    hooks:
+      - id: shellcheck
+  # - repo: https://github.com/cheshirekow/cmake-format-precommit
+  #   rev: v0.6.10
+  #   hooks:
+  #     - id: cmake-format
+  #     - id: cmake-lint
diff --git a/3rdparty/dlpack b/3rdparty/dlpack
diff --git a/3rdparty/googletest b/3rdparty/googletest
diff --git a/3rdparty/libbacktrace b/3rdparty/libbacktrace
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -0,0 +1,104 @@
+cmake_minimum_required(VERSION 3.15)
+
+project(
+  mlc
+  VERSION 1.0
+  DESCRIPTION "MLC's FFI system"
+  LANGUAGES C CXX
+)
+
+option(MLC_BUILD_TESTS "Build tests. This option will enable a test target `mlc_tests`." OFF)
+option(MLC_BUILD_PY "Build Python bindings." OFF)
+option(MLC_BUILD_REGISTRY
+  "Support for objects with non-static type indices. When turned on, \
+  targets linked against `mlc` will allow objects that comes with non-pre-defined type indices, \
+  so that the object hierarchy could expand without limitation. \
+  This will require the downstream targets to link against target `mlc_registry` to be effective."
+  OFF
+)
+
+include(cmake/Utils/CxxWarning.cmake)
+include(cmake/Utils/Sanitizer.cmake)
+include(cmake/Utils/Library.cmake)
+include(cmake/Utils/AddLibbacktrace.cmake)
+include(cmake/Utils/DebugSymbol.cmake)
+
+########## Target: `dlpack_header` ##########
+
+add_library(dlpack_header INTERFACE)
+target_include_directories(dlpack_header INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dlpack/include")
+
+########## Target: `mlc` ##########
+
+add_library(mlc INTERFACE)
+target_link_libraries(mlc INTERFACE dlpack_header)
+target_compile_features(mlc INTERFACE cxx_std_17)
+target_include_directories(mlc INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include")
+
+########## Target: `mlc_registry` ##########
+
+if (MLC_BUILD_REGISTRY)
+  add_library(mlc_registry_objs OBJECT
+    "${CMAKE_CURRENT_SOURCE_DIR}/cpp/c_api.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/cpp/c_api_tests.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/cpp/traceback.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/cpp/traceback_win.cc"
+  )
+  set_target_properties(
+    mlc_registry_objs PROPERTIES
+    POSITION_INDEPENDENT_CODE ON
+    CXX_STANDARD 17
+    CXX_EXTENSIONS OFF
+    CXX_STANDARD_REQUIRED ON
+    CXX_VISIBILITY_PRESET hidden
+    VISIBILITY_INLINES_HIDDEN ON
+    PREFIX "lib"
+  )
+  add_cxx_warning(mlc_registry_objs)
+  target_link_libraries(mlc_registry_objs PRIVATE dlpack_header)
+  target_include_directories(mlc_registry_objs PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include")
+  add_target_from_obj(mlc_registry mlc_registry_objs)
+  if (TARGET libbacktrace)
+    target_link_libraries(mlc_registry_objs PRIVATE libbacktrace)
+    target_link_libraries(mlc_registry_shared PRIVATE libbacktrace)
+    target_link_libraries(mlc_registry_static PRIVATE libbacktrace)
+    add_debug_symbol_apple(mlc_registry_shared "lib/")
+  endif ()
+  install(TARGETS mlc_registry_static DESTINATION "lib/")
+  install(TARGETS mlc_registry_shared DESTINATION "lib/")
+endif (MLC_BUILD_REGISTRY)
+
+########## Target: `mlc_py`  ##########
+
+if (MLC_BUILD_PY)
+  find_package(Python COMPONENTS Interpreter Development.Module REQUIRED)
+  file(GLOB _cython_sources CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/python/mlc/_cython/*.pyx")
+  set(_cython_outputs "")
+  foreach(_in IN LISTS _cython_sources)
+    get_filename_component(_file ${_in} NAME_WLE)
+    set(_out "${CMAKE_BINARY_DIR}/_cython/${_file}_cython.cc")
+    message(STATUS "Cythonize: ${_in} -> ${_out}")
+    add_custom_command(
+      OUTPUT "${_out}" DEPENDS "${_in}"
+      COMMENT "Making `${_out}` from `${_in}"
+      COMMAND Python::Interpreter -m cython "${_in}" --output-file "${_out}" --cplus
+      VERBATIM
+    )
+    list(APPEND _cython_outputs "${_out}")
+  endforeach()
+  Python_add_library(mlc_py MODULE ${_cython_outputs} WITH_SOABI)
+  set_target_properties(mlc_py PROPERTIES OUTPUT_NAME "core")
+  target_link_libraries(mlc_py PUBLIC mlc)
+  install(TARGETS mlc_py DESTINATION _cython/)
+endif (MLC_BUILD_PY)
+
+########## Adding tests ##########
+
+if (${PROJECT_NAME} STREQUAL ${CMAKE_PROJECT_NAME})
+  if (MLC_BUILD_TESTS)
+    enable_testing()
+    message(STATUS "Enable Testing")
+    include(cmake/Utils/AddGoogleTest.cmake)
+    add_subdirectory(tests/cpp/)
+  endif()
+endif ()