diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 5ade1231..cc7a750e 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -28,13 +28,25 @@ jobs: include: - build: 'noavx' defines: '-DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF' + os: ubuntu-24.04 + arch: x64 - build: 'avx2' defines: '' + os: ubuntu-24.04 + arch: x64 - build: 'avx' defines: '-DGGML_AVX2=OFF' + os: ubuntu-24.04 + arch: x64 - build: 'avx512' defines: '-DGGML_AVX512=ON' - runs-on: ubuntu-24.04 + os: ubuntu-24.04 + arch: x64 + - build: 'aarch64' + defines: '-DGGML_NATIVE=OFF -DGGML_CPU_AARCH64=ON -DGGML_CPU_ARM_ARCH=armv8-a' + os: ubuntu-24.04-arm + arch: arm64 + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 with: @@ -52,28 +64,28 @@ jobs: - uses: actions/upload-artifact@v4 with: path: ./build/bin/libllama.so - name: llama-bin-linux-${{ matrix.build }}-x64.so + name: llama-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so if-no-files-found: error - uses: actions/upload-artifact@v4 with: path: ./build/bin/libggml.so - name: ggml-bin-linux-${{ matrix.build }}-x64.so + name: ggml-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so if-no-files-found: error - uses: actions/upload-artifact@v4 with: path: ./build/bin/libggml-base.so - name: ggml-base-bin-linux-${{ matrix.build }}-x64.so + name: ggml-base-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so if-no-files-found: error - uses: actions/upload-artifact@v4 with: path: ./build/bin/libggml-cpu.so - name: ggml-cpu-bin-linux-${{ matrix.build }}-x64.so + name: ggml-cpu-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so if-no-files-found: error - name: Upload Llava uses: actions/upload-artifact@v4 with: path: ./build/bin/libllava_shared.so - name: llava-bin-linux-${{ matrix.build }}-x64.so + name: llava-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so if-no-files-found: error compile-musl: @@ -601,7 +613,7 @@ jobs: - name: Rearrange Files run: | # Make all directories at once - mkdir --parents deps/{noavx,avx,avx2,avx512,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu11.7.1,cu12.2.0,vulkan,android-arm64-v8a,android-x86,android-x86_64} + mkdir --parents deps/{noavx,avx,avx2,avx512,linux-arm64,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu11.7.1,cu12.2.0,vulkan,android-arm64-v8a,android-x86,android-x86_64} # Linux cp artifacts/ggml-bin-linux-noavx-x64.so/libggml.so deps/noavx/libggml.so @@ -628,6 +640,13 @@ jobs: cp artifacts/llama-bin-linux-avx512-x64.so/libllama.so deps/avx512/libllama.so cp artifacts/llava-bin-linux-avx512-x64.so/libllava_shared.so deps/avx512/libllava_shared.so + # Arm64 + cp artifacts/ggml-bin-linux-aarch64-arm64.so/libggml.so deps/linux-arm64/libggml.so + cp artifacts/ggml-base-bin-linux-aarch64-arm64.so/libggml-base.so deps/linux-arm64/libggml-base.so + cp artifacts/ggml-cpu-bin-linux-aarch64-arm64.so/libggml-cpu.so deps/linux-arm64/libggml-cpu.so + cp artifacts/llama-bin-linux-aarch64-arm64.so/libllama.so deps/linux-arm64/libllama.so + cp artifacts/llava-bin-linux-aarch64-arm64.so/libllava_shared.so deps/linux-arm64/libllava_shared.so + # Musl cp artifacts/ggml-bin-musl-noavx-x64.so/libggml.so deps/musl-noavx/libggml.so cp artifacts/ggml-base-bin-musl-noavx-x64.so/libggml-base.so deps/musl-noavx/libggml-base.so diff --git a/LLama/LLamaSharp.Runtime.targets b/LLama/LLamaSharp.Runtime.targets index 22a3e04e..2972bcad 100644 --- a/LLama/LLamaSharp.Runtime.targets +++ b/LLama/LLamaSharp.Runtime.targets @@ -202,6 +202,28 @@ + + PreserveNewest + runtimes/linux-arm64/native/libllama.so + + + PreserveNewest + runtimes/linux-arm64/native/libggml.so + + + PreserveNewest + runtimes/linux-arm64/native/libggml-base.so + + + PreserveNewest + runtimes/linux-arm64/native/libggml-cpu.so + + + PreserveNewest + runtimes/linux-arm64/native/libllava_shared.so + + + PreserveNewest runtimes/linux-x64/native/cuda11/libllama.so diff --git a/LLama/Native/Load/NativeLibraryUtils.cs b/LLama/Native/Load/NativeLibraryUtils.cs index b0e8a792..9f6457cd 100644 --- a/LLama/Native/Load/NativeLibraryUtils.cs +++ b/LLama/Native/Load/NativeLibraryUtils.cs @@ -88,19 +88,28 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config, out INativeLib // On other platforms (Windows, Linux), we need to load the CPU backend from the specified AVX level directory // We are using the AVX level supplied by NativeLibraryConfig, which automatically detects the highest supported AVX level for us - // ggml-cpu - dependencyPaths.Add(Path.Combine( - $"runtimes/{os}/native/{NativeLibraryConfig.AvxLevelToString(library.Metadata.AvxLevel)}", - $"{libPrefix}ggml-cpu{ext}" - )); - - // ggml-cuda - if (library.Metadata.UseCuda) - dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-cuda{ext}")); - - // ggml-vulkan - if (library.Metadata.UseVulkan) - dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-vulkan{ext}")); + if (os == "linux-arm64"){ + dependencyPaths.Add(Path.Combine( + $"runtimes/{os}/native", + $"{libPrefix}ggml-cpu{ext}" + )); + } + else{ + // ggml-cpu + dependencyPaths.Add(Path.Combine( + $"runtimes/{os}/native/{NativeLibraryConfig.AvxLevelToString(library.Metadata.AvxLevel)}", + $"{libPrefix}ggml-cpu{ext}" + )); + + // ggml-cuda + if (library.Metadata.UseCuda) + dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-cuda{ext}")); + + // ggml-vulkan + if (library.Metadata.UseVulkan) + dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-vulkan{ext}")); + } + } } @@ -218,6 +227,13 @@ public static void GetPlatformPathParts(OSPlatform platform, out string os, out if (platform == OSPlatform.Linux) { + if(System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported){ + // linux arm64 + os = "linux-arm64"; + fileExtension = ".so"; + libPrefix = "lib"; + return; + } if(RuntimeInformation.RuntimeIdentifier.ToLower().StartsWith("alpine")) { // alpine linux distro diff --git a/LLama/Native/Load/NativeLibraryWithAvx.cs b/LLama/Native/Load/NativeLibraryWithAvx.cs index 932c4986..e6cbd86f 100644 --- a/LLama/Native/Load/NativeLibraryWithAvx.cs +++ b/LLama/Native/Load/NativeLibraryWithAvx.cs @@ -50,11 +50,17 @@ public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaL private string? GetAvxPath(SystemInfo systemInfo, AvxLevel avxLevel, NativeLogConfig.LLamaLogCallback? logCallback) { NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); - var avxStr = NativeLibraryConfig.AvxLevelToString(avxLevel); - if (!string.IsNullOrEmpty(avxStr)) - avxStr += "/"; - var relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; - return relativePath; + if (os != "linux-arm64"){ + var avxStr = NativeLibraryConfig.AvxLevelToString(avxLevel); + if (!string.IsNullOrEmpty(avxStr)) + avxStr += "/"; + var relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; + return relativePath; + } else { + var relativePath = $"runtimes/{os}/native/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; + return relativePath; + } + } } #endif diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec index 7c69534d..d173039a 100644 --- a/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec +++ b/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec @@ -66,7 +66,13 @@ - + + + + + + +