diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml
index 5ade12316..cc7a750ec 100644
--- a/.github/workflows/compile.yml
+++ b/.github/workflows/compile.yml
@@ -28,13 +28,25 @@ jobs:
include:
- build: 'noavx'
defines: '-DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF'
+ os: ubuntu-24.04
+ arch: x64
- build: 'avx2'
defines: ''
+ os: ubuntu-24.04
+ arch: x64
- build: 'avx'
defines: '-DGGML_AVX2=OFF'
+ os: ubuntu-24.04
+ arch: x64
- build: 'avx512'
defines: '-DGGML_AVX512=ON'
- runs-on: ubuntu-24.04
+ os: ubuntu-24.04
+ arch: x64
+ - build: 'aarch64'
+ defines: '-DGGML_NATIVE=OFF -DGGML_CPU_AARCH64=ON -DGGML_CPU_ARM_ARCH=armv8-a'
+ os: ubuntu-24.04-arm
+ arch: arm64
+ runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
@@ -52,28 +64,28 @@ jobs:
- uses: actions/upload-artifact@v4
with:
path: ./build/bin/libllama.so
- name: llama-bin-linux-${{ matrix.build }}-x64.so
+ name: llama-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
if-no-files-found: error
- uses: actions/upload-artifact@v4
with:
path: ./build/bin/libggml.so
- name: ggml-bin-linux-${{ matrix.build }}-x64.so
+ name: ggml-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
if-no-files-found: error
- uses: actions/upload-artifact@v4
with:
path: ./build/bin/libggml-base.so
- name: ggml-base-bin-linux-${{ matrix.build }}-x64.so
+ name: ggml-base-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
if-no-files-found: error
- uses: actions/upload-artifact@v4
with:
path: ./build/bin/libggml-cpu.so
- name: ggml-cpu-bin-linux-${{ matrix.build }}-x64.so
+ name: ggml-cpu-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
if-no-files-found: error
- name: Upload Llava
uses: actions/upload-artifact@v4
with:
path: ./build/bin/libllava_shared.so
- name: llava-bin-linux-${{ matrix.build }}-x64.so
+ name: llava-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
if-no-files-found: error
compile-musl:
@@ -601,7 +613,7 @@ jobs:
- name: Rearrange Files
run: |
# Make all directories at once
- mkdir --parents deps/{noavx,avx,avx2,avx512,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu11.7.1,cu12.2.0,vulkan,android-arm64-v8a,android-x86,android-x86_64}
+ mkdir --parents deps/{noavx,avx,avx2,avx512,linux-arm64,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu11.7.1,cu12.2.0,vulkan,android-arm64-v8a,android-x86,android-x86_64}
# Linux
cp artifacts/ggml-bin-linux-noavx-x64.so/libggml.so deps/noavx/libggml.so
@@ -628,6 +640,13 @@ jobs:
cp artifacts/llama-bin-linux-avx512-x64.so/libllama.so deps/avx512/libllama.so
cp artifacts/llava-bin-linux-avx512-x64.so/libllava_shared.so deps/avx512/libllava_shared.so
+ # Arm64
+ cp artifacts/ggml-bin-linux-aarch64-arm64.so/libggml.so deps/linux-arm64/libggml.so
+ cp artifacts/ggml-base-bin-linux-aarch64-arm64.so/libggml-base.so deps/linux-arm64/libggml-base.so
+ cp artifacts/ggml-cpu-bin-linux-aarch64-arm64.so/libggml-cpu.so deps/linux-arm64/libggml-cpu.so
+ cp artifacts/llama-bin-linux-aarch64-arm64.so/libllama.so deps/linux-arm64/libllama.so
+ cp artifacts/llava-bin-linux-aarch64-arm64.so/libllava_shared.so deps/linux-arm64/libllava_shared.so
+
# Musl
cp artifacts/ggml-bin-musl-noavx-x64.so/libggml.so deps/musl-noavx/libggml.so
cp artifacts/ggml-base-bin-musl-noavx-x64.so/libggml-base.so deps/musl-noavx/libggml-base.so
diff --git a/LLama/LLamaSharp.Runtime.targets b/LLama/LLamaSharp.Runtime.targets
index 22a3e04e1..2972bcadd 100644
--- a/LLama/LLamaSharp.Runtime.targets
+++ b/LLama/LLamaSharp.Runtime.targets
@@ -202,6 +202,28 @@
+
+ PreserveNewest
+ runtimes/linux-arm64/native/libllama.so
+
+
+ PreserveNewest
+ runtimes/linux-arm64/native/libggml.so
+
+
+ PreserveNewest
+ runtimes/linux-arm64/native/libggml-base.so
+
+
+ PreserveNewest
+ runtimes/linux-arm64/native/libggml-cpu.so
+
+
+ PreserveNewest
+ runtimes/linux-arm64/native/libllava_shared.so
+
+
+
PreserveNewest
runtimes/linux-x64/native/cuda11/libllama.so
diff --git a/LLama/Native/Load/NativeLibraryUtils.cs b/LLama/Native/Load/NativeLibraryUtils.cs
index b0e8a792a..9f6457cd1 100644
--- a/LLama/Native/Load/NativeLibraryUtils.cs
+++ b/LLama/Native/Load/NativeLibraryUtils.cs
@@ -88,19 +88,28 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config, out INativeLib
// On other platforms (Windows, Linux), we need to load the CPU backend from the specified AVX level directory
// We are using the AVX level supplied by NativeLibraryConfig, which automatically detects the highest supported AVX level for us
- // ggml-cpu
- dependencyPaths.Add(Path.Combine(
- $"runtimes/{os}/native/{NativeLibraryConfig.AvxLevelToString(library.Metadata.AvxLevel)}",
- $"{libPrefix}ggml-cpu{ext}"
- ));
-
- // ggml-cuda
- if (library.Metadata.UseCuda)
- dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-cuda{ext}"));
-
- // ggml-vulkan
- if (library.Metadata.UseVulkan)
- dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-vulkan{ext}"));
+ if (os == "linux-arm64"){
+ dependencyPaths.Add(Path.Combine(
+ $"runtimes/{os}/native",
+ $"{libPrefix}ggml-cpu{ext}"
+ ));
+ }
+ else{
+ // ggml-cpu
+ dependencyPaths.Add(Path.Combine(
+ $"runtimes/{os}/native/{NativeLibraryConfig.AvxLevelToString(library.Metadata.AvxLevel)}",
+ $"{libPrefix}ggml-cpu{ext}"
+ ));
+
+ // ggml-cuda
+ if (library.Metadata.UseCuda)
+ dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-cuda{ext}"));
+
+ // ggml-vulkan
+ if (library.Metadata.UseVulkan)
+ dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-vulkan{ext}"));
+ }
+
}
}
@@ -218,6 +227,13 @@ public static void GetPlatformPathParts(OSPlatform platform, out string os, out
if (platform == OSPlatform.Linux)
{
+ if(System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported){
+ // linux arm64
+ os = "linux-arm64";
+ fileExtension = ".so";
+ libPrefix = "lib";
+ return;
+ }
if(RuntimeInformation.RuntimeIdentifier.ToLower().StartsWith("alpine"))
{
// alpine linux distro
diff --git a/LLama/Native/Load/NativeLibraryWithAvx.cs b/LLama/Native/Load/NativeLibraryWithAvx.cs
index 932c49866..e6cbd86f3 100644
--- a/LLama/Native/Load/NativeLibraryWithAvx.cs
+++ b/LLama/Native/Load/NativeLibraryWithAvx.cs
@@ -50,11 +50,17 @@ public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaL
private string? GetAvxPath(SystemInfo systemInfo, AvxLevel avxLevel, NativeLogConfig.LLamaLogCallback? logCallback)
{
NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix);
- var avxStr = NativeLibraryConfig.AvxLevelToString(avxLevel);
- if (!string.IsNullOrEmpty(avxStr))
- avxStr += "/";
- var relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
- return relativePath;
+ if (os != "linux-arm64"){
+ var avxStr = NativeLibraryConfig.AvxLevelToString(avxLevel);
+ if (!string.IsNullOrEmpty(avxStr))
+ avxStr += "/";
+ var relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
+ return relativePath;
+ } else {
+ var relativePath = $"runtimes/{os}/native/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
+ return relativePath;
+ }
+
}
}
#endif
diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec
index 7c69534da..d173039a9 100644
--- a/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec
+++ b/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec
@@ -66,7 +66,13 @@
-
+
+
+
+
+
+
+