Linux build + C++17 compatibility option

.gitignore

@@ -1,6 +1,6 @@
 # Files to tell git to ignore
 
 /src/.vs
-/src/ort
-/src/build
+/ort
+/build
 /src/models/files/*.onnx

src/CMakeLists.txt → CMakeLists.txt

@@ -29,14 +29,18 @@ FetchContent_MakeAvailable(safeint)
 endif()
 
 project (Generators LANGUAGES CXX)
-set(GENERATORS_ROOT ${PROJECT_SOURCE_DIR})
-set(MODELS_ROOT ${PROJECT_SOURCE_DIR}/models)
-set(TESTS_ROOT ${PROJECT_SOURCE_DIR}/tests)
-set(PYBIND_ROOT ${PROJECT_SOURCE_DIR}/pybind)
+
+set(CMAKE_CXX_STANDARD 20)
+# add_compile_definitions(_DUSE_CXX17=1)
+
+set(GENERATORS_ROOT ${PROJECT_SOURCE_DIR}/src)
+set(MODELS_ROOT ${PROJECT_SOURCE_DIR}/src/models)
+set(TESTS_ROOT ${PROJECT_SOURCE_DIR}/src/tests)
+set(PYBIND_ROOT ${PROJECT_SOURCE_DIR}/src/pybind)
 
 set(USE_CUDA 1)
 
-add_subdirectory(${PROJECT_SOURCE_DIR}/../extern/pybind11 ${PROJECT_SOURCE_DIR}/build/pybind)
+add_subdirectory(${PROJECT_SOURCE_DIR}/extern/pybind11 ${PROJECT_SOURCE_DIR}/build/pybind)
 
 # CUDA Being enabled will make it not a debug build without this option, so all of the C++ headers will complain
 # about a mismatch with the actual debug headers and it'll fail to link. I don't know why this happens, or if this is the best fix.
@@ -95,15 +99,26 @@ target_include_directories(Models PRIVATE ${CMAKE_SOURCE_DIR}/ort)
 add_library (Generators ${generator_srcs})
 target_include_directories(Generators PRIVATE ${CMAKE_SOURCE_DIR}/ort)
 
+if(WIN32)
+  set(ONNXRUNTIME_LIB "onnxruntime.lib")
+  set(ONNXRUNTIME_FILES "onnxruntime*.dll")
+elseif(APPLE)
+  set(ONNXRUNTIME_LIB "libonnxruntime.dylib")
+  # TODO: Figure ONNXRUNTIME_FILES for MacOS
+else()
+  set(ONNXRUNTIME_LIB "libonnxruntime.so")
+  set(ONNXRUNTIME_FILES "libonnxruntime*.so*")
+endif()
+
 add_executable (Tests ${test_srcs})
 target_include_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort)
 target_link_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort)
-target_link_libraries(Tests PRIVATE Generators Models "onnxruntime.lib")
+target_link_libraries(Tests PRIVATE Generators Models ${ONNXRUNTIME_LIB})
 
 pybind11_add_module(ort_generators ${pybind_srcs})
 target_include_directories(ort_generators PRIVATE ${CMAKE_SOURCE_DIR}/ort)
 target_link_directories(ort_generators PRIVATE ${CMAKE_SOURCE_DIR}/ort)
-target_link_libraries(ort_generators PRIVATE Generators Models "onnxruntime.lib")
+target_link_libraries(ort_generators PRIVATE Generators Models ${ONNXRUNTIME_LIB})
 
 # Microsoft.GSL::GSL
 # target_include_directories(Generators PRIVATE ${safeint_SOURCE_DIR})
@@ -117,7 +132,7 @@ if(MSVC)
 endif()
 
  # Copy the onnxruntime binaries into the build folder so it's found on launch
-file(GLOB onnxruntime_libs "${CMAKE_SOURCE_DIR}/ort/onnxruntime*.dll")
+file(GLOB onnxruntime_libs "${CMAKE_SOURCE_DIR}/ort/${ONNXRUNTIME_FILES}")
 foreach(DLL_FILE ${onnxruntime_libs})
     add_custom_command(
         TARGET Generators POST_BUILD
@@ -127,13 +142,6 @@ foreach(DLL_FILE ${onnxruntime_libs})
     )
 endforeach()
 
-if (CMAKE_VERSION VERSION_GREATER 3.12)
-  set_property(TARGET Generators PROPERTY CXX_STANDARD 20)
-  set_property(TARGET Models PROPERTY CXX_STANDARD 20)
-  set_property(TARGET Tests PROPERTY CXX_STANDARD 20)
-  set_property(TARGET ort_generators PROPERTY CXX_STANDARD 20)
-endif()
-
 # Have visual studio put all files into one single 'Sources' folder vs the default split of header files into a separate folder
 source_group("Sources" FILES ${generator_srcs})
 source_group("Sources" FILES ${model_srcs})

build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+git submodule update --init --recursive
+
+cuda_version="cuda"
+cuda_arch=70
+
+export CUDA_HOME="/usr/local/${cuda_version}"
+export CUDNN_HOME="/usr/local/${cuda_version}"
+cuda_compiler="/usr/local/${cuda_version}/bin/nvcc"
+
+cd src
+cmake -S . -B build -DCMAKE_CUDA_ARCHITECTURES=$cuda_arch -DCMAKE_CUDA_COMPILER=$cuda_compiler -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+cd build
+make

install_ort.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+mkdir -p src/ort
+
+cp ~/onnxruntime/include/onnxruntime/core/session/onnxruntime_c*.h src/ort/
+cp ~/onnxruntime/build/Linux/Release/libonnxruntime*.so* src/ort/

src/beam_search_scorer_cuda.cu

@@ -1,6 +1,7 @@
 #include <cuda_runtime.h>
 #include <assert.h>
-#include <span>
+#include <algorithm>
+#include "span.h"
 #include "beam_search_scorer_cuda.cuh"
 
 namespace Generators

src/generators.h

@@ -4,8 +4,10 @@
 
 #include <algorithm>
 #include <assert.h>
+#include <cmath>
+#include <cstring>
 #include <functional>
-#include <span>
+#include "span.h"
 #include <memory>
 #include <numeric>
 #include <queue>

src/models/gpt_cpu.cpp

@@ -196,13 +196,13 @@ void Gpt::Run(std::span<const int32_t> next_tokens, std::span<const int32_t> nex
     UpdateInputs(next_tokens, next_indices, current_length);
 
 #if 0
-  printf("**Inputs:\r\n");
-  DumpTensors(inputs_.data(), input_names_.data(), input_names_.size(), true);
-  printf("**Outputs:\r\n");
-  DumpTensors(outputs_.data(), output_names_.data(), output_names_.size(), false);
+    printf("**Inputs:\r\n");
+    DumpTensors(inputs_.data(), input_names_.data(), input_names_.size(), true);
+    printf("**Outputs:\r\n");
+    DumpTensors(outputs_.data(), output_names_.data(), output_names_.size(), false);
 #endif
 
-try {
+  try {
     session_decode_->Run(nullptr, input_names_.data(), inputs_.data(), input_names_.size(), output_names_.data(), outputs_.data(), output_names_.size());
 //    session_decode_->Run(nullptr, *io_binding_decode_);
 //    logits_ = std::move(io_binding_decode_->GetOutputValues()[0]);

src/models/onnxruntime_cxx_inline_2.h

@@ -28,7 +28,7 @@ struct StandardAllocator : OrtAllocator
   {
     version = ORT_API_VERSION;
     OrtAllocator::Alloc = [](OrtAllocator* this_, size_t size) -> void* { return new std::byte[size]; };
-    OrtAllocator::Free = [](OrtAllocator* this_, void *p) { delete p; };
+    OrtAllocator::Free = [](OrtAllocator* this_, void *p) { delete reinterpret_cast<std::byte*>(p); };
   }
 };
 

src/pybind/python.cpp

@@ -28,7 +28,7 @@ struct ORTCHAR_String {
   std::wstring string_;
 };
 #else
-using ORTCHAR_String = std::string;
+#define ORTchar_String(string) string
 #endif
 
 struct float16 {

src/pybind/test_greedy.py

@@ -2,7 +2,7 @@
 import numpy as np
 from transformers import GPT2Tokenizer
 
-text = "best hotel in bay area"
+text = "The best hotel in bay area"
 
 # Generate input tokens from the text prompt
 tokenizer = GPT2Tokenizer.from_pretrained('gpt2')

src/search_cuda.cu

@@ -1,6 +1,6 @@
 #include <cuda_runtime.h>
 #include <algorithm>
-#include <span>
+#include "span.h"
 
 using ScoreType = float; // TODO: Move to header includable by cuda
 

src/sequences_cuda.cu

@@ -1,6 +1,6 @@
 #include <cuda_runtime.h>
 #include <assert.h>
-#include <span>
+#include "span.h"
 
 namespace Generators {
 namespace cuda {

src/smartptrs.h

@@ -6,7 +6,7 @@ namespace Generators {
 template <typename T>
 void copy(std::span<const T> source, std::span<T> dest) {
   assert(source.size() == dest.size());
-  copy(source.begin(), source.end(), dest.begin());
+  std::copy(source.begin(), source.end(), dest.begin());
 }
 
 template <typename T>

src/span.h

@@ -0,0 +1,41 @@
+#pragma once
+#include <vector>
+
+#ifndef USE_CXX17
+#include <span>
+#else
+namespace std {
+
+template <typename T>
+struct span {
+  span() = default;
+  span(T* p, size_t length) : p_{p}, length_{length} {}
+
+  span(const span<std::remove_const_t<T>>& s) : p_{const_cast<T*>(s.data())}, length_{s.size()} {}
+  span(std::vector<std::remove_const_t<T>>& s) : p_{const_cast<T*>(s.data())}, length_{s.size()} {}
+
+  bool empty() const { return length_ == 0; }
+
+  size_t size() const { return length_; }
+  size_t size_bytes() const { return length_ * sizeof(T); }
+
+  T* data() { return p_; }
+  const T* data() const { return p_; }
+
+  T& operator[](size_t index) { return p_[index]; }
+
+  T& back() { return p_[length_ - 1]; }
+  T back() const { return p_[length_ - 1]; }
+
+  T* begin() { return p_; }
+  T* end() { return p_ + length_; }
+
+  span subspan(size_t index, size_t length) { return span(p_ + index, length); }
+  span<const T> subspan(size_t index, size_t length) const { return span(p_ + index, length); }
+
+ private:
+  T* p_{};
+  size_t length_{};
+};
+}  // namespace std
+#endif

src/tests/tests.cpp

@@ -7,8 +7,8 @@
 #endif
 #include <iostream>
 
-// Our working directory is generators/src/build so one up puts us in the src directory:
-#define MODEL_PATH "../models/files/"
+// Our working directory is generators/build so one up puts us in the src directory:
+#define MODEL_PATH "../src/models/files/"
 
 #define ASSERT_EQ(a, b) assert((a) == (b))
 #define ASSERT_TRUE(a) assert(a)