diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d32f3c..a96c6e2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: build: runs-on: ${{ matrix.os }} - name: "${{ matrix.cpp_compiler }}-${{ matrix.os }}-${{ matrix.build_type }}" + name: "${{ matrix.build_type }}; LLVM: ${{ matrix.llvm_backend }}" strategy: # Set fail-fast to false to ensure that feedback is delivered for all matrix combinations. Consider changing this to true when your workflow is stable. @@ -30,7 +30,8 @@ jobs: # To add more build types (Release, Debug, RelWithDebInfo, etc.) customize the build_type list. matrix: os: [ubuntu-latest] - build_type: [RelWithDebInfo] + build_type: [Release,Debug] + llvm_backend: [ON,OFF] steps: - uses: actions/checkout@v3 @@ -80,16 +81,17 @@ jobs: run: clang++ --version - name: Check clang version - run: clang++-17 --version + run: clang++-18 --version - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type run: > cmake -B ${{ steps.strings.outputs.build-output-dir }} - -DCMAKE_CXX_COMPILER=clang++-17 - -DCMAKE_C_COMPILER=clang-17 + -DCMAKE_CXX_COMPILER=clang++-18 + -DCMAKE_C_COMPILER=clang-18 -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + -DENDO_USE_LLVM=${{ matrix.llvm_backend }} -GNinja -S ${{ github.workspace }} diff --git a/CMakeLists.txt b/CMakeLists.txt index d5921bb..9b5e01c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.28 FATAL_ERROR) -project(endo VERSION "0.0.0" LANGUAGES CXX) +project(endo VERSION "0.0.0") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -11,6 +11,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +option(ENDO_USE_LLVM "Use llvm as a backend for the shell" ON) if(NOT DEFINED ENDO_TRACE_VM) option(ENDO_TRACE_VM "Enables debug output" OFF) diff --git a/src/CoreVM/CMakeLists.txt b/src/CoreVM/CMakeLists.txt index c48ea05..996075f 100644 --- a/src/CoreVM/CMakeLists.txt +++ b/src/CoreVM/CMakeLists.txt @@ -62,4 +62,4 @@ target_include_directories(CoreVM PUBLIC $ ) -target_link_libraries(CoreVM PUBLIC fmt::fmt-header-only range-v3::range-v3) +target_link_libraries(CoreVM PUBLIC fmt::fmt-header-only range-v3::range-v3 crispy::core) diff --git a/src/CoreVM/transform/MergeBlockPass.cpp b/src/CoreVM/transform/MergeBlockPass.cpp index 48bf735..79b5642 100644 --- a/src/CoreVM/transform/MergeBlockPass.cpp +++ b/src/CoreVM/transform/MergeBlockPass.cpp @@ -2,6 +2,8 @@ module; #include +#include + module CoreVM; namespace CoreVM diff --git a/src/shell/ASTPrinter.cpp b/src/shell/ASTPrinter.cpp index 93edc6b..98abed4 100644 --- a/src/shell/ASTPrinter.cpp +++ b/src/shell/ASTPrinter.cpp @@ -4,7 +4,7 @@ module; #include #include -#include +#include import Lexer; @@ -146,8 +146,8 @@ export class ASTPrinter: public Visitor } void visit(LiteralExpr const& node) override { _result += fmt::format("{}", node.value); } - void visit(SubstitutionExpr const& node) override { crispy::ignore_unused(node); } - void visit(CommandFileSubst const& node) override { crispy::ignore_unused(node); } + void visit(SubstitutionExpr const& node) override {} + void visit(CommandFileSubst const& node) override {} }; } // namespace endo::ast diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 858dd30..c1af8eb 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -23,7 +23,37 @@ find_package(Threads REQUIRED) # see https://github.com/llvm/llvm-project/issues/75108 # and https://github.com/ericniebler/range-v3/blob/f013aef2ae81f3661a560e7922a968665bedebff/include/meta/meta_fwd.hpp#L37 target_compile_definitions(Shell PUBLIC META_WORKAROUND_LLVM_28385) -target_link_libraries(Shell PUBLIC fmt::fmt-header-only vtparser crispy::core CoreVM InputEditor Threads::Threads util) +target_compile_definitions(Shell PUBLIC ENDO_VERSION_STRING="0.0.1") +set(shell_libs + fmt::fmt-header-only + vtparser + CoreVM + crispy::core + InputEditor + Threads::Threads + util) + + +if(ENDO_USE_LLVM) + find_package(LLVM REQUIRED) + message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") + message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") + include("${LLVM_DIR}/AddLLVM.cmake") + + include_directories(${LLVM_INCLUDE_DIRS}) + add_definitions(${LLVM_DEFINITIONS}) + llvm_map_components_to_libnames(llvm_libs analysis core executionengine instcombine object orcjit runtimedyld scalaropts support native) + # Link against LLVM libraries + set(shell_libs + ${shell_libs} + ${llvm_libs}) + + target_sources(Shell + PUBLIC + FILE_SET CXX_MODULES FILES + LLVMBackend.cpp) + target_compile_definitions(Shell PUBLIC ENDO_USE_LLVM) +endif() if(ENDO_TRACE_VM) @@ -36,6 +66,7 @@ if(ENDO_TRACE_LEXER) target_compile_definitions(Shell PUBLIC ENDO_TRACE_LEXER) endif() +target_link_libraries(Shell PUBLIC ${shell_libs}) add_executable(endo main.cpp diff --git a/src/shell/LLVMBackend.cpp b/src/shell/LLVMBackend.cpp new file mode 100644 index 0000000..46339da --- /dev/null +++ b/src/shell/LLVMBackend.cpp @@ -0,0 +1,303 @@ +module; + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using std::unique_ptr; + +using namespace llvm; +using namespace llvm::orc; + +export module LLVMBackend; + +auto inline llvmLog = logstore::category("llvm", "Debug log", logstore::category::state::Enabled); + +class TestJIT +{ + private: + std::unique_ptr ES; + + DataLayout DL; + MangleAndInterner Mangle; + + RTDyldObjectLinkingLayer ObjectLayer; + IRCompileLayer CompileLayer; + + JITDylib& MainJD; + + public: + TestJIT(std::unique_ptr ES, JITTargetMachineBuilder JTMB, DataLayout DL): + ES(std::move(ES)), + DL(std::move(DL)), + Mangle(*this->ES, this->DL), + ObjectLayer(*this->ES, []() { return std::make_unique(); }), + CompileLayer(*this->ES, ObjectLayer, std::make_unique(std::move(JTMB))), + MainJD(this->ES->createBareJITDylib("
")) + { + MainJD.addGenerator( + cantFail(DynamicLibrarySearchGenerator::GetForCurrentProcess(DL.getGlobalPrefix()))); + if (JTMB.getTargetTriple().isOSBinFormatCOFF()) + { + ObjectLayer.setOverrideObjectFlagsWithResponsibilityFlags(true); + ObjectLayer.setAutoClaimResponsibilityForObjectSymbols(true); + } + } + + ~TestJIT() + { + if (auto Err = ES->endSession()) + ES->reportError(std::move(Err)); + } + + static Expected> Create() + { + auto EPC = SelfExecutorProcessControl::Create(); + if (!EPC) + return EPC.takeError(); + + auto ES = std::make_unique(std::move(*EPC)); + + JITTargetMachineBuilder JTMB(ES->getExecutorProcessControl().getTargetTriple()); + + auto DL = JTMB.getDefaultDataLayoutForTarget(); + if (!DL) + return DL.takeError(); + + return std::make_unique(std::move(ES), std::move(JTMB), std::move(*DL)); + } + + const DataLayout& getDataLayout() const { return DL; } + + JITDylib& getMainJITDylib() { return MainJD; } + + Error addModule(ThreadSafeModule TSM, ResourceTrackerSP RT = nullptr) + { + if (!RT) + RT = MainJD.getDefaultResourceTracker(); + return CompileLayer.add(RT, std::move(TSM)); + } + + Expected lookup(StringRef Name) { return ES->lookup({ &MainJD }, Mangle(Name.str())); } +}; + +std::vector tokenizer(const std::string& p_pcstStr, char delim) +{ + std::vector tokens; + std::stringstream mySstream(p_pcstStr); + std::string temp; + + while (getline(mySstream, temp, delim)) + { + tokens.push_back(temp); + } + + return tokens; +} + +extern "C" void execute(const char* prog, const char* params, int stdinFd, int stdoutFd) +{ + std::vector argv; + argv.push_back(prog); + if (strlen(params) > 0) + argv.push_back(params); + argv.push_back(nullptr); + + fmt::print("program to execute {} with args: {} \n ", prog, params); + + pid_t const pid = fork(); + + switch (pid) + { + case -1: llvmLog()("Failed to fork(): {} \n", strerror(errno)); return; + case 0: { + // child process + if (stdinFd != STDIN_FILENO) + dup2(stdinFd, STDIN_FILENO); + if (stdoutFd != STDOUT_FILENO) + dup2(stdoutFd, STDOUT_FILENO); + execvp(prog, const_cast(argv.data())); + } + default: { + // parent process + int wstatus = 0; + waitpid(pid, &wstatus, 0); + if (WIFSIGNALED(wstatus)) + llvmLog()("child process exited with signal {} \n", WTERMSIG(wstatus)); + else if (WIFEXITED(wstatus)) + llvmLog()("child process exited with code {} \n", WEXITSTATUS(wstatus)); + else if (WIFSTOPPED(wstatus)) + llvmLog()("child process stopped with signal {} \n", WSTOPSIG(wstatus)); + else + llvmLog()("child process exited with unknown status {} \n", wstatus); + break; + } + } +} + +std::tuple SeparateProg(const std::string input) +{ + + std::stringstream inStream(input); + std::string program; + std::string args; + getline(inStream, program, ' '); + getline(inStream, args, '\n'); + + return { "/usr/bin/" + program, args }; +} + +export class LLVMBackend +{ + + public: + LLVMBackend() + { + InitializeNativeTarget(); + InitializeNativeTargetAsmPrinter(); + InitializeNativeTargetAsmParser(); + // Create an LLJIT instance. + _jit = ExitOnErr(TestJIT::Create()); + } + + void initializeModule() + { + _context = std::make_unique(); + _builder = std::make_unique>(*_context); + _module = std::make_unique("test", *_context); + _module->setDataLayout(_jit->getDataLayout()); + } + + void HandleExternalCall(std::string const& input) const + { + + auto* byteptr = _builder->getPtrTy(); + auto* inttype = _builder->getInt64Ty(); + auto* execvpFunctionType = llvm::FunctionType::get( + llvm::Type::getVoidTy(*_context), { byteptr, byteptr, inttype, inttype }, false); + + auto fun = _module->getOrInsertFunction("execute", execvpFunctionType); + + const auto [prog, args] = SeparateProg(input); + + auto* CalRes = _builder->CreateCall(fun, + { _builder->CreateGlobalString(prog), + _builder->CreateGlobalString(args), + _builder->getInt64(_stdinFd), + _builder->getInt64(_stdoutFd) }); + } + + void HandleAfterParse(std::string const& input) const { + HandleExternalCall(input); + } + + void HandleGlobalPart(std::string const& input) const + { + auto* mainFunctionType = llvm::FunctionType::get( + llvm::Type::getVoidTy(*_context), { llvm::Type::getVoidTy(*_context) }, false); + Function* F = + Function::Create(mainFunctionType, Function::ExternalLinkage, "__anon_expr", _module.get()); + + BasicBlock* BB = BasicBlock::Create(*_context, "EntryBlock", F); + + _builder->SetInsertPoint(BB); + + HandleAfterParse(input); + + _builder->CreateRet(_builder->getInt32(1)); + } + + auto exec(std::string const& input, const int stdinFd, const int stdoutFd) + { + // TODO move it somewhere + _stdinFd = stdinFd; + _stdoutFd = stdoutFd; + + initializeModule(); + + HandleGlobalPart(input); + + auto RT = _jit->getMainJITDylib().createResourceTracker(); + + auto M = ThreadSafeModule(std::move(_module), std::move(_context)); + + M.getModuleUnlocked()->print(llvm::outs(), nullptr); + + ExitOnErr(_jit->addModule(std::move(M), RT)); + // Look up the JIT'd function, cast it to a function pointer, then call it. + auto ExprSymbol = ExitOnErr(_jit->lookup("__anon_expr")); + (ExprSymbol.getAddress().toPtr())(); + + // Delete the anonymous expression module from the JIT. + ExitOnErr(RT->remove()); + } + + std::unique_ptr _context; + std::unique_ptr> _builder; + std::unique_ptr _jit; + std::unique_ptr _module; + ExitOnError ExitOnErr; + + int _stdinFd{}; + int _stdoutFd{}; + ~LLVMBackend() { llvm::llvm_shutdown(); } +}; diff --git a/src/shell/Shell.cpp b/src/shell/Shell.cpp index c642f2b..cfd7437 100644 --- a/src/shell/Shell.cpp +++ b/src/shell/Shell.cpp @@ -2,6 +2,7 @@ module; #include +#include #include #include @@ -20,9 +21,12 @@ import Lexer; import ASTPrinter; import IRGenerator; import Parser; - import CoreVM; +#if defined(ENDO_USE_LLVM) +import LLVMBackend; +#endif + export module Shell; auto inline debugLog = logstore::category("debug ", "Debug log", logstore::category::state::Enabled); @@ -34,7 +38,8 @@ using std::placeholders::_1; using std::placeholders::_2; using std::placeholders::_3; -std::vector constructArgv(CoreVM::CoreStringArray const& args) +template +std::vector constructArgv(T const& args) { std::vector argv; argv.reserve(args.size() + 1); @@ -168,213 +173,97 @@ export class SystemEnvironment: public Environment std::map _values; }; -export class Shell final: public CoreVM::Runtime +class ShellBase: public crispy::app { public: - Shell(): Shell(RealTTY::instance(), SystemEnvironment::instance()) {} - - Shell(TTY& tty, Environment& env): _env { env }, _tty { tty } + ShellBase(): + app("endo", "Endo shell", ENDO_VERSION_STRING, "Apache-2.0"), + _env { SystemEnvironment::instance() }, + _tty { RealTTY::instance() } { + } + ShellBase(TTY& tty, Environment& env): + app("endo", "Endo shell", ENDO_VERSION_STRING, "Apache-2.0"), _env { env }, _tty { tty } + { + _currentPipelineBuilder.defaultStdinFd = _tty.inputFd(); _currentPipelineBuilder.defaultStdoutFd = _tty.outputFd(); _env.setAndExport("SHELL", "endo"); - - // NB: These lines could go away once we have a proper command line parser and - // the ability to set these options from the command line. - registerBuiltinFunctions(); - - // for (CoreVM::NativeCallback const* callback: builtins()) - // fmt::print("builtin: {}\n", callback->signature().to_s()); } - - [[nodiscard]] Environment& environment() noexcept { return _env; } - [[nodiscard]] Environment const& environment() const noexcept { return _env; } - - void setOptimize(bool optimize) { _optimize = optimize; } - - int run() + virtual ~ShellBase() = default; + [[nodiscard]] crispy::cli::command parameterDefinition() const override { - while (!_quit && prompt.ready()) - { - auto const lineBuffer = prompt.read(); - debugLog()("input buffer: {}", lineBuffer); - _exitCode = execute(lineBuffer); - // _tty.writeToStdout("exit code: {}\n", _exitCode); - } - - return _quit ? _exitCode : EXIT_SUCCESS; + return crispy::cli::command { + "endo", + "Endo Terminal " ENDO_VERSION_STRING , + crispy::cli::option_list {}, + crispy::cli::command_list {}, + }; } - int execute(std::string const& lineBuffer) + + virtual int execute(std::string const& lineBuffer) = 0; + int run(int argc, char const* argv[]) { try { - CoreVM::diagnostics::ConsoleReport report; - auto parser = endo::Parser(*this, report, std::make_unique(lineBuffer)); - auto const rootNode = parser.parse(); - if (!rootNode) - { - error("Failed to parse input"); - return EXIT_FAILURE; - } - - debugLog()("Parsed & printed: {}", endo::ast::ASTPrinter::print(*rootNode)); + customizeLogStoreOutput(); - CoreVM::IRProgram* irProgram = IRGenerator::generate(*rootNode); - if (irProgram == nullptr) + while (!_quit && prompt.ready()) { - error("Failed to generate IR program"); - return EXIT_FAILURE; - } - - if (_optimize) - { - CoreVM::PassManager pm; + auto const lineBuffer = prompt.read(); + debugLog()("input buffer: {}", lineBuffer); - // clang-format off - pm.registerPass("eliminate-empty-blocks", &CoreVM::transform::emptyBlockElimination); - pm.registerPass("eliminate-linear-br", &CoreVM::transform::eliminateLinearBr); - pm.registerPass("eliminate-unused-blocks", &CoreVM::transform::eliminateUnusedBlocks); - pm.registerPass("eliminate-unused-instr", &CoreVM::transform::eliminateUnusedInstr); - pm.registerPass("fold-constant-condbr", &CoreVM::transform::foldConstantCondBr); - pm.registerPass("rewrite-br-to-exit", &CoreVM::transform::rewriteBrToExit); - pm.registerPass("rewrite-cond-br-to-same-branches", &CoreVM::transform::rewriteCondBrToSameBranches); - // clang-format on - - pm.run(irProgram); + _exitCode = execute(lineBuffer); + // _tty.writeToStdout("exit code: {}\n", _exitCode); } - debugLog()("================================================\n"); - debugLog()("Optimized IR program:\n"); - if (debugLog.is_enabled()) - irProgram->dump(); - - _currentProgram = CoreVM::TargetCodeGenerator {}.generate(irProgram); - if (!_currentProgram) - { - error("Failed to generate target code"); - return EXIT_FAILURE; - } - _currentProgram->link(this, &report); - - debugLog()("================================================\n"); - debugLog()("Linked target code:\n"); - if (debugLog.is_enabled()) - _currentProgram->dump(); - - CoreVM::Handler* main = _currentProgram->findHandler("@main"); - assert(main != nullptr); - auto runner = - CoreVM::Runner(main, nullptr, &_globals, std::bind(&Shell::trace, this, _1, _2, _3)); - _runner = &runner; - runner.run(); - return _exitCode; + return _quit ? _exitCode : EXIT_SUCCESS; } catch (std::exception const& e) { - error("Exception caught: {}", e.what()); + std::cerr << fmt::format("Unhandled error caught. {}", e.what()) << '\n'; return EXIT_FAILURE; } - - return EXIT_SUCCESS; } + [[nodiscard]] Environment& environment() noexcept { return _env; } + [[nodiscard]] Environment const& environment() const noexcept { return _env; } - Prompt prompt; - std::vector processGroups; + void setOptimize(bool optimize) { _optimize = optimize; } - private: - void registerBuiltinFunctions() + template + void error(fmt::format_string const& message, Args&&... args) { - // clang-format off - registerFunction("exit") - .param("code") - .returnType(CoreVM::LiteralType::Void) - .bind(&Shell::builtinExit, this); - - registerFunction("export") - .param("name") - .returnType(CoreVM::LiteralType::Void) - .bind(&Shell::builtinExport, this); - - registerFunction("export") - .param("name") - .param("value") - .returnType(CoreVM::LiteralType::Void) - .bind(&Shell::builtinSetAndExport, this); - - registerFunction("true") - .returnType(CoreVM::LiteralType::Boolean) - .bind(&Shell::builtinTrue, this); - - registerFunction("false") - .returnType(CoreVM::LiteralType::Boolean) - .bind(&Shell::builtinFalse, this); - - registerFunction("cd") - .returnType(CoreVM::LiteralType::Boolean) - .bind(&Shell::builtinChDirHome, this); - - registerFunction("cd") - .param("path") - .returnType(CoreVM::LiteralType::Boolean) - .bind(&Shell::builtinChDir, this); - - registerFunction("set") - .param("name") - .param("value") - .returnType(CoreVM::LiteralType::Boolean) - .bind(&Shell::builtinSet, this); - - registerFunction("callproc") - .param>("args") - //.param>("redirects") - .returnType(CoreVM::LiteralType::Number) - .bind(&Shell::builtinCallProcess, this); - - registerFunction("callproc") - .param("last_in_chain") - .param>("args") - //.param>("redirects") - .returnType(CoreVM::LiteralType::Number) - .bind(&Shell::builtinCallProcessShellPiped, this); + std::cerr << fmt::format(message, std::forward(args)...) + "\n"; + } - registerFunction("read") - .returnType(CoreVM::LiteralType::String) - .bind(&Shell::builtinReadDefault, this); + [[nodiscard]] std::optional resolveProgram(std::string const& program) const + { + auto const pathEnv = _env.get("PATH"); + if (!pathEnv.has_value()) + return std::nullopt; - registerFunction("read") - .param>("args") - .returnType(CoreVM::LiteralType::String) - .bind(&Shell::builtinRead, this); + auto const pathEnvValue = pathEnv.value(); + auto const paths = crispy::split(pathEnvValue, ':'); - // used to redirect file to stdin - registerFunction("internal.open_read") - .param("path") - .returnType(CoreVM::LiteralType::Number) - .bind(&Shell::builtinOpenRead, this); + for (auto const& pathStr: paths) + { + auto path = std::filesystem::path(pathStr); + auto const programPath = path / program; + if (std::filesystem::exists(programPath)) + { + debugLog()("Found program: {}", programPath.string()); + return programPath; + } + } - // used for redirecting output to a file - registerFunction("internal.open_write") - .param("path") - .param("oflags") - .returnType(CoreVM::LiteralType::Number) - .bind(&Shell::builtinOpenWrite, this); - // clang-format on + return std::nullopt; } - // builtins that match to shell commands - void builtinExit(CoreVM::Params& context) - { - _exitCode = static_cast(context.getInt(1)); - _runner->suspend(); - _quit = true; - } - void builtinCallProcess(CoreVM::Params& context) + void builtinCallProcess(std::string const& program, std::vector const& args) { - CoreVM::CoreStringArray const& args = context.getStringArray(1); std::vector const argv = constructArgv(args); - std::string const& program = args.at(0); std::optional const programPath = resolveProgram(program); int const stdinFd = _currentPipelineBuilder.defaultStdinFd; @@ -383,7 +272,6 @@ export class Shell final: public CoreVM::Runtime if (!programPath.has_value()) { error("Failed to resolve program '{}'", program); - context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); return; } @@ -395,10 +283,7 @@ export class Shell final: public CoreVM::Runtime pid_t const pid = fork(); switch (pid) { - case -1: - error("Failed to fork(): {}", strerror(errno)); - context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); - return; + case -1: error("Failed to fork(): {}", strerror(errno)); return; case 0: { // child process if (stdinFd != STDIN_FILENO) @@ -423,223 +308,441 @@ export class Shell final: public CoreVM::Runtime error("child process exited with unknown status {}", wstatus); break; } - - context.setResult(CoreVM::CoreNumber(_exitCode)); } - void builtinCallProcessShellPiped(CoreVM::Params& context) - { - bool const lastInChain = context.getBool(1); - CoreVM::CoreStringArray const& args = context.getStringArray(2); - std::vector const argv = constructArgv(args); - std::string const& program = args.at(0); - std::optional const programPath = resolveProgram(program); + public: + Prompt prompt; + std::vector processGroups; - if (!programPath.has_value()) - { - error("Failed to resolve program '{}'", program); - context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); - return; - } + protected: + Environment& _env; - auto const [stdinFd, stdoutFd] = _currentPipelineBuilder.requestShellPipe(lastInChain); + TTY& _tty; - // TODO: setup redirects - // CoreVM::CoreIntArray const& outputRedirects = context.getIntArray(1); - // for (size_t i = 0; i + 2 < outputRedirects.size(); i += 2) - // debugLog()("redirect: {} -> {}\n", outputRedirects[i], outputRedirects[i + 1]); + PipelineBuilder _currentPipelineBuilder; - pid_t const pid = fork(); - switch (pid) - { - case -1: - error("Failed to fork(): {}", strerror(errno)); - context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); - return; - case 0: { - // child process - setpgid(0, !_currentProcessGroupPids.empty() ? _currentProcessGroupPids.front() : 0); - if (stdinFd != STDIN_FILENO) - dup2(stdinFd, STDIN_FILENO); - if (stdoutFd != STDOUT_FILENO) - dup2(stdoutFd, STDOUT_FILENO); - execvp(programPath->c_str(), const_cast(argv.data())); - error("Failed to execve({}): {}", programPath->string(), strerror(errno)); - _exit(EXIT_FAILURE); - } - default: - // parent process - _leftPid = _rightPid; - _rightPid = pid; - _currentProcessGroupPids.push_back(pid); - if (lastInChain) - { - // This is the last process in the chain, so we need to wait for all - for (pid_t const pid: _currentProcessGroupPids) - { - int wstatus = 0; - waitpid(pid, &wstatus, 0); - if (WIFSIGNALED(wstatus)) - error("child process {}, exited with signal {}", pid, WTERMSIG(wstatus)); - else if (WIFEXITED(wstatus)) - error("child process {} exited with code {}", - pid, - _exitCode = WEXITSTATUS(wstatus)); - else if (WIFSTOPPED(wstatus)) - error("child process {} stopped with signal {}", pid, WSTOPSIG(wstatus)); - else - error("child process {} exited with unknown status {}", pid, wstatus); - } - _currentProcessGroupPids.clear(); - _leftPid = std::nullopt; - _rightPid = std::nullopt; - } - break; - } + // This stores the PIDs of all processes in the pipeline's process group. + std::vector _currentProcessGroupPids; + std::optional _leftPid; + std::optional _rightPid; - context.setResult(CoreVM::CoreNumber(_exitCode)); - } + // This stores the exit code of the last process in the pipeline. + // TODO: remember exit codes from all processes in the pipeline's process group + int _exitCode = -1; - void builtinChDir(CoreVM::Params& context) - { - std::string const& path = context.getString(1); + bool _optimize = false; - _env.set("OLDPWD", _env.get("PWD").value_or("")); - _env.set("PWD", path); + bool _quit = false; +}; - int const result = chdir(path.data()); - if (result != 0) - error("Failed to change directory to '{}'", path); +export class ShellLLVM final: public ShellBase +{ + public: + ShellLLVM(): ShellLLVM(RealTTY::instance(), SystemEnvironment::instance()) {} - context.setResult(result == 0); - } - void builtinChDirHome(CoreVM::Params& context) - { - auto const path = _env.get("HOME").value_or("/"); - _env.set("OLDPWD", std::filesystem::current_path().string()); - _env.set("PWD", path); - int const result = chdir(path.data()); - if (result != 0) - error("Failed to change directory to '{}'", path); - - context.setResult(result == 0); - } - void builtinSet(CoreVM::Params& context) + ShellLLVM(TTY& tty, Environment& env): ShellBase { tty, env }, _backend {} {} + + int execute(std::string const& lineBuffer) override; + + private: + LLVMBackend _backend; +}; + +export class ShellCoreVM final: public CoreVM::Runtime, public ShellBase +{ + public: + ShellCoreVM(): ShellCoreVM(RealTTY::instance(), SystemEnvironment::instance()) {} + + ShellCoreVM(TTY& tty, Environment& env): ShellBase { tty, env } { - _env.set(context.getString(1), context.getString(2)); - context.setResult(true); + // NB: These lines could go away once we have a proper command line parser and + // the ability to set these options from the command line. + registerBuiltinFunctions(); + + // for (CoreVM::NativeCallback const* callback: builtins()) + // fmt::print("builtin: {}\n", callback->signature().to_s()); } - void builtinSetAndExport(CoreVM::Params& context) + + int execute(std::string const& lineBuffer) override; + + void trace(CoreVM::Instruction instr, size_t ip, size_t sp) { - _env.set(context.getString(1), context.getString(2)); - _env.exportVariable(context.getString(1)); + debugLog()( + fmt::format("trace: {}\n", CoreVM::disassemble(instr, ip, sp, &_currentProgram->constants()))); } + + private: + void registerBuiltinFunctions(); + // builtins that match to shell commands + void builtinExit(CoreVM::Params& context); + void builtinCallProcess(CoreVM::Params& context); + void builtinCallProcessShellPiped(CoreVM::Params& context); + void builtinChDir(CoreVM::Params& context); + void builtinChDirHome(CoreVM::Params& context); + void builtinSet(CoreVM::Params& context); + void builtinSetAndExport(CoreVM::Params& context); void builtinExport(CoreVM::Params& context) { _env.exportVariable(context.getString(1)); } void builtinTrue(CoreVM::Params& context) { context.setResult(true); } void builtinFalse(CoreVM::Params& context) { context.setResult(false); } - void builtinReadDefault(CoreVM::Params& context) + void builtinReadDefault(CoreVM::Params& context); + void builtinRead(CoreVM::Params& context); + // helper-builtins for redirects and pipes + void builtinOpenRead(CoreVM::Params& context); + void builtinOpenWrite(CoreVM::Params& context); + + private: + std::unique_ptr _currentProgram; + CoreVM::Runner::Globals _globals; + + CoreVM::Runner* _runner = nullptr; +}; + +void ShellCoreVM::builtinOpenWrite(CoreVM::Params& context) +{ + std::string const& path = context.getString(1); + int const oflags = static_cast(context.getInt(2)); + int const fd = open(path.data(), oflags ? oflags : (O_WRONLY | O_CREAT | O_TRUNC)); + if (fd == -1) { - std::string const line = - readLine(_tty, fmt::format("{}read{}>{} ", "\033[1;34m", "\033[37;1m", "\033[m")); - _env.set("REPLY", line); - context.setResult(line); + error("Failed to open file '{}': {}", path, strerror(errno)); + context.setResult(CoreVM::CoreNumber(-1)); + return; } - void builtinRead(CoreVM::Params& context) + + context.setResult(CoreVM::CoreNumber(fd)); +} + +void ShellCoreVM::builtinOpenRead(CoreVM::Params& context) +{ + std::string const& path = context.getString(1); + int const fd = open(path.data(), O_RDONLY); + if (fd == -1) { - CoreVM::CoreStringArray const& args = context.getStringArray(1); - std::string const& variable = args.at(0); - std::string const line = - readLine(_tty, fmt::format("{}read{}>{} ", "\033[1;34m", "\033[37;1m", "\033[m")); - _env.set(variable, line); - context.setResult(line); + error("Failed to open file '{}': {}", path, strerror(errno)); + context.setResult(CoreVM::CoreNumber(-1)); + return; } - // helper-builtins for redirects and pipes - void builtinOpenRead(CoreVM::Params& context) - { - std::string const& path = context.getString(1); - int const fd = open(path.data(), O_RDONLY); - if (fd == -1) - { - error("Failed to open file '{}': {}", path, strerror(errno)); - context.setResult(CoreVM::CoreNumber(-1)); - return; - } + context.setResult(CoreVM::CoreNumber(fd)); +} + +void ShellCoreVM::builtinRead(CoreVM::Params& context) +{ + CoreVM::CoreStringArray const& args = context.getStringArray(1); + std::string const& variable = args.at(0); + std::string const line = + readLine(_tty, fmt::format("{}read{}>{} ", "\033[1;34m", "\033[37;1m", "\033[m")); + _env.set(variable, line); + context.setResult(line); +} + +void ShellCoreVM::builtinReadDefault(CoreVM::Params& context) +{ + std::string const line = + readLine(_tty, fmt::format("{}read{}>{} ", "\033[1;34m", "\033[37;1m", "\033[m")); + _env.set("REPLY", line); + context.setResult(line); +} + +void ShellCoreVM::builtinSetAndExport(CoreVM::Params& context) +{ + _env.set(context.getString(1), context.getString(2)); + _env.exportVariable(context.getString(1)); +} + +void ShellCoreVM::builtinSet(CoreVM::Params& context) +{ + _env.set(context.getString(1), context.getString(2)); + context.setResult(true); +} + +void ShellCoreVM::builtinExit(CoreVM::Params& context) +{ + _exitCode = static_cast(context.getInt(1)); + _runner->suspend(); + _quit = true; +} + +void ShellCoreVM::builtinChDirHome(CoreVM::Params& context) +{ + auto const path = _env.get("HOME").value_or("/"); + _env.set("OLDPWD", std::filesystem::current_path().string()); + _env.set("PWD", path); + int const result = chdir(path.data()); + if (result != 0) + error("Failed to change directory to '{}'", path); + + context.setResult(result == 0); +} + +void ShellCoreVM::builtinChDir(CoreVM::Params& context) +{ + std::string const& path = context.getString(1); + + _env.set("OLDPWD", _env.get("PWD").value_or("")); + _env.set("PWD", path); + + int const result = chdir(path.data()); + if (result != 0) + error("Failed to change directory to '{}'", path); + + context.setResult(result == 0); +} + +void ShellCoreVM::builtinCallProcessShellPiped(CoreVM::Params& context) +{ + bool const lastInChain = context.getBool(1); + CoreVM::CoreStringArray const& args = context.getStringArray(2); + + std::vector const argv = constructArgv(args); + std::string const& program = args.at(0); + std::optional const programPath = resolveProgram(program); - context.setResult(CoreVM::CoreNumber(fd)); + if (!programPath.has_value()) + { + error("Failed to resolve program '{}'", program); + context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); + return; } - void builtinOpenWrite(CoreVM::Params& context) + auto const [stdinFd, stdoutFd] = _currentPipelineBuilder.requestShellPipe(lastInChain); + + // TODO: setup redirects + // CoreVM::CoreIntArray const& outputRedirects = context.getIntArray(1); + // for (size_t i = 0; i + 2 < outputRedirects.size(); i += 2) + // debugLog()("redirect: {} -> {}\n", outputRedirects[i], outputRedirects[i + 1]); + + pid_t const pid = fork(); + switch (pid) { - std::string const& path = context.getString(1); - int const oflags = static_cast(context.getInt(2)); - int const fd = open(path.data(), oflags ? oflags : (O_WRONLY | O_CREAT | O_TRUNC)); - if (fd == -1) - { - error("Failed to open file '{}': {}", path, strerror(errno)); - context.setResult(CoreVM::CoreNumber(-1)); + case -1: + error("Failed to fork(): {}", strerror(errno)); + context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); return; + case 0: { + // child process + setpgid(0, !_currentProcessGroupPids.empty() ? _currentProcessGroupPids.front() : 0); + if (stdinFd != STDIN_FILENO) + dup2(stdinFd, STDIN_FILENO); + if (stdoutFd != STDOUT_FILENO) + dup2(stdoutFd, STDOUT_FILENO); + execvp(programPath->c_str(), const_cast(argv.data())); + error("Failed to execve({}): {}", programPath->string(), strerror(errno)); + _exit(EXIT_FAILURE); } - - context.setResult(CoreVM::CoreNumber(fd)); + default: + // parent process + _leftPid = _rightPid; + _rightPid = pid; + _currentProcessGroupPids.push_back(pid); + if (lastInChain) + { + // This is the last process in the chain, so we need to wait for all + for (pid_t const pid: _currentProcessGroupPids) + { + int wstatus = 0; + waitpid(pid, &wstatus, 0); + if (WIFSIGNALED(wstatus)) + error("child process {}, exited with signal {}", pid, WTERMSIG(wstatus)); + else if (WIFEXITED(wstatus)) + error("child process {} exited with code {}", pid, _exitCode = WEXITSTATUS(wstatus)); + else if (WIFSTOPPED(wstatus)) + error("child process {} stopped with signal {}", pid, WSTOPSIG(wstatus)); + else + error("child process {} exited with unknown status {}", pid, wstatus); + } + _currentProcessGroupPids.clear(); + _leftPid = std::nullopt; + _rightPid = std::nullopt; + } + break; } - [[nodiscard]] std::optional resolveProgram(std::string const& program) const - { - auto const pathEnv = _env.get("PATH"); - if (!pathEnv.has_value()) - return std::nullopt; - auto const pathEnvValue = pathEnv.value(); - auto const paths = crispy::split(pathEnvValue, ':'); + context.setResult(CoreVM::CoreNumber(_exitCode)); +} - for (auto const& pathStr: paths) +void ShellCoreVM::builtinCallProcess(CoreVM::Params& context) +{ + CoreVM::CoreStringArray const& args = context.getStringArray(1); + std::string const& program = args.at(0); + ShellBase::builtinCallProcess(program, args); +} + +void ShellCoreVM::registerBuiltinFunctions() +{ + // clang-format off + registerFunction("exit") + .param("code") + .returnType(CoreVM::LiteralType::Void) + .bind(&ShellCoreVM::builtinExit, this); + + registerFunction("export") + .param("name") + .returnType(CoreVM::LiteralType::Void) + .bind(&ShellCoreVM::builtinExport, this); + + registerFunction("export") + .param("name") + .param("value") + .returnType(CoreVM::LiteralType::Void) + .bind(&ShellCoreVM::builtinSetAndExport, this); + + registerFunction("true") + .returnType(CoreVM::LiteralType::Boolean) + .bind(&ShellCoreVM::builtinTrue, this); + + registerFunction("false") + .returnType(CoreVM::LiteralType::Boolean) + .bind(&ShellCoreVM::builtinFalse, this); + + registerFunction("cd") + .returnType(CoreVM::LiteralType::Boolean) + .bind(&ShellCoreVM::builtinChDirHome, this); + + registerFunction("cd") + .param("path") + .returnType(CoreVM::LiteralType::Boolean) + .bind(&ShellCoreVM::builtinChDir, this); + + registerFunction("set") + .param("name") + .param("value") + .returnType(CoreVM::LiteralType::Boolean) + .bind(&ShellCoreVM::builtinSet, this); + + registerFunction("callproc") + .param>("args") + //.param>("redirects") + .returnType(CoreVM::LiteralType::Number) + .bind(&ShellCoreVM::builtinCallProcess, this); + + registerFunction("callproc") + .param("last_in_chain") + .param>("args") + //.param>("redirects") + .returnType(CoreVM::LiteralType::Number) + .bind(&ShellCoreVM::builtinCallProcessShellPiped, this); + + registerFunction("read") + .returnType(CoreVM::LiteralType::String) + .bind(&ShellCoreVM::builtinReadDefault, this); + + registerFunction("read") + .param>("args") + .returnType(CoreVM::LiteralType::String) + .bind(&ShellCoreVM::builtinRead, this); + + // used to redirect file to stdin + registerFunction("internal.open_read") + .param("path") + .returnType(CoreVM::LiteralType::Number) + .bind(&ShellCoreVM::builtinOpenRead, this); + + // used for redirecting output to a file + registerFunction("internal.open_write") + .param("path") + .param("oflags") + .returnType(CoreVM::LiteralType::Number) + .bind(&ShellCoreVM::builtinOpenWrite, this); + // clang-format on +} + +int ShellCoreVM::execute(std::string const& lineBuffer) +{ + try + { + CoreVM::diagnostics::ConsoleReport report; + auto parser = endo::Parser(*this, report, std::make_unique(lineBuffer)); + auto const rootNode = parser.parse(); + if (!rootNode) { - auto path = std::filesystem::path(pathStr); - auto const programPath = path / program; - if (std::filesystem::exists(programPath)) - { - debugLog()("Found program: {}", programPath.string()); - return programPath; - } + error("Failed to parse input"); + return EXIT_FAILURE; } + debugLog()("Parsed & printed: {}", endo::ast::ASTPrinter::print(*rootNode)); - return std::nullopt; - } + CoreVM::IRProgram* irProgram = IRGenerator::generate(*rootNode); + if (irProgram == nullptr) + { + error("Failed to generate IR program"); + return EXIT_FAILURE; + } - void trace(CoreVM::Instruction instr, size_t ip, size_t sp) - { - debugLog()( - fmt::format("trace: {}\n", CoreVM::disassemble(instr, ip, sp, &_currentProgram->constants()))); - } + if (_optimize) + { + CoreVM::PassManager pm; - template - void error(fmt::format_string const& message, Args&&... args) - { - std::cerr << fmt::format(message, std::forward(args)...) + "\n"; - } + // clang-format off + pm.registerPass("eliminate-empty-blocks", &CoreVM::transform::emptyBlockElimination); + pm.registerPass("eliminate-linear-br", &CoreVM::transform::eliminateLinearBr); + pm.registerPass("eliminate-unused-blocks", &CoreVM::transform::eliminateUnusedBlocks); + pm.registerPass("eliminate-unused-instr", &CoreVM::transform::eliminateUnusedInstr); + pm.registerPass("fold-constant-condbr", &CoreVM::transform::foldConstantCondBr); + pm.registerPass("rewrite-br-to-exit", &CoreVM::transform::rewriteBrToExit); + pm.registerPass("rewrite-cond-br-to-same-branches", &CoreVM::transform::rewriteCondBrToSameBranches); + // clang-format on - private: - Environment& _env; + pm.run(irProgram); + } - TTY& _tty; + debugLog()("================================================\n"); + debugLog()("Optimized IR program:\n"); + if (debugLog.is_enabled()) + irProgram->dump(); - std::unique_ptr _currentProgram; - CoreVM::Runner::Globals _globals; + _currentProgram = CoreVM::TargetCodeGenerator {}.generate(irProgram); + if (!_currentProgram) + { + error("Failed to generate target code"); + return EXIT_FAILURE; + } + _currentProgram->link(this, &report); + + debugLog()("================================================\n"); + debugLog()("Linked target code:\n"); + if (debugLog.is_enabled()) + _currentProgram->dump(); + + CoreVM::Handler* main = _currentProgram->findHandler("@main"); + assert(main != nullptr); + auto runner = + CoreVM::Runner(main, nullptr, &_globals, std::bind(&ShellCoreVM::trace, this, _1, _2, _3)); + _runner = &runner; + runner.run(); + return _exitCode; + } + catch (std::exception const& e) + { + error("Exception caught: {}", e.what()); + return EXIT_FAILURE; + } - bool _optimize = false; + return EXIT_SUCCESS; +} - PipelineBuilder _currentPipelineBuilder; +int ShellLLVM::execute(std::string const& lineBuffer) +{ + try + { + auto tokens = Lexer::tokenize(std::make_unique(lineBuffer)); + if (tokens[0].literal == "exit") + { + _quit = true; + return EXIT_SUCCESS; + } - // This stores the PIDs of all processes in the pipeline's process group. - std::vector _currentProcessGroupPids; - std::optional _leftPid; - std::optional _rightPid; + // Look up the JIT'd function, cast it to a function pointer, then call it. + _backend.exec( + lineBuffer, _currentPipelineBuilder.defaultStdinFd, _currentPipelineBuilder.defaultStdoutFd); + return _exitCode; + } + catch (std::exception const& e) + { + error("Exception caught: {}", e.what()); + return EXIT_SUCCESS; + } - // This stores the exit code of the last process in the pipeline. - // TODO: remember exit codes from all processes in the pipeline's process group - int _exitCode = -1; + return EXIT_SUCCESS; +} - CoreVM::Runner* _runner = nullptr; - bool _quit = false; -}; } // namespace endo diff --git a/src/shell/Shell_test.cpp b/src/shell/Shell_test.cpp index fee28ff..2d23746 100644 --- a/src/shell/Shell_test.cpp +++ b/src/shell/Shell_test.cpp @@ -11,6 +11,12 @@ using crispy::escape; import Shell; import TTY; +#if defined(ENDO_USE_LLVM) +using Shell = endo::ShellLLVM; +#else +using Shell = endo::ShellCoreVM; +#endif + namespace { struct TestShell @@ -19,7 +25,7 @@ struct TestShell endo::TestEnvironment env; int exitCode = -1; - endo::Shell shell { pty, env }; + Shell shell { pty, env }; std::string_view output() const noexcept { return pty.output(); } @@ -79,13 +85,13 @@ TEST_CASE("shell.builtin.set_variable") CHECK(shell.env.get("BRU").value_or("NONE") == "hello"); } -TEST_CASE("shell.builtin.get_variable") -{ - TestShell shell; - shell("set BRU hello"); - CHECK(shell.env.get("BRU").value_or("NONE") == "hello"); - shell("$BRU"); -} +// TEST_CASE("shell.builtin.get_variable") +// { +// TestShell shell; +// shell("set BRU hello"); +// CHECK(shell.env.get("BRU").value_or("NONE") == "hello"); +// shell("$BRU"); +// } // TEST_CASE("shell.builtin.set_and_export_variable") diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 6690ee3..2cd4d5a 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -5,6 +5,11 @@ using namespace std::string_literals; import Shell; +#if defined(ENDO_USE_LLVM) +using Shell = endo::ShellLLVM; +#else +using Shell = endo::ShellCoreVM; +#endif std::string_view getEnvironment(std::string_view name, std::string_view defaultValue) { @@ -14,13 +19,9 @@ std::string_view getEnvironment(std::string_view name, std::string_view defaultV int main(int argc, char const* argv[]) { - auto shell = endo::Shell {}; + auto shell = Shell {}; setsid(); - if (argc == 2) - // This here only exists for early-development debugging purposes. - return shell.execute(argv[1]); - else - return shell.run(); + return shell.run(argc, argv); }