From 1c9ee518690fdbd3e929535ada847bc40ccfa30a Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sun, 10 Apr 2022 13:56:48 +0200 Subject: [PATCH 1/4] First quick experiment to switch to `nanoserde` --- Cargo.lock | 23 +++++++++--- crates/rustc_codegen_spirv-types/Cargo.toml | 2 +- .../src/compile_result.rs | 20 +++++------ crates/rustc_codegen_spirv/Cargo.toml | 6 +--- crates/rustc_codegen_spirv/src/decorations.rs | 36 ++++++++++--------- crates/rustc_codegen_spirv/src/link.rs | 11 +++--- crates/spirv-builder/Cargo.toml | 3 +- crates/spirv-builder/src/lib.rs | 17 +++++---- examples/runners/ash/src/main.rs | 4 +-- 9 files changed, 65 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index de1fa33cba..6f92aa2782 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1379,6 +1379,21 @@ dependencies = [ "thiserror", ] +[[package]] +name = "nanoserde" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2786e3e6331deef0ed595643fb9266686369917bdf99b4169701399c9d262868" +dependencies = [ + "nanoserde-derive", +] + +[[package]] +name = "nanoserde-derive" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290eec6719d68aef1f5ca0e695f8ad6421adcf8791fc17641c4ccc6c5388fb39" + [[package]] name = "ndk" version = "0.4.0" @@ -2001,6 +2016,7 @@ dependencies = [ "hashbrown", "indexmap", "libc", + "nanoserde", "num-traits", "pipe", "pretty_assertions", @@ -2008,8 +2024,6 @@ dependencies = [ "rustc-demangle", "rustc_codegen_spirv-types", "sanitize-filename", - "serde", - "serde_json", "smallvec", "spirv-tools", "syn", @@ -2020,8 +2034,8 @@ dependencies = [ name = "rustc_codegen_spirv-types" version = "0.4.0-alpha.12" dependencies = [ + "nanoserde", "rspirv", - "serde", ] [[package]] @@ -2229,12 +2243,11 @@ name = "spirv-builder" version = "0.4.0-alpha.12" dependencies = [ "memchr", + "nanoserde", "notify", "raw-string", "rustc_codegen_spirv", "rustc_codegen_spirv-types", - "serde", - "serde_json", ] [[package]] diff --git a/crates/rustc_codegen_spirv-types/Cargo.toml b/crates/rustc_codegen_spirv-types/Cargo.toml index 1cde4e5105..d867807b1a 100644 --- a/crates/rustc_codegen_spirv-types/Cargo.toml +++ b/crates/rustc_codegen_spirv-types/Cargo.toml @@ -9,4 +9,4 @@ repository = "https://github.com/EmbarkStudios/rust-gpu" [dependencies] rspirv = "0.11" -serde = { version = "1.0", features = ["derive"] } +nanoserde = "0.1.9" diff --git a/crates/rustc_codegen_spirv-types/src/compile_result.rs b/crates/rustc_codegen_spirv-types/src/compile_result.rs index 90eb948e02..6e81203435 100644 --- a/crates/rustc_codegen_spirv-types/src/compile_result.rs +++ b/crates/rustc_codegen_spirv-types/src/compile_result.rs @@ -1,17 +1,15 @@ -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use nanoserde::{DeJson, SerJson}; +use std::collections::HashMap; use std::fmt::Write; -use std::path::{Path, PathBuf}; -#[derive(Debug, Serialize, Deserialize)] -#[serde(untagged)] +#[derive(Debug, DeJson, SerJson)] pub enum ModuleResult { - SingleModule(PathBuf), - MultiModule(BTreeMap), + SingleModule(String), + MultiModule(HashMap), } impl ModuleResult { - pub fn unwrap_single(&self) -> &Path { + pub fn unwrap_single(&self) -> &String { match self { ModuleResult::SingleModule(result) => result, ModuleResult::MultiModule(_) => { @@ -20,7 +18,7 @@ impl ModuleResult { } } - pub fn unwrap_multi(&self) -> &BTreeMap { + pub fn unwrap_multi(&self) -> &HashMap { match self { ModuleResult::MultiModule(result) => result, ModuleResult::SingleModule(_) => { @@ -30,7 +28,7 @@ impl ModuleResult { } } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, DeJson, SerJson)] pub struct CompileResult { pub module: ModuleResult, pub entry_points: Vec, @@ -48,7 +46,7 @@ impl CompileResult { #[derive(Default)] struct Trie<'a> { present: bool, - children: BTreeMap<&'a str, Trie<'a>>, + children: HashMap<&'a str, Trie<'a>>, } impl<'a> Trie<'a> { diff --git a/crates/rustc_codegen_spirv/Cargo.toml b/crates/rustc_codegen_spirv/Cargo.toml index 4b5648254c..061fa212fa 100644 --- a/crates/rustc_codegen_spirv/Cargo.toml +++ b/crates/rustc_codegen_spirv/Cargo.toml @@ -9,9 +9,6 @@ repository = "https://github.com/EmbarkStudios/rust-gpu" #categories = [] #readme = "README.md" -[lib] -crate-type = ["dylib"] - [features] # By default, the use-compiled-tools is enabled, as doesn't require additional # setup steps for the user. This does however mean that you will need to disable @@ -41,8 +38,7 @@ indexmap = "1.6.0" rspirv = "0.11" rustc-demangle = "0.1.21" sanitize-filename = "0.3" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +nanoserde = "0.1.9" smallvec = "1.6.1" spirv-tools = { version = "0.8", default-features = false } rustc_codegen_spirv-types = { path = "../rustc_codegen_spirv-types", version = "0.4.0-alpha.12" } diff --git a/crates/rustc_codegen_spirv/src/decorations.rs b/crates/rustc_codegen_spirv/src/decorations.rs index 0fd8dedfe0..6a94d24306 100644 --- a/crates/rustc_codegen_spirv/src/decorations.rs +++ b/crates/rustc_codegen_spirv/src/decorations.rs @@ -1,13 +1,12 @@ //! SPIR-V decorations specific to `rustc_codegen_spirv`, produced during //! the original codegen of a crate, and consumed by the `linker`. +use nanoserde::{DeJson, SerJson}; use rspirv::dr::{Instruction, Module, Operand}; use rspirv::spirv::{Decoration, Op, Word}; use rustc_span::{source_map::SourceMap, FileName, Pos, Span}; -use serde::{Deserialize, Serialize}; -use std::marker::PhantomData; -use std::path::PathBuf; use std::{iter, slice}; +use std::{marker::PhantomData, path::Path}; /// Decorations not native to SPIR-V require some form of encoding into existing /// SPIR-V constructs, for which we use `OpDecorateString` with decoration type @@ -22,7 +21,7 @@ use std::{iter, slice}; /// /// TODO: uses `non_semantic` instead of piggybacking off of `UserTypeGOOGLE` /// -pub trait CustomDecoration: for<'de> Deserialize<'de> + Serialize { +pub trait CustomDecoration: DeJson + SerJson { const ENCODING_PREFIX: &'static str; fn encode(self, id: Word) -> Instruction { @@ -30,7 +29,7 @@ pub trait CustomDecoration: for<'de> Deserialize<'de> + Serialize { // in `serde_json` for writing to something that impls `fmt::Write`, // only for `io::Write`, which would require performing redundant UTF-8 // (re)validation, or relying on `unsafe` code, to use with `String`. - let json = serde_json::to_string(&self).unwrap(); + let json = self.serialize_json(); let encoded = [Self::ENCODING_PREFIX, &json].concat(); Instruction::new( @@ -93,27 +92,26 @@ pub struct LazilyDeserialized<'a, D> { _marker: PhantomData, } -impl<'a, D: Deserialize<'a>> LazilyDeserialized<'a, D> { +impl<'a, D: DeJson> LazilyDeserialized<'a, D> { pub fn deserialize(self) -> D { - serde_json::from_str(self.json).unwrap() + D::deserialize_json(self.json).unwrap() } } /// An `OpFunction` with `#[spirv(unroll_loops)]` on the Rust `fn` definition, /// which should get `LoopControl::UNROLL` applied to all of its loops' /// `OpLoopMerge` instructions, during structuralization. -#[derive(Deserialize, Serialize)] +#[derive(DeJson, SerJson)] pub struct UnrollLoopsDecoration {} impl CustomDecoration for UnrollLoopsDecoration { const ENCODING_PREFIX: &'static str = "U"; } -#[derive(Deserialize, Serialize)] +#[derive(DeJson, SerJson)] pub struct ZombieDecoration { pub reason: String, - #[serde(flatten)] pub span: Option, } @@ -124,9 +122,9 @@ impl CustomDecoration for ZombieDecoration { /// Representation of a `rustc` `Span` that can be turned into a `Span` again /// in another compilation, by reloading the file. However, note that this will /// fail if the file changed since, which is detected using the serialized `hash`. -#[derive(Deserialize, Serialize)] +#[derive(DeJson, SerJson)] pub struct SerializedSpan { - file: PathBuf, + file: String, hash: serde_adapters::SourceFileHash, lo: u32, hi: u32, @@ -135,9 +133,9 @@ pub struct SerializedSpan { // HACK(eddyb) `rustc_span` types implement only `rustc_serialize` traits, but // not `serde` traits, and the easiest workaround is to have our own types. mod serde_adapters { - use serde::{Deserialize, Serialize}; + use nanoserde::{DeJson, SerJson}; - #[derive(Copy, Clone, PartialEq, Eq, Deserialize, Serialize)] + #[derive(Copy, Clone, PartialEq, Eq, DeJson, SerJson)] pub enum SourceFileHashAlgorithm { Md5, Sha1, @@ -154,7 +152,7 @@ mod serde_adapters { } } - #[derive(Copy, Clone, PartialEq, Eq, Deserialize, Serialize)] + #[derive(Copy, Clone, PartialEq, Eq, DeJson, SerJson)] pub struct SourceFileHash { kind: SourceFileHashAlgorithm, value: [u8; 32], @@ -197,7 +195,11 @@ impl SerializedSpan { file: match &file.name { // We can only support real files, not "synthetic" ones (which // are almost never exposed to the compiler backend anyway). - FileName::Real(real_name) => real_name.local_path()?.to_path_buf(), + FileName::Real(real_name) => real_name + .local_path()? + .to_path_buf() + .to_string_lossy() + .to_string(), _ => return None, }, hash: file.src_hash.into(), @@ -207,7 +209,7 @@ impl SerializedSpan { } pub fn to_rustc(&self, source_map: &SourceMap) -> Option { - let file = source_map.load_file(&self.file).ok()?; + let file = source_map.load_file(Path::new(&self.file)).ok()?; // If the file has changed since serializing, there's not much we can do, // other than avoid creating invalid/confusing `Span`s. diff --git a/crates/rustc_codegen_spirv/src/link.rs b/crates/rustc_codegen_spirv/src/link.rs index 73a8e397a4..81a3c4eef6 100644 --- a/crates/rustc_codegen_spirv/src/link.rs +++ b/crates/rustc_codegen_spirv/src/link.rs @@ -1,6 +1,7 @@ use crate::codegen_cx::{CodegenArgs, ModuleOutputType, SpirvMetadata}; use crate::{linker, SpirvCodegenBackend, SpirvModuleBuffer, SpirvThinBuffer}; use ar::{Archive, GnuBuilder, Header}; +use nanoserde::SerJson; use rspirv::binary::Assemble; use rustc_codegen_spirv_types::{CompileResult, ModuleResult}; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; @@ -18,7 +19,7 @@ use rustc_session::Session; use std::env; use std::ffi::CString; use std::fs::File; -use std::io::{BufWriter, Read}; +use std::io::Read; use std::iter; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -148,7 +149,8 @@ fn link_exe( module_filename.push("module"); post_link_single_module(sess, &cg_args, spv_binary.assemble(), &module_filename); cg_args.do_disassemble(&spv_binary); - let module_result = ModuleResult::SingleModule(module_filename); + let module_result = + ModuleResult::SingleModule(module_filename.to_string_lossy().to_string()); CompileResult { module: module_result, entry_points: entry_points(&spv_binary), @@ -167,7 +169,7 @@ fn link_exe( spv_binary.assemble(), &module_filename, ); - (name, module_filename) + (name, module_filename.to_string_lossy().to_string()) }) .collect(); let module_result = ModuleResult::MultiModule(map); @@ -178,8 +180,7 @@ fn link_exe( } }; - let file = File::create(out_filename).unwrap(); - serde_json::to_writer(BufWriter::new(file), &compile_result).unwrap(); + std::fs::write(out_filename, compile_result.serialize_json()).unwrap(); } fn entry_points(module: &rspirv::dr::Module) -> Vec { diff --git a/crates/spirv-builder/Cargo.toml b/crates/spirv-builder/Cargo.toml index f8eaa5b758..886050bd30 100644 --- a/crates/spirv-builder/Cargo.toml +++ b/crates/spirv-builder/Cargo.toml @@ -15,8 +15,7 @@ watch = ["notify"] [dependencies] memchr = "2.4" raw-string = "0.3.5" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +nanoserde = "0.1.9" rustc_codegen_spirv-types = { path = "../rustc_codegen_spirv-types" } # See comment in lib.rs invoke_rustc for why this is here rustc_codegen_spirv = { path = "../rustc_codegen_spirv", default-features = false } diff --git a/crates/spirv-builder/src/lib.rs b/crates/spirv-builder/src/lib.rs index 998c662a1b..0f2a4ea70c 100644 --- a/crates/spirv-builder/src/lib.rs +++ b/crates/spirv-builder/src/lib.rs @@ -74,15 +74,13 @@ mod depfile; #[cfg(feature = "watch")] mod watch; +use nanoserde::DeJson; use raw_string::{RawStr, RawString}; -use serde::Deserialize; use std::borrow::Borrow; use std::collections::HashMap; use std::env; use std::error::Error; use std::fmt; -use std::fs::File; -use std::io::BufReader; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -97,7 +95,7 @@ pub enum SpirvBuilderError { MultiModuleWithPrintMetadata, WatchWithPrintMetadata, MetadataFileMissing(std::io::Error), - MetadataFileMalformed(serde_json::Error), + MetadataFileMalformed(nanoserde::DeJsonErr), } impl fmt::Display for SpirvBuilderError { @@ -339,15 +337,16 @@ impl SpirvBuilder { &self, at: &Path, ) -> Result { - let metadata_contents = File::open(&at).map_err(SpirvBuilderError::MetadataFileMissing)?; - let metadata: CompileResult = serde_json::from_reader(BufReader::new(metadata_contents)) + let metadata_contents = + std::fs::read_to_string(&at).map_err(SpirvBuilderError::MetadataFileMissing)?; + let metadata = CompileResult::deserialize_json(&metadata_contents) .map_err(SpirvBuilderError::MetadataFileMalformed)?; match &metadata.module { ModuleResult::SingleModule(spirv_module) => { assert!(!self.multimodule); let env_var = at.file_name().unwrap().to_str().unwrap(); if self.print_metadata == MetadataPrintout::Full { - println!("cargo:rustc-env={}={}", env_var, spirv_module.display()); + println!("cargo:rustc-env={env_var}={spirv_module}"); } } ModuleResult::MultiModule(_) => { @@ -535,7 +534,7 @@ fn invoke_rustc(builder: &SpirvBuilder) -> Result { } } -#[derive(Deserialize)] +#[derive(DeJson)] struct RustcOutput { reason: String, filenames: Option>, @@ -544,7 +543,7 @@ struct RustcOutput { fn get_last_artifact(out: &str) -> Option { let last = out .lines() - .filter_map(|line| match serde_json::from_str::(line) { + .filter_map(|line| match RustcOutput::deserialize_json(line) { Ok(line) => Some(line), Err(_) => { // Pass through invalid lines diff --git a/examples/runners/ash/src/main.rs b/examples/runners/ash/src/main.rs index 841fa8fc67..b43b636054 100644 --- a/examples/runners/ash/src/main.rs +++ b/examples/runners/ash/src/main.rs @@ -204,10 +204,10 @@ pub fn compile_shaders() -> Vec { .unwrap() .module .unwrap_single() - .to_path_buf(); + .clone(); let sky_shader = SpvFile { name: "sky_shader".to_string(), - data: read_spv(&mut File::open(sky_shader_path).unwrap()).unwrap(), + data: read_spv(&mut File::open(&sky_shader_path).unwrap()).unwrap(), }; vec![sky_shader] } From 57c56bcb0f8df6a4392284e74ed1394c1b74a3d5 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sun, 17 Apr 2022 11:58:15 +0200 Subject: [PATCH 2/4] Don't compile/run tests on example doc comments --- crates/rustc_codegen_spirv/src/linker/peephole_opts.rs | 4 ++-- crates/rustc_codegen_spirv/src/linker/specializer.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/rustc_codegen_spirv/src/linker/peephole_opts.rs b/crates/rustc_codegen_spirv/src/linker/peephole_opts.rs index 668f6f50e9..e534bea20b 100644 --- a/crates/rustc_codegen_spirv/src/linker/peephole_opts.rs +++ b/crates/rustc_codegen_spirv/src/linker/peephole_opts.rs @@ -400,7 +400,7 @@ fn process_instruction( } /// Fuse a sequence of scalar operations into a single vector operation. For example: -/// ``` +/// ```ignore /// %x_0 = OpCompositeExtract %x 0 /// %x_1 = OpCompositeExtract %x 1 /// %y_0 = OpCompositeExtract %y 0 @@ -410,7 +410,7 @@ fn process_instruction( /// %r = OpCompositeConstruct %r_0 %r_1 /// ``` /// into -/// ``` +/// ```ignore /// %r = OpAdd %x %y /// ``` /// (We don't remove the intermediate instructions, however, in case they're used elsewhere - we diff --git a/crates/rustc_codegen_spirv/src/linker/specializer.rs b/crates/rustc_codegen_spirv/src/linker/specializer.rs index 9c058d484e..be5cb1ea47 100644 --- a/crates/rustc_codegen_spirv/src/linker/specializer.rs +++ b/crates/rustc_codegen_spirv/src/linker/specializer.rs @@ -27,7 +27,7 @@ //! "generic" parameters to every (indirect) mention of any pointer type in //! the group, using an additional parameter remapping, for which `Generic`: //! * requires this extra documentation: -//! ``` +//! ```ignore //! /// The one exception are `OpTypePointer`s involved in recursive data types //! /// (i.e. they were declared by `OpTypeForwardPointer`s, and their pointees are //! /// `OpTypeStruct`s that have the same pointer type as a leaf). @@ -36,7 +36,7 @@ //! /// a mapping (`expand_params`) indicates how to create the flattened list. //! ``` //! * and this extra field: -//! ``` +//! ```ignore //! /// For every entry in the regular flattened list of parameters expected by //! /// operands, this contains the parameter index (i.e. `0..self.param_count`) //! /// to use for that parameter. From d1d6b4ad6e91dc0e07d511e8e76fdcb4aed3fa0a Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sun, 17 Apr 2022 12:13:20 +0200 Subject: [PATCH 3/4] Fix clippy warnings from inside nanoserde macro --- crates/rustc_codegen_spirv/src/decorations.rs | 2 ++ crates/spirv-builder/src/lib.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/crates/rustc_codegen_spirv/src/decorations.rs b/crates/rustc_codegen_spirv/src/decorations.rs index 6a94d24306..f92634776a 100644 --- a/crates/rustc_codegen_spirv/src/decorations.rs +++ b/crates/rustc_codegen_spirv/src/decorations.rs @@ -1,6 +1,8 @@ //! SPIR-V decorations specific to `rustc_codegen_spirv`, produced during //! the original codegen of a crate, and consumed by the `linker`. +#![allow(clippy::question_mark)] // warning from inside the nanoserde macro + use nanoserde::{DeJson, SerJson}; use rspirv::dr::{Instruction, Module, Operand}; use rspirv::spirv::{Decoration, Op, Word}; diff --git a/crates/spirv-builder/src/lib.rs b/crates/spirv-builder/src/lib.rs index 0f2a4ea70c..f11b857f28 100644 --- a/crates/spirv-builder/src/lib.rs +++ b/crates/spirv-builder/src/lib.rs @@ -69,6 +69,7 @@ // END - Embark standard lints v0.4 // crate-specific exceptions: // #![allow()] +#![allow(clippy::question_mark)] // warning from inside the nanoserde macro mod depfile; #[cfg(feature = "watch")] From e66e4147356e3074fa33834b00a04b4ec26380d2 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sun, 17 Apr 2022 12:37:07 +0200 Subject: [PATCH 4/4] Revert dylib change --- crates/rustc_codegen_spirv/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/rustc_codegen_spirv/Cargo.toml b/crates/rustc_codegen_spirv/Cargo.toml index 061fa212fa..cfacd26deb 100644 --- a/crates/rustc_codegen_spirv/Cargo.toml +++ b/crates/rustc_codegen_spirv/Cargo.toml @@ -9,6 +9,9 @@ repository = "https://github.com/EmbarkStudios/rust-gpu" #categories = [] #readme = "README.md" +[lib] +crate-type = ["dylib"] + [features] # By default, the use-compiled-tools is enabled, as doesn't require additional # setup steps for the user. This does however mean that you will need to disable