Move jaq-syn into jaq-interpret.

Cargo.toml

@@ -1,9 +1,9 @@
 [workspace]
 members = [
-    "jaq-syn",
     "jaq-interpret",
     "jaq-core",
     "jaq-std",
+    "jaq-json",
     "jaq",
     "jaq-play",
 ]

jaq-core/Cargo.toml

@@ -30,5 +30,4 @@ base64 = { version = "0.22", optional = true }
 urlencoding = { version = "2.1.3", optional = true }
 
 [dev-dependencies]
-jaq-syn = { version = "1.6.0", path = "../jaq-syn" }
 serde_json = "1.0"

jaq-core/tests/common/mod.rs

@@ -2,7 +2,7 @@ use jaq_interpret::json::{Error, Val, ValR};
 use serde_json::Value;
 
 fn yields(x: Val, code: &str, ys: impl Iterator<Item = ValR>) {
-    use jaq_syn::load::{Arena, File, Loader};
+    use jaq_interpret::load::{Arena, File, Loader};
 
     let arena = Arena::default();
     let loader = Loader::new([]);

jaq-interpret/Cargo.toml

@@ -16,10 +16,11 @@ default = ["std", "hifijson", "serde_json"]
 std = []
 
 [dependencies]
-jaq-syn = { version = "1.1.0", path = "../jaq-syn" }
-ahash = "0.8.6"
 dyn-clone = "1.0"
+once_cell = "1.16.0"
+typed-arena = "2.0.2"
+
+ahash = "0.8.6"
 hifijson = { version = "0.2.0", optional = true }
 indexmap = "2.0"
-once_cell = "1.16.0"
 serde_json = { version = "1.0.81", optional = true }

jaq-interpret/src/compile.rs

@@ -1,8 +1,9 @@
 //! Program compilation.
 
+use crate::load::{self, lex, parse};
 use crate::{Bind, Filter};
+use crate::{MathOp, OrdOp};
 use alloc::{boxed::Box, string::String, vec::Vec};
-use jaq_syn::{load, parse, MathOp, OrdOp};
 
 type NativeId = usize;
 type ModId = usize;
@@ -332,7 +333,7 @@ impl<'s, F> Compiler<&'s str, F> {
         y
     }
 
-    fn module(&mut self, m: jaq_syn::load::Module<&'s str>) {
+    fn module(&mut self, m: load::Module<&'s str>) {
         self.imported_mods.clear();
         self.included_mods.clear();
         for (mid, as_) in m.mods {
@@ -458,7 +459,7 @@ impl<'s, F> Compiler<&'s str, F> {
                 Term::Fold(fold, xs, init, update)
             }
             BinOp(l, op, r) => {
-                use jaq_syn::parse::BinaryOp::*;
+                use parse::BinaryOp::*;
                 let (l, r) = match op {
                     Comma => (self.iterm_tr(*l), self.iterm_tr(*r)),
                     Alt => (self.iterm(*l), self.iterm_tr(*r)),
@@ -478,21 +479,13 @@ impl<'s, F> Compiler<&'s str, F> {
                 }
             }
             Path(t, path) => {
-                use crate::path::Part;
-                use jaq_syn::path::Part::{Index, Range};
                 let t = self.iterm(*t);
-                let path = path.into_iter().map(|(p, opt)| match p {
-                    Index(i) => (Part::Index(self.iterm(i)), opt),
-                    Range(lower, upper) => {
-                        let lower = lower.map(|f| self.iterm(f));
-                        let upper = upper.map(|f| self.iterm(f));
-                        (Part::Range(lower, upper), opt)
-                    }
-                });
+                let path = path.0.into_iter();
+                let path = path.map(|(p, opt)| (p.map(|f| self.iterm(f)), opt));
                 Term::Path(t, crate::path::Path(path.collect()))
             }
             Str(fmt, parts) => {
-                use jaq_syn::lex::StrPart;
+                use lex::StrPart;
                 let fmt = match fmt {
                     Some(fmt) => self.iterm(Call(fmt, Vec::new())),
                     None => self.lut.insert_term(Term::ToString),

jaq-interpret/src/error.rs

@@ -15,7 +15,7 @@ pub enum Error<V> {
     /// Expected a value of given type, but got something else
     Type(V, Type),
     /// `1 - "a"`
-    MathOp(V, jaq_syn::MathOp, V),
+    MathOp(V, crate::MathOp, V),
     /// `{} | .[0]` or `[] | has("a")` or `{} | has(0)`
     Index(V, V),
 

jaq-interpret/src/json.rs

@@ -4,13 +4,13 @@ use crate::box_iter::{box_once, BoxIter};
 use crate::error::Type;
 use crate::val::{Range, ValT};
 use crate::Exn;
+use crate::{path::Opt, MathOp};
 use alloc::string::{String, ToString};
 use alloc::{boxed::Box, rc::Rc, vec::Vec};
 use core::cmp::Ordering;
 use core::fmt::{self, Debug};
 #[cfg(feature = "hifijson")]
 use hifijson::{LexAlloc, Token};
-use jaq_syn::{path::Opt, MathOp};
 
 /// JSON value with sharing.
 ///

jaq-interpret/src/lib.rs

@@ -56,7 +56,9 @@ pub(crate) mod exn;
 mod filter;
 mod into_iter;
 pub mod json;
-mod path;
+pub mod load;
+mod ops;
+pub mod path;
 mod rc_iter;
 mod rc_lazy_list;
 mod rc_list;
@@ -73,6 +75,7 @@ pub use rc_iter::RcIter;
 pub use val::{ValR, ValT, ValX, ValXs};
 
 use alloc::string::String;
+use ops::{MathOp, OrdOp};
 use rc_list::List as RcList;
 use stack::Stack;
 

jaq-syn/src/load.rs → jaq-interpret/src/load/mod.rs

@@ -1,9 +1,20 @@
 //! Combined file loading, lexing, and parsing for multiple modules.
 
-use crate::lex::{self, Token};
-use crate::parse::{self, Def, Term};
+pub mod lex;
+pub mod parse;
+mod prec_climb;
+pub mod test;
+
+pub use lex::Lexer;
+pub use parse::Parser;
+
+use crate::ops::{MathOp, OrdOp};
+use crate::path;
+
 use alloc::string::String;
 use alloc::vec::Vec;
+use lex::Token;
+use parse::{Def, Term};
 
 #[cfg(feature = "std")]
 extern crate std;
@@ -278,3 +289,26 @@ fn parse_defs(code: &str) -> Result<parse::Module<&str, Vec<Def<&str>>>, Error<&
         .parse(|p| p.module(|p| p.defs()))
         .map_err(|e| Error::Parse(e.into_iter().map(conv_err).collect()))
 }
+
+/// Lex a string and parse resulting tokens, returning [`None`] if any error occurred.
+///
+/// Example:
+///
+/// ~~~
+/// # use jaq_syn::parse;
+/// let t = parse("[] | .[]", |p| p.term());
+/// ~~~
+pub fn parse<'s, T: Default, F>(s: &'s str, f: F) -> Option<T>
+where
+    F: for<'t> FnOnce(&mut Parser<'s, 't>) -> parse::Result<'s, 't, T>,
+{
+    Parser::new(&Lexer::new(s).lex().ok()?).parse(f).ok()
+}
+
+/// Return the span of a string slice `part` relative to a string slice `whole`.
+///
+/// The caller must ensure that `part` is fully contained inside `whole`.
+pub fn span(whole: &str, part: &str) -> core::ops::Range<usize> {
+    let start = part.as_ptr() as usize - whole.as_ptr() as usize;
+    start..start + part.len()
+}

jaq-syn/src/parse.rs → jaq-interpret/src/load/parse.rs

@@ -1,8 +1,8 @@
 //! Parsing.
 
-use crate::lex::{StrPart, Tok, Token};
-use crate::path::{self, Path};
-use crate::{prec_climb, MathOp, OrdOp};
+use super::lex::{StrPart, Tok, Token};
+use super::path::{self, Path};
+use super::{prec_climb, MathOp, OrdOp};
 use alloc::{boxed::Box, vec::Vec};
 
 /// Parse error, storing what we expected and what we got instead.
@@ -156,18 +156,17 @@ impl<S> Term<S> {
 
     /// `..`, also known as `recurse/0`, is defined as `., (.[]? | ..)`.
     pub(crate) fn recurse(recurse: S) -> Self {
-        use Term::*;
         // `[]?`
         let path = (path::Part::Range(None, None), path::Opt::Optional);
         // `.[]?` (returns array/object elements or nothing instead)
-        let path = Term::Path(Id.into(), Vec::from([path]));
+        let path = Term::Path(Term::Id.into(), Path(Vec::from([path])));
 
         // `..`
         let f = Term::Call(recurse, Vec::new());
         // .[]? | ..
         let pipe = Term::Pipe(path.into(), None, f.into());
         // ., (.[]? | ..)
-        Term::BinOp(Id.into(), BinaryOp::Comma, pipe.into())
+        Term::BinOp(Term::Id.into(), BinaryOp::Comma, pipe.into())
     }
 
     /// `{}[]` returns zero values.
@@ -177,7 +176,7 @@ impl<S> Term<S> {
         // `{}`
         let obj = Term::Obj(Vec::new());
         // `{}[]`
-        Term::Path(obj.into(), Vec::from([path]))
+        Term::Path(obj.into(), Path(Vec::from([path])))
     }
 }
 
@@ -505,8 +504,8 @@ impl<'s, 't> Parser<'s, 't> {
 
                 if let Some(key) = key {
                     let head = (path::Part::Index(key), self.opt());
-                    let path = core::iter::once(head).chain(self.path()?).collect();
-                    Term::Path(Box::new(Term::Id), path)
+                    let path = core::iter::once(head).chain(self.path()?.0).collect();
+                    Term::Path(Box::new(Term::Id), Path(path))
                 } else {
                     Term::Id
                 }
@@ -530,7 +529,7 @@ impl<'s, 't> Parser<'s, 't> {
         };
 
         let path = self.path()?;
-        Ok(if path.is_empty() {
+        Ok(if path.0.is_empty() {
             tm
         } else {
             Term::Path(Box::new(tm), path)
@@ -592,7 +591,7 @@ impl<'s, 't> Parser<'s, 't> {
             path.push((path::Part::Index(key), self.opt()));
             path.extend(core::iter::from_fn(|| self.path_part_opt()));
         }
-        Ok(path)
+        Ok(Path(path))
     }
 
     /// Parse `[]`, `[t]`, `[t:]`, `[t:t]`, `[:t]` (all without brackets).

jaq-interpret/src/path.rs

@@ -1,25 +1,55 @@
+//! Paths and their parts.
+
 use crate::box_iter::{box_once, flat_map_with, map_with, BoxIter};
 use crate::results::then;
 use crate::val::{ValR, ValT, ValX, ValXs};
 use alloc::{boxed::Box, vec::Vec};
-use jaq_syn::path::Opt;
 
+/// Path such as `.[].a?[1:]`.
 #[derive(Clone, Debug)]
 pub struct Path<F>(pub Vec<(Part<F>, Opt)>);
 
-/// Part of a path.
-///
-/// This is identical to [`jaq_syn::path::Part`], but we cannot use that here directly
-/// because that way, we could not implement new methods for that type.
+/// Part of a path, such as `[]`, `a`, and `[1:]` in `.[].a?[1:]`.
 #[derive(Clone, Debug)]
 pub enum Part<I> {
+    /// Access arrays with integer and objects with string indices
     Index(I),
-    /// if both are `None`, return iterator over whole array/object
+    /// Iterate over arrays with optional range bounds and over objects without bounds
+    /// If both are `None`, return iterator over whole array/object
     Range(Option<I>, Option<I>),
 }
 
+/// Optionality of a path part, i.e. whether `?` is present.
+///
+/// For example, `[] | .a` fails with an error, while `[] | .a?` returns nothing.
+/// By default, path parts are *essential*, meaning that they fail.
+/// Annotating them with `?` makes them *optional*.
+#[derive(Copy, Clone, Debug)]
+pub enum Opt {
+    /// Return nothing if the input cannot be accessed with the path
+    Optional,
+    /// Fail if the input cannot be accessed with the path
+    Essential,
+}
+
+impl<I> Default for Part<I> {
+    fn default() -> Self {
+        Self::Range(None, None)
+    }
+}
+
+impl Opt {
+    /// If `self` is optional, return `x`, else fail with `f(x)`.
+    pub(crate) fn fail<T, E>(self, x: T, f: impl FnOnce(T) -> E) -> Result<T, E> {
+        match self {
+            Self::Optional => Ok(x),
+            Self::Essential => Err(f(x)),
+        }
+    }
+}
+
 impl<'a, U: Clone + 'a, E: Clone + 'a, T: Clone + IntoIterator<Item = Result<U, E>> + 'a> Path<T> {
-    pub fn explode(self) -> impl Iterator<Item = Result<Path<U>, E>> + 'a {
+    pub(crate) fn explode(self) -> impl Iterator<Item = Result<Path<U>, E>> + 'a {
         Path(Vec::new())
             .combinations(self.0.into_iter())
             .map(Path::transpose)
@@ -45,11 +75,11 @@ impl<'a, U: Clone + 'a> Path<U> {
 }
 
 impl<'a, V: ValT + 'a> Path<V> {
-    pub fn run(self, v: V) -> BoxIter<'a, ValR<V>> {
+    pub(crate) fn run(self, v: V) -> BoxIter<'a, ValR<V>> {
         run(self.0.into_iter(), v)
     }
 
-    pub fn update<F>(mut self, v: V, f: F) -> ValX<'a, V>
+    pub(crate) fn update<F>(mut self, v: V, f: F) -> ValX<'a, V>
     where
         F: Fn(V) -> ValXs<'a, V>,
     {
@@ -134,15 +164,16 @@ impl<'a, U: Clone + 'a, F: IntoIterator<Item = U> + Clone + 'a> Part<F> {
 }
 
 impl<T> Path<T> {
-    pub fn map_ref<'a, U>(&'a self, mut f: impl FnMut(&'a T) -> U) -> Path<U> {
+    pub(crate) fn map_ref<'a, U>(&'a self, mut f: impl FnMut(&'a T) -> U) -> Path<U> {
         let path = self.0.iter();
         let path = path.map(move |(part, opt)| (part.as_ref().map(&mut f), *opt));
         Path(path.collect())
     }
 }
 
 impl<T> Part<T> {
-    fn map<U, F: FnMut(T) -> U>(self, mut f: F) -> Part<U> {
+    /// Apply a function to the contained indices.
+    pub(crate) fn map<U, F: FnMut(T) -> U>(self, mut f: F) -> Part<U> {
         use Part::{Index, Range};
         match self {
             Index(i) => Index(f(i)),