Merge branch 'zesterer:main' into fixes/recursive-leak

Author: Zij-IT

Cargo.lock

@@ -85,7 +85,7 @@ unicode-ident =  "1.0.10"
 ariadne = "0.2"
 pom = "3.2"
 nom = "7.1"
-winnow = "0.5.19"
+winnow = "0.6.0"
 serde_json = { version = "1.0", features = ["preserve_order"] }
 ciborium = { version = "0.2" }
 criterion = "0.4.0"
@@ -137,3 +137,7 @@ required-features = ["std"]
 [[example]]
 name = "io"
 required-features = ["std"]
+
+[[example]]
+name = "foo"
+required-features = ["std"]

Cargo.toml

@@ -29,6 +29,7 @@ you may need. It also has `no_std` support, making it suitable for embedded envi
 - 🏷️ **Pattern labelling** for dynamic, user-friendly error messages
 - 🗃️ **Caching** allows parsers to be created once and reused many times
 - ↔️ **Pratt parsing** support for unary and binary operators
+- 🪛 **no_std** support, allowing chumsky to run in embedded environments
 
 *Note: Error diagnostic rendering is performed by [Ariadne](https://github.com/zesterer/ariadne)*
 

README2.md

@@ -361,7 +361,7 @@ mod winnow {
         combinator::{preceded, separated_pair, terminated},
         error::{InputError, ParserError},
         prelude::*,
-        token::{any, none_of, one_of, tag, take_while},
+        token::{any, none_of, one_of, take_while},
     };
 
     use super::JsonZero;
@@ -441,9 +441,9 @@ mod winnow {
         preceded(
             space,
             dispatch!(peek(any);
-                b'n' => tag("null").value(JsonZero::Null),
-                b't' => tag("true").value(JsonZero::Bool(true)),
-                b'f' => tag("false").value(JsonZero::Bool(false)),
+                b'n' => "null".value(JsonZero::Null),
+                b't' => "true".value(JsonZero::Bool(true)),
+                b'f' => "false".value(JsonZero::Bool(false)),
                 b'-' | b'0'..=b'9' => number.map(JsonZero::Num),
                 b'"' => string.map(JsonZero::Str),
                 b'[' => array.map(JsonZero::Array),
@@ -463,6 +463,7 @@ mod winnow {
     }
 }
 
+#[allow(clippy::empty_docs)] // TODO: Remove, pest does things clippy doesn't like for some reason
 mod pest {
     use super::JsonZero;
 

benches/json.rs

@@ -180,7 +180,8 @@ fn eval<'a>(
 }
 
 fn main() {
-    let src = std::fs::read_to_string(std::env::args().nth(1).unwrap()).unwrap();
+    let usage = "Run `cargo run --example foo -- examples/sample.foo`";
+    let src = std::fs::read_to_string(std::env::args().nth(1).expect(usage)).expect(usage);
 
     match parser().parse(&src).into_result() {
         Ok(ast) => match eval(&ast, &mut Vec::new(), &mut Vec::new()) {

examples/foo.rs

@@ -1,7 +1,7 @@
 use chumsky::prelude::*;
 
 #[derive(Clone, Debug)]
-enum Stmt {
+pub enum Stmt {
     Expr,
     Loop(Vec<Stmt>),
 }

examples/indent.rs

@@ -7,7 +7,7 @@ use chumsky::prelude::*;
 use std::{collections::HashMap, env, fs};
 
 #[derive(Clone, Debug)]
-enum Json {
+pub enum Json {
     Invalid,
     Null,
     Bool(bool),

examples/json.rs

@@ -354,7 +354,7 @@ where
     I: Input<'a>,
     E: ParserExtra<'a, I>,
     A: Parser<'a, I, OA, E>,
-    F: Fn(OA, &mut MapExtra<'a, '_, I, E>) -> O,
+    F: Fn(OA, &mut MapExtra<'a, '_, '_, I, E>) -> O,
 {
     #[inline(always)]
     fn go<M: Mode>(&self, inp: &mut InputRef<'a, '_, I, E>) -> PResult<M, O> {
@@ -373,7 +373,7 @@ where
     I: Input<'a>,
     E: ParserExtra<'a, I>,
     A: IterParser<'a, I, OA, E>,
-    F: Fn(OA, &mut MapExtra<'a, '_, I, E>) -> O,
+    F: Fn(OA, &mut MapExtra<'a, '_, '_, I, E>) -> O,
 {
     type IterState<M: Mode> = A::IterState<M>
     where
@@ -546,7 +546,7 @@ where
         match (self.mapper)(out, span) {
             Ok(out) => Ok(M::bind(|| out)),
             Err(err) => {
-                inp.add_alt_err(inp.offset().offset, err);
+                inp.add_alt_err(before.offset, err);
                 Err(())
             }
         }
@@ -579,7 +579,7 @@ where
     I: Input<'a>,
     E: ParserExtra<'a, I>,
     A: Parser<'a, I, OA, E>,
-    F: Fn(OA, &mut MapExtra<'a, '_, I, E>) -> Result<O, E::Error>,
+    F: Fn(OA, &mut MapExtra<'a, '_, '_, I, E>) -> Result<O, E::Error>,
 {
     #[inline(always)]
     fn go<M: Mode>(&self, inp: &mut InputRef<'a, '_, I, E>) -> PResult<M, O> {
@@ -1427,6 +1427,7 @@ where
     A: Parser<'a, I, OA, E>,
 {
     #[inline(always)]
+    #[allow(clippy::nonminimal_bool)] // TODO: Remove this, lint is currently buggy
     fn go<M: Mode>(&self, inp: &mut InputRef<'a, '_, I, E>) -> PResult<M, ()> {
         if self.at_most == !0 && self.at_least == 0 {
             loop {
@@ -2362,7 +2363,7 @@ where
     A: IterParser<'a, I, OA, E>,
     B: Parser<'a, I, O, E>,
     E: ParserExtra<'a, I>,
-    F: Fn(OA, O, &mut MapExtra<'a, '_, I, E>) -> O,
+    F: Fn(OA, O, &mut MapExtra<'a, '_, '_, I, E>) -> O,
 {
     #[inline(always)]
     fn go<M: Mode>(&self, inp: &mut InputRef<'a, '_, I, E>) -> PResult<M, O>
@@ -2497,7 +2498,7 @@ where
     A: Parser<'a, I, O, E>,
     B: IterParser<'a, I, OB, E>,
     E: ParserExtra<'a, I>,
-    F: Fn(O, OB, &mut MapExtra<'a, '_, I, E>) -> O,
+    F: Fn(O, OB, &mut MapExtra<'a, '_, '_, I, E>) -> O,
 {
     #[inline(always)]
     fn go<M: Mode>(&self, inp: &mut InputRef<'a, '_, I, E>) -> PResult<M, O>
@@ -2688,7 +2689,7 @@ where
     I: Input<'a>,
     E: ParserExtra<'a, I>,
     A: Parser<'a, I, OA, E>,
-    F: Fn(OA, &mut MapExtra<'a, '_, I, E>, &mut Emitter<E::Error>) -> U,
+    F: Fn(OA, &mut MapExtra<'a, '_, '_, I, E>, &mut Emitter<E::Error>) -> U,
 {
     #[inline(always)]
     fn go<M: Mode>(&self, inp: &mut InputRef<'a, '_, I, E>) -> PResult<M, U>

src/combinator.rs

@@ -7,6 +7,7 @@
 //! like [`Cheap`], [`Simple`] or [`Rich`].
 
 use super::*;
+#[cfg(not(feature = "std"))]
 use alloc::string::ToString;
 
 /// A trait that describes parser error types.
@@ -426,6 +427,7 @@ impl<'a, T, L> RichReason<'a, T, L> {
         mut fmt_span: impl FnMut(&S, &mut fmt::Formatter<'_>) -> fmt::Result,
         mut fmt_label: impl FnMut(&L, &mut fmt::Formatter<'_>) -> fmt::Result,
         span: Option<&S>,
+        #[cfg(feature = "label")] context: &[(L, S)],
     ) -> fmt::Result {
         match self {
             RichReason::ExpectedFound { expected, found } => {
@@ -467,6 +469,13 @@ impl<'a, T, L> RichReason<'a, T, L> {
                 }
             }
         }
+        #[cfg(feature = "label")]
+        for (l, s) in context {
+            write!(f, " in ")?;
+            fmt_label(l, f)?;
+            write!(f, " at ")?;
+            fmt_span(s, f)?;
+        }
         Ok(())
     }
 }
@@ -526,7 +535,15 @@ where
     L: fmt::Display,
 {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.inner_fmt(f, T::fmt, |_: &(), _| Ok(()), L::fmt, None)
+        self.inner_fmt(
+            f,
+            T::fmt,
+            |_: &(), _| Ok(()),
+            L::fmt,
+            None,
+            #[cfg(feature = "label")]
+            &[],
+        )
     }
 }
 
@@ -557,6 +574,8 @@ impl<'a, T, S, L> Rich<'a, T, S, L> {
             fmt_span,
             fmt_label,
             if with_spans { Some(&self.span) } else { None },
+            #[cfg(feature = "label")]
+            &self.context,
         )
     }
 }

src/error.rs

@@ -8,13 +8,8 @@
 pub use crate::stream::{BoxedExactSizeStream, BoxedStream, Stream};
 
 use super::*;
-#[cfg(feature = "memoization")]
-use hashbrown::HashMap;
 #[cfg(feature = "std")]
-use std::{
-    cell::RefCell,
-    io::{BufReader, Read, Seek},
-};
+use std::io::{BufReader, Read, Seek};
 
 /// A trait for types that represents a stream of input tokens. Unlike [`Iterator`], this type
 /// supports backtracking and a few other features required by the crate.
@@ -1518,36 +1513,28 @@ impl<E> Emitter<E> {
 }
 
 /// See [`Parser::map_with`].
-pub struct MapExtra<'a, 'b, I: Input<'a>, E: ParserExtra<'a, I>> {
-    before: I::Offset,
-    after: I::Offset,
-    inp: &'b I,
-    state: &'b mut E::State,
-    ctx: &'b E::Context,
+/// Note: 'a is the lifetime of the Input, 'b is the lifetime of the embedded InputRef, 'parse if the lifetime of the parser and corresponding state.
+pub struct MapExtra<'a, 'b, 'parse, I: Input<'a>, E: ParserExtra<'a, I>> {
+    before: Offset<'a, 'parse, I>,
+    inp: &'b mut InputRef<'a, 'parse, I, E>,
 }
 
-impl<'a, 'b, I: Input<'a>, E: ParserExtra<'a, I>> MapExtra<'a, 'b, I, E> {
-    #[inline(always)]
-    pub(crate) fn new<'parse>(
+impl<'a, 'b, 'parse, I: Input<'a>, E: ParserExtra<'a, I>> MapExtra<'a, 'b, 'parse, I, E> {
+    /// Create new MapExtra
+    /// SAFETY: `before` Offset must be from a previous call to inp.offset().
+    pub(crate) fn new(
         before: Offset<'a, 'parse, I>,
         inp: &'b mut InputRef<'a, 'parse, I, E>,
     ) -> Self {
-        Self {
-            before: before.offset,
-            after: inp.offset,
-            ctx: inp.ctx,
-            state: inp.state,
-            inp: inp.input,
-        }
+        MapExtra { before, inp }
     }
-
     /// Get the span corresponding to the output.
+    // SAFETY: The offsets both came from the same input
+    // TODO: Should this make `MapExtra::new` unsafe? Probably, but it's an internal API and we simply wouldn't
+    // ever abuse it in this way, even accidentally.
     #[inline(always)]
     pub fn span(&self) -> I::Span {
-        // SAFETY: The offsets both came from the same input
-        // TODO: Should this make `MapExtra::new` unsafe? Probably, but it's an internal API and we simply wouldn't
-        // ever abuse it in this way, even accidentally.
-        unsafe { self.inp.span(self.before..self.after) }
+        self.inp.span(self.before..self.inp.offset())
     }
 
     /// Get the slice corresponding to the output.
@@ -1556,18 +1543,18 @@ impl<'a, 'b, I: Input<'a>, E: ParserExtra<'a, I>> MapExtra<'a, 'b, I, E> {
     where
         I: SliceInput<'a>,
     {
-        self.inp.slice(self.before..self.after)
+        self.inp.slice(self.before..self.inp.offset())
     }
 
     /// Get the parser state.
     #[inline(always)]
     pub fn state(&mut self) -> &mut E::State {
-        self.state
+        self.inp.state()
     }
 
     /// Get the current parser context.
     #[inline(always)]
     pub fn ctx(&self) -> &E::Context {
-        self.ctx
+        self.inp.ctx()
     }
 }