granit-parser - cargo Package Compare versions

Comparing version 0.0.3 to 0.0.4

tests/comments_possible.rs

+use granit_parser::{
+    input::SkipTabs, BorrowedInput, Event, Input, Parser, ScanError, Scanner, Span, StrInput, Token,
+};
+
+struct CommentEnabledStrInput<'input> {
+    inner: StrInput<'input>,
+}
+
+impl<'input> CommentEnabledStrInput<'input> {
+    #[must_use]
+    fn new(source: &'input str) -> Self {
+        Self {
+            inner: StrInput::new(source),
+        }
+    }
+}
+
+impl Input for CommentEnabledStrInput<'_> {
+    fn lookahead(&mut self, count: usize) {
+        self.inner.lookahead(count);
+    }
+
+    fn buflen(&self) -> usize {
+        self.inner.buflen()
+    }
+
+    fn bufmaxlen(&self) -> usize {
+        self.inner.bufmaxlen()
+    }
+
+    fn raw_read_ch(&mut self) -> char {
+        self.inner.raw_read_ch()
+    }
+
+    fn raw_read_non_breakz_ch(&mut self) -> Option<char> {
+        self.inner.raw_read_non_breakz_ch()
+    }
+
+    fn skip(&mut self) {
+        self.inner.skip();
+    }
+
+    fn skip_n(&mut self, count: usize) {
+        self.inner.skip_n(count);
+    }
+
+    fn peek(&self) -> char {
+        self.inner.peek()
+    }
+
+    fn peek_nth(&self, n: usize) -> char {
+        self.inner.peek_nth(n)
+    }
+
+    fn byte_offset(&self) -> Option<usize> {
+        self.inner.byte_offset()
+    }
+
+    fn slice_bytes(&self, start: usize, end: usize) -> Option<&str> {
+        self.inner.slice_bytes(start, end)
+    }
+
+    fn may_contain_comments(&self) -> bool {
+        true
+    }
+
+    fn skip_ws_to_eol(&mut self, skip_tabs: SkipTabs) -> (usize, Result<SkipTabs, &'static str>) {
+        self.inner.skip_ws_to_eol(skip_tabs)
+    }
+
+    fn skip_ws_to_eol_blanks(&mut self, skip_tabs: SkipTabs) -> (usize, SkipTabs) {
+        self.inner.skip_ws_to_eol_blanks(skip_tabs)
+    }
+}
+
+impl<'input> BorrowedInput<'input> for CommentEnabledStrInput<'input> {
+    fn slice_borrowed(&self, start: usize, end: usize) -> Option<&'input str> {
+        self.inner.slice_borrowed(start, end)
+    }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct ParseTrace<'input> {
+    events: Vec<(Event<'input>, Span)>,
+    error: Option<ScanError>,
+}
+
+fn scanner_tokens_fast_path(source: &str) -> Vec<Token<'_>> {
+    Scanner::new(StrInput::new(source)).collect()
+}
+
+fn scanner_tokens_comment_enabled(source: &str) -> Vec<Token<'_>> {
+    Scanner::new(CommentEnabledStrInput::new(source)).collect()
+}
+
+fn parser_trace_fast_path(source: &str) -> ParseTrace<'_> {
+    let mut parser = Parser::new_from_str(source);
+    parser_trace(&mut parser)
+}
+
+fn parser_trace_comment_enabled(source: &str) -> ParseTrace<'_> {
+    let mut parser = Parser::new(CommentEnabledStrInput::new(source));
+    parser_trace(&mut parser)
+}
+
+fn parser_trace<'input, T>(parser: &mut Parser<'input, T>) -> ParseTrace<'input>
+where
+    T: BorrowedInput<'input>,
+{
+    let mut events = Vec::new();
+    let mut error = None;
+
+    while let Some(next) = parser.next_event() {
+        match next {
+            Ok(event) => events.push(event),
+            Err(err) => {
+                error = Some(err);
+                break;
+            }
+        }
+    }
+
+    ParseTrace { events, error }
+}
+
+fn assert_no_hash(source: &str, name: &str) {
+    assert!(
+        !source.contains('#'),
+        "{name} fixture must exercise the no-comment fast path"
+    );
+    assert!(
+        !StrInput::new(source).may_contain_comments(),
+        "{name} should make StrInput disable comment probing"
+    );
+    assert!(
+        CommentEnabledStrInput::new(source).may_contain_comments(),
+        "{name} control input should keep comment probing enabled"
+    );
+}
+
+#[test]
+fn no_comment_fast_path_matches_comment_enabled_tokens_and_events() {
+    for (name, source) in [
+        ("block mapping", "a: b\n"),
+        ("block sequence value", "a:\n  - b\n"),
+        ("flow sequence", "[a, b]\n"),
+        ("flow mapping", "{a: b}\n"),
+        ("explicit document", "---\na: b\n...\n"),
+    ] {
+        assert_no_hash(source, name);
+
+        assert_eq!(
+            scanner_tokens_fast_path(source),
+            scanner_tokens_comment_enabled(source),
+            "{name} scanner token output differed"
+        );
+        assert_eq!(
+            parser_trace_fast_path(source),
+            parser_trace_comment_enabled(source),
+            "{name} parser event output differed"
+        );
+    }
+}
+
+#[test]
+fn invalid_no_comment_fast_path_matches_comment_enabled_event_prefixes() {
+    for (name, source) in [
+        ("unclosed flow sequence", "a: [1, 2\n"),
+        ("extra flow sequence end", "key: [1, 2]]\n"),
+        ("tab after value indicator", "a:\tb\n"),
+        (
+            "directive after implicit document",
+            "a: b\n%YAML 1.2\n---\nc: d\n",
+        ),
+        ("reserved indicator", "@\n"),
+    ] {
+        assert_no_hash(source, name);
+
+        let fast_path = parser_trace_fast_path(source);
+        let comment_enabled = parser_trace_comment_enabled(source);
+
+        assert!(fast_path.error.is_some(), "{name} should fail");
+        assert_eq!(
+            fast_path, comment_enabled,
+            "{name} parser event prefix or error differed"
+        );
+    }
+}

.cargo_vcs_info.json

 {
   "git": {
-    "sha1": "c7df7015b8eae3aa1b7ab5abc44b3a3eb7a2524e"
+    "sha1": "ca296b7858b3fbd531794e7ecda5732f0bd8206f"
   },
   "path_in_vcs": ""
 }

Cargo.lock

@@ -115,3 +115,3 @@ # This file is automatically @generated by Cargo.
 name = "granit-parser"
-version = "0.0.3"
+version = "0.0.4"
 dependencies = [
@@ -118,0 +118,0 @@ "arraydeque",

Cargo.toml

@@ -16,3 +16,3 @@ # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
 name = "granit-parser"
-version = "0.0.3"
+version = "0.0.4"
 authors = [
@@ -34,3 +34,3 @@ "Ethiraric <[e-mail]>",
 autobenches = false
-description = "A YAML parser in pure Rust with comment and style support"
+description = "A YAML parser with comment and style support, written in pure Rust"
 documentation = "https://docs.rs/granit-parser/latest/granit_parser"
@@ -42,2 +42,3 @@ readme = "README.md"
     "no-std",
+    "cst",
 ]
@@ -47,2 +48,3 @@ categories = [
     "parser-implementations",
+    "text-processing",
 ]
@@ -104,2 +106,6 @@ license = "MIT OR Apache-2.0"
 [[test]]
+name = "comments_possible"
+path = "tests/comments_possible.rs"
+
+[[test]]
 name = "coverage_edges"
@@ -106,0 +112,0 @@ path = "tests/coverage_edges.rs"

CHANGELOG.md

 # Changelog
+## v0.0.4
+- Performance improvements on comment parsing
 
+
 ## v0.0.3
@@ -4,0 +7,0 @@

src/input.rs

@@ -161,2 +161,12 @@ //! Utilities to create a source of input to the parser.
 
+    /// Return whether this input may contain a `#` character.
+    ///
+    /// This is a conservative performance hint. Inputs that cannot answer cheaply should return
+    /// `true`, which keeps full comment handling enabled.
+    #[inline]
+    #[must_use]
+    fn may_contain_comments(&self) -> bool {
+        true
+    }
+
     /// Look for the next character and return it.
@@ -302,2 +312,40 @@ ///
 
+    /// Skip YAML blank characters, stopping before comments, line breaks, or other content.
+    ///
+    /// This is the comment-aware counterpart to [`Input::skip_ws_to_eol`]: it preserves a
+    /// following `#` for the scanner to tokenize while still letting input implementations batch
+    /// the common run of spaces and tabs.
+    ///
+    /// # Return
+    /// Returns the number of consumed characters and a [`SkipTabs::Result`] describing whether
+    /// tabs and valid YAML whitespace (` `) were encountered.
+    fn skip_ws_to_eol_blanks(&mut self, skip_tabs: SkipTabs) -> (usize, SkipTabs) {
+        assert!(!matches!(skip_tabs, SkipTabs::Result(..)));
+
+        let mut encountered_tab = false;
+        let mut has_yaml_ws = false;
+        let mut chars_consumed = 0;
+
+        loop {
+            match self.look_ch() {
+                ' ' => {
+                    has_yaml_ws = true;
+                    chars_consumed += 1;
+                    self.skip();
+                }
+                '\t' if skip_tabs != SkipTabs::No => {
+                    encountered_tab = true;
+                    chars_consumed += 1;
+                    self.skip();
+                }
+                _ => break,
+            }
+        }
+
+        (
+            chars_consumed,
+            SkipTabs::Result(encountered_tab, has_yaml_ws),
+        )
+    }
+
     /// Check whether the next characters may be part of a plain scalar.
@@ -304,0 +352,0 @@ ///

src/input/str.rs

@@ -166,2 +166,7 @@ use crate::{
     #[inline]
+    fn may_contain_comments(&self) -> bool {
+        self.original.as_bytes().contains(&b'#')
+    }
+
+    #[inline]
     fn look_ch(&mut self) -> char {
@@ -293,2 +298,36 @@ self.lookahead(1);
 
+    fn skip_ws_to_eol_blanks(&mut self, skip_tabs: SkipTabs) -> (usize, SkipTabs) {
+        assert!(!matches!(skip_tabs, SkipTabs::Result(..)));
+
+        let bytes = self.buffer.as_bytes();
+        let mut i = 0;
+        let mut encountered_tab = false;
+        let mut has_yaml_ws = false;
+
+        if skip_tabs == SkipTabs::Yes {
+            while i < bytes.len() {
+                match bytes[i] {
+                    b' ' => {
+                        has_yaml_ws = true;
+                        i += 1;
+                    }
+                    b'\t' => {
+                        encountered_tab = true;
+                        i += 1;
+                    }
+                    _ => break,
+                }
+            }
+        } else {
+            while i < bytes.len() && bytes[i] == b' ' {
+                has_yaml_ws = true;
+                i += 1;
+            }
+        }
+
+        self.buffer = &self.buffer[i..];
+
+        (i, SkipTabs::Result(encountered_tab, has_yaml_ws))
+    }
+
     #[allow(clippy::inline_always)]
@@ -295,0 +334,0 @@ #[inline(always)]

src/lib.rs

@@ -57,2 +57,8 @@ // Copyright 2015, Yuheng Chen.
 //!
+//! # Limits
+//!
+//! To keep streaming parsing memory bounded, syntactically ambiguous collection-entry positions
+//! that require comment lookahead accept at most 32 consecutive comments before the following node
+//! is resolved. Longer runs return a [`ScanError`] instead of being buffered without bound.
+//!
 //! # Features
@@ -59,0 +65,0 @@ //! **Note:** This crate's MSRV is `1.81.0`.

src/parser_stack.rs

@@ -12,3 +12,2 @@ use crate::{
     index: usize,
-    current: Option<(Event<'input>, Span)>,
     anchor_offset: usize,
@@ -24,3 +23,2 @@ }
             index: 0,
-            current: None,
             anchor_offset,
@@ -57,14 +55,6 @@ }
     fn peek(&mut self) -> Option<Result<&(Event<'input>, Span), ScanError>> {
-        if self.current.is_none() {
-            self.current = self.events.get(self.index).cloned();
-        }
-        self.current.as_ref().map(Ok)
+        self.events.get(self.index).map(Ok)
     }
 
     fn next_event(&mut self) -> Option<ParseResult<'input>> {
-        if let Some(current) = self.current.take() {
-            self.index += 1;
-            self.advance_anchor_offset(&current.0);
-            return Some(Ok(current));
-        }
         let event = self.events.get(self.index).cloned()?;
@@ -71,0 +61,0 @@ self.index += 1;

tests/comment.rs

@@ -1,2 +0,2 @@
-use std::borrow::Cow;
+use std::{borrow::Cow, cell::Cell, rc::Rc};
 
@@ -12,2 +12,98 @@ use granit_parser::{
 
+fn scanner_tokens(source: &str) -> Result<Vec<Token<'_>>, ScanError> {
+    let mut scanner = Scanner::new(StrInput::new(source));
+    let mut tokens = Vec::new();
+
+    while let Some(token) = scanner.next_token()? {
+        tokens.push(token);
+    }
+
+    Ok(tokens)
+}
+
+/// Iterator wrapper that records how many characters the parser pulls from streaming input.
+struct CountingChars<I> {
+    /// Wrapped character iterator consumed by `Parser::new_from_iter`.
+    iter: I,
+    /// Shared count of successfully yielded characters.
+    ///
+    /// `Rc` lets the test keep a readable handle after this iterator is moved into the parser.
+    /// `Cell` gives that shared handle single-threaded interior mutability, so `next` can update
+    /// the count while the test can still read it later.
+    read: Rc<Cell<usize>>,
+}
+
+impl<I> Iterator for CountingChars<I>
+where
+    I: Iterator<Item = char>,
+{
+    type Item = char;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let next = self.iter.next();
+        if next.is_some() {
+            self.read.set(self.read.get() + 1);
+        }
+        next
+    }
+}
+
+fn chars_pulled_until_error(source: &str) -> (usize, String) {
+    let read = Rc::new(Cell::new(0));
+    let iter = CountingChars {
+        iter: source.chars(),
+        read: Rc::clone(&read),
+    };
+    let mut parser = Parser::new_from_iter(iter);
+
+    loop {
+        match parser
+            .next_event()
+            .expect("parser should emit an event before EOF")
+        {
+            Ok(_) => {}
+            Err(error) => return (read.get(), error.info().to_owned()),
+        }
+    }
+}
+
+fn chars_pulled_before_first_comment(source: &str) -> usize {
+    let read = Rc::new(Cell::new(0));
+    let iter = CountingChars {
+        iter: source.chars(),
+        read: Rc::clone(&read),
+    };
+    let mut parser = Parser::new_from_iter(iter);
+
+    loop {
+        let (event, _) = parser
+            .next_event()
+            .expect("parser should emit an event before EOF")
+            .expect("parser should accept generated comment run");
+        if matches!(event, Event::Comment(..)) {
+            return read.get();
+        }
+    }
+}
+
+fn long_comment_run(start: usize, count: usize) -> String {
+    let mut comments = String::new();
+    for index in start..start + count {
+        comments.push_str("# c");
+        comments.push_str(&index.to_string());
+        comments.push('\n');
+    }
+    comments
+}
+
+fn long_indented_comment_run(start: usize, count: usize) -> String {
+    let mut comments = String::new();
+    for index in start..start + count {
+        comments.push_str("  # c");
+        comments.push_str(&index.to_string());
+        comments.push('\n');
+    }
+    comments
+}
+
 fn first_empty_scalar_span(events: &[(Event<'_>, Span)]) -> Span {
@@ -479,2 +575,82 @@ events
 #[test]
+fn scanner_preserves_comment_adjacent_plain_scalar_whitespace_rules() {
+    struct Case<'input> {
+        yaml: &'input str,
+        expected_value: &'input str,
+        expected_comment: Option<(&'input str, &'input str)>,
+    }
+
+    let cases = [
+        Case {
+            yaml: "a: b#bad\n",
+            expected_value: "b#bad",
+            expected_comment: None,
+        },
+        Case {
+            yaml: "a: b # good\n",
+            expected_value: "b",
+            expected_comment: Some((" good", "# good")),
+        },
+        Case {
+            yaml: "a: b\t# tab-before-comment\n",
+            expected_value: "b",
+            expected_comment: Some((" tab-before-comment", "# tab-before-comment")),
+        },
+    ];
+
+    for case in cases {
+        let tokens = scanner_tokens(case.yaml).expect("scanner should accept YAML");
+        let scalars = tokens
+            .iter()
+            .filter_map(|Token(_, token)| match token {
+                TokenType::Scalar(ScalarStyle::Plain, value) => Some(value.as_ref()),
+                _ => None,
+            })
+            .collect::<Vec<_>>();
+        let comments = tokens
+            .iter()
+            .filter_map(|Token(span, token)| match token {
+                TokenType::Comment(comment) => Some((
+                    comment.text.as_ref(),
+                    comment.placement,
+                    span.slice(case.yaml),
+                )),
+                _ => None,
+            })
+            .collect::<Vec<_>>();
+
+        assert_eq!(scalars, vec!["a", case.expected_value], "{:?}", case.yaml);
+
+        match case.expected_comment {
+            Some((text, slice)) => {
+                assert_eq!(
+                    comments,
+                    vec![(text, Placement::Right, Some(slice))],
+                    "{:?}",
+                    case.yaml
+                );
+            }
+            None => assert!(comments.is_empty(), "{:?}", case.yaml),
+        }
+    }
+}
+
+#[test]
+fn scanner_rejects_tab_immediately_after_explicit_key_indicator() {
+    let mut scanner = Scanner::new(StrInput::new("?\tkey\n"));
+
+    let error = loop {
+        match scanner.next_token() {
+            Ok(Some(_)) => {}
+            Ok(None) => panic!("expected scanner error"),
+            Err(error) => break error,
+        }
+    };
+
+    assert_eq!(error.info(), "expected whitespace");
+    assert_eq!(error.marker().line(), 1);
+    assert_eq!(error.marker().col(), 1);
+}
+
+#[test]
 fn parser_emits_full_line_indented_and_trailing_comment_events() {
@@ -725,2 +901,91 @@ let yaml = "# top\n  # indented\nkey: value # trailing\n#eof";
 #[test]
+fn max_buffered_empty_node_comment_runs_preserve_span_order() {
+    let trailing_comments = long_comment_run(1, 31);
+    let cases = [
+        format!("key: # c0\n{trailing_comments}next: value\n"),
+        format!("- # c0\n{trailing_comments}- value\n"),
+        format!("key:\n- # c0\n{trailing_comments}next: value\n"),
+        format!("root: {{key: # c0\n{trailing_comments}}}\n"),
+    ];
+
+    for yaml in cases {
+        let events = parser_events(&yaml).expect("parser should accept capped comment run");
+        assert_monotonic_spans(&events);
+    }
+}
+
+#[test]
+fn parser_streams_explicit_key_comment_runs_before_reading_tail() {
+    let trailing_comments = long_indented_comment_run(1, 128);
+    let yaml = format!("? # c0\n{trailing_comments}  key\n: value\n");
+
+    let total = yaml.chars().count();
+    let pulled = chars_pulled_before_first_comment(&yaml);
+
+    assert!(
+        pulled < total / 2,
+        "parser read {pulled} of {total} chars before first explicit-key comment event",
+    );
+}
+
+#[test]
+fn explicit_key_comment_does_not_hide_tab_indentation_error() {
+    let yaml = "? # c\n\tkey\n: value\n";
+
+    let err = Parser::new_from_str(yaml)
+        .find_map(Result::err)
+        .expect("parser should reject tab indentation after explicit key comment");
+
+    assert_eq!(err.info(), "tabs disallowed in this context");
+}
+
+#[test]
+fn explicit_key_comment_run_does_not_hide_tab_indentation_error() {
+    let yaml = "? # c0\n  # c1\n  # c2\n\tkey\n: value\n";
+
+    let err = Parser::new_from_str(yaml)
+        .find_map(Result::err)
+        .expect("parser should reject tab indentation after explicit key comment run");
+
+    assert_eq!(err.info(), "tabs disallowed in this context");
+}
+
+#[test]
+fn parser_rejects_ambiguous_large_comment_runs_before_reading_tail() {
+    let trailing_comments = long_comment_run(1, 128);
+    let cases = [
+        (
+            "block mapping value",
+            format!("key: # c0\n{trailing_comments}next: value\n"),
+        ),
+        (
+            "block sequence entry",
+            format!("- # c0\n{trailing_comments}- value\n"),
+        ),
+        (
+            "indentless sequence entry",
+            format!("key:\n- # c0\n{trailing_comments}next: value\n"),
+        ),
+        (
+            "flow mapping value",
+            format!("root: {{key: # c0\n{trailing_comments}}}\n"),
+        ),
+    ];
+
+    for (name, yaml) in cases {
+        let total = yaml.chars().count();
+        let (pulled, info) = chars_pulled_until_error(&yaml);
+
+        assert_eq!(
+            info, "too many consecutive comments before resolving collection entry",
+            "{name}: unexpected parser error",
+        );
+        assert!(
+            pulled < total / 2,
+            "{name}: parser read {pulled} of {total} chars before rejecting",
+        );
+    }
+}
+
+#[test]
 fn later_empty_indentless_sequence_entry_after_comment_is_queued_before_next_key() {
@@ -784,2 +1049,30 @@ let yaml = "key:\n- first\n- # empty\nnext: value\n";
 #[test]
+fn first_indentless_sequence_entry_after_comment_keeps_value_in_sequence() {
+    let yaml = "key:\n- # value\n  first\nnext: value\n";
+    let events = parser_events(yaml).expect("indentless sequence should parse");
+
+    let names: Vec<_> = events
+        .iter()
+        .filter_map(|(event, _)| match event {
+            Event::Scalar(value, ScalarStyle::Plain, ..) => Some(format!("Scalar({value})")),
+            Event::Comment(text, _) => Some(format!("Comment({text})")),
+            Event::SequenceEnd => Some("SequenceEnd".into()),
+            _ => None,
+        })
+        .collect();
+
+    assert_eq!(
+        names,
+        vec![
+            "Scalar(key)",
+            "Comment( value)",
+            "Scalar(first)",
+            "SequenceEnd",
+            "Scalar(next)",
+            "Scalar(value)",
+        ]
+    );
+}
+
+#[test]
 fn parser_handles_comments_in_flow_mapping_key_and_separator_states() {
@@ -786,0 +1079,0 @@ let yaml = "{? # key\n  key\n: value, # comma\nnext: value}\n";

No alert changes

No metric changes