winnow/token/
mod.rs

Help
1//! Parsers extracting tokens from the stream
2
3#[cfg(test)]
4mod tests;
5
6use crate::combinator::trace;
7use crate::combinator::DisplayDebug;
8use crate::error::ErrMode;
9use crate::error::ErrorKind;
10use crate::error::Needed;
11use crate::error::ParserError;
12use crate::lib::std::result::Result::Ok;
13use crate::stream::Range;
14use crate::stream::{Compare, CompareResult, ContainsToken, FindSlice, SliceLen, Stream};
15use crate::stream::{StreamIsPartial, ToUsize};
16use crate::PResult;
17use crate::Parser;
18
19/// Matches one token
20///
21/// *Complete version*: Will return an error if there's not enough input data.
22///
23/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
24///
25/// # Effective Signature
26///
27/// Assuming you are parsing a `&str` [Stream]:
28/// ```rust
29/// # use winnow::prelude::*;;
30/// pub fn any(input: &mut &str) -> PResult<char>
31/// # {
32/// #     winnow::token::any.parse_next(input)
33/// # }
34/// ```
35///
36/// # Example
37///
38/// ```rust
39/// # use winnow::{token::any, error::ErrMode, error::{InputError, ErrorKind}};
40/// # use winnow::prelude::*;
41/// fn parser(input: &str) -> IResult<&str, char> {
42///     any.parse_peek(input)
43/// }
44///
45/// assert_eq!(parser("abc"), Ok(("bc",'a')));
46/// assert_eq!(parser(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
47/// ```
48///
49/// ```rust
50/// # use winnow::{token::any, error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
51/// # use winnow::prelude::*;
52/// # use winnow::Partial;
53/// assert_eq!(any::<_, InputError<_>>.parse_peek(Partial::new("abc")), Ok((Partial::new("bc"),'a')));
54/// assert_eq!(any::<_, InputError<_>>.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
55/// ```
56#[inline(always)]
57#[doc(alias = "token")]
58pub fn any<Input, Error>(input: &mut Input) -> PResult<<Input as Stream>::Token, Error>
59where
60    Input: StreamIsPartial + Stream,
61    Error: ParserError<Input>,
62{
63    trace("any", move |input: &mut Input| {
64        if <Input as StreamIsPartial>::is_partial_supported() {
65            any_::<_, _, true>(input)
66        } else {
67            any_::<_, _, false>(input)
68        }
69    })
70    .parse_next(input)
71}
72
73fn any_<I, E: ParserError<I>, const PARTIAL: bool>(
74    input: &mut I,
75) -> PResult<<I as Stream>::Token, E>
76where
77    I: StreamIsPartial,
78    I: Stream,
79{
80    input.next_token().ok_or_else(|| {
81        if PARTIAL && input.is_partial() {
82            ErrMode::Incomplete(Needed::new(1))
83        } else {
84            ErrMode::from_error_kind(input, ErrorKind::Token)
85        }
86    })
87}
88
89/// Recognizes a literal
90///
91/// The input data will be compared to the literal combinator's argument and will return the part of
92/// the input that matches the argument
93///
94/// It will return `Err(ErrMode::Backtrack(InputError::new(_, ErrorKind::Tag)))` if the input doesn't match the literal
95///
96/// <div class="warning">
97///
98/// **Note:** [`Parser`] is implemented for strings and byte strings as a convenience (complete
99/// only)
100///
101/// </div>
102///
103/// # Effective Signature
104///
105/// Assuming you are parsing a `&str` [Stream]:
106/// ```rust
107/// # use winnow::prelude::*;;
108/// # use winnow::error::ContextError;
109/// pub fn literal(literal: &str) -> impl Parser<&str, &str, ContextError>
110/// # {
111/// #     winnow::token::literal(literal)
112/// # }
113/// ```
114///
115/// # Example
116/// ```rust
117/// # use winnow::prelude::*;
118/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
119/// #
120/// fn parser(s: &str) -> IResult<&str, &str> {
121///   "Hello".parse_peek(s)
122/// }
123///
124/// assert_eq!(parser("Hello, World!"), Ok((", World!", "Hello")));
125/// assert_eq!(parser("Something"), Err(ErrMode::Backtrack(InputError::new("Something", ErrorKind::Tag))));
126/// assert_eq!(parser(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Tag))));
127/// ```
128///
129/// ```rust
130/// # use winnow::prelude::*;
131/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
132/// # use winnow::Partial;
133///
134/// fn parser(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
135///   "Hello".parse_peek(s)
136/// }
137///
138/// assert_eq!(parser(Partial::new("Hello, World!")), Ok((Partial::new(", World!"), "Hello")));
139/// assert_eq!(parser(Partial::new("Something")), Err(ErrMode::Backtrack(InputError::new(Partial::new("Something"), ErrorKind::Tag))));
140/// assert_eq!(parser(Partial::new("S")), Err(ErrMode::Backtrack(InputError::new(Partial::new("S"), ErrorKind::Tag))));
141/// assert_eq!(parser(Partial::new("H")), Err(ErrMode::Incomplete(Needed::new(4))));
142/// ```
143///
144/// ```rust
145/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
146/// # use winnow::prelude::*;
147/// use winnow::token::literal;
148/// use winnow::ascii::Caseless;
149///
150/// fn parser(s: &str) -> IResult<&str, &str> {
151///   literal(Caseless("hello")).parse_peek(s)
152/// }
153///
154/// assert_eq!(parser("Hello, World!"), Ok((", World!", "Hello")));
155/// assert_eq!(parser("hello, World!"), Ok((", World!", "hello")));
156/// assert_eq!(parser("HeLlO, World!"), Ok((", World!", "HeLlO")));
157/// assert_eq!(parser("Something"), Err(ErrMode::Backtrack(InputError::new("Something", ErrorKind::Tag))));
158/// assert_eq!(parser(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Tag))));
159/// ```
160#[inline(always)]
161#[doc(alias = "tag")]
162#[doc(alias = "bytes")]
163#[doc(alias = "just")]
164pub fn literal<Literal, Input, Error>(
165    literal: Literal,
166) -> impl Parser<Input, <Input as Stream>::Slice, Error>
167where
168    Input: StreamIsPartial + Stream + Compare<Literal>,
169    Literal: SliceLen + Clone + crate::lib::std::fmt::Debug,
170    Error: ParserError<Input>,
171{
172    trace(DisplayDebug(literal.clone()), move |i: &mut Input| {
173        let t = literal.clone();
174        if <Input as StreamIsPartial>::is_partial_supported() {
175            literal_::<_, _, _, true>(i, t)
176        } else {
177            literal_::<_, _, _, false>(i, t)
178        }
179    })
180}
181
182fn literal_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
183    i: &mut I,
184    t: T,
185) -> PResult<<I as Stream>::Slice, Error>
186where
187    I: StreamIsPartial,
188    I: Stream + Compare<T>,
189    T: SliceLen + crate::lib::std::fmt::Debug,
190{
191    let literal_len = t.slice_len();
192    match i.compare(t) {
193        CompareResult::Ok(len) => Ok(i.next_slice(len)),
194        CompareResult::Incomplete if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(
195            Needed::new(literal_len - i.eof_offset()),
196        )),
197        CompareResult::Incomplete | CompareResult::Error => {
198            let e: ErrorKind = ErrorKind::Tag;
199            Err(ErrMode::from_error_kind(i, e))
200        }
201    }
202}
203
204/// Recognize a token that matches a [set of tokens][ContainsToken]
205///
206/// <div class="warning">
207///
208/// **Note:** [`Parser`] is implemented as a convenience (complete
209/// only) for
210/// - `u8`
211/// - `char`
212///
213/// </div>
214///
215/// *Complete version*: Will return an error if there's not enough input data.
216///
217/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
218///
219/// # Effective Signature
220///
221/// Assuming you are parsing a `&str` [Stream]:
222/// ```rust
223/// # use winnow::prelude::*;;
224/// # use winnow::stream::ContainsToken;
225/// # use winnow::error::ContextError;
226/// pub fn one_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
227/// # {
228/// #     winnow::token::one_of(set)
229/// # }
230/// ```
231///
232/// # Example
233///
234/// ```rust
235/// # use winnow::prelude::*;
236/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError};
237/// # use winnow::token::one_of;
238/// assert_eq!(one_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek("b"), Ok(("", 'b')));
239/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek("bc"), Err(ErrMode::Backtrack(InputError::new("bc", ErrorKind::Verify))));
240/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
241///
242/// fn parser_fn(i: &str) -> IResult<&str, char> {
243///     one_of(|c| c == 'a' || c == 'b').parse_peek(i)
244/// }
245/// assert_eq!(parser_fn("abc"), Ok(("bc", 'a')));
246/// assert_eq!(parser_fn("cd"), Err(ErrMode::Backtrack(InputError::new("cd", ErrorKind::Verify))));
247/// assert_eq!(parser_fn(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
248/// ```
249///
250/// ```
251/// # use winnow::prelude::*;
252/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
253/// # use winnow::Partial;
254/// # use winnow::token::one_of;
255/// assert_eq!(one_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek(Partial::new("b")), Ok((Partial::new(""), 'b')));
256/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek(Partial::new("bc")), Err(ErrMode::Backtrack(InputError::new(Partial::new("bc"), ErrorKind::Verify))));
257/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
258///
259/// fn parser_fn(i: Partial<&str>) -> IResult<Partial<&str>, char> {
260///     one_of(|c| c == 'a' || c == 'b').parse_peek(i)
261/// }
262/// assert_eq!(parser_fn(Partial::new("abc")), Ok((Partial::new("bc"), 'a')));
263/// assert_eq!(parser_fn(Partial::new("cd")), Err(ErrMode::Backtrack(InputError::new(Partial::new("cd"), ErrorKind::Verify))));
264/// assert_eq!(parser_fn(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
265/// ```
266#[inline(always)]
267#[doc(alias = "char")]
268#[doc(alias = "token")]
269#[doc(alias = "satisfy")]
270pub fn one_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
271where
272    Input: StreamIsPartial + Stream,
273    <Input as Stream>::Token: Clone,
274    Set: ContainsToken<<Input as Stream>::Token>,
275    Error: ParserError<Input>,
276{
277    trace(
278        "one_of",
279        any.verify(move |t: &<Input as Stream>::Token| set.contains_token(t.clone())),
280    )
281}
282
283/// Recognize a token that does not match a [set of tokens][ContainsToken]
284///
285/// *Complete version*: Will return an error if there's not enough input data.
286///
287/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
288///
289/// # Effective Signature
290///
291/// Assuming you are parsing a `&str` [Stream]:
292/// ```rust
293/// # use winnow::prelude::*;;
294/// # use winnow::stream::ContainsToken;
295/// # use winnow::error::ContextError;
296/// pub fn none_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
297/// # {
298/// #     winnow::token::none_of(set)
299/// # }
300/// ```
301///
302/// # Example
303///
304/// ```rust
305/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError};
306/// # use winnow::prelude::*;
307/// # use winnow::token::none_of;
308/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek("z"), Ok(("", 'z')));
309/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b']).parse_peek("a"), Err(ErrMode::Backtrack(InputError::new("a", ErrorKind::Verify))));
310/// assert_eq!(none_of::<_, _, InputError<_>>('a').parse_peek(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
311/// ```
312///
313/// ```
314/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
315/// # use winnow::prelude::*;
316/// # use winnow::Partial;
317/// # use winnow::token::none_of;
318/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek(Partial::new("z")), Ok((Partial::new(""), 'z')));
319/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b']).parse_peek(Partial::new("a")), Err(ErrMode::Backtrack(InputError::new(Partial::new("a"), ErrorKind::Verify))));
320/// assert_eq!(none_of::<_, _, InputError<_>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
321/// ```
322#[inline(always)]
323pub fn none_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
324where
325    Input: StreamIsPartial + Stream,
326    <Input as Stream>::Token: Clone,
327    Set: ContainsToken<<Input as Stream>::Token>,
328    Error: ParserError<Input>,
329{
330    trace(
331        "none_of",
332        any.verify(move |t: &<Input as Stream>::Token| !set.contains_token(t.clone())),
333    )
334}
335
336/// Recognize the longest (m <= len <= n) input slice that matches a [set of tokens][ContainsToken]
337///
338/// It will return an `ErrMode::Backtrack(InputError::new(_, ErrorKind::Slice))` if the set of tokens wasn't met or is out
339/// of range (m <= len <= n).
340///
341/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if a member of the set of tokens reaches the end of the input or is too short.
342///
343/// To take a series of tokens, use [`repeat`][crate::combinator::repeat] to [`Accumulate`][crate::stream::Accumulate] into a `()` and then [`Parser::take`].
344///
345/// # Effective Signature
346///
347/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
348/// ```rust
349/// # use std::ops::RangeFrom;
350/// # use winnow::prelude::*;
351/// # use winnow::stream::ContainsToken;
352/// # use winnow::error::ContextError;
353/// pub fn take_while<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
354/// # {
355/// #     winnow::token::take_while(occurrences, set)
356/// # }
357/// ```
358///
359/// # Example
360///
361/// Zero or more tokens:
362/// ```rust
363/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
364/// # use winnow::prelude::*;
365/// use winnow::token::take_while;
366/// use winnow::stream::AsChar;
367///
368/// fn alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {
369///   take_while(0.., AsChar::is_alpha).parse_peek(s)
370/// }
371///
372/// assert_eq!(alpha(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
373/// assert_eq!(alpha(b"12345"), Ok((&b"12345"[..], &b""[..])));
374/// assert_eq!(alpha(b"latin"), Ok((&b""[..], &b"latin"[..])));
375/// assert_eq!(alpha(b""), Ok((&b""[..], &b""[..])));
376/// ```
377///
378/// ```rust
379/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
380/// # use winnow::prelude::*;
381/// # use winnow::Partial;
382/// use winnow::token::take_while;
383/// use winnow::stream::AsChar;
384///
385/// fn alpha(s: Partial<&[u8]>) -> IResult<Partial<&[u8]>, &[u8]> {
386///   take_while(0.., AsChar::is_alpha).parse_peek(s)
387/// }
388///
389/// assert_eq!(alpha(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
390/// assert_eq!(alpha(Partial::new(b"12345")), Ok((Partial::new(&b"12345"[..]), &b""[..])));
391/// assert_eq!(alpha(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
392/// assert_eq!(alpha(Partial::new(b"")), Err(ErrMode::Incomplete(Needed::new(1))));
393/// ```
394///
395/// One or more tokens:
396/// ```rust
397/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
398/// # use winnow::prelude::*;
399/// use winnow::token::take_while;
400/// use winnow::stream::AsChar;
401///
402/// fn alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {
403///   take_while(1.., AsChar::is_alpha).parse_peek(s)
404/// }
405///
406/// assert_eq!(alpha(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
407/// assert_eq!(alpha(b"latin"), Ok((&b""[..], &b"latin"[..])));
408/// assert_eq!(alpha(b"12345"), Err(ErrMode::Backtrack(InputError::new(&b"12345"[..], ErrorKind::Slice))));
409///
410/// fn hex(s: &str) -> IResult<&str, &str> {
411///   take_while(1.., ('0'..='9', 'A'..='F')).parse_peek(s)
412/// }
413///
414/// assert_eq!(hex("123 and voila"), Ok((" and voila", "123")));
415/// assert_eq!(hex("DEADBEEF and others"), Ok((" and others", "DEADBEEF")));
416/// assert_eq!(hex("BADBABEsomething"), Ok(("something", "BADBABE")));
417/// assert_eq!(hex("D15EA5E"), Ok(("", "D15EA5E")));
418/// assert_eq!(hex(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
419/// ```
420///
421/// ```rust
422/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
423/// # use winnow::prelude::*;
424/// # use winnow::Partial;
425/// use winnow::token::take_while;
426/// use winnow::stream::AsChar;
427///
428/// fn alpha(s: Partial<&[u8]>) -> IResult<Partial<&[u8]>, &[u8]> {
429///   take_while(1.., AsChar::is_alpha).parse_peek(s)
430/// }
431///
432/// assert_eq!(alpha(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
433/// assert_eq!(alpha(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
434/// assert_eq!(alpha(Partial::new(b"12345")), Err(ErrMode::Backtrack(InputError::new(Partial::new(&b"12345"[..]), ErrorKind::Slice))));
435///
436/// fn hex(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
437///   take_while(1.., ('0'..='9', 'A'..='F')).parse_peek(s)
438/// }
439///
440/// assert_eq!(hex(Partial::new("123 and voila")), Ok((Partial::new(" and voila"), "123")));
441/// assert_eq!(hex(Partial::new("DEADBEEF and others")), Ok((Partial::new(" and others"), "DEADBEEF")));
442/// assert_eq!(hex(Partial::new("BADBABEsomething")), Ok((Partial::new("something"), "BADBABE")));
443/// assert_eq!(hex(Partial::new("D15EA5E")), Err(ErrMode::Incomplete(Needed::new(1))));
444/// assert_eq!(hex(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
445/// ```
446///
447/// Arbitrary amount of tokens:
448/// ```rust
449/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
450/// # use winnow::prelude::*;
451/// use winnow::token::take_while;
452/// use winnow::stream::AsChar;
453///
454/// fn short_alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {
455///   take_while(3..=6, AsChar::is_alpha).parse_peek(s)
456/// }
457///
458/// assert_eq!(short_alpha(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
459/// assert_eq!(short_alpha(b"lengthy"), Ok((&b"y"[..], &b"length"[..])));
460/// assert_eq!(short_alpha(b"latin"), Ok((&b""[..], &b"latin"[..])));
461/// assert_eq!(short_alpha(b"ed"), Err(ErrMode::Backtrack(InputError::new(&b"ed"[..], ErrorKind::Slice))));
462/// assert_eq!(short_alpha(b"12345"), Err(ErrMode::Backtrack(InputError::new(&b"12345"[..], ErrorKind::Slice))));
463/// ```
464///
465/// ```rust
466/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
467/// # use winnow::prelude::*;
468/// # use winnow::Partial;
469/// use winnow::token::take_while;
470/// use winnow::stream::AsChar;
471///
472/// fn short_alpha(s: Partial<&[u8]>) -> IResult<Partial<&[u8]>, &[u8]> {
473///   take_while(3..=6, AsChar::is_alpha).parse_peek(s)
474/// }
475///
476/// assert_eq!(short_alpha(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
477/// assert_eq!(short_alpha(Partial::new(b"lengthy")), Ok((Partial::new(&b"y"[..]), &b"length"[..])));
478/// assert_eq!(short_alpha(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
479/// assert_eq!(short_alpha(Partial::new(b"ed")), Err(ErrMode::Incomplete(Needed::new(1))));
480/// assert_eq!(short_alpha(Partial::new(b"12345")), Err(ErrMode::Backtrack(InputError::new(Partial::new(&b"12345"[..]), ErrorKind::Slice))));
481/// ```
482#[inline(always)]
483#[doc(alias = "is_a")]
484#[doc(alias = "take_while0")]
485#[doc(alias = "take_while1")]
486pub fn take_while<Set, Input, Error>(
487    occurrences: impl Into<Range>,
488    set: Set,
489) -> impl Parser<Input, <Input as Stream>::Slice, Error>
490where
491    Input: StreamIsPartial + Stream,
492    Set: ContainsToken<<Input as Stream>::Token>,
493    Error: ParserError<Input>,
494{
495    let Range {
496        start_inclusive,
497        end_inclusive,
498    } = occurrences.into();
499    trace("take_while", move |i: &mut Input| {
500        match (start_inclusive, end_inclusive) {
501            (0, None) => {
502                if <Input as StreamIsPartial>::is_partial_supported() {
503                    take_till0::<_, _, _, true>(i, |c| !set.contains_token(c))
504                } else {
505                    take_till0::<_, _, _, false>(i, |c| !set.contains_token(c))
506                }
507            }
508            (1, None) => {
509                if <Input as StreamIsPartial>::is_partial_supported() {
510                    take_till1::<_, _, _, true>(i, |c| !set.contains_token(c))
511                } else {
512                    take_till1::<_, _, _, false>(i, |c| !set.contains_token(c))
513                }
514            }
515            (start, end) => {
516                let end = end.unwrap_or(usize::MAX);
517                if <Input as StreamIsPartial>::is_partial_supported() {
518                    take_till_m_n::<_, _, _, true>(i, start, end, |c| !set.contains_token(c))
519                } else {
520                    take_till_m_n::<_, _, _, false>(i, start, end, |c| !set.contains_token(c))
521                }
522            }
523        }
524    })
525}
526
527fn take_till0<P, I: StreamIsPartial + Stream, E: ParserError<I>, const PARTIAL: bool>(
528    input: &mut I,
529    predicate: P,
530) -> PResult<<I as Stream>::Slice, E>
531where
532    P: Fn(I::Token) -> bool,
533{
534    let offset = match input.offset_for(predicate) {
535        Some(offset) => offset,
536        None if PARTIAL && input.is_partial() => {
537            return Err(ErrMode::Incomplete(Needed::new(1)));
538        }
539        None => input.eof_offset(),
540    };
541    Ok(input.next_slice(offset))
542}
543
544fn take_till1<P, I: StreamIsPartial + Stream, E: ParserError<I>, const PARTIAL: bool>(
545    input: &mut I,
546    predicate: P,
547) -> PResult<<I as Stream>::Slice, E>
548where
549    P: Fn(I::Token) -> bool,
550{
551    let e: ErrorKind = ErrorKind::Slice;
552    let offset = match input.offset_for(predicate) {
553        Some(offset) => offset,
554        None if PARTIAL && input.is_partial() => {
555            return Err(ErrMode::Incomplete(Needed::new(1)));
556        }
557        None => input.eof_offset(),
558    };
559    if offset == 0 {
560        Err(ErrMode::from_error_kind(input, e))
561    } else {
562        Ok(input.next_slice(offset))
563    }
564}
565
566fn take_till_m_n<P, I, Error: ParserError<I>, const PARTIAL: bool>(
567    input: &mut I,
568    m: usize,
569    n: usize,
570    predicate: P,
571) -> PResult<<I as Stream>::Slice, Error>
572where
573    I: StreamIsPartial,
574    I: Stream,
575    P: Fn(I::Token) -> bool,
576{
577    if n < m {
578        return Err(ErrMode::assert(
579            input,
580            "`occurrences` should be ascending, rather than descending",
581        ));
582    }
583
584    let mut final_count = 0;
585    for (processed, (offset, token)) in input.iter_offsets().enumerate() {
586        if predicate(token) {
587            if processed < m {
588                return Err(ErrMode::from_error_kind(input, ErrorKind::Slice));
589            } else {
590                return Ok(input.next_slice(offset));
591            }
592        } else {
593            if processed == n {
594                return Ok(input.next_slice(offset));
595            }
596            final_count = processed + 1;
597        }
598    }
599    if PARTIAL && input.is_partial() {
600        if final_count == n {
601            Ok(input.finish())
602        } else {
603            let needed = if m > input.eof_offset() {
604                m - input.eof_offset()
605            } else {
606                1
607            };
608            Err(ErrMode::Incomplete(Needed::new(needed)))
609        }
610    } else {
611        if m <= final_count {
612            Ok(input.finish())
613        } else {
614            Err(ErrMode::from_error_kind(input, ErrorKind::Slice))
615        }
616    }
617}
618
619/// Recognize the longest input slice (if any) till a member of a [set of tokens][ContainsToken] is found.
620///
621/// It doesn't consume the terminating token from the set.
622///
623/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if the match reaches the
624/// end of input or if there was not match.
625///
626/// See also
627/// - [`take_until`] for recognizing up-to a [`literal`] (w/ optional simd optimizations)
628/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
629///
630/// # Effective Signature
631///
632/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
633/// ```rust
634/// # use std::ops::RangeFrom;
635/// # use winnow::prelude::*;
636/// # use winnow::stream::ContainsToken;
637/// # use winnow::error::ContextError;
638/// pub fn take_till<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
639/// # {
640/// #     winnow::token::take_till(occurrences, set)
641/// # }
642/// ```
643///
644/// # Example
645///
646/// ```rust
647/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
648/// # use winnow::prelude::*;
649/// use winnow::token::take_till;
650///
651/// fn till_colon(s: &str) -> IResult<&str, &str> {
652///   take_till(0.., |c| c == ':').parse_peek(s)
653/// }
654///
655/// assert_eq!(till_colon("latin:123"), Ok((":123", "latin")));
656/// assert_eq!(till_colon(":empty matched"), Ok((":empty matched", ""))); //allowed
657/// assert_eq!(till_colon("12345"), Ok(("", "12345")));
658/// assert_eq!(till_colon(""), Ok(("", "")));
659/// ```
660///
661/// ```rust
662/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
663/// # use winnow::prelude::*;
664/// # use winnow::Partial;
665/// use winnow::token::take_till;
666///
667/// fn till_colon(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
668///   take_till(0.., |c| c == ':').parse_peek(s)
669/// }
670///
671/// assert_eq!(till_colon(Partial::new("latin:123")), Ok((Partial::new(":123"), "latin")));
672/// assert_eq!(till_colon(Partial::new(":empty matched")), Ok((Partial::new(":empty matched"), ""))); //allowed
673/// assert_eq!(till_colon(Partial::new("12345")), Err(ErrMode::Incomplete(Needed::new(1))));
674/// assert_eq!(till_colon(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
675/// ```
676#[inline(always)]
677#[doc(alias = "is_not")]
678pub fn take_till<Set, Input, Error>(
679    occurrences: impl Into<Range>,
680    set: Set,
681) -> impl Parser<Input, <Input as Stream>::Slice, Error>
682where
683    Input: StreamIsPartial + Stream,
684    Set: ContainsToken<<Input as Stream>::Token>,
685    Error: ParserError<Input>,
686{
687    let Range {
688        start_inclusive,
689        end_inclusive,
690    } = occurrences.into();
691    trace("take_till", move |i: &mut Input| {
692        match (start_inclusive, end_inclusive) {
693            (0, None) => {
694                if <Input as StreamIsPartial>::is_partial_supported() {
695                    take_till0::<_, _, _, true>(i, |c| set.contains_token(c))
696                } else {
697                    take_till0::<_, _, _, false>(i, |c| set.contains_token(c))
698                }
699            }
700            (1, None) => {
701                if <Input as StreamIsPartial>::is_partial_supported() {
702                    take_till1::<_, _, _, true>(i, |c| set.contains_token(c))
703                } else {
704                    take_till1::<_, _, _, false>(i, |c| set.contains_token(c))
705                }
706            }
707            (start, end) => {
708                let end = end.unwrap_or(usize::MAX);
709                if <Input as StreamIsPartial>::is_partial_supported() {
710                    take_till_m_n::<_, _, _, true>(i, start, end, |c| set.contains_token(c))
711                } else {
712                    take_till_m_n::<_, _, _, false>(i, start, end, |c| set.contains_token(c))
713                }
714            }
715        }
716    })
717}
718
719/// Recognize an input slice containing the first N input elements (I[..N]).
720///
721/// *Complete version*: It will return `Err(ErrMode::Backtrack(InputError::new(_, ErrorKind::Slice)))` if the input is shorter than the argument.
722///
723/// *[Partial version][crate::_topic::partial]*: if the input has less than N elements, `take` will
724/// return a `ErrMode::Incomplete(Needed::new(M))` where M is the number of
725/// additional bytes the parser would need to succeed.
726/// It is well defined for `&[u8]` as the number of elements is the byte size,
727/// but for types like `&str`, we cannot know how many bytes correspond for
728/// the next few chars, so the result will be `ErrMode::Incomplete(Needed::Unknown)`
729///
730/// # Effective Signature
731///
732/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` ranges:
733/// ```rust
734/// # use std::ops::RangeFrom;
735/// # use winnow::prelude::*;
736/// # use winnow::stream::ContainsToken;
737/// # use winnow::error::ContextError;
738/// pub fn take<'i>(token_count: usize) -> impl Parser<&'i str, &'i str, ContextError>
739/// # {
740/// #     winnow::token::take(token_count)
741/// # }
742/// ```
743///
744/// # Example
745///
746/// ```rust
747/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
748/// # use winnow::prelude::*;
749/// use winnow::token::take;
750///
751/// fn take6(s: &str) -> IResult<&str, &str> {
752///   take(6usize).parse_peek(s)
753/// }
754///
755/// assert_eq!(take6("1234567"), Ok(("7", "123456")));
756/// assert_eq!(take6("things"), Ok(("", "things")));
757/// assert_eq!(take6("short"), Err(ErrMode::Backtrack(InputError::new("short", ErrorKind::Slice))));
758/// assert_eq!(take6(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
759/// ```
760///
761/// The units that are taken will depend on the input type. For example, for a
762/// `&str` it will take a number of `char`'s, whereas for a `&[u8]` it will
763/// take that many `u8`'s:
764///
765/// ```rust
766/// # use winnow::prelude::*;
767/// use winnow::error::InputError;
768/// use winnow::token::take;
769///
770/// assert_eq!(take::<_, _, InputError<_>>(1usize).parse_peek("💙"), Ok(("", "💙")));
771/// assert_eq!(take::<_, _, InputError<_>>(1usize).parse_peek("💙".as_bytes()), Ok((b"\x9F\x92\x99".as_ref(), b"\xF0".as_ref())));
772/// ```
773///
774/// ```rust
775/// # use winnow::prelude::*;
776/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
777/// # use winnow::Partial;
778/// use winnow::token::take;
779///
780/// fn take6(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
781///   take(6usize).parse_peek(s)
782/// }
783///
784/// assert_eq!(take6(Partial::new("1234567")), Ok((Partial::new("7"), "123456")));
785/// assert_eq!(take6(Partial::new("things")), Ok((Partial::new(""), "things")));
786/// // `Unknown` as we don't know the number of bytes that `count` corresponds to
787/// assert_eq!(take6(Partial::new("short")), Err(ErrMode::Incomplete(Needed::Unknown)));
788/// ```
789#[inline(always)]
790pub fn take<UsizeLike, Input, Error>(
791    token_count: UsizeLike,
792) -> impl Parser<Input, <Input as Stream>::Slice, Error>
793where
794    Input: StreamIsPartial + Stream,
795    UsizeLike: ToUsize,
796    Error: ParserError<Input>,
797{
798    let c = token_count.to_usize();
799    trace("take", move |i: &mut Input| {
800        if <Input as StreamIsPartial>::is_partial_supported() {
801            take_::<_, _, true>(i, c)
802        } else {
803            take_::<_, _, false>(i, c)
804        }
805    })
806}
807
808fn take_<I, Error: ParserError<I>, const PARTIAL: bool>(
809    i: &mut I,
810    c: usize,
811) -> PResult<<I as Stream>::Slice, Error>
812where
813    I: StreamIsPartial,
814    I: Stream,
815{
816    match i.offset_at(c) {
817        Ok(offset) => Ok(i.next_slice(offset)),
818        Err(e) if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(e)),
819        Err(_needed) => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
820    }
821}
822
823/// Recognize the input slice up to the first occurrence of a [literal].
824///
825/// Feature `simd` will enable the use of [`memchr`](https://docs.rs/memchr/latest/memchr/).
826///
827/// It doesn't consume the literal.
828///
829/// *Complete version*: It will return `Err(ErrMode::Backtrack(InputError::new(_, ErrorKind::Slice)))`
830/// if the literal wasn't met.
831///
832/// *[Partial version][crate::_topic::partial]*: will return a `ErrMode::Incomplete(Needed::new(N))` if the input doesn't
833/// contain the literal or if the input is smaller than the literal.
834///
835/// See also
836/// - [`take_till`] for recognizing up-to a [set of tokens][ContainsToken]
837/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
838///
839/// # Effective Signature
840///
841/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
842/// ```rust
843/// # use std::ops::RangeFrom;
844/// # use winnow::prelude::*;;
845/// # use winnow::error::ContextError;
846/// pub fn take_until(occurrences: RangeFrom<usize>, literal: &str) -> impl Parser<&str, &str, ContextError>
847/// # {
848/// #     winnow::token::take_until(occurrences, literal)
849/// # }
850/// ```
851///
852/// # Example
853///
854/// ```rust
855/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
856/// # use winnow::prelude::*;
857/// use winnow::token::take_until;
858///
859/// fn until_eof(s: &str) -> IResult<&str, &str> {
860///   take_until(0.., "eof").parse_peek(s)
861/// }
862///
863/// assert_eq!(until_eof("hello, worldeof"), Ok(("eof", "hello, world")));
864/// assert_eq!(until_eof("hello, world"), Err(ErrMode::Backtrack(InputError::new("hello, world", ErrorKind::Slice))));
865/// assert_eq!(until_eof(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
866/// assert_eq!(until_eof("1eof2eof"), Ok(("eof2eof", "1")));
867/// ```
868///
869/// ```rust
870/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
871/// # use winnow::prelude::*;
872/// # use winnow::Partial;
873/// use winnow::token::take_until;
874///
875/// fn until_eof(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
876///   take_until(0.., "eof").parse_peek(s)
877/// }
878///
879/// assert_eq!(until_eof(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
880/// assert_eq!(until_eof(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
881/// assert_eq!(until_eof(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
882/// assert_eq!(until_eof(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
883/// ```
884///
885/// ```rust
886/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
887/// # use winnow::prelude::*;
888/// use winnow::token::take_until;
889///
890/// fn until_eof(s: &str) -> IResult<&str, &str> {
891///   take_until(1.., "eof").parse_peek(s)
892/// }
893///
894/// assert_eq!(until_eof("hello, worldeof"), Ok(("eof", "hello, world")));
895/// assert_eq!(until_eof("hello, world"), Err(ErrMode::Backtrack(InputError::new("hello, world", ErrorKind::Slice))));
896/// assert_eq!(until_eof(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
897/// assert_eq!(until_eof("1eof2eof"), Ok(("eof2eof", "1")));
898/// assert_eq!(until_eof("eof"), Err(ErrMode::Backtrack(InputError::new("eof", ErrorKind::Slice))));
899/// ```
900///
901/// ```rust
902/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
903/// # use winnow::prelude::*;
904/// # use winnow::Partial;
905/// use winnow::token::take_until;
906///
907/// fn until_eof(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
908///   take_until(1.., "eof").parse_peek(s)
909/// }
910///
911/// assert_eq!(until_eof(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
912/// assert_eq!(until_eof(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
913/// assert_eq!(until_eof(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
914/// assert_eq!(until_eof(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
915/// assert_eq!(until_eof(Partial::new("eof")), Err(ErrMode::Backtrack(InputError::new(Partial::new("eof"), ErrorKind::Slice))));
916/// ```
917#[inline(always)]
918pub fn take_until<Literal, Input, Error>(
919    occurrences: impl Into<Range>,
920    literal: Literal,
921) -> impl Parser<Input, <Input as Stream>::Slice, Error>
922where
923    Input: StreamIsPartial + Stream + FindSlice<Literal>,
924    Literal: Clone,
925    Error: ParserError<Input>,
926{
927    let Range {
928        start_inclusive,
929        end_inclusive,
930    } = occurrences.into();
931    trace("take_until", move |i: &mut Input| {
932        match (start_inclusive, end_inclusive) {
933            (0, None) => {
934                if <Input as StreamIsPartial>::is_partial_supported() {
935                    take_until0_::<_, _, _, true>(i, literal.clone())
936                } else {
937                    take_until0_::<_, _, _, false>(i, literal.clone())
938                }
939            }
940            (1, None) => {
941                if <Input as StreamIsPartial>::is_partial_supported() {
942                    take_until1_::<_, _, _, true>(i, literal.clone())
943                } else {
944                    take_until1_::<_, _, _, false>(i, literal.clone())
945                }
946            }
947            (start, end) => {
948                let end = end.unwrap_or(usize::MAX);
949                if <Input as StreamIsPartial>::is_partial_supported() {
950                    take_until_m_n_::<_, _, _, true>(i, start, end, literal.clone())
951                } else {
952                    take_until_m_n_::<_, _, _, false>(i, start, end, literal.clone())
953                }
954            }
955        }
956    })
957}
958
959fn take_until0_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
960    i: &mut I,
961    t: T,
962) -> PResult<<I as Stream>::Slice, Error>
963where
964    I: StreamIsPartial,
965    I: Stream + FindSlice<T>,
966{
967    match i.find_slice(t) {
968        Some(range) => Ok(i.next_slice(range.start)),
969        None if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(Needed::Unknown)),
970        None => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
971    }
972}
973
974fn take_until1_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
975    i: &mut I,
976    t: T,
977) -> PResult<<I as Stream>::Slice, Error>
978where
979    I: StreamIsPartial,
980    I: Stream + FindSlice<T>,
981{
982    match i.find_slice(t) {
983        None if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(Needed::Unknown)),
984        None => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
985        Some(range) => {
986            if range.start == 0 {
987                Err(ErrMode::from_error_kind(i, ErrorKind::Slice))
988            } else {
989                Ok(i.next_slice(range.start))
990            }
991        }
992    }
993}
994
995fn take_until_m_n_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
996    i: &mut I,
997    start: usize,
998    end: usize,
999    t: T,
1000) -> PResult<<I as Stream>::Slice, Error>
1001where
1002    I: StreamIsPartial,
1003    I: Stream + FindSlice<T>,
1004{
1005    if end < start {
1006        return Err(ErrMode::assert(
1007            i,
1008            "`occurrences` should be ascending, rather than descending",
1009        ));
1010    }
1011
1012    match i.find_slice(t) {
1013        Some(range) => {
1014            let start_offset = i.offset_at(start);
1015            let end_offset = i.offset_at(end).unwrap_or_else(|_err| i.eof_offset());
1016            if start_offset.map(|s| range.start < s).unwrap_or(true) {
1017                if PARTIAL && i.is_partial() {
1018                    return Err(ErrMode::Incomplete(Needed::Unknown));
1019                } else {
1020                    return Err(ErrMode::from_error_kind(i, ErrorKind::Slice));
1021                }
1022            }
1023            if end_offset < range.start {
1024                return Err(ErrMode::from_error_kind(i, ErrorKind::Slice));
1025            }
1026            Ok(i.next_slice(range.start))
1027        }
1028        None if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(Needed::Unknown)),
1029        None => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
1030    }
1031}
1032
1033/// Return the remaining input.
1034///
1035/// # Effective Signature
1036///
1037/// Assuming you are parsing a `&str` [Stream]:
1038/// ```rust
1039/// # use winnow::prelude::*;;
1040/// pub fn rest<'i>(input: &mut &'i str) -> PResult<&'i str>
1041/// # {
1042/// #     winnow::token::rest.parse_next(input)
1043/// # }
1044/// ```
1045///
1046/// # Example
1047///
1048/// ```rust
1049/// # use winnow::prelude::*;
1050/// # use winnow::error::ErrorKind;
1051/// # use winnow::error::InputError;
1052/// use winnow::token::rest;
1053/// assert_eq!(rest::<_,InputError<_>>.parse_peek("abc"), Ok(("", "abc")));
1054/// assert_eq!(rest::<_,InputError<_>>.parse_peek(""), Ok(("", "")));
1055/// ```
1056#[inline]
1057pub fn rest<Input, Error>(input: &mut Input) -> PResult<<Input as Stream>::Slice, Error>
1058where
1059    Input: Stream,
1060    Error: ParserError<Input>,
1061{
1062    trace("rest", move |input: &mut Input| Ok(input.finish())).parse_next(input)
1063}
1064
1065/// Return the length of the remaining input.
1066///
1067/// <div class="warning">
1068///
1069/// Note: this does not advance the [`Stream`]
1070///
1071/// </div>
1072///
1073/// # Effective Signature
1074///
1075/// Assuming you are parsing a `&str` [Stream]:
1076/// ```rust
1077/// # use winnow::prelude::*;;
1078/// pub fn rest_len(input: &mut &str) -> PResult<usize>
1079/// # {
1080/// #     winnow::token::rest_len.parse_next(input)
1081/// # }
1082/// ```
1083///
1084/// # Example
1085///
1086/// ```rust
1087/// # use winnow::prelude::*;
1088/// # use winnow::error::ErrorKind;
1089/// # use winnow::error::InputError;
1090/// use winnow::token::rest_len;
1091/// assert_eq!(rest_len::<_,InputError<_>>.parse_peek("abc"), Ok(("abc", 3)));
1092/// assert_eq!(rest_len::<_,InputError<_>>.parse_peek(""), Ok(("", 0)));
1093/// ```
1094#[inline]
1095pub fn rest_len<Input, Error>(input: &mut Input) -> PResult<usize, Error>
1096where
1097    Input: Stream,
1098    Error: ParserError<Input>,
1099{
1100    trace("rest_len", move |input: &mut Input| {
1101        let len = input.eof_offset();
1102        Ok(len)
1103    })
1104    .parse_next(input)
1105}
winnow/token/mod.rs

winnow/token/
mod.rs
