winnow/token/mod.rs
1//! Parsers extracting tokens from the stream
2
3#[cfg(test)]
4mod tests;
5
6use crate::combinator::trace;
7use crate::combinator::DisplayDebug;
8use crate::error::ErrMode;
9use crate::error::ErrorKind;
10use crate::error::Needed;
11use crate::error::ParserError;
12use crate::lib::std::result::Result::Ok;
13use crate::stream::Range;
14use crate::stream::{Compare, CompareResult, ContainsToken, FindSlice, SliceLen, Stream};
15use crate::stream::{StreamIsPartial, ToUsize};
16use crate::PResult;
17use crate::Parser;
18
19/// Matches one token
20///
21/// *Complete version*: Will return an error if there's not enough input data.
22///
23/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
24///
25/// # Effective Signature
26///
27/// Assuming you are parsing a `&str` [Stream]:
28/// ```rust
29/// # use winnow::prelude::*;;
30/// pub fn any(input: &mut &str) -> PResult<char>
31/// # {
32/// # winnow::token::any.parse_next(input)
33/// # }
34/// ```
35///
36/// # Example
37///
38/// ```rust
39/// # use winnow::{token::any, error::ErrMode, error::{InputError, ErrorKind}};
40/// # use winnow::prelude::*;
41/// fn parser(input: &str) -> IResult<&str, char> {
42/// any.parse_peek(input)
43/// }
44///
45/// assert_eq!(parser("abc"), Ok(("bc",'a')));
46/// assert_eq!(parser(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
47/// ```
48///
49/// ```rust
50/// # use winnow::{token::any, error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
51/// # use winnow::prelude::*;
52/// # use winnow::Partial;
53/// assert_eq!(any::<_, InputError<_>>.parse_peek(Partial::new("abc")), Ok((Partial::new("bc"),'a')));
54/// assert_eq!(any::<_, InputError<_>>.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
55/// ```
56#[inline(always)]
57#[doc(alias = "token")]
58pub fn any<Input, Error>(input: &mut Input) -> PResult<<Input as Stream>::Token, Error>
59where
60 Input: StreamIsPartial + Stream,
61 Error: ParserError<Input>,
62{
63 trace("any", move |input: &mut Input| {
64 if <Input as StreamIsPartial>::is_partial_supported() {
65 any_::<_, _, true>(input)
66 } else {
67 any_::<_, _, false>(input)
68 }
69 })
70 .parse_next(input)
71}
72
73fn any_<I, E: ParserError<I>, const PARTIAL: bool>(
74 input: &mut I,
75) -> PResult<<I as Stream>::Token, E>
76where
77 I: StreamIsPartial,
78 I: Stream,
79{
80 input.next_token().ok_or_else(|| {
81 if PARTIAL && input.is_partial() {
82 ErrMode::Incomplete(Needed::new(1))
83 } else {
84 ErrMode::from_error_kind(input, ErrorKind::Token)
85 }
86 })
87}
88
89/// Recognizes a literal
90///
91/// The input data will be compared to the literal combinator's argument and will return the part of
92/// the input that matches the argument
93///
94/// It will return `Err(ErrMode::Backtrack(InputError::new(_, ErrorKind::Tag)))` if the input doesn't match the literal
95///
96/// <div class="warning">
97///
98/// **Note:** [`Parser`] is implemented for strings and byte strings as a convenience (complete
99/// only)
100///
101/// </div>
102///
103/// # Effective Signature
104///
105/// Assuming you are parsing a `&str` [Stream]:
106/// ```rust
107/// # use winnow::prelude::*;;
108/// # use winnow::error::ContextError;
109/// pub fn literal(literal: &str) -> impl Parser<&str, &str, ContextError>
110/// # {
111/// # winnow::token::literal(literal)
112/// # }
113/// ```
114///
115/// # Example
116/// ```rust
117/// # use winnow::prelude::*;
118/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
119/// #
120/// fn parser(s: &str) -> IResult<&str, &str> {
121/// "Hello".parse_peek(s)
122/// }
123///
124/// assert_eq!(parser("Hello, World!"), Ok((", World!", "Hello")));
125/// assert_eq!(parser("Something"), Err(ErrMode::Backtrack(InputError::new("Something", ErrorKind::Tag))));
126/// assert_eq!(parser(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Tag))));
127/// ```
128///
129/// ```rust
130/// # use winnow::prelude::*;
131/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
132/// # use winnow::Partial;
133///
134/// fn parser(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
135/// "Hello".parse_peek(s)
136/// }
137///
138/// assert_eq!(parser(Partial::new("Hello, World!")), Ok((Partial::new(", World!"), "Hello")));
139/// assert_eq!(parser(Partial::new("Something")), Err(ErrMode::Backtrack(InputError::new(Partial::new("Something"), ErrorKind::Tag))));
140/// assert_eq!(parser(Partial::new("S")), Err(ErrMode::Backtrack(InputError::new(Partial::new("S"), ErrorKind::Tag))));
141/// assert_eq!(parser(Partial::new("H")), Err(ErrMode::Incomplete(Needed::new(4))));
142/// ```
143///
144/// ```rust
145/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
146/// # use winnow::prelude::*;
147/// use winnow::token::literal;
148/// use winnow::ascii::Caseless;
149///
150/// fn parser(s: &str) -> IResult<&str, &str> {
151/// literal(Caseless("hello")).parse_peek(s)
152/// }
153///
154/// assert_eq!(parser("Hello, World!"), Ok((", World!", "Hello")));
155/// assert_eq!(parser("hello, World!"), Ok((", World!", "hello")));
156/// assert_eq!(parser("HeLlO, World!"), Ok((", World!", "HeLlO")));
157/// assert_eq!(parser("Something"), Err(ErrMode::Backtrack(InputError::new("Something", ErrorKind::Tag))));
158/// assert_eq!(parser(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Tag))));
159/// ```
160#[inline(always)]
161#[doc(alias = "tag")]
162#[doc(alias = "bytes")]
163#[doc(alias = "just")]
164pub fn literal<Literal, Input, Error>(
165 literal: Literal,
166) -> impl Parser<Input, <Input as Stream>::Slice, Error>
167where
168 Input: StreamIsPartial + Stream + Compare<Literal>,
169 Literal: SliceLen + Clone + crate::lib::std::fmt::Debug,
170 Error: ParserError<Input>,
171{
172 trace(DisplayDebug(literal.clone()), move |i: &mut Input| {
173 let t = literal.clone();
174 if <Input as StreamIsPartial>::is_partial_supported() {
175 literal_::<_, _, _, true>(i, t)
176 } else {
177 literal_::<_, _, _, false>(i, t)
178 }
179 })
180}
181
182fn literal_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
183 i: &mut I,
184 t: T,
185) -> PResult<<I as Stream>::Slice, Error>
186where
187 I: StreamIsPartial,
188 I: Stream + Compare<T>,
189 T: SliceLen + crate::lib::std::fmt::Debug,
190{
191 let literal_len = t.slice_len();
192 match i.compare(t) {
193 CompareResult::Ok(len) => Ok(i.next_slice(len)),
194 CompareResult::Incomplete if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(
195 Needed::new(literal_len - i.eof_offset()),
196 )),
197 CompareResult::Incomplete | CompareResult::Error => {
198 let e: ErrorKind = ErrorKind::Tag;
199 Err(ErrMode::from_error_kind(i, e))
200 }
201 }
202}
203
204/// Recognize a token that matches a [set of tokens][ContainsToken]
205///
206/// <div class="warning">
207///
208/// **Note:** [`Parser`] is implemented as a convenience (complete
209/// only) for
210/// - `u8`
211/// - `char`
212///
213/// </div>
214///
215/// *Complete version*: Will return an error if there's not enough input data.
216///
217/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
218///
219/// # Effective Signature
220///
221/// Assuming you are parsing a `&str` [Stream]:
222/// ```rust
223/// # use winnow::prelude::*;;
224/// # use winnow::stream::ContainsToken;
225/// # use winnow::error::ContextError;
226/// pub fn one_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
227/// # {
228/// # winnow::token::one_of(set)
229/// # }
230/// ```
231///
232/// # Example
233///
234/// ```rust
235/// # use winnow::prelude::*;
236/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError};
237/// # use winnow::token::one_of;
238/// assert_eq!(one_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek("b"), Ok(("", 'b')));
239/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek("bc"), Err(ErrMode::Backtrack(InputError::new("bc", ErrorKind::Verify))));
240/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
241///
242/// fn parser_fn(i: &str) -> IResult<&str, char> {
243/// one_of(|c| c == 'a' || c == 'b').parse_peek(i)
244/// }
245/// assert_eq!(parser_fn("abc"), Ok(("bc", 'a')));
246/// assert_eq!(parser_fn("cd"), Err(ErrMode::Backtrack(InputError::new("cd", ErrorKind::Verify))));
247/// assert_eq!(parser_fn(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
248/// ```
249///
250/// ```
251/// # use winnow::prelude::*;
252/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
253/// # use winnow::Partial;
254/// # use winnow::token::one_of;
255/// assert_eq!(one_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek(Partial::new("b")), Ok((Partial::new(""), 'b')));
256/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek(Partial::new("bc")), Err(ErrMode::Backtrack(InputError::new(Partial::new("bc"), ErrorKind::Verify))));
257/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
258///
259/// fn parser_fn(i: Partial<&str>) -> IResult<Partial<&str>, char> {
260/// one_of(|c| c == 'a' || c == 'b').parse_peek(i)
261/// }
262/// assert_eq!(parser_fn(Partial::new("abc")), Ok((Partial::new("bc"), 'a')));
263/// assert_eq!(parser_fn(Partial::new("cd")), Err(ErrMode::Backtrack(InputError::new(Partial::new("cd"), ErrorKind::Verify))));
264/// assert_eq!(parser_fn(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
265/// ```
266#[inline(always)]
267#[doc(alias = "char")]
268#[doc(alias = "token")]
269#[doc(alias = "satisfy")]
270pub fn one_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
271where
272 Input: StreamIsPartial + Stream,
273 <Input as Stream>::Token: Clone,
274 Set: ContainsToken<<Input as Stream>::Token>,
275 Error: ParserError<Input>,
276{
277 trace(
278 "one_of",
279 any.verify(move |t: &<Input as Stream>::Token| set.contains_token(t.clone())),
280 )
281}
282
283/// Recognize a token that does not match a [set of tokens][ContainsToken]
284///
285/// *Complete version*: Will return an error if there's not enough input data.
286///
287/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
288///
289/// # Effective Signature
290///
291/// Assuming you are parsing a `&str` [Stream]:
292/// ```rust
293/// # use winnow::prelude::*;;
294/// # use winnow::stream::ContainsToken;
295/// # use winnow::error::ContextError;
296/// pub fn none_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
297/// # {
298/// # winnow::token::none_of(set)
299/// # }
300/// ```
301///
302/// # Example
303///
304/// ```rust
305/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError};
306/// # use winnow::prelude::*;
307/// # use winnow::token::none_of;
308/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek("z"), Ok(("", 'z')));
309/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b']).parse_peek("a"), Err(ErrMode::Backtrack(InputError::new("a", ErrorKind::Verify))));
310/// assert_eq!(none_of::<_, _, InputError<_>>('a').parse_peek(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
311/// ```
312///
313/// ```
314/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
315/// # use winnow::prelude::*;
316/// # use winnow::Partial;
317/// # use winnow::token::none_of;
318/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek(Partial::new("z")), Ok((Partial::new(""), 'z')));
319/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b']).parse_peek(Partial::new("a")), Err(ErrMode::Backtrack(InputError::new(Partial::new("a"), ErrorKind::Verify))));
320/// assert_eq!(none_of::<_, _, InputError<_>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
321/// ```
322#[inline(always)]
323pub fn none_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
324where
325 Input: StreamIsPartial + Stream,
326 <Input as Stream>::Token: Clone,
327 Set: ContainsToken<<Input as Stream>::Token>,
328 Error: ParserError<Input>,
329{
330 trace(
331 "none_of",
332 any.verify(move |t: &<Input as Stream>::Token| !set.contains_token(t.clone())),
333 )
334}
335
336/// Recognize the longest (m <= len <= n) input slice that matches a [set of tokens][ContainsToken]
337///
338/// It will return an `ErrMode::Backtrack(InputError::new(_, ErrorKind::Slice))` if the set of tokens wasn't met or is out
339/// of range (m <= len <= n).
340///
341/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if a member of the set of tokens reaches the end of the input or is too short.
342///
343/// To take a series of tokens, use [`repeat`][crate::combinator::repeat] to [`Accumulate`][crate::stream::Accumulate] into a `()` and then [`Parser::take`].
344///
345/// # Effective Signature
346///
347/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
348/// ```rust
349/// # use std::ops::RangeFrom;
350/// # use winnow::prelude::*;
351/// # use winnow::stream::ContainsToken;
352/// # use winnow::error::ContextError;
353/// pub fn take_while<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
354/// # {
355/// # winnow::token::take_while(occurrences, set)
356/// # }
357/// ```
358///
359/// # Example
360///
361/// Zero or more tokens:
362/// ```rust
363/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
364/// # use winnow::prelude::*;
365/// use winnow::token::take_while;
366/// use winnow::stream::AsChar;
367///
368/// fn alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {
369/// take_while(0.., AsChar::is_alpha).parse_peek(s)
370/// }
371///
372/// assert_eq!(alpha(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
373/// assert_eq!(alpha(b"12345"), Ok((&b"12345"[..], &b""[..])));
374/// assert_eq!(alpha(b"latin"), Ok((&b""[..], &b"latin"[..])));
375/// assert_eq!(alpha(b""), Ok((&b""[..], &b""[..])));
376/// ```
377///
378/// ```rust
379/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
380/// # use winnow::prelude::*;
381/// # use winnow::Partial;
382/// use winnow::token::take_while;
383/// use winnow::stream::AsChar;
384///
385/// fn alpha(s: Partial<&[u8]>) -> IResult<Partial<&[u8]>, &[u8]> {
386/// take_while(0.., AsChar::is_alpha).parse_peek(s)
387/// }
388///
389/// assert_eq!(alpha(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
390/// assert_eq!(alpha(Partial::new(b"12345")), Ok((Partial::new(&b"12345"[..]), &b""[..])));
391/// assert_eq!(alpha(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
392/// assert_eq!(alpha(Partial::new(b"")), Err(ErrMode::Incomplete(Needed::new(1))));
393/// ```
394///
395/// One or more tokens:
396/// ```rust
397/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
398/// # use winnow::prelude::*;
399/// use winnow::token::take_while;
400/// use winnow::stream::AsChar;
401///
402/// fn alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {
403/// take_while(1.., AsChar::is_alpha).parse_peek(s)
404/// }
405///
406/// assert_eq!(alpha(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
407/// assert_eq!(alpha(b"latin"), Ok((&b""[..], &b"latin"[..])));
408/// assert_eq!(alpha(b"12345"), Err(ErrMode::Backtrack(InputError::new(&b"12345"[..], ErrorKind::Slice))));
409///
410/// fn hex(s: &str) -> IResult<&str, &str> {
411/// take_while(1.., ('0'..='9', 'A'..='F')).parse_peek(s)
412/// }
413///
414/// assert_eq!(hex("123 and voila"), Ok((" and voila", "123")));
415/// assert_eq!(hex("DEADBEEF and others"), Ok((" and others", "DEADBEEF")));
416/// assert_eq!(hex("BADBABEsomething"), Ok(("something", "BADBABE")));
417/// assert_eq!(hex("D15EA5E"), Ok(("", "D15EA5E")));
418/// assert_eq!(hex(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
419/// ```
420///
421/// ```rust
422/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
423/// # use winnow::prelude::*;
424/// # use winnow::Partial;
425/// use winnow::token::take_while;
426/// use winnow::stream::AsChar;
427///
428/// fn alpha(s: Partial<&[u8]>) -> IResult<Partial<&[u8]>, &[u8]> {
429/// take_while(1.., AsChar::is_alpha).parse_peek(s)
430/// }
431///
432/// assert_eq!(alpha(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
433/// assert_eq!(alpha(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
434/// assert_eq!(alpha(Partial::new(b"12345")), Err(ErrMode::Backtrack(InputError::new(Partial::new(&b"12345"[..]), ErrorKind::Slice))));
435///
436/// fn hex(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
437/// take_while(1.., ('0'..='9', 'A'..='F')).parse_peek(s)
438/// }
439///
440/// assert_eq!(hex(Partial::new("123 and voila")), Ok((Partial::new(" and voila"), "123")));
441/// assert_eq!(hex(Partial::new("DEADBEEF and others")), Ok((Partial::new(" and others"), "DEADBEEF")));
442/// assert_eq!(hex(Partial::new("BADBABEsomething")), Ok((Partial::new("something"), "BADBABE")));
443/// assert_eq!(hex(Partial::new("D15EA5E")), Err(ErrMode::Incomplete(Needed::new(1))));
444/// assert_eq!(hex(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
445/// ```
446///
447/// Arbitrary amount of tokens:
448/// ```rust
449/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
450/// # use winnow::prelude::*;
451/// use winnow::token::take_while;
452/// use winnow::stream::AsChar;
453///
454/// fn short_alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {
455/// take_while(3..=6, AsChar::is_alpha).parse_peek(s)
456/// }
457///
458/// assert_eq!(short_alpha(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
459/// assert_eq!(short_alpha(b"lengthy"), Ok((&b"y"[..], &b"length"[..])));
460/// assert_eq!(short_alpha(b"latin"), Ok((&b""[..], &b"latin"[..])));
461/// assert_eq!(short_alpha(b"ed"), Err(ErrMode::Backtrack(InputError::new(&b"ed"[..], ErrorKind::Slice))));
462/// assert_eq!(short_alpha(b"12345"), Err(ErrMode::Backtrack(InputError::new(&b"12345"[..], ErrorKind::Slice))));
463/// ```
464///
465/// ```rust
466/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
467/// # use winnow::prelude::*;
468/// # use winnow::Partial;
469/// use winnow::token::take_while;
470/// use winnow::stream::AsChar;
471///
472/// fn short_alpha(s: Partial<&[u8]>) -> IResult<Partial<&[u8]>, &[u8]> {
473/// take_while(3..=6, AsChar::is_alpha).parse_peek(s)
474/// }
475///
476/// assert_eq!(short_alpha(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
477/// assert_eq!(short_alpha(Partial::new(b"lengthy")), Ok((Partial::new(&b"y"[..]), &b"length"[..])));
478/// assert_eq!(short_alpha(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
479/// assert_eq!(short_alpha(Partial::new(b"ed")), Err(ErrMode::Incomplete(Needed::new(1))));
480/// assert_eq!(short_alpha(Partial::new(b"12345")), Err(ErrMode::Backtrack(InputError::new(Partial::new(&b"12345"[..]), ErrorKind::Slice))));
481/// ```
482#[inline(always)]
483#[doc(alias = "is_a")]
484#[doc(alias = "take_while0")]
485#[doc(alias = "take_while1")]
486pub fn take_while<Set, Input, Error>(
487 occurrences: impl Into<Range>,
488 set: Set,
489) -> impl Parser<Input, <Input as Stream>::Slice, Error>
490where
491 Input: StreamIsPartial + Stream,
492 Set: ContainsToken<<Input as Stream>::Token>,
493 Error: ParserError<Input>,
494{
495 let Range {
496 start_inclusive,
497 end_inclusive,
498 } = occurrences.into();
499 trace("take_while", move |i: &mut Input| {
500 match (start_inclusive, end_inclusive) {
501 (0, None) => {
502 if <Input as StreamIsPartial>::is_partial_supported() {
503 take_till0::<_, _, _, true>(i, |c| !set.contains_token(c))
504 } else {
505 take_till0::<_, _, _, false>(i, |c| !set.contains_token(c))
506 }
507 }
508 (1, None) => {
509 if <Input as StreamIsPartial>::is_partial_supported() {
510 take_till1::<_, _, _, true>(i, |c| !set.contains_token(c))
511 } else {
512 take_till1::<_, _, _, false>(i, |c| !set.contains_token(c))
513 }
514 }
515 (start, end) => {
516 let end = end.unwrap_or(usize::MAX);
517 if <Input as StreamIsPartial>::is_partial_supported() {
518 take_till_m_n::<_, _, _, true>(i, start, end, |c| !set.contains_token(c))
519 } else {
520 take_till_m_n::<_, _, _, false>(i, start, end, |c| !set.contains_token(c))
521 }
522 }
523 }
524 })
525}
526
527fn take_till0<P, I: StreamIsPartial + Stream, E: ParserError<I>, const PARTIAL: bool>(
528 input: &mut I,
529 predicate: P,
530) -> PResult<<I as Stream>::Slice, E>
531where
532 P: Fn(I::Token) -> bool,
533{
534 let offset = match input.offset_for(predicate) {
535 Some(offset) => offset,
536 None if PARTIAL && input.is_partial() => {
537 return Err(ErrMode::Incomplete(Needed::new(1)));
538 }
539 None => input.eof_offset(),
540 };
541 Ok(input.next_slice(offset))
542}
543
544fn take_till1<P, I: StreamIsPartial + Stream, E: ParserError<I>, const PARTIAL: bool>(
545 input: &mut I,
546 predicate: P,
547) -> PResult<<I as Stream>::Slice, E>
548where
549 P: Fn(I::Token) -> bool,
550{
551 let e: ErrorKind = ErrorKind::Slice;
552 let offset = match input.offset_for(predicate) {
553 Some(offset) => offset,
554 None if PARTIAL && input.is_partial() => {
555 return Err(ErrMode::Incomplete(Needed::new(1)));
556 }
557 None => input.eof_offset(),
558 };
559 if offset == 0 {
560 Err(ErrMode::from_error_kind(input, e))
561 } else {
562 Ok(input.next_slice(offset))
563 }
564}
565
566fn take_till_m_n<P, I, Error: ParserError<I>, const PARTIAL: bool>(
567 input: &mut I,
568 m: usize,
569 n: usize,
570 predicate: P,
571) -> PResult<<I as Stream>::Slice, Error>
572where
573 I: StreamIsPartial,
574 I: Stream,
575 P: Fn(I::Token) -> bool,
576{
577 if n < m {
578 return Err(ErrMode::assert(
579 input,
580 "`occurrences` should be ascending, rather than descending",
581 ));
582 }
583
584 let mut final_count = 0;
585 for (processed, (offset, token)) in input.iter_offsets().enumerate() {
586 if predicate(token) {
587 if processed < m {
588 return Err(ErrMode::from_error_kind(input, ErrorKind::Slice));
589 } else {
590 return Ok(input.next_slice(offset));
591 }
592 } else {
593 if processed == n {
594 return Ok(input.next_slice(offset));
595 }
596 final_count = processed + 1;
597 }
598 }
599 if PARTIAL && input.is_partial() {
600 if final_count == n {
601 Ok(input.finish())
602 } else {
603 let needed = if m > input.eof_offset() {
604 m - input.eof_offset()
605 } else {
606 1
607 };
608 Err(ErrMode::Incomplete(Needed::new(needed)))
609 }
610 } else {
611 if m <= final_count {
612 Ok(input.finish())
613 } else {
614 Err(ErrMode::from_error_kind(input, ErrorKind::Slice))
615 }
616 }
617}
618
619/// Recognize the longest input slice (if any) till a member of a [set of tokens][ContainsToken] is found.
620///
621/// It doesn't consume the terminating token from the set.
622///
623/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if the match reaches the
624/// end of input or if there was not match.
625///
626/// See also
627/// - [`take_until`] for recognizing up-to a [`literal`] (w/ optional simd optimizations)
628/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
629///
630/// # Effective Signature
631///
632/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
633/// ```rust
634/// # use std::ops::RangeFrom;
635/// # use winnow::prelude::*;
636/// # use winnow::stream::ContainsToken;
637/// # use winnow::error::ContextError;
638/// pub fn take_till<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
639/// # {
640/// # winnow::token::take_till(occurrences, set)
641/// # }
642/// ```
643///
644/// # Example
645///
646/// ```rust
647/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
648/// # use winnow::prelude::*;
649/// use winnow::token::take_till;
650///
651/// fn till_colon(s: &str) -> IResult<&str, &str> {
652/// take_till(0.., |c| c == ':').parse_peek(s)
653/// }
654///
655/// assert_eq!(till_colon("latin:123"), Ok((":123", "latin")));
656/// assert_eq!(till_colon(":empty matched"), Ok((":empty matched", ""))); //allowed
657/// assert_eq!(till_colon("12345"), Ok(("", "12345")));
658/// assert_eq!(till_colon(""), Ok(("", "")));
659/// ```
660///
661/// ```rust
662/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
663/// # use winnow::prelude::*;
664/// # use winnow::Partial;
665/// use winnow::token::take_till;
666///
667/// fn till_colon(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
668/// take_till(0.., |c| c == ':').parse_peek(s)
669/// }
670///
671/// assert_eq!(till_colon(Partial::new("latin:123")), Ok((Partial::new(":123"), "latin")));
672/// assert_eq!(till_colon(Partial::new(":empty matched")), Ok((Partial::new(":empty matched"), ""))); //allowed
673/// assert_eq!(till_colon(Partial::new("12345")), Err(ErrMode::Incomplete(Needed::new(1))));
674/// assert_eq!(till_colon(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
675/// ```
676#[inline(always)]
677#[doc(alias = "is_not")]
678pub fn take_till<Set, Input, Error>(
679 occurrences: impl Into<Range>,
680 set: Set,
681) -> impl Parser<Input, <Input as Stream>::Slice, Error>
682where
683 Input: StreamIsPartial + Stream,
684 Set: ContainsToken<<Input as Stream>::Token>,
685 Error: ParserError<Input>,
686{
687 let Range {
688 start_inclusive,
689 end_inclusive,
690 } = occurrences.into();
691 trace("take_till", move |i: &mut Input| {
692 match (start_inclusive, end_inclusive) {
693 (0, None) => {
694 if <Input as StreamIsPartial>::is_partial_supported() {
695 take_till0::<_, _, _, true>(i, |c| set.contains_token(c))
696 } else {
697 take_till0::<_, _, _, false>(i, |c| set.contains_token(c))
698 }
699 }
700 (1, None) => {
701 if <Input as StreamIsPartial>::is_partial_supported() {
702 take_till1::<_, _, _, true>(i, |c| set.contains_token(c))
703 } else {
704 take_till1::<_, _, _, false>(i, |c| set.contains_token(c))
705 }
706 }
707 (start, end) => {
708 let end = end.unwrap_or(usize::MAX);
709 if <Input as StreamIsPartial>::is_partial_supported() {
710 take_till_m_n::<_, _, _, true>(i, start, end, |c| set.contains_token(c))
711 } else {
712 take_till_m_n::<_, _, _, false>(i, start, end, |c| set.contains_token(c))
713 }
714 }
715 }
716 })
717}
718
719/// Recognize an input slice containing the first N input elements (I[..N]).
720///
721/// *Complete version*: It will return `Err(ErrMode::Backtrack(InputError::new(_, ErrorKind::Slice)))` if the input is shorter than the argument.
722///
723/// *[Partial version][crate::_topic::partial]*: if the input has less than N elements, `take` will
724/// return a `ErrMode::Incomplete(Needed::new(M))` where M is the number of
725/// additional bytes the parser would need to succeed.
726/// It is well defined for `&[u8]` as the number of elements is the byte size,
727/// but for types like `&str`, we cannot know how many bytes correspond for
728/// the next few chars, so the result will be `ErrMode::Incomplete(Needed::Unknown)`
729///
730/// # Effective Signature
731///
732/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` ranges:
733/// ```rust
734/// # use std::ops::RangeFrom;
735/// # use winnow::prelude::*;
736/// # use winnow::stream::ContainsToken;
737/// # use winnow::error::ContextError;
738/// pub fn take<'i>(token_count: usize) -> impl Parser<&'i str, &'i str, ContextError>
739/// # {
740/// # winnow::token::take(token_count)
741/// # }
742/// ```
743///
744/// # Example
745///
746/// ```rust
747/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
748/// # use winnow::prelude::*;
749/// use winnow::token::take;
750///
751/// fn take6(s: &str) -> IResult<&str, &str> {
752/// take(6usize).parse_peek(s)
753/// }
754///
755/// assert_eq!(take6("1234567"), Ok(("7", "123456")));
756/// assert_eq!(take6("things"), Ok(("", "things")));
757/// assert_eq!(take6("short"), Err(ErrMode::Backtrack(InputError::new("short", ErrorKind::Slice))));
758/// assert_eq!(take6(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
759/// ```
760///
761/// The units that are taken will depend on the input type. For example, for a
762/// `&str` it will take a number of `char`'s, whereas for a `&[u8]` it will
763/// take that many `u8`'s:
764///
765/// ```rust
766/// # use winnow::prelude::*;
767/// use winnow::error::InputError;
768/// use winnow::token::take;
769///
770/// assert_eq!(take::<_, _, InputError<_>>(1usize).parse_peek("💙"), Ok(("", "💙")));
771/// assert_eq!(take::<_, _, InputError<_>>(1usize).parse_peek("💙".as_bytes()), Ok((b"\x9F\x92\x99".as_ref(), b"\xF0".as_ref())));
772/// ```
773///
774/// ```rust
775/// # use winnow::prelude::*;
776/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
777/// # use winnow::Partial;
778/// use winnow::token::take;
779///
780/// fn take6(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
781/// take(6usize).parse_peek(s)
782/// }
783///
784/// assert_eq!(take6(Partial::new("1234567")), Ok((Partial::new("7"), "123456")));
785/// assert_eq!(take6(Partial::new("things")), Ok((Partial::new(""), "things")));
786/// // `Unknown` as we don't know the number of bytes that `count` corresponds to
787/// assert_eq!(take6(Partial::new("short")), Err(ErrMode::Incomplete(Needed::Unknown)));
788/// ```
789#[inline(always)]
790pub fn take<UsizeLike, Input, Error>(
791 token_count: UsizeLike,
792) -> impl Parser<Input, <Input as Stream>::Slice, Error>
793where
794 Input: StreamIsPartial + Stream,
795 UsizeLike: ToUsize,
796 Error: ParserError<Input>,
797{
798 let c = token_count.to_usize();
799 trace("take", move |i: &mut Input| {
800 if <Input as StreamIsPartial>::is_partial_supported() {
801 take_::<_, _, true>(i, c)
802 } else {
803 take_::<_, _, false>(i, c)
804 }
805 })
806}
807
808fn take_<I, Error: ParserError<I>, const PARTIAL: bool>(
809 i: &mut I,
810 c: usize,
811) -> PResult<<I as Stream>::Slice, Error>
812where
813 I: StreamIsPartial,
814 I: Stream,
815{
816 match i.offset_at(c) {
817 Ok(offset) => Ok(i.next_slice(offset)),
818 Err(e) if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(e)),
819 Err(_needed) => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
820 }
821}
822
823/// Recognize the input slice up to the first occurrence of a [literal].
824///
825/// Feature `simd` will enable the use of [`memchr`](https://docs.rs/memchr/latest/memchr/).
826///
827/// It doesn't consume the literal.
828///
829/// *Complete version*: It will return `Err(ErrMode::Backtrack(InputError::new(_, ErrorKind::Slice)))`
830/// if the literal wasn't met.
831///
832/// *[Partial version][crate::_topic::partial]*: will return a `ErrMode::Incomplete(Needed::new(N))` if the input doesn't
833/// contain the literal or if the input is smaller than the literal.
834///
835/// See also
836/// - [`take_till`] for recognizing up-to a [set of tokens][ContainsToken]
837/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
838///
839/// # Effective Signature
840///
841/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
842/// ```rust
843/// # use std::ops::RangeFrom;
844/// # use winnow::prelude::*;;
845/// # use winnow::error::ContextError;
846/// pub fn take_until(occurrences: RangeFrom<usize>, literal: &str) -> impl Parser<&str, &str, ContextError>
847/// # {
848/// # winnow::token::take_until(occurrences, literal)
849/// # }
850/// ```
851///
852/// # Example
853///
854/// ```rust
855/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
856/// # use winnow::prelude::*;
857/// use winnow::token::take_until;
858///
859/// fn until_eof(s: &str) -> IResult<&str, &str> {
860/// take_until(0.., "eof").parse_peek(s)
861/// }
862///
863/// assert_eq!(until_eof("hello, worldeof"), Ok(("eof", "hello, world")));
864/// assert_eq!(until_eof("hello, world"), Err(ErrMode::Backtrack(InputError::new("hello, world", ErrorKind::Slice))));
865/// assert_eq!(until_eof(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
866/// assert_eq!(until_eof("1eof2eof"), Ok(("eof2eof", "1")));
867/// ```
868///
869/// ```rust
870/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
871/// # use winnow::prelude::*;
872/// # use winnow::Partial;
873/// use winnow::token::take_until;
874///
875/// fn until_eof(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
876/// take_until(0.., "eof").parse_peek(s)
877/// }
878///
879/// assert_eq!(until_eof(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
880/// assert_eq!(until_eof(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
881/// assert_eq!(until_eof(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
882/// assert_eq!(until_eof(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
883/// ```
884///
885/// ```rust
886/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
887/// # use winnow::prelude::*;
888/// use winnow::token::take_until;
889///
890/// fn until_eof(s: &str) -> IResult<&str, &str> {
891/// take_until(1.., "eof").parse_peek(s)
892/// }
893///
894/// assert_eq!(until_eof("hello, worldeof"), Ok(("eof", "hello, world")));
895/// assert_eq!(until_eof("hello, world"), Err(ErrMode::Backtrack(InputError::new("hello, world", ErrorKind::Slice))));
896/// assert_eq!(until_eof(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
897/// assert_eq!(until_eof("1eof2eof"), Ok(("eof2eof", "1")));
898/// assert_eq!(until_eof("eof"), Err(ErrMode::Backtrack(InputError::new("eof", ErrorKind::Slice))));
899/// ```
900///
901/// ```rust
902/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
903/// # use winnow::prelude::*;
904/// # use winnow::Partial;
905/// use winnow::token::take_until;
906///
907/// fn until_eof(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
908/// take_until(1.., "eof").parse_peek(s)
909/// }
910///
911/// assert_eq!(until_eof(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
912/// assert_eq!(until_eof(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
913/// assert_eq!(until_eof(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
914/// assert_eq!(until_eof(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
915/// assert_eq!(until_eof(Partial::new("eof")), Err(ErrMode::Backtrack(InputError::new(Partial::new("eof"), ErrorKind::Slice))));
916/// ```
917#[inline(always)]
918pub fn take_until<Literal, Input, Error>(
919 occurrences: impl Into<Range>,
920 literal: Literal,
921) -> impl Parser<Input, <Input as Stream>::Slice, Error>
922where
923 Input: StreamIsPartial + Stream + FindSlice<Literal>,
924 Literal: Clone,
925 Error: ParserError<Input>,
926{
927 let Range {
928 start_inclusive,
929 end_inclusive,
930 } = occurrences.into();
931 trace("take_until", move |i: &mut Input| {
932 match (start_inclusive, end_inclusive) {
933 (0, None) => {
934 if <Input as StreamIsPartial>::is_partial_supported() {
935 take_until0_::<_, _, _, true>(i, literal.clone())
936 } else {
937 take_until0_::<_, _, _, false>(i, literal.clone())
938 }
939 }
940 (1, None) => {
941 if <Input as StreamIsPartial>::is_partial_supported() {
942 take_until1_::<_, _, _, true>(i, literal.clone())
943 } else {
944 take_until1_::<_, _, _, false>(i, literal.clone())
945 }
946 }
947 (start, end) => {
948 let end = end.unwrap_or(usize::MAX);
949 if <Input as StreamIsPartial>::is_partial_supported() {
950 take_until_m_n_::<_, _, _, true>(i, start, end, literal.clone())
951 } else {
952 take_until_m_n_::<_, _, _, false>(i, start, end, literal.clone())
953 }
954 }
955 }
956 })
957}
958
959fn take_until0_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
960 i: &mut I,
961 t: T,
962) -> PResult<<I as Stream>::Slice, Error>
963where
964 I: StreamIsPartial,
965 I: Stream + FindSlice<T>,
966{
967 match i.find_slice(t) {
968 Some(range) => Ok(i.next_slice(range.start)),
969 None if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(Needed::Unknown)),
970 None => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
971 }
972}
973
974fn take_until1_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
975 i: &mut I,
976 t: T,
977) -> PResult<<I as Stream>::Slice, Error>
978where
979 I: StreamIsPartial,
980 I: Stream + FindSlice<T>,
981{
982 match i.find_slice(t) {
983 None if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(Needed::Unknown)),
984 None => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
985 Some(range) => {
986 if range.start == 0 {
987 Err(ErrMode::from_error_kind(i, ErrorKind::Slice))
988 } else {
989 Ok(i.next_slice(range.start))
990 }
991 }
992 }
993}
994
995fn take_until_m_n_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
996 i: &mut I,
997 start: usize,
998 end: usize,
999 t: T,
1000) -> PResult<<I as Stream>::Slice, Error>
1001where
1002 I: StreamIsPartial,
1003 I: Stream + FindSlice<T>,
1004{
1005 if end < start {
1006 return Err(ErrMode::assert(
1007 i,
1008 "`occurrences` should be ascending, rather than descending",
1009 ));
1010 }
1011
1012 match i.find_slice(t) {
1013 Some(range) => {
1014 let start_offset = i.offset_at(start);
1015 let end_offset = i.offset_at(end).unwrap_or_else(|_err| i.eof_offset());
1016 if start_offset.map(|s| range.start < s).unwrap_or(true) {
1017 if PARTIAL && i.is_partial() {
1018 return Err(ErrMode::Incomplete(Needed::Unknown));
1019 } else {
1020 return Err(ErrMode::from_error_kind(i, ErrorKind::Slice));
1021 }
1022 }
1023 if end_offset < range.start {
1024 return Err(ErrMode::from_error_kind(i, ErrorKind::Slice));
1025 }
1026 Ok(i.next_slice(range.start))
1027 }
1028 None if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(Needed::Unknown)),
1029 None => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
1030 }
1031}
1032
1033/// Return the remaining input.
1034///
1035/// # Effective Signature
1036///
1037/// Assuming you are parsing a `&str` [Stream]:
1038/// ```rust
1039/// # use winnow::prelude::*;;
1040/// pub fn rest<'i>(input: &mut &'i str) -> PResult<&'i str>
1041/// # {
1042/// # winnow::token::rest.parse_next(input)
1043/// # }
1044/// ```
1045///
1046/// # Example
1047///
1048/// ```rust
1049/// # use winnow::prelude::*;
1050/// # use winnow::error::ErrorKind;
1051/// # use winnow::error::InputError;
1052/// use winnow::token::rest;
1053/// assert_eq!(rest::<_,InputError<_>>.parse_peek("abc"), Ok(("", "abc")));
1054/// assert_eq!(rest::<_,InputError<_>>.parse_peek(""), Ok(("", "")));
1055/// ```
1056#[inline]
1057pub fn rest<Input, Error>(input: &mut Input) -> PResult<<Input as Stream>::Slice, Error>
1058where
1059 Input: Stream,
1060 Error: ParserError<Input>,
1061{
1062 trace("rest", move |input: &mut Input| Ok(input.finish())).parse_next(input)
1063}
1064
1065/// Return the length of the remaining input.
1066///
1067/// <div class="warning">
1068///
1069/// Note: this does not advance the [`Stream`]
1070///
1071/// </div>
1072///
1073/// # Effective Signature
1074///
1075/// Assuming you are parsing a `&str` [Stream]:
1076/// ```rust
1077/// # use winnow::prelude::*;;
1078/// pub fn rest_len(input: &mut &str) -> PResult<usize>
1079/// # {
1080/// # winnow::token::rest_len.parse_next(input)
1081/// # }
1082/// ```
1083///
1084/// # Example
1085///
1086/// ```rust
1087/// # use winnow::prelude::*;
1088/// # use winnow::error::ErrorKind;
1089/// # use winnow::error::InputError;
1090/// use winnow::token::rest_len;
1091/// assert_eq!(rest_len::<_,InputError<_>>.parse_peek("abc"), Ok(("abc", 3)));
1092/// assert_eq!(rest_len::<_,InputError<_>>.parse_peek(""), Ok(("", 0)));
1093/// ```
1094#[inline]
1095pub fn rest_len<Input, Error>(input: &mut Input) -> PResult<usize, Error>
1096where
1097 Input: Stream,
1098 Error: ParserError<Input>,
1099{
1100 trace("rest_len", move |input: &mut Input| {
1101 let len = input.eof_offset();
1102 Ok(len)
1103 })
1104 .parse_next(input)
1105}