paste - cargo Package Compare versions

Comparing version 1.0.1 to 1.0.2

src/segment.rs

+use crate::error::{Error, Result};
+use proc_macro::{token_stream, Delimiter, Ident, Span, TokenTree};
+use std::iter::Peekable;
+
+pub(crate) enum Segment {
+    String(LitStr),
+    Apostrophe(Span),
+    Env(LitStr),
+    Modifier(Colon, Ident),
+}
+
+pub(crate) struct LitStr {
+    pub value: String,
+    pub span: Span,
+}
+
+pub(crate) struct Colon {
+    pub span: Span,
+}
+
+pub(crate) fn parse(tokens: &mut Peekable<token_stream::IntoIter>) -> Result<Vec<Segment>> {
+    let mut segments = Vec::new();
+    while match tokens.peek() {
+        None => false,
+        Some(TokenTree::Punct(punct)) => punct.as_char() != '>',
+        Some(_) => true,
+    } {
+        match tokens.next().unwrap() {
+            TokenTree::Ident(ident) => {
+                let mut fragment = ident.to_string();
+                if fragment.starts_with("r#") {
+                    fragment = fragment.split_off(2);
+                }
+                if fragment == "env"
+                    && match tokens.peek() {
+                        Some(TokenTree::Punct(punct)) => punct.as_char() == '!',
+                        _ => false,
+                    }
+                {
+                    let bang = tokens.next().unwrap(); // `!`
+                    let expect_group = tokens.next();
+                    let parenthesized = match &expect_group {
+                        Some(TokenTree::Group(group))
+                            if group.delimiter() == Delimiter::Parenthesis =>
+                        {
+                            group
+                        }
+                        Some(wrong) => return Err(Error::new(wrong.span(), "expected `(`")),
+                        None => {
+                            return Err(Error::new2(
+                                ident.span(),
+                                bang.span(),
+                                "expected `(` after `env!`",
+                            ));
+                        }
+                    };
+                    let mut inner = parenthesized.stream().into_iter();
+                    let lit = match inner.next() {
+                        Some(TokenTree::Literal(lit)) => lit,
+                        Some(wrong) => {
+                            return Err(Error::new(wrong.span(), "expected string literal"))
+                        }
+                        None => {
+                            return Err(Error::new2(
+                                ident.span(),
+                                parenthesized.span(),
+                                "expected string literal as argument to env! macro",
+                            ))
+                        }
+                    };
+                    let lit_string = lit.to_string();
+                    if lit_string.starts_with('"')
+                        && lit_string.ends_with('"')
+                        && lit_string.len() >= 2
+                    {
+                        // TODO: maybe handle escape sequences in the string if
+                        // someone has a use case.
+                        segments.push(Segment::Env(LitStr {
+                            value: lit_string[1..lit_string.len() - 1].to_owned(),
+                            span: lit.span(),
+                        }));
+                    } else {
+                        return Err(Error::new(lit.span(), "expected string literal"));
+                    }
+                    if let Some(unexpected) = inner.next() {
+                        return Err(Error::new(
+                            unexpected.span(),
+                            "unexpected token in env! macro",
+                        ));
+                    }
+                } else {
+                    segments.push(Segment::String(LitStr {
+                        value: fragment,
+                        span: ident.span(),
+                    }));
+                }
+            }
+            TokenTree::Literal(lit) => {
+                segments.push(Segment::String(LitStr {
+                    value: lit.to_string(),
+                    span: lit.span(),
+                }));
+            }
+            TokenTree::Punct(punct) => match punct.as_char() {
+                '_' => segments.push(Segment::String(LitStr {
+                    value: "_".to_owned(),
+                    span: punct.span(),
+                })),
+                '\'' => segments.push(Segment::Apostrophe(punct.span())),
+                ':' => {
+                    let colon_span = punct.span();
+                    let colon = Colon { span: colon_span };
+                    let ident = match tokens.next() {
+                        Some(TokenTree::Ident(ident)) => ident,
+                        wrong => {
+                            let span = wrong.as_ref().map_or(colon_span, TokenTree::span);
+                            return Err(Error::new(span, "expected identifier after `:`"));
+                        }
+                    };
+                    segments.push(Segment::Modifier(colon, ident));
+                }
+                _ => return Err(Error::new(punct.span(), "unexpected punct")),
+            },
+            TokenTree::Group(group) => {
+                if group.delimiter() == Delimiter::None {
+                    let mut inner = group.stream().into_iter().peekable();
+                    let nested = parse(&mut inner)?;
+                    if let Some(unexpected) = inner.next() {
+                        return Err(Error::new(unexpected.span(), "unexpected token"));
+                    }
+                    segments.extend(nested);
+                } else {
+                    return Err(Error::new(group.span(), "unexpected token"));
+                }
+            }
+        }
+    }
+    Ok(segments)
+}
+
+pub(crate) fn paste(segments: &[Segment]) -> Result<String> {
+    let mut evaluated = Vec::new();
+    let mut is_lifetime = false;
+
+    for segment in segments {
+        match segment {
+            Segment::String(segment) => {
+                evaluated.push(segment.value.clone());
+            }
+            Segment::Apostrophe(span) => {
+                if is_lifetime {
+                    return Err(Error::new(*span, "unexpected lifetime"));
+                }
+                is_lifetime = true;
+            }
+            Segment::Env(var) => {
+                let resolved = match std::env::var(&var.value) {
+                    Ok(resolved) => resolved,
+                    Err(_) => {
+                        return Err(Error::new(
+                            var.span,
+                            &format!("no such env var: {:?}", var.value),
+                        ));
+                    }
+                };
+                let resolved = resolved.replace('-', "_");
+                evaluated.push(resolved);
+            }
+            Segment::Modifier(colon, ident) => {
+                let last = match evaluated.pop() {
+                    Some(last) => last,
+                    None => {
+                        return Err(Error::new2(colon.span, ident.span(), "unexpected modifier"))
+                    }
+                };
+                match ident.to_string().as_str() {
+                    "lower" => {
+                        evaluated.push(last.to_lowercase());
+                    }
+                    "upper" => {
+                        evaluated.push(last.to_uppercase());
+                    }
+                    "snake" => {
+                        let mut acc = String::new();
+                        let mut prev = '_';
+                        for ch in last.chars() {
+                            if ch.is_uppercase() && prev != '_' {
+                                acc.push('_');
+                            }
+                            acc.push(ch);
+                            prev = ch;
+                        }
+                        evaluated.push(acc.to_lowercase());
+                    }
+                    "camel" => {
+                        let mut acc = String::new();
+                        let mut prev = '_';
+                        for ch in last.chars() {
+                            if ch != '_' {
+                                if prev == '_' {
+                                    for chu in ch.to_uppercase() {
+                                        acc.push(chu);
+                                    }
+                                } else if prev.is_uppercase() {
+                                    for chl in ch.to_lowercase() {
+                                        acc.push(chl);
+                                    }
+                                } else {
+                                    acc.push(ch);
+                                }
+                            }
+                            prev = ch;
+                        }
+                        evaluated.push(acc);
+                    }
+                    _ => {
+                        return Err(Error::new2(
+                            colon.span,
+                            ident.span(),
+                            "unsupported modifier",
+                        ));
+                    }
+                }
+            }
+        }
+    }
+
+    let mut pasted = evaluated.into_iter().collect::<String>();
+    if is_lifetime {
+        pasted.insert(0, '\'');
+    }
+    Ok(pasted)
+}

.cargo_vcs_info.json

 {
   "git": {
-    "sha1": "ead8998a76e6b28a0ade8574490e18f7bb52877b"
+    "sha1": "6a5265f7a937412fb1da72fb72fd32bbaffecebc"
   }
 }

Cargo.toml

@@ -16,3 +16,3 @@ # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
 name = "paste"
-version = "1.0.1"
+version = "1.0.2"
 authors = ["David Tolnay <[e-mail]>"]
@@ -19,0 +19,0 @@ description = "Macros for all your token pasting needs"

src/doc.rs

@@ -0,3 +1,6 @@
+use crate::error::Result;
+use crate::segment::{self, Segment};
 use proc_macro::{Delimiter, Span, TokenStream, TokenTree};
 use std::iter;
+use std::mem;
 use std::str::FromStr;
@@ -29,12 +32,29 @@
 
-pub fn do_paste_doc(attr: &TokenStream, span: Span) -> TokenStream {
+pub fn do_paste_doc(attr: &TokenStream, span: Span) -> Result<TokenStream> {
     let mut expanded = TokenStream::new();
-    let mut tokens = attr.clone().into_iter();
+    let mut tokens = attr.clone().into_iter().peekable();
     expanded.extend(tokens.by_ref().take(2)); // `doc =`
 
-    let mut lit = String::new();
-    lit.push('"');
-    for token in tokens {
-        lit += &escaped_string_value(&token).unwrap();
+    let mut segments = segment::parse(&mut tokens)?;
+
+    for segment in &mut segments {
+        if let Segment::String(string) = segment {
+            if let Some(open_quote) = string.value.find('"') {
+                if open_quote == 0 {
+                    string.value.truncate(string.value.len() - 1);
+                    string.value.remove(0);
+                } else {
+                    let begin = open_quote + 1;
+                    let end = string.value.rfind('"').unwrap();
+                    let raw_string = mem::replace(&mut string.value, String::new());
+                    for ch in raw_string[begin..end].chars() {
+                        string.value.extend(ch.escape_default());
+                    }
+                }
+            }
+        }
     }
+
+    let mut lit = segment::paste(&segments)?;
+    lit.insert(0, '"');
     lit.push('"');
@@ -49,46 +69,24 @@
     expanded.extend(iter::once(lit));
-    expanded
+    Ok(expanded)
 }
 
 fn is_stringlike(token: &TokenTree) -> bool {
-    escaped_string_value(token).is_some()
-}
-
-fn escaped_string_value(token: &TokenTree) -> Option<String> {
     match token {
-        TokenTree::Ident(ident) => Some(ident.to_string()),
+        TokenTree::Ident(_) => true,
         TokenTree::Literal(literal) => {
-            let mut repr = literal.to_string();
-            if repr.starts_with('b') || repr.starts_with('\'') {
-                None
-            } else if repr.starts_with('"') {
-                repr.truncate(repr.len() - 1);
-                repr.remove(0);
-                Some(repr)
-            } else if repr.starts_with('r') {
-                let begin = repr.find('"').unwrap() + 1;
-                let end = repr.rfind('"').unwrap();
-                let mut escaped = String::new();
-                for ch in repr[begin..end].chars() {
-                    escaped.extend(ch.escape_default());
-                }
-                Some(escaped)
-            } else {
-                Some(repr)
-            }
+            let repr = literal.to_string();
+            !repr.starts_with('b') && !repr.starts_with('\'')
         }
         TokenTree::Group(group) => {
             if group.delimiter() != Delimiter::None {
-                return None;
+                return false;
             }
             let mut inner = group.stream().into_iter();
-            let first = inner.next()?;
-            if inner.next().is_none() {
-                escaped_string_value(&first)
-            } else {
-                None
+            match inner.next() {
+                Some(first) => inner.next().is_none() && is_stringlike(&first),
+                None => false,
             }
         }
-        TokenTree::Punct(_) => None,
+        TokenTree::Punct(punct) => punct.as_char() == '\'' || punct.as_char() == ':',
     }
 }

src/lib.rs

@@ -145,9 +145,9 @@ //! [![github]](https://github.com/dtolnay/paste) [![crates-io]](https://crates.io/crates/paste) [![docs-rs]](https://docs.rs/paste)
 mod error;
+mod segment;
 
 use crate::doc::{do_paste_doc, is_pasted_doc};
 use crate::error::{Error, Result};
-use proc_macro::{
-    token_stream, Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree,
-};
-use std::iter::{self, FromIterator, Peekable};
+use crate::segment::Segment;
+use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree};
+use std::iter;
 use std::panic;
@@ -203,4 +203,5 @@
                     let segments = parse_bracket_as_segments(content, span)?;
-                    let pasted = paste_segments(span, &segments)?;
-                    expanded.extend(pasted);
+                    let pasted = segment::paste(&segments)?;
+                    let tokens = pasted_to_tokens(pasted, span)?;
+                    expanded.extend(tokens);
                     *contains_paste = true;
@@ -214,3 +215,3 @@ } else if delimiter == Delimiter::None && is_flat_group(&content) {
                 {
-                    let pasted = do_paste_doc(&content, span);
+                    let pasted = do_paste_doc(&content, span)?;
                     let mut group = Group::new(delimiter, pasted);
@@ -308,18 +309,2 @@ group.set_span(span);
 
-struct LitStr {
-    value: String,
-    span: Span,
-}
-
-struct Colon {
-    span: Span,
-}
-
-enum Segment {
-    String(String),
-    Apostrophe(Span),
-    Env(LitStr),
-    Modifier(Colon, Ident),
-}
-
 fn is_paste_operation(input: &TokenStream) -> bool {
@@ -354,3 +339,3 @@ let mut tokens = input.clone().into_iter();
 
-    let segments = parse_segments(&mut tokens, scope)?;
+    let mut segments = segment::parse(&mut tokens)?;
 
@@ -363,214 +348,35 @@ match &tokens.next() {
 
-    match tokens.next() {
-        Some(unexpected) => Err(Error::new(
+    if let Some(unexpected) = tokens.next() {
+        return Err(Error::new(
             unexpected.span(),
             "unexpected input, expected `[< ... >]`",
-        )),
-        None => Ok(segments),
+        ));
     }
-}
 
-fn parse_segments(
-    tokens: &mut Peekable<token_stream::IntoIter>,
-    scope: Span,
-) -> Result<Vec<Segment>> {
-    let mut segments = Vec::new();
-    while match tokens.peek() {
-        None => false,
-        Some(TokenTree::Punct(punct)) => punct.as_char() != '>',
-        Some(_) => true,
-    } {
-        match tokens.next().unwrap() {
-            TokenTree::Ident(ident) => {
-                let mut fragment = ident.to_string();
-                if fragment.starts_with("r#") {
-                    fragment = fragment.split_off(2);
-                }
-                if fragment == "env"
-                    && match tokens.peek() {
-                        Some(TokenTree::Punct(punct)) => punct.as_char() == '!',
-                        _ => false,
-                    }
-                {
-                    tokens.next().unwrap(); // `!`
-                    let expect_group = tokens.next();
-                    let parenthesized = match &expect_group {
-                        Some(TokenTree::Group(group))
-                            if group.delimiter() == Delimiter::Parenthesis =>
-                        {
-                            group
-                        }
-                        Some(wrong) => return Err(Error::new(wrong.span(), "expected `(`")),
-                        None => return Err(Error::new(scope, "expected `(` after `env!`")),
-                    };
-                    let mut inner = parenthesized.stream().into_iter();
-                    let lit = match inner.next() {
-                        Some(TokenTree::Literal(lit)) => lit,
-                        Some(wrong) => {
-                            return Err(Error::new(wrong.span(), "expected string literal"))
-                        }
-                        None => {
-                            return Err(Error::new2(
-                                ident.span(),
-                                parenthesized.span(),
-                                "expected string literal as argument to env! macro",
-                            ))
-                        }
-                    };
-                    let lit_string = lit.to_string();
-                    if lit_string.starts_with('"')
-                        && lit_string.ends_with('"')
-                        && lit_string.len() >= 2
-                    {
-                        // TODO: maybe handle escape sequences in the string if
-                        // someone has a use case.
-                        segments.push(Segment::Env(LitStr {
-                            value: lit_string[1..lit_string.len() - 1].to_owned(),
-                            span: lit.span(),
-                        }));
-                    } else {
-                        return Err(Error::new(lit.span(), "expected string literal"));
-                    }
-                    if let Some(unexpected) = inner.next() {
-                        return Err(Error::new(
-                            unexpected.span(),
-                            "unexpected token in env! macro",
-                        ));
-                    }
-                } else {
-                    segments.push(Segment::String(fragment));
-                }
+    for segment in &mut segments {
+        if let Segment::String(string) = segment {
+            if string.value.contains(&['#', '\\', '.', '+'][..]) {
+                return Err(Error::new(string.span, "unsupported literal"));
             }
-            TokenTree::Literal(lit) => {
-                let mut lit_string = lit.to_string();
-                if lit_string.contains(&['#', '\\', '.', '+'][..]) {
-                    return Err(Error::new(lit.span(), "unsupported literal"));
-                }
-                lit_string = lit_string
-                    .replace('"', "")
-                    .replace('\'', "")
-                    .replace('-', "_");
-                segments.push(Segment::String(lit_string));
-            }
-            TokenTree::Punct(punct) => match punct.as_char() {
-                '_' => segments.push(Segment::String("_".to_owned())),
-                '\'' => segments.push(Segment::Apostrophe(punct.span())),
-                ':' => {
-                    let colon = Colon { span: punct.span() };
-                    let ident = match tokens.next() {
-                        Some(TokenTree::Ident(ident)) => ident,
-                        wrong => {
-                            let span = wrong.as_ref().map_or(scope, TokenTree::span);
-                            return Err(Error::new(span, "expected identifier after `:`"));
-                        }
-                    };
-                    segments.push(Segment::Modifier(colon, ident));
-                }
-                _ => return Err(Error::new(punct.span(), "unexpected punct")),
-            },
-            TokenTree::Group(group) => {
-                if group.delimiter() == Delimiter::None {
-                    let mut inner = group.stream().into_iter().peekable();
-                    let nested = parse_segments(&mut inner, group.span())?;
-                    if let Some(unexpected) = inner.next() {
-                        return Err(Error::new(unexpected.span(), "unexpected token"));
-                    }
-                    segments.extend(nested);
-                } else {
-                    return Err(Error::new(group.span(), "unexpected token"));
-                }
-            }
+            string.value = string
+                .value
+                .replace('"', "")
+                .replace('\'', "")
+                .replace('-', "_");
         }
     }
+
     Ok(segments)
 }
 
-fn paste_segments(span: Span, segments: &[Segment]) -> Result<TokenStream> {
-    let mut evaluated = Vec::new();
-    let mut is_lifetime = false;
+fn pasted_to_tokens(mut pasted: String, span: Span) -> Result<TokenStream> {
+    let mut tokens = TokenStream::new();
 
-    for segment in segments {
-        match segment {
-            Segment::String(segment) => {
-                evaluated.push(segment.clone());
-            }
-            Segment::Apostrophe(span) => {
-                if is_lifetime {
-                    return Err(Error::new(*span, "unexpected lifetime"));
-                }
-                is_lifetime = true;
-            }
-            Segment::Env(var) => {
-                let resolved = match std::env::var(&var.value) {
-                    Ok(resolved) => resolved,
-                    Err(_) => {
-                        return Err(Error::new(
-                            var.span,
-                            &format!("no such env var: {:?}", var.value),
-                        ));
-                    }
-                };
-                let resolved = resolved.replace('-', "_");
-                evaluated.push(resolved);
-            }
-            Segment::Modifier(colon, ident) => {
-                let last = match evaluated.pop() {
-                    Some(last) => last,
-                    None => {
-                        return Err(Error::new2(colon.span, ident.span(), "unexpected modifier"))
-                    }
-                };
-                match ident.to_string().as_str() {
-                    "lower" => {
-                        evaluated.push(last.to_lowercase());
-                    }
-                    "upper" => {
-                        evaluated.push(last.to_uppercase());
-                    }
-                    "snake" => {
-                        let mut acc = String::new();
-                        let mut prev = '_';
-                        for ch in last.chars() {
-                            if ch.is_uppercase() && prev != '_' {
-                                acc.push('_');
-                            }
-                            acc.push(ch);
-                            prev = ch;
-                        }
-                        evaluated.push(acc.to_lowercase());
-                    }
-                    "camel" => {
-                        let mut acc = String::new();
-                        let mut prev = '_';
-                        for ch in last.chars() {
-                            if ch != '_' {
-                                if prev == '_' {
-                                    for chu in ch.to_uppercase() {
-                                        acc.push(chu);
-                                    }
-                                } else if prev.is_uppercase() {
-                                    for chl in ch.to_lowercase() {
-                                        acc.push(chl);
-                                    }
-                                } else {
-                                    acc.push(ch);
-                                }
-                            }
-                            prev = ch;
-                        }
-                        evaluated.push(acc);
-                    }
-                    _ => {
-                        return Err(Error::new2(
-                            colon.span,
-                            ident.span(),
-                            "unsupported modifier",
-                        ));
-                    }
-                }
-            }
-        }
+    if pasted.starts_with('\'') {
+        let mut apostrophe = TokenTree::Punct(Punct::new('\'', Spacing::Joint));
+        apostrophe.set_span(span);
+        tokens.extend(iter::once(apostrophe));
+        pasted.remove(0);
     }
 
-    let pasted = evaluated.into_iter().collect::<String>();
     let ident = match panic::catch_unwind(|| Ident::new(&pasted, span)) {
@@ -585,9 +391,5 @@ Ok(ident) => TokenTree::Ident(ident),
     };
-    let tokens = if is_lifetime {
-        let apostrophe = TokenTree::Punct(Punct::new('\'', Spacing::Joint));
-        vec![apostrophe, ident]
-    } else {
-        vec![ident]
-    };
-    Ok(TokenStream::from_iter(tokens))
+
+    tokens.extend(iter::once(ident));
+    Ok(tokens)
 }

tests/test_doc.rs

@@ -45,1 +45,11 @@ use paste::paste;
 }
+
+#[test]
+fn test_case() {
+    let doc = paste! {
+        get_doc!(#[doc = "HTTP " get:upper "!"])
+    };
+
+    let expected = "HTTP GET!";
+    assert_eq!(doc, expected);
+}

No alert changes

No metric changes