@lezer/python - npm Package Compare versions

Comparing version 1.1.4 to 1.1.5

CHANGELOG.md

@@ -0,1 +1,7 @@
+## 1.1.5 (2023-04-28)
+
+### Bug fixes
+
+Fix a bug that caused triple-quoted format strings with quotes in them to be parsed incorrectly.
+
 ## 1.1.4 (2023-03-30)
@@ -2,0 +8,0 @@

package.json

 {
   "name": "@lezer/python",
-  "version": "1.1.4",
+  "version": "1.1.5",
   "description": "Lezer-based Python grammar",
@@ -5,0 +5,0 @@ "main": "dist/index.cjs",

src/tokens.js

@@ -5,9 +5,14 @@ import {ExternalTokenizer, ContextTracker} from "@lezer/lr"
   ParenthesizedExpression, TupleExpression, ComprehensionExpression,
-  PatternArgList, SequencePattern, MappingPattern,
+  PatternArgList, SequencePattern, MappingPattern, FormatString,
   ArrayExpression, ArrayComprehensionExpression, ArgList, ParamList, importList, subscript,
-  DictionaryExpression, DictionaryComprehensionExpression, SetExpression, SetComprehensionExpression, FormatReplacement,
+  DictionaryExpression, DictionaryComprehensionExpression, SetExpression, SetComprehensionExpression,
+  formatString1Content, formatString1Brace, formatString1End,
+  formatString2Content, formatString2Brace, formatString2End,
+  formatString1lContent, formatString1lBrace, formatString1lEnd,
+  formatString2lContent, formatString2lBrace, formatString2lEnd,
   ParenL, BraceL, BracketL
 } from "./parser.terms.js"
 
-const newline = 10, carriageReturn = 13, space = 32, tab = 9, hash = 35, parenOpen = 40, dot = 46
+const newline = 10, carriageReturn = 13, space = 32, tab = 9, hash = 35, parenOpen = 40, dot = 46,
+      braceOpen = 123, singleQuote = 39, doubleQuote = 34
 
@@ -17,4 +22,4 @@ const bracketed = new Set([
   ArrayExpression, ArrayComprehensionExpression, subscript,
-  SetExpression, SetComprehensionExpression,
-  DictionaryExpression, DictionaryComprehensionExpression, FormatReplacement,
+  SetExpression, SetComprehensionExpression, FormatString,
+  DictionaryExpression, DictionaryComprehensionExpression,
   SequencePattern, MappingPattern, PatternArgList
@@ -38,3 +43,3 @@ ])
   } else if (isLineBreak(input.next)) {
-    input.acceptToken(stack.context.depth < 0 ? newlineBracketed : newlineToken, 1)
+    input.acceptToken(newlineToken, 1)
   }
@@ -47,3 +52,3 @@ }, {contextual: true})
   let prev = input.peek(-1), depth
-  if ((prev == newline || prev == carriageReturn) && stack.context.depth >= 0) {
+  if (prev == newline || prev == carriageReturn) {
     let depth = 0, chars = 0
@@ -109,1 +114,39 @@ for (;;) {
 })
+
+function formatString(quote, len, content, brace, end) {
+  return new ExternalTokenizer(input => {
+    let start = input.pos
+    for (;;) {
+      if (input.next < 0) {
+        break
+      } else if (input.next == braceOpen) {
+        if (input.peek(1) == braceOpen) {
+          input.advance(2)
+        } else {
+          if (input.pos == start) {
+            input.acceptToken(brace, 1)
+            return
+          }
+          break
+        }
+      } else if (input.next == "\\") {
+        input.advance()
+        if (input.next >= 0) input.advance()
+      } else if (input.next == quote && (len == 1 || input.peek(1) == quote && input.peek(2) == quote)) {
+        if (input.pos == start) {
+          input.acceptToken(end, len)
+          return
+        }
+        break
+      } else {
+        input.advance()
+      }
+    }
+    if (input.pos > start) input.acceptToken(content)
+  })
+}
+
+export const formatString1 = formatString(singleQuote, 1, formatString1Content, formatString1Brace, formatString1End)
+export const formatString2 = formatString(doubleQuote, 1, formatString2Content, formatString2Brace, formatString2End)
+export const formatString1l = formatString(singleQuote, 3, formatString1lContent, formatString1lBrace, formatString1lEnd)
+export const formatString2l = formatString(doubleQuote, 3, formatString2lContent, formatString2lBrace, formatString2lEnd)

test/expression.txt

@@ -53,2 +53,3 @@ # Operator precedence
 f'''well {{ \x }} {2 :{bar}}'''
+f"""one"{two}"three"""
 
@@ -61,3 +62,4 @@ ==>
        ExpressionStatement(FormatString(FormatReplacement(VariableName, FormatSpec))),
-       ExpressionStatement(FormatString(FormatReplacement(Number, FormatSpec(FormatReplacement(VariableName))))))
+       ExpressionStatement(FormatString(FormatReplacement(Number, FormatSpec(FormatReplacement(VariableName))))),
+       ExpressionStatement(FormatString(FormatReplacement(VariableName))))
 
@@ -64,0 +66,0 @@ # Lambda

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

Found 1 instance in 1 package

Improved metrics

Lines of code

1012

increased by1.3%

Worsened metrics

Total package byte prevSize

144233

decreased by-15.65%

No dependency changes