coffee-lex - npm Package Compare versions

Comparing version 1.3.11 to 1.4.0

dist/coffee-lex.es.js

@@ -82,2 +82,52 @@ var babelHelpers = {};
 
+var BufferedStream = function () {
+  function BufferedStream(stream) {
+    babelHelpers.classCallCheck(this, BufferedStream);
+    this.pending = [];
+
+    this._getNextLocation = stream;
+  }
+
+  babelHelpers.createClass(BufferedStream, [{
+    key: 'shift',
+    value: function shift() {
+      return this.pending.shift() || this._getNextLocation();
+    }
+  }, {
+    key: 'hasNext',
+    value: function hasNext() {
+      var _this = this;
+
+      var locationsToPutBack = [];
+
+      for (var _len = arguments.length, types = Array(_len), _key = 0; _key < _len; _key++) {
+        types[_key] = arguments[_key];
+      }
+
+      var result = types.every(function (type) {
+        var next = _this.shift();
+        locationsToPutBack.push(next);
+        return next.type === type;
+      });
+      this.unshift.apply(this, locationsToPutBack);
+      return result;
+    }
+  }, {
+    key: 'peek',
+    value: function peek() {
+      var result = this.shift();
+      this.unshift(result);
+      return result;
+    }
+  }, {
+    key: 'unshift',
+    value: function unshift() {
+      var _pending;
+
+      (_pending = this.pending).unshift.apply(_pending, arguments);
+    }
+  }]);
+  return BufferedStream;
+}();
+
 /**
@@ -191,8 +241,18 @@ * Represents a change in source code type at a particular index.
 
-var SourceType = function SourceType(name) {
-  babelHelpers.classCallCheck(this, SourceType);
+var SourceType = function () {
+  function SourceType(name) {
+    babelHelpers.classCallCheck(this, SourceType);
 
-  this.name = name;
-};
+    this.name = name;
+  }
 
+  babelHelpers.createClass(SourceType, [{
+    key: "toString",
+    value: function toString() {
+      return this.name;
+    }
+  }]);
+  return SourceType;
+}();
+
 /**
@@ -616,2 +676,294 @@ * Represents a list of tokens and provides various utility functions for
 
+var IndexRangeList = function () {
+  function IndexRangeList() {
+    var ranges = arguments.length <= 0 || arguments[0] === undefined ? [] : arguments[0];
+    babelHelpers.classCallCheck(this, IndexRangeList);
+
+    this.ranges = ranges;
+    this.length = ranges.length;
+  }
+
+  babelHelpers.createClass(IndexRangeList, [{
+    key: "addRange",
+    value: function addRange(start, end) {
+      return new IndexRangeList(rangeListAddingRange(this.ranges, { start: start, end: end }));
+    }
+  }, {
+    key: "forEach",
+    value: function forEach(iterator) {
+      var _this = this;
+
+      var context = arguments.length <= 1 || arguments[1] === undefined ? null : arguments[1];
+
+      this.ranges.forEach(function (range, index) {
+        iterator.call(context, range, index, _this);
+      });
+    }
+  }, {
+    key: "invert",
+    value: function invert(start, end) {
+      var ranges = this.ranges;
+      var inverted = new IndexRangeList();
+
+      if (ranges.length === 0) {
+        return inverted.addRange(start, end);
+      } else if (ranges.length === 1) {
+        return inverted.addRange(start, ranges[0].start).addRange(ranges[0].end, end);
+      }
+
+      var nextStart = start;
+
+      for (var i = 0; i < ranges.length; i++) {
+        var _range = ranges[i];
+
+        inverted = inverted.addRange(nextStart, _range.start);
+
+        nextStart = _range.end;
+      }
+
+      return inverted.addRange(nextStart, end);
+    }
+  }, {
+    key: "toArray",
+    value: function toArray() {
+      return this.ranges.slice();
+    }
+  }]);
+  return IndexRangeList;
+}();
+
+function rangeListAddingRange(ranges, range) {
+  if (range.start === range.end) {
+    return ranges;
+  }
+
+  var result = [];
+
+  ranges
+  // Add the new range.
+  .concat([range])
+  // Sort by start of range.
+  .sort(function (left, right) {
+    return left.start - right.start;
+  })
+  // Add ranges to the results.
+  .forEach(function (_ref, i) {
+    var start = _ref.start;
+    var end = _ref.end;
+
+    if (i === 0) {
+      result.push({ start: start, end: end });
+    } else {
+      var lastRange = result[result.length - 1];
+
+      if (start <= lastRange.end) {
+        lastRange.end = Math.max(end, lastRange.end);
+      } else {
+        result.push({ start: start, end: end });
+      }
+    }
+  });
+
+  return result;
+}
+
+var QUOTE_LENGTH = 3;
+
+function tripleQuotedStringSourceLocations(source, stream) {
+  if (!stream.hasNext(TSSTRING) && !stream.hasNext(TDSTRING)) {
+    return [];
+  }
+
+  var result = [];
+  var ignoredRanges = new IndexRangeList();
+  var firstLocation = stream.shift();
+
+  result.push(new SourceLocation(STRING_START, firstLocation.index));
+
+  var leadingMarginStart = firstLocation.index + QUOTE_LENGTH;
+  var leadingMarginEnd = getLeadingMarginEnd(source, leadingMarginStart);
+
+  ignoredRanges = ignoredRanges.addRange(leadingMarginStart, leadingMarginEnd);
+
+  if (stream.peek().type === INTERPOLATION_START) {
+    var interpolationDepth = 0;
+    var start = null;
+
+    for (;;) {
+      var location = stream.shift();
+
+      if (location.type === INTERPOLATION_START) {
+        result.push(location);
+        if (interpolationDepth === 0) {
+          start = location.index;
+        }
+        interpolationDepth += 1;
+      } else if (location.type === INTERPOLATION_END) {
+        result.push(location);
+        interpolationDepth -= 1;
+        if (interpolationDepth === 0 && start !== null) {
+          ignoredRanges = ignoredRanges.addRange(start, stream.peek().index);
+        }
+      } else if (location.type === firstLocation.type) {
+        if (interpolationDepth === 0 && stream.peek().type !== INTERPOLATION_START) {
+          break;
+        }
+      } else {
+        result.push(location);
+      }
+    }
+  }
+
+  var trailingMarginEnd = stream.peek().index - QUOTE_LENGTH;
+  var trailingMarginStart = getTrailingMarginStart(source, trailingMarginEnd);
+
+  ignoredRanges = ignoredRanges.addRange(trailingMarginStart, trailingMarginEnd);
+
+  var _getIndentInfoForRang = getIndentInfoForRanges(source, ignoredRanges.invert(leadingMarginStart, trailingMarginEnd));
+
+  var sharedIndent = _getIndentInfoForRang.sharedIndent;
+  var indexes = _getIndentInfoForRang.indexes;
+
+
+  var spaceRanges = new IndexRangeList().addRange(leadingMarginStart, leadingMarginEnd).addRange(trailingMarginStart, trailingMarginEnd);
+  indexes.forEach(function (index) {
+    spaceRanges = spaceRanges.addRange(index, index + sharedIndent.length);
+    ignoredRanges = ignoredRanges.addRange(index, index + sharedIndent.length);
+  });
+
+  result.push(new SourceLocation(STRING_END, trailingMarginEnd));
+
+  var resultIndex = 0;
+
+  spaceRanges.forEach(function (_ref) {
+    var start = _ref.start;
+    var end = _ref.end;
+
+    while (result[resultIndex].index < start) {
+      resultIndex++;
+    }
+    result.splice(resultIndex, 0, new SourceLocation(SPACE, start));
+  });
+
+  var contentRanges = ignoredRanges.invert(leadingMarginStart, trailingMarginEnd);
+
+  if (contentRanges.length === 0) {
+    // It's all content.
+    result.splice(result.length - 1, 0, new SourceLocation(STRING_CONTENT, leadingMarginEnd));
+  }
+
+  resultIndex = 0;
+  contentRanges.forEach(function (_ref2) {
+    var start = _ref2.start;
+
+    while (result[resultIndex].index < start) {
+      resultIndex++;
+    }
+    result.splice(resultIndex, 0, new SourceLocation(STRING_CONTENT, start));
+  });
+
+  return result;
+}
+
+function getLeadingMarginEnd(source, marginStart) {
+  for (var i = marginStart; i < source.length; i++) {
+    var char = source.charAt(i);
+
+    if (char === ' ' || char === '\t') {
+      // Just part of the margin.
+    } else if (char === '\n') {
+        // End of the margin.
+        return i + '\n'.length;
+      } else if (char === '\r') {
+        if (source.charAt(i + '\r'.length) === '\n') {
+          // Ends with \r\n.
+          return i + '\r\n'.length;
+        } else {
+          // Only ends with \r.
+          return i + '\r'.length;
+        }
+      } else {
+        // Non-space before a newline, so there is no margin.
+        return marginStart;
+      }
+  }
+
+  throw new Error('unexpected EOF while looking for end of margin at offset ' + marginStart);
+}
+
+function getTrailingMarginStart(source, marginEnd) {
+  for (var i = marginEnd - 1; i >= 0; i--) {
+    var char = source.charAt(i);
+
+    if (char === ' ' || char === '\t') {
+      // Just part of the margin.
+    } else if (char === '\n') {
+        if (source.charAt(i - '\n'.length) === '\r') {
+          // Starts with \r\n.
+          return i - '\r'.length;
+        } else {
+          // Only ends with \n.
+          return i;
+        }
+      } else if (char === '\r') {
+        // Start of the margin.
+        return i;
+      } else {
+        // Non-space before the ending, so there is no margin.
+        return marginEnd;
+      }
+  }
+
+  throw new Error('unexpected SOF while looking for stat of margin at offset ' + marginEnd);
+}
+
+function getIndentInfoForRanges(source, ranges) {
+  var sharedIndent = null;
+  var isAtStartOfLine = true;
+  var indexes = [];
+
+  ranges.forEach(function (_ref3) {
+    var start = _ref3.start;
+    var end = _ref3.end;
+
+    for (var i = start; i < end; i++) {
+      var char = source[i];
+
+      if (char === '\n' || char === '\r') {
+        isAtStartOfLine = true;
+      } else if (isAtStartOfLine) {
+        isAtStartOfLine = false;
+        indexes.push(i);
+
+        var endOfPossibleIndent = sharedIndent ? i + sharedIndent.length : end;
+
+        for (var j = i; j < endOfPossibleIndent; j++) {
+          var maybeIndentChar = source[j];
+
+          if (sharedIndent) {
+            if (maybeIndentChar === sharedIndent[j - i]) {
+              // Matches the known indent so far.
+              continue;
+            }
+          } else if (maybeIndentChar === ' ' || maybeIndentChar === '\t') {
+            // Looks like a valid indent character.
+            continue;
+          }
+
+          // Set or shrink the known indent.
+          sharedIndent = source.slice(i, j);
+          i = j;
+          break;
+        }
+      }
+    }
+  });
+
+  if (sharedIndent === null) {
+    return { sharedIndent: '', indexes: [] };
+  }
+
+  return { sharedIndent: sharedIndent, indexes: indexes };
+}
+
 /**
@@ -626,2 +978,3 @@ * Generate a list of tokens from CoffeeScript source code.
   do {
+    pending.unshift.apply(pending, babelHelpers.toConsumableArray(tripleQuotedStringSourceLocations(source, pending)));
     pending.unshift.apply(pending, babelHelpers.toConsumableArray(stringLocationsFromStream(pending)));
@@ -639,3 +992,3 @@ pending.unshift.apply(pending, babelHelpers.toConsumableArray(combinedLocationsForMultiwordOperators(pending, source)));
 function stringLocationsFromStream(stream) {
-  var startOfStringInterpolation = stream.hasNext(DSTRING, INTERPOLATION_START) || stream.hasNext(TDSTRING, INTERPOLATION_START);
+  var startOfStringInterpolation = stream.hasNext(DSTRING, INTERPOLATION_START);
 
@@ -648,3 +1001,3 @@ if (!startOfStringInterpolation) {
   var first = stream.shift();
-  var quote = first.type === DSTRING ? '"' : '"""';
+  var quote = '"';
 
@@ -722,52 +1075,2 @@ result.push(
 
-var BufferedStream = function () {
-  function BufferedStream(stream) {
-    babelHelpers.classCallCheck(this, BufferedStream);
-    this.pending = [];
-
-    this._getNextLocation = stream;
-  }
-
-  babelHelpers.createClass(BufferedStream, [{
-    key: 'shift',
-    value: function shift() {
-      return this.pending.shift() || this._getNextLocation();
-    }
-  }, {
-    key: 'hasNext',
-    value: function hasNext() {
-      var _this = this;
-
-      var locationsToPutBack = [];
-
-      for (var _len = arguments.length, types = Array(_len), _key = 0; _key < _len; _key++) {
-        types[_key] = arguments[_key];
-      }
-
-      var result = types.every(function (type) {
-        var next = _this.shift();
-        locationsToPutBack.push(next);
-        return next.type === type;
-      });
-      this.unshift.apply(this, locationsToPutBack);
-      return result;
-    }
-  }, {
-    key: 'peek',
-    value: function peek() {
-      var result = this.shift();
-      this.unshift(result);
-      return result;
-    }
-  }, {
-    key: 'unshift',
-    value: function unshift() {
-      var _pending;
-
-      (_pending = this.pending).unshift.apply(_pending, arguments);
-    }
-  }]);
-  return BufferedStream;
-}();
-
 var REGEXP_FLAGS = ['i', 'g', 'm', 'y'];
@@ -839,2 +1142,4 @@
 var IDENTIFIER = new SourceType('IDENTIFIER');
+var YIELD = new SourceType('YIELD');
+var YIELDFROM = new SourceType('YIELDFROM');
 
@@ -853,2 +1158,3 @@ /**
 var REGEXP_PATTERN = /^\/(?!\/)((?:[^[\/\n\\]|\\[^\n]|\[(?:\\[^\n]|[^\]\n\\])*\])*)(\/)?/;
+var YIELDFROM_PATTERN = /^yield[^\n\r\S]+from/;
 
@@ -947,2 +1253,4 @@ var OPERATORS = [
         case DO:
+        case YIELD:
+        case YIELDFROM:
         case CONTINUATION:
@@ -1033,2 +1341,4 @@ if (consume(SPACE_PATTERN)) {
             }
+          } else if (consume(YIELDFROM_PATTERN)) {
+            setType(YIELDFROM);
           } else if (consume(IDENTIFIER_PATTERN)) {
@@ -1153,2 +1463,6 @@ var prev = locations[locations.length - 1];
 
+                case 'yield':
+                  setType(YIELD);
+                  break;
+
                 default:
@@ -1379,2 +1693,2 @@ setType(IDENTIFIER);
 
-export { AT, BOOL, BREAK, CATCH, CALL_END, CALL_START, CLASS, COLON, COMMA, COMMENT, CONTINUATION, CONTINUE, DELETE, DO, DOT, DSTRING, ELSE, EOF, EXISTENCE, FINALLY, FOR, FUNCTION, HERECOMMENT, HEREGEXP, IF, INTERPOLATION_START, INTERPOLATION_END, JS, LBRACE, LBRACKET, LOOP, LPAREN, NEWLINE, NORMAL, NULL, NUMBER, OPERATOR, OWN, PROTO, RANGE, REGEXP, RBRACE, RBRACKET, RELATION, RETURN, RPAREN, SEMICOLON, SPACE, SSTRING, SUPER, SWITCH, TDSTRING, STRING_CONTENT, STRING_END, STRING_START, THEN, THIS, TRY, TSSTRING, UNDEFINED, UNKNOWN, WHEN, WHILE, IDENTIFIER, stream, consumeStream };export default lex;
+export { AT, BOOL, BREAK, CATCH, CALL_END, CALL_START, CLASS, COLON, COMMA, COMMENT, CONTINUATION, CONTINUE, DELETE, DO, DOT, DSTRING, ELSE, EOF, EXISTENCE, FINALLY, FOR, FUNCTION, HERECOMMENT, HEREGEXP, IF, INTERPOLATION_START, INTERPOLATION_END, JS, LBRACE, LBRACKET, LOOP, LPAREN, NEWLINE, NORMAL, NULL, NUMBER, OPERATOR, OWN, PROTO, RANGE, REGEXP, RBRACE, RBRACKET, RELATION, RETURN, RPAREN, SEMICOLON, SPACE, SSTRING, SUPER, SWITCH, TDSTRING, STRING_CONTENT, STRING_END, STRING_START, THEN, THIS, TRY, TSSTRING, UNDEFINED, UNKNOWN, WHEN, WHILE, IDENTIFIER, YIELD, YIELDFROM, stream, consumeStream };export default lex;

dist/coffee-lex.umd.js

@@ -88,2 +88,52 @@ (function (global, factory) {
 
+  var BufferedStream = function () {
+    function BufferedStream(stream) {
+      babelHelpers.classCallCheck(this, BufferedStream);
+      this.pending = [];
+
+      this._getNextLocation = stream;
+    }
+
+    babelHelpers.createClass(BufferedStream, [{
+      key: 'shift',
+      value: function shift() {
+        return this.pending.shift() || this._getNextLocation();
+      }
+    }, {
+      key: 'hasNext',
+      value: function hasNext() {
+        var _this = this;
+
+        var locationsToPutBack = [];
+
+        for (var _len = arguments.length, types = Array(_len), _key = 0; _key < _len; _key++) {
+          types[_key] = arguments[_key];
+        }
+
+        var result = types.every(function (type) {
+          var next = _this.shift();
+          locationsToPutBack.push(next);
+          return next.type === type;
+        });
+        this.unshift.apply(this, locationsToPutBack);
+        return result;
+      }
+    }, {
+      key: 'peek',
+      value: function peek() {
+        var result = this.shift();
+        this.unshift(result);
+        return result;
+      }
+    }, {
+      key: 'unshift',
+      value: function unshift() {
+        var _pending;
+
+        (_pending = this.pending).unshift.apply(_pending, arguments);
+      }
+    }]);
+    return BufferedStream;
+  }();
+
   /**
@@ -197,8 +247,18 @@ * Represents a change in source code type at a particular index.
 
-  var SourceType = function SourceType(name) {
-    babelHelpers.classCallCheck(this, SourceType);
+  var SourceType = function () {
+    function SourceType(name) {
+      babelHelpers.classCallCheck(this, SourceType);
 
-    this.name = name;
-  };
+      this.name = name;
+    }
 
+    babelHelpers.createClass(SourceType, [{
+      key: "toString",
+      value: function toString() {
+        return this.name;
+      }
+    }]);
+    return SourceType;
+  }();
+
   /**
@@ -622,2 +682,294 @@ * Represents a list of tokens and provides various utility functions for
 
+  var IndexRangeList = function () {
+    function IndexRangeList() {
+      var ranges = arguments.length <= 0 || arguments[0] === undefined ? [] : arguments[0];
+      babelHelpers.classCallCheck(this, IndexRangeList);
+
+      this.ranges = ranges;
+      this.length = ranges.length;
+    }
+
+    babelHelpers.createClass(IndexRangeList, [{
+      key: "addRange",
+      value: function addRange(start, end) {
+        return new IndexRangeList(rangeListAddingRange(this.ranges, { start: start, end: end }));
+      }
+    }, {
+      key: "forEach",
+      value: function forEach(iterator) {
+        var _this = this;
+
+        var context = arguments.length <= 1 || arguments[1] === undefined ? null : arguments[1];
+
+        this.ranges.forEach(function (range, index) {
+          iterator.call(context, range, index, _this);
+        });
+      }
+    }, {
+      key: "invert",
+      value: function invert(start, end) {
+        var ranges = this.ranges;
+        var inverted = new IndexRangeList();
+
+        if (ranges.length === 0) {
+          return inverted.addRange(start, end);
+        } else if (ranges.length === 1) {
+          return inverted.addRange(start, ranges[0].start).addRange(ranges[0].end, end);
+        }
+
+        var nextStart = start;
+
+        for (var i = 0; i < ranges.length; i++) {
+          var _range = ranges[i];
+
+          inverted = inverted.addRange(nextStart, _range.start);
+
+          nextStart = _range.end;
+        }
+
+        return inverted.addRange(nextStart, end);
+      }
+    }, {
+      key: "toArray",
+      value: function toArray() {
+        return this.ranges.slice();
+      }
+    }]);
+    return IndexRangeList;
+  }();
+
+  function rangeListAddingRange(ranges, range) {
+    if (range.start === range.end) {
+      return ranges;
+    }
+
+    var result = [];
+
+    ranges
+    // Add the new range.
+    .concat([range])
+    // Sort by start of range.
+    .sort(function (left, right) {
+      return left.start - right.start;
+    })
+    // Add ranges to the results.
+    .forEach(function (_ref, i) {
+      var start = _ref.start;
+      var end = _ref.end;
+
+      if (i === 0) {
+        result.push({ start: start, end: end });
+      } else {
+        var lastRange = result[result.length - 1];
+
+        if (start <= lastRange.end) {
+          lastRange.end = Math.max(end, lastRange.end);
+        } else {
+          result.push({ start: start, end: end });
+        }
+      }
+    });
+
+    return result;
+  }
+
+  var QUOTE_LENGTH = 3;
+
+  function tripleQuotedStringSourceLocations(source, stream) {
+    if (!stream.hasNext(TSSTRING) && !stream.hasNext(TDSTRING)) {
+      return [];
+    }
+
+    var result = [];
+    var ignoredRanges = new IndexRangeList();
+    var firstLocation = stream.shift();
+
+    result.push(new SourceLocation(STRING_START, firstLocation.index));
+
+    var leadingMarginStart = firstLocation.index + QUOTE_LENGTH;
+    var leadingMarginEnd = getLeadingMarginEnd(source, leadingMarginStart);
+
+    ignoredRanges = ignoredRanges.addRange(leadingMarginStart, leadingMarginEnd);
+
+    if (stream.peek().type === INTERPOLATION_START) {
+      var interpolationDepth = 0;
+      var start = null;
+
+      for (;;) {
+        var location = stream.shift();
+
+        if (location.type === INTERPOLATION_START) {
+          result.push(location);
+          if (interpolationDepth === 0) {
+            start = location.index;
+          }
+          interpolationDepth += 1;
+        } else if (location.type === INTERPOLATION_END) {
+          result.push(location);
+          interpolationDepth -= 1;
+          if (interpolationDepth === 0 && start !== null) {
+            ignoredRanges = ignoredRanges.addRange(start, stream.peek().index);
+          }
+        } else if (location.type === firstLocation.type) {
+          if (interpolationDepth === 0 && stream.peek().type !== INTERPOLATION_START) {
+            break;
+          }
+        } else {
+          result.push(location);
+        }
+      }
+    }
+
+    var trailingMarginEnd = stream.peek().index - QUOTE_LENGTH;
+    var trailingMarginStart = getTrailingMarginStart(source, trailingMarginEnd);
+
+    ignoredRanges = ignoredRanges.addRange(trailingMarginStart, trailingMarginEnd);
+
+    var _getIndentInfoForRang = getIndentInfoForRanges(source, ignoredRanges.invert(leadingMarginStart, trailingMarginEnd));
+
+    var sharedIndent = _getIndentInfoForRang.sharedIndent;
+    var indexes = _getIndentInfoForRang.indexes;
+
+
+    var spaceRanges = new IndexRangeList().addRange(leadingMarginStart, leadingMarginEnd).addRange(trailingMarginStart, trailingMarginEnd);
+    indexes.forEach(function (index) {
+      spaceRanges = spaceRanges.addRange(index, index + sharedIndent.length);
+      ignoredRanges = ignoredRanges.addRange(index, index + sharedIndent.length);
+    });
+
+    result.push(new SourceLocation(STRING_END, trailingMarginEnd));
+
+    var resultIndex = 0;
+
+    spaceRanges.forEach(function (_ref) {
+      var start = _ref.start;
+      var end = _ref.end;
+
+      while (result[resultIndex].index < start) {
+        resultIndex++;
+      }
+      result.splice(resultIndex, 0, new SourceLocation(SPACE, start));
+    });
+
+    var contentRanges = ignoredRanges.invert(leadingMarginStart, trailingMarginEnd);
+
+    if (contentRanges.length === 0) {
+      // It's all content.
+      result.splice(result.length - 1, 0, new SourceLocation(STRING_CONTENT, leadingMarginEnd));
+    }
+
+    resultIndex = 0;
+    contentRanges.forEach(function (_ref2) {
+      var start = _ref2.start;
+
+      while (result[resultIndex].index < start) {
+        resultIndex++;
+      }
+      result.splice(resultIndex, 0, new SourceLocation(STRING_CONTENT, start));
+    });
+
+    return result;
+  }
+
+  function getLeadingMarginEnd(source, marginStart) {
+    for (var i = marginStart; i < source.length; i++) {
+      var char = source.charAt(i);
+
+      if (char === ' ' || char === '\t') {
+        // Just part of the margin.
+      } else if (char === '\n') {
+          // End of the margin.
+          return i + '\n'.length;
+        } else if (char === '\r') {
+          if (source.charAt(i + '\r'.length) === '\n') {
+            // Ends with \r\n.
+            return i + '\r\n'.length;
+          } else {
+            // Only ends with \r.
+            return i + '\r'.length;
+          }
+        } else {
+          // Non-space before a newline, so there is no margin.
+          return marginStart;
+        }
+    }
+
+    throw new Error('unexpected EOF while looking for end of margin at offset ' + marginStart);
+  }
+
+  function getTrailingMarginStart(source, marginEnd) {
+    for (var i = marginEnd - 1; i >= 0; i--) {
+      var char = source.charAt(i);
+
+      if (char === ' ' || char === '\t') {
+        // Just part of the margin.
+      } else if (char === '\n') {
+          if (source.charAt(i - '\n'.length) === '\r') {
+            // Starts with \r\n.
+            return i - '\r'.length;
+          } else {
+            // Only ends with \n.
+            return i;
+          }
+        } else if (char === '\r') {
+          // Start of the margin.
+          return i;
+        } else {
+          // Non-space before the ending, so there is no margin.
+          return marginEnd;
+        }
+    }
+
+    throw new Error('unexpected SOF while looking for stat of margin at offset ' + marginEnd);
+  }
+
+  function getIndentInfoForRanges(source, ranges) {
+    var sharedIndent = null;
+    var isAtStartOfLine = true;
+    var indexes = [];
+
+    ranges.forEach(function (_ref3) {
+      var start = _ref3.start;
+      var end = _ref3.end;
+
+      for (var i = start; i < end; i++) {
+        var char = source[i];
+
+        if (char === '\n' || char === '\r') {
+          isAtStartOfLine = true;
+        } else if (isAtStartOfLine) {
+          isAtStartOfLine = false;
+          indexes.push(i);
+
+          var endOfPossibleIndent = sharedIndent ? i + sharedIndent.length : end;
+
+          for (var j = i; j < endOfPossibleIndent; j++) {
+            var maybeIndentChar = source[j];
+
+            if (sharedIndent) {
+              if (maybeIndentChar === sharedIndent[j - i]) {
+                // Matches the known indent so far.
+                continue;
+              }
+            } else if (maybeIndentChar === ' ' || maybeIndentChar === '\t') {
+              // Looks like a valid indent character.
+              continue;
+            }
+
+            // Set or shrink the known indent.
+            sharedIndent = source.slice(i, j);
+            i = j;
+            break;
+          }
+        }
+      }
+    });
+
+    if (sharedIndent === null) {
+      return { sharedIndent: '', indexes: [] };
+    }
+
+    return { sharedIndent: sharedIndent, indexes: indexes };
+  }
+
   /**
@@ -632,2 +984,3 @@ * Generate a list of tokens from CoffeeScript source code.
     do {
+      pending.unshift.apply(pending, babelHelpers.toConsumableArray(tripleQuotedStringSourceLocations(source, pending)));
       pending.unshift.apply(pending, babelHelpers.toConsumableArray(stringLocationsFromStream(pending)));
@@ -645,3 +998,3 @@ pending.unshift.apply(pending, babelHelpers.toConsumableArray(combinedLocationsForMultiwordOperators(pending, source)));
   function stringLocationsFromStream(stream) {
-    var startOfStringInterpolation = stream.hasNext(DSTRING, INTERPOLATION_START) || stream.hasNext(TDSTRING, INTERPOLATION_START);
+    var startOfStringInterpolation = stream.hasNext(DSTRING, INTERPOLATION_START);
 
@@ -654,3 +1007,3 @@ if (!startOfStringInterpolation) {
     var first = stream.shift();
-    var quote = first.type === DSTRING ? '"' : '"""';
+    var quote = '"';
 
@@ -728,52 +1081,2 @@ result.push(
 
-  var BufferedStream = function () {
-    function BufferedStream(stream) {
-      babelHelpers.classCallCheck(this, BufferedStream);
-      this.pending = [];
-
-      this._getNextLocation = stream;
-    }
-
-    babelHelpers.createClass(BufferedStream, [{
-      key: 'shift',
-      value: function shift() {
-        return this.pending.shift() || this._getNextLocation();
-      }
-    }, {
-      key: 'hasNext',
-      value: function hasNext() {
-        var _this = this;
-
-        var locationsToPutBack = [];
-
-        for (var _len = arguments.length, types = Array(_len), _key = 0; _key < _len; _key++) {
-          types[_key] = arguments[_key];
-        }
-
-        var result = types.every(function (type) {
-          var next = _this.shift();
-          locationsToPutBack.push(next);
-          return next.type === type;
-        });
-        this.unshift.apply(this, locationsToPutBack);
-        return result;
-      }
-    }, {
-      key: 'peek',
-      value: function peek() {
-        var result = this.shift();
-        this.unshift(result);
-        return result;
-      }
-    }, {
-      key: 'unshift',
-      value: function unshift() {
-        var _pending;
-
-        (_pending = this.pending).unshift.apply(_pending, arguments);
-      }
-    }]);
-    return BufferedStream;
-  }();
-
   var REGEXP_FLAGS = ['i', 'g', 'm', 'y'];
@@ -845,2 +1148,4 @@
   var IDENTIFIER = new SourceType('IDENTIFIER');
+  var YIELD = new SourceType('YIELD');
+  var YIELDFROM = new SourceType('YIELDFROM');
 
@@ -859,2 +1164,3 @@ /**
   var REGEXP_PATTERN = /^\/(?!\/)((?:[^[\/\n\\]|\\[^\n]|\[(?:\\[^\n]|[^\]\n\\])*\])*)(\/)?/;
+  var YIELDFROM_PATTERN = /^yield[^\n\r\S]+from/;
 
@@ -953,2 +1259,4 @@ var OPERATORS = [
           case DO:
+          case YIELD:
+          case YIELDFROM:
           case CONTINUATION:
@@ -1039,2 +1347,4 @@ if (consume(SPACE_PATTERN)) {
               }
+            } else if (consume(YIELDFROM_PATTERN)) {
+              setType(YIELDFROM);
             } else if (consume(IDENTIFIER_PATTERN)) {
@@ -1159,2 +1469,6 @@ var prev = locations[locations.length - 1];
 
+                  case 'yield':
+                    setType(YIELD);
+                    break;
+
                   default:
@@ -1451,2 +1765,4 @@ setType(IDENTIFIER);
   exports.IDENTIFIER = IDENTIFIER;
+  exports.YIELD = YIELD;
+  exports.YIELDFROM = YIELDFROM;
   exports.stream = stream;
@@ -1453,0 +1769,0 @@ exports.consumeStream = consumeStream;

package.json

 {
   "name": "coffee-lex",
-  "version": "1.3.11",
+  "version": "1.4.0",
   "description": "Stupid lexer for CoffeeScript.",
@@ -5,0 +5,0 @@ "main": "dist/coffee-lex.umd.js",

No alert changes

Improved metrics

Total package byte prevSize

103447

increased by21.04%

Lines of code

2951

increased by20.99%

No dependency changes