tokenizer.js 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630
  1. "use strict";
  2. // https://www.w3.org/TR/css-syntax-3
  3. Object.defineProperty(exports, "__esModule", { value: true });
  4. exports.Tokenizer = exports.EOF_TOKEN = exports.FLAG_NUMBER = exports.FLAG_INTEGER = exports.FLAG_ID = exports.FLAG_UNRESTRICTED = void 0;
  5. var css_line_break_1 = require("css-line-break");
  6. exports.FLAG_UNRESTRICTED = 1 << 0;
  7. exports.FLAG_ID = 1 << 1;
  8. exports.FLAG_INTEGER = 1 << 2;
  9. exports.FLAG_NUMBER = 1 << 3;
  10. var LINE_FEED = 0x000a;
  11. var SOLIDUS = 0x002f;
  12. var REVERSE_SOLIDUS = 0x005c;
  13. var CHARACTER_TABULATION = 0x0009;
  14. var SPACE = 0x0020;
  15. var QUOTATION_MARK = 0x0022;
  16. var EQUALS_SIGN = 0x003d;
  17. var NUMBER_SIGN = 0x0023;
  18. var DOLLAR_SIGN = 0x0024;
  19. var PERCENTAGE_SIGN = 0x0025;
  20. var APOSTROPHE = 0x0027;
  21. var LEFT_PARENTHESIS = 0x0028;
  22. var RIGHT_PARENTHESIS = 0x0029;
  23. var LOW_LINE = 0x005f;
  24. var HYPHEN_MINUS = 0x002d;
  25. var EXCLAMATION_MARK = 0x0021;
  26. var LESS_THAN_SIGN = 0x003c;
  27. var GREATER_THAN_SIGN = 0x003e;
  28. var COMMERCIAL_AT = 0x0040;
  29. var LEFT_SQUARE_BRACKET = 0x005b;
  30. var RIGHT_SQUARE_BRACKET = 0x005d;
  31. var CIRCUMFLEX_ACCENT = 0x003d;
  32. var LEFT_CURLY_BRACKET = 0x007b;
  33. var QUESTION_MARK = 0x003f;
  34. var RIGHT_CURLY_BRACKET = 0x007d;
  35. var VERTICAL_LINE = 0x007c;
  36. var TILDE = 0x007e;
  37. var CONTROL = 0x0080;
  38. var REPLACEMENT_CHARACTER = 0xfffd;
  39. var ASTERISK = 0x002a;
  40. var PLUS_SIGN = 0x002b;
  41. var COMMA = 0x002c;
  42. var COLON = 0x003a;
  43. var SEMICOLON = 0x003b;
  44. var FULL_STOP = 0x002e;
  45. var NULL = 0x0000;
  46. var BACKSPACE = 0x0008;
  47. var LINE_TABULATION = 0x000b;
  48. var SHIFT_OUT = 0x000e;
  49. var INFORMATION_SEPARATOR_ONE = 0x001f;
  50. var DELETE = 0x007f;
  51. var EOF = -1;
  52. var ZERO = 0x0030;
  53. var a = 0x0061;
  54. var e = 0x0065;
  55. var f = 0x0066;
  56. var u = 0x0075;
  57. var z = 0x007a;
  58. var A = 0x0041;
  59. var E = 0x0045;
  60. var F = 0x0046;
  61. var U = 0x0055;
  62. var Z = 0x005a;
  63. var isDigit = function (codePoint) { return codePoint >= ZERO && codePoint <= 0x0039; };
  64. var isSurrogateCodePoint = function (codePoint) { return codePoint >= 0xd800 && codePoint <= 0xdfff; };
  65. var isHex = function (codePoint) {
  66. return isDigit(codePoint) || (codePoint >= A && codePoint <= F) || (codePoint >= a && codePoint <= f);
  67. };
  68. var isLowerCaseLetter = function (codePoint) { return codePoint >= a && codePoint <= z; };
  69. var isUpperCaseLetter = function (codePoint) { return codePoint >= A && codePoint <= Z; };
  70. var isLetter = function (codePoint) { return isLowerCaseLetter(codePoint) || isUpperCaseLetter(codePoint); };
  71. var isNonASCIICodePoint = function (codePoint) { return codePoint >= CONTROL; };
  72. var isWhiteSpace = function (codePoint) {
  73. return codePoint === LINE_FEED || codePoint === CHARACTER_TABULATION || codePoint === SPACE;
  74. };
  75. var isNameStartCodePoint = function (codePoint) {
  76. return isLetter(codePoint) || isNonASCIICodePoint(codePoint) || codePoint === LOW_LINE;
  77. };
  78. var isNameCodePoint = function (codePoint) {
  79. return isNameStartCodePoint(codePoint) || isDigit(codePoint) || codePoint === HYPHEN_MINUS;
  80. };
  81. var isNonPrintableCodePoint = function (codePoint) {
  82. return ((codePoint >= NULL && codePoint <= BACKSPACE) ||
  83. codePoint === LINE_TABULATION ||
  84. (codePoint >= SHIFT_OUT && codePoint <= INFORMATION_SEPARATOR_ONE) ||
  85. codePoint === DELETE);
  86. };
  87. var isValidEscape = function (c1, c2) {
  88. if (c1 !== REVERSE_SOLIDUS) {
  89. return false;
  90. }
  91. return c2 !== LINE_FEED;
  92. };
  93. var isIdentifierStart = function (c1, c2, c3) {
  94. if (c1 === HYPHEN_MINUS) {
  95. return isNameStartCodePoint(c2) || isValidEscape(c2, c3);
  96. }
  97. else if (isNameStartCodePoint(c1)) {
  98. return true;
  99. }
  100. else if (c1 === REVERSE_SOLIDUS && isValidEscape(c1, c2)) {
  101. return true;
  102. }
  103. return false;
  104. };
  105. var isNumberStart = function (c1, c2, c3) {
  106. if (c1 === PLUS_SIGN || c1 === HYPHEN_MINUS) {
  107. if (isDigit(c2)) {
  108. return true;
  109. }
  110. return c2 === FULL_STOP && isDigit(c3);
  111. }
  112. if (c1 === FULL_STOP) {
  113. return isDigit(c2);
  114. }
  115. return isDigit(c1);
  116. };
  117. var stringToNumber = function (codePoints) {
  118. var c = 0;
  119. var sign = 1;
  120. if (codePoints[c] === PLUS_SIGN || codePoints[c] === HYPHEN_MINUS) {
  121. if (codePoints[c] === HYPHEN_MINUS) {
  122. sign = -1;
  123. }
  124. c++;
  125. }
  126. var integers = [];
  127. while (isDigit(codePoints[c])) {
  128. integers.push(codePoints[c++]);
  129. }
  130. var int = integers.length ? parseInt(css_line_break_1.fromCodePoint.apply(void 0, integers), 10) : 0;
  131. if (codePoints[c] === FULL_STOP) {
  132. c++;
  133. }
  134. var fraction = [];
  135. while (isDigit(codePoints[c])) {
  136. fraction.push(codePoints[c++]);
  137. }
  138. var fracd = fraction.length;
  139. var frac = fracd ? parseInt(css_line_break_1.fromCodePoint.apply(void 0, fraction), 10) : 0;
  140. if (codePoints[c] === E || codePoints[c] === e) {
  141. c++;
  142. }
  143. var expsign = 1;
  144. if (codePoints[c] === PLUS_SIGN || codePoints[c] === HYPHEN_MINUS) {
  145. if (codePoints[c] === HYPHEN_MINUS) {
  146. expsign = -1;
  147. }
  148. c++;
  149. }
  150. var exponent = [];
  151. while (isDigit(codePoints[c])) {
  152. exponent.push(codePoints[c++]);
  153. }
  154. var exp = exponent.length ? parseInt(css_line_break_1.fromCodePoint.apply(void 0, exponent), 10) : 0;
  155. return sign * (int + frac * Math.pow(10, -fracd)) * Math.pow(10, expsign * exp);
  156. };
  157. var LEFT_PARENTHESIS_TOKEN = {
  158. type: 2 /* LEFT_PARENTHESIS_TOKEN */
  159. };
  160. var RIGHT_PARENTHESIS_TOKEN = {
  161. type: 3 /* RIGHT_PARENTHESIS_TOKEN */
  162. };
  163. var COMMA_TOKEN = { type: 4 /* COMMA_TOKEN */ };
  164. var SUFFIX_MATCH_TOKEN = { type: 13 /* SUFFIX_MATCH_TOKEN */ };
  165. var PREFIX_MATCH_TOKEN = { type: 8 /* PREFIX_MATCH_TOKEN */ };
  166. var COLUMN_TOKEN = { type: 21 /* COLUMN_TOKEN */ };
  167. var DASH_MATCH_TOKEN = { type: 9 /* DASH_MATCH_TOKEN */ };
  168. var INCLUDE_MATCH_TOKEN = { type: 10 /* INCLUDE_MATCH_TOKEN */ };
  169. var LEFT_CURLY_BRACKET_TOKEN = {
  170. type: 11 /* LEFT_CURLY_BRACKET_TOKEN */
  171. };
  172. var RIGHT_CURLY_BRACKET_TOKEN = {
  173. type: 12 /* RIGHT_CURLY_BRACKET_TOKEN */
  174. };
  175. var SUBSTRING_MATCH_TOKEN = { type: 14 /* SUBSTRING_MATCH_TOKEN */ };
  176. var BAD_URL_TOKEN = { type: 23 /* BAD_URL_TOKEN */ };
  177. var BAD_STRING_TOKEN = { type: 1 /* BAD_STRING_TOKEN */ };
  178. var CDO_TOKEN = { type: 25 /* CDO_TOKEN */ };
  179. var CDC_TOKEN = { type: 24 /* CDC_TOKEN */ };
  180. var COLON_TOKEN = { type: 26 /* COLON_TOKEN */ };
  181. var SEMICOLON_TOKEN = { type: 27 /* SEMICOLON_TOKEN */ };
  182. var LEFT_SQUARE_BRACKET_TOKEN = {
  183. type: 28 /* LEFT_SQUARE_BRACKET_TOKEN */
  184. };
  185. var RIGHT_SQUARE_BRACKET_TOKEN = {
  186. type: 29 /* RIGHT_SQUARE_BRACKET_TOKEN */
  187. };
  188. var WHITESPACE_TOKEN = { type: 31 /* WHITESPACE_TOKEN */ };
  189. exports.EOF_TOKEN = { type: 32 /* EOF_TOKEN */ };
  190. var Tokenizer = /** @class */ (function () {
  191. function Tokenizer() {
  192. this._value = [];
  193. }
  194. Tokenizer.prototype.write = function (chunk) {
  195. this._value = this._value.concat(css_line_break_1.toCodePoints(chunk));
  196. };
  197. Tokenizer.prototype.read = function () {
  198. var tokens = [];
  199. var token = this.consumeToken();
  200. while (token !== exports.EOF_TOKEN) {
  201. tokens.push(token);
  202. token = this.consumeToken();
  203. }
  204. return tokens;
  205. };
  206. Tokenizer.prototype.consumeToken = function () {
  207. var codePoint = this.consumeCodePoint();
  208. switch (codePoint) {
  209. case QUOTATION_MARK:
  210. return this.consumeStringToken(QUOTATION_MARK);
  211. case NUMBER_SIGN:
  212. var c1 = this.peekCodePoint(0);
  213. var c2 = this.peekCodePoint(1);
  214. var c3 = this.peekCodePoint(2);
  215. if (isNameCodePoint(c1) || isValidEscape(c2, c3)) {
  216. var flags = isIdentifierStart(c1, c2, c3) ? exports.FLAG_ID : exports.FLAG_UNRESTRICTED;
  217. var value = this.consumeName();
  218. return { type: 5 /* HASH_TOKEN */, value: value, flags: flags };
  219. }
  220. break;
  221. case DOLLAR_SIGN:
  222. if (this.peekCodePoint(0) === EQUALS_SIGN) {
  223. this.consumeCodePoint();
  224. return SUFFIX_MATCH_TOKEN;
  225. }
  226. break;
  227. case APOSTROPHE:
  228. return this.consumeStringToken(APOSTROPHE);
  229. case LEFT_PARENTHESIS:
  230. return LEFT_PARENTHESIS_TOKEN;
  231. case RIGHT_PARENTHESIS:
  232. return RIGHT_PARENTHESIS_TOKEN;
  233. case ASTERISK:
  234. if (this.peekCodePoint(0) === EQUALS_SIGN) {
  235. this.consumeCodePoint();
  236. return SUBSTRING_MATCH_TOKEN;
  237. }
  238. break;
  239. case PLUS_SIGN:
  240. if (isNumberStart(codePoint, this.peekCodePoint(0), this.peekCodePoint(1))) {
  241. this.reconsumeCodePoint(codePoint);
  242. return this.consumeNumericToken();
  243. }
  244. break;
  245. case COMMA:
  246. return COMMA_TOKEN;
  247. case HYPHEN_MINUS:
  248. var e1 = codePoint;
  249. var e2 = this.peekCodePoint(0);
  250. var e3 = this.peekCodePoint(1);
  251. if (isNumberStart(e1, e2, e3)) {
  252. this.reconsumeCodePoint(codePoint);
  253. return this.consumeNumericToken();
  254. }
  255. if (isIdentifierStart(e1, e2, e3)) {
  256. this.reconsumeCodePoint(codePoint);
  257. return this.consumeIdentLikeToken();
  258. }
  259. if (e2 === HYPHEN_MINUS && e3 === GREATER_THAN_SIGN) {
  260. this.consumeCodePoint();
  261. this.consumeCodePoint();
  262. return CDC_TOKEN;
  263. }
  264. break;
  265. case FULL_STOP:
  266. if (isNumberStart(codePoint, this.peekCodePoint(0), this.peekCodePoint(1))) {
  267. this.reconsumeCodePoint(codePoint);
  268. return this.consumeNumericToken();
  269. }
  270. break;
  271. case SOLIDUS:
  272. if (this.peekCodePoint(0) === ASTERISK) {
  273. this.consumeCodePoint();
  274. while (true) {
  275. var c = this.consumeCodePoint();
  276. if (c === ASTERISK) {
  277. c = this.consumeCodePoint();
  278. if (c === SOLIDUS) {
  279. return this.consumeToken();
  280. }
  281. }
  282. if (c === EOF) {
  283. return this.consumeToken();
  284. }
  285. }
  286. }
  287. break;
  288. case COLON:
  289. return COLON_TOKEN;
  290. case SEMICOLON:
  291. return SEMICOLON_TOKEN;
  292. case LESS_THAN_SIGN:
  293. if (this.peekCodePoint(0) === EXCLAMATION_MARK &&
  294. this.peekCodePoint(1) === HYPHEN_MINUS &&
  295. this.peekCodePoint(2) === HYPHEN_MINUS) {
  296. this.consumeCodePoint();
  297. this.consumeCodePoint();
  298. return CDO_TOKEN;
  299. }
  300. break;
  301. case COMMERCIAL_AT:
  302. var a1 = this.peekCodePoint(0);
  303. var a2 = this.peekCodePoint(1);
  304. var a3 = this.peekCodePoint(2);
  305. if (isIdentifierStart(a1, a2, a3)) {
  306. var value = this.consumeName();
  307. return { type: 7 /* AT_KEYWORD_TOKEN */, value: value };
  308. }
  309. break;
  310. case LEFT_SQUARE_BRACKET:
  311. return LEFT_SQUARE_BRACKET_TOKEN;
  312. case REVERSE_SOLIDUS:
  313. if (isValidEscape(codePoint, this.peekCodePoint(0))) {
  314. this.reconsumeCodePoint(codePoint);
  315. return this.consumeIdentLikeToken();
  316. }
  317. break;
  318. case RIGHT_SQUARE_BRACKET:
  319. return RIGHT_SQUARE_BRACKET_TOKEN;
  320. case CIRCUMFLEX_ACCENT:
  321. if (this.peekCodePoint(0) === EQUALS_SIGN) {
  322. this.consumeCodePoint();
  323. return PREFIX_MATCH_TOKEN;
  324. }
  325. break;
  326. case LEFT_CURLY_BRACKET:
  327. return LEFT_CURLY_BRACKET_TOKEN;
  328. case RIGHT_CURLY_BRACKET:
  329. return RIGHT_CURLY_BRACKET_TOKEN;
  330. case u:
  331. case U:
  332. var u1 = this.peekCodePoint(0);
  333. var u2 = this.peekCodePoint(1);
  334. if (u1 === PLUS_SIGN && (isHex(u2) || u2 === QUESTION_MARK)) {
  335. this.consumeCodePoint();
  336. this.consumeUnicodeRangeToken();
  337. }
  338. this.reconsumeCodePoint(codePoint);
  339. return this.consumeIdentLikeToken();
  340. case VERTICAL_LINE:
  341. if (this.peekCodePoint(0) === EQUALS_SIGN) {
  342. this.consumeCodePoint();
  343. return DASH_MATCH_TOKEN;
  344. }
  345. if (this.peekCodePoint(0) === VERTICAL_LINE) {
  346. this.consumeCodePoint();
  347. return COLUMN_TOKEN;
  348. }
  349. break;
  350. case TILDE:
  351. if (this.peekCodePoint(0) === EQUALS_SIGN) {
  352. this.consumeCodePoint();
  353. return INCLUDE_MATCH_TOKEN;
  354. }
  355. break;
  356. case EOF:
  357. return exports.EOF_TOKEN;
  358. }
  359. if (isWhiteSpace(codePoint)) {
  360. this.consumeWhiteSpace();
  361. return WHITESPACE_TOKEN;
  362. }
  363. if (isDigit(codePoint)) {
  364. this.reconsumeCodePoint(codePoint);
  365. return this.consumeNumericToken();
  366. }
  367. if (isNameStartCodePoint(codePoint)) {
  368. this.reconsumeCodePoint(codePoint);
  369. return this.consumeIdentLikeToken();
  370. }
  371. return { type: 6 /* DELIM_TOKEN */, value: css_line_break_1.fromCodePoint(codePoint) };
  372. };
  373. Tokenizer.prototype.consumeCodePoint = function () {
  374. var value = this._value.shift();
  375. return typeof value === 'undefined' ? -1 : value;
  376. };
  377. Tokenizer.prototype.reconsumeCodePoint = function (codePoint) {
  378. this._value.unshift(codePoint);
  379. };
  380. Tokenizer.prototype.peekCodePoint = function (delta) {
  381. if (delta >= this._value.length) {
  382. return -1;
  383. }
  384. return this._value[delta];
  385. };
  386. Tokenizer.prototype.consumeUnicodeRangeToken = function () {
  387. var digits = [];
  388. var codePoint = this.consumeCodePoint();
  389. while (isHex(codePoint) && digits.length < 6) {
  390. digits.push(codePoint);
  391. codePoint = this.consumeCodePoint();
  392. }
  393. var questionMarks = false;
  394. while (codePoint === QUESTION_MARK && digits.length < 6) {
  395. digits.push(codePoint);
  396. codePoint = this.consumeCodePoint();
  397. questionMarks = true;
  398. }
  399. if (questionMarks) {
  400. var start_1 = parseInt(css_line_break_1.fromCodePoint.apply(void 0, digits.map(function (digit) { return (digit === QUESTION_MARK ? ZERO : digit); })), 16);
  401. var end = parseInt(css_line_break_1.fromCodePoint.apply(void 0, digits.map(function (digit) { return (digit === QUESTION_MARK ? F : digit); })), 16);
  402. return { type: 30 /* UNICODE_RANGE_TOKEN */, start: start_1, end: end };
  403. }
  404. var start = parseInt(css_line_break_1.fromCodePoint.apply(void 0, digits), 16);
  405. if (this.peekCodePoint(0) === HYPHEN_MINUS && isHex(this.peekCodePoint(1))) {
  406. this.consumeCodePoint();
  407. codePoint = this.consumeCodePoint();
  408. var endDigits = [];
  409. while (isHex(codePoint) && endDigits.length < 6) {
  410. endDigits.push(codePoint);
  411. codePoint = this.consumeCodePoint();
  412. }
  413. var end = parseInt(css_line_break_1.fromCodePoint.apply(void 0, endDigits), 16);
  414. return { type: 30 /* UNICODE_RANGE_TOKEN */, start: start, end: end };
  415. }
  416. else {
  417. return { type: 30 /* UNICODE_RANGE_TOKEN */, start: start, end: start };
  418. }
  419. };
  420. Tokenizer.prototype.consumeIdentLikeToken = function () {
  421. var value = this.consumeName();
  422. if (value.toLowerCase() === 'url' && this.peekCodePoint(0) === LEFT_PARENTHESIS) {
  423. this.consumeCodePoint();
  424. return this.consumeUrlToken();
  425. }
  426. else if (this.peekCodePoint(0) === LEFT_PARENTHESIS) {
  427. this.consumeCodePoint();
  428. return { type: 19 /* FUNCTION_TOKEN */, value: value };
  429. }
  430. return { type: 20 /* IDENT_TOKEN */, value: value };
  431. };
  432. Tokenizer.prototype.consumeUrlToken = function () {
  433. var value = [];
  434. this.consumeWhiteSpace();
  435. if (this.peekCodePoint(0) === EOF) {
  436. return { type: 22 /* URL_TOKEN */, value: '' };
  437. }
  438. var next = this.peekCodePoint(0);
  439. if (next === APOSTROPHE || next === QUOTATION_MARK) {
  440. var stringToken = this.consumeStringToken(this.consumeCodePoint());
  441. if (stringToken.type === 0 /* STRING_TOKEN */) {
  442. this.consumeWhiteSpace();
  443. if (this.peekCodePoint(0) === EOF || this.peekCodePoint(0) === RIGHT_PARENTHESIS) {
  444. this.consumeCodePoint();
  445. return { type: 22 /* URL_TOKEN */, value: stringToken.value };
  446. }
  447. }
  448. this.consumeBadUrlRemnants();
  449. return BAD_URL_TOKEN;
  450. }
  451. while (true) {
  452. var codePoint = this.consumeCodePoint();
  453. if (codePoint === EOF || codePoint === RIGHT_PARENTHESIS) {
  454. return { type: 22 /* URL_TOKEN */, value: css_line_break_1.fromCodePoint.apply(void 0, value) };
  455. }
  456. else if (isWhiteSpace(codePoint)) {
  457. this.consumeWhiteSpace();
  458. if (this.peekCodePoint(0) === EOF || this.peekCodePoint(0) === RIGHT_PARENTHESIS) {
  459. this.consumeCodePoint();
  460. return { type: 22 /* URL_TOKEN */, value: css_line_break_1.fromCodePoint.apply(void 0, value) };
  461. }
  462. this.consumeBadUrlRemnants();
  463. return BAD_URL_TOKEN;
  464. }
  465. else if (codePoint === QUOTATION_MARK ||
  466. codePoint === APOSTROPHE ||
  467. codePoint === LEFT_PARENTHESIS ||
  468. isNonPrintableCodePoint(codePoint)) {
  469. this.consumeBadUrlRemnants();
  470. return BAD_URL_TOKEN;
  471. }
  472. else if (codePoint === REVERSE_SOLIDUS) {
  473. if (isValidEscape(codePoint, this.peekCodePoint(0))) {
  474. value.push(this.consumeEscapedCodePoint());
  475. }
  476. else {
  477. this.consumeBadUrlRemnants();
  478. return BAD_URL_TOKEN;
  479. }
  480. }
  481. else {
  482. value.push(codePoint);
  483. }
  484. }
  485. };
  486. Tokenizer.prototype.consumeWhiteSpace = function () {
  487. while (isWhiteSpace(this.peekCodePoint(0))) {
  488. this.consumeCodePoint();
  489. }
  490. };
  491. Tokenizer.prototype.consumeBadUrlRemnants = function () {
  492. while (true) {
  493. var codePoint = this.consumeCodePoint();
  494. if (codePoint === RIGHT_PARENTHESIS || codePoint === EOF) {
  495. return;
  496. }
  497. if (isValidEscape(codePoint, this.peekCodePoint(0))) {
  498. this.consumeEscapedCodePoint();
  499. }
  500. }
  501. };
  502. Tokenizer.prototype.consumeStringSlice = function (count) {
  503. var SLICE_STACK_SIZE = 50000;
  504. var value = '';
  505. while (count > 0) {
  506. var amount = Math.min(SLICE_STACK_SIZE, count);
  507. value += css_line_break_1.fromCodePoint.apply(void 0, this._value.splice(0, amount));
  508. count -= amount;
  509. }
  510. this._value.shift();
  511. return value;
  512. };
  513. Tokenizer.prototype.consumeStringToken = function (endingCodePoint) {
  514. var value = '';
  515. var i = 0;
  516. do {
  517. var codePoint = this._value[i];
  518. if (codePoint === EOF || codePoint === undefined || codePoint === endingCodePoint) {
  519. value += this.consumeStringSlice(i);
  520. return { type: 0 /* STRING_TOKEN */, value: value };
  521. }
  522. if (codePoint === LINE_FEED) {
  523. this._value.splice(0, i);
  524. return BAD_STRING_TOKEN;
  525. }
  526. if (codePoint === REVERSE_SOLIDUS) {
  527. var next = this._value[i + 1];
  528. if (next !== EOF && next !== undefined) {
  529. if (next === LINE_FEED) {
  530. value += this.consumeStringSlice(i);
  531. i = -1;
  532. this._value.shift();
  533. }
  534. else if (isValidEscape(codePoint, next)) {
  535. value += this.consumeStringSlice(i);
  536. value += css_line_break_1.fromCodePoint(this.consumeEscapedCodePoint());
  537. i = -1;
  538. }
  539. }
  540. }
  541. i++;
  542. } while (true);
  543. };
  544. Tokenizer.prototype.consumeNumber = function () {
  545. var repr = [];
  546. var type = exports.FLAG_INTEGER;
  547. var c1 = this.peekCodePoint(0);
  548. if (c1 === PLUS_SIGN || c1 === HYPHEN_MINUS) {
  549. repr.push(this.consumeCodePoint());
  550. }
  551. while (isDigit(this.peekCodePoint(0))) {
  552. repr.push(this.consumeCodePoint());
  553. }
  554. c1 = this.peekCodePoint(0);
  555. var c2 = this.peekCodePoint(1);
  556. if (c1 === FULL_STOP && isDigit(c2)) {
  557. repr.push(this.consumeCodePoint(), this.consumeCodePoint());
  558. type = exports.FLAG_NUMBER;
  559. while (isDigit(this.peekCodePoint(0))) {
  560. repr.push(this.consumeCodePoint());
  561. }
  562. }
  563. c1 = this.peekCodePoint(0);
  564. c2 = this.peekCodePoint(1);
  565. var c3 = this.peekCodePoint(2);
  566. if ((c1 === E || c1 === e) && (((c2 === PLUS_SIGN || c2 === HYPHEN_MINUS) && isDigit(c3)) || isDigit(c2))) {
  567. repr.push(this.consumeCodePoint(), this.consumeCodePoint());
  568. type = exports.FLAG_NUMBER;
  569. while (isDigit(this.peekCodePoint(0))) {
  570. repr.push(this.consumeCodePoint());
  571. }
  572. }
  573. return [stringToNumber(repr), type];
  574. };
  575. Tokenizer.prototype.consumeNumericToken = function () {
  576. var _a = this.consumeNumber(), number = _a[0], flags = _a[1];
  577. var c1 = this.peekCodePoint(0);
  578. var c2 = this.peekCodePoint(1);
  579. var c3 = this.peekCodePoint(2);
  580. if (isIdentifierStart(c1, c2, c3)) {
  581. var unit = this.consumeName();
  582. return { type: 15 /* DIMENSION_TOKEN */, number: number, flags: flags, unit: unit };
  583. }
  584. if (c1 === PERCENTAGE_SIGN) {
  585. this.consumeCodePoint();
  586. return { type: 16 /* PERCENTAGE_TOKEN */, number: number, flags: flags };
  587. }
  588. return { type: 17 /* NUMBER_TOKEN */, number: number, flags: flags };
  589. };
  590. Tokenizer.prototype.consumeEscapedCodePoint = function () {
  591. var codePoint = this.consumeCodePoint();
  592. if (isHex(codePoint)) {
  593. var hex = css_line_break_1.fromCodePoint(codePoint);
  594. while (isHex(this.peekCodePoint(0)) && hex.length < 6) {
  595. hex += css_line_break_1.fromCodePoint(this.consumeCodePoint());
  596. }
  597. if (isWhiteSpace(this.peekCodePoint(0))) {
  598. this.consumeCodePoint();
  599. }
  600. var hexCodePoint = parseInt(hex, 16);
  601. if (hexCodePoint === 0 || isSurrogateCodePoint(hexCodePoint) || hexCodePoint > 0x10ffff) {
  602. return REPLACEMENT_CHARACTER;
  603. }
  604. return hexCodePoint;
  605. }
  606. if (codePoint === EOF) {
  607. return REPLACEMENT_CHARACTER;
  608. }
  609. return codePoint;
  610. };
  611. Tokenizer.prototype.consumeName = function () {
  612. var result = '';
  613. while (true) {
  614. var codePoint = this.consumeCodePoint();
  615. if (isNameCodePoint(codePoint)) {
  616. result += css_line_break_1.fromCodePoint(codePoint);
  617. }
  618. else if (isValidEscape(codePoint, this.peekCodePoint(0))) {
  619. result += css_line_break_1.fromCodePoint(this.consumeEscapedCodePoint());
  620. }
  621. else {
  622. this.reconsumeCodePoint(codePoint);
  623. return result;
  624. }
  625. }
  626. };
  627. return Tokenizer;
  628. }());
  629. exports.Tokenizer = Tokenizer;
  630. //# sourceMappingURL=tokenizer.js.map