Merge pull request #3149 from marc1006/issue_3148

Update `markedjs` package

.meteor/versions

@@ -191,7 +191,7 @@ webapp-hashing@1.0.9
 wekan-accounts-cas@0.1.0
 wekan-accounts-oidc@1.0.10
 wekan-ldap@0.0.2
-wekan-markdown@1.0.7
+wekan-markdown@1.0.8
 wekan-oidc@1.0.12
 wekan-scrollbar@3.1.3
 yasaricli:slugify@0.0.7

packages/markdown/marked/docs/USING_ADVANCED.md

@@ -43,7 +43,7 @@ console.log(marked(markdownString));
 |Member      |Type      |Default  |Since    |Notes         |
 |:-----------|:---------|:--------|:--------|:-------------|
 |baseUrl     |`string`  |`null`   |0.3.9    |A prefix url for any relative link. |
-|breaks      |`boolean` |`false`  |v0.2.7   |If true, add `<br>` on a single line break (copies GitHub). Requires `gfm` be `true`.|
+|breaks      |`boolean` |`false`  |v0.2.7   |If true, add `<br>` on a single line break (copies GitHub behavior on comments, but not on rendered markdown files). Requires `gfm` be `true`.|
 |gfm         |`boolean` |`true`   |v0.2.1   |If true, use approved [GitHub Flavored Markdown (GFM) specification](https://github.github.com/gfm/).|
 |headerIds   |`boolean` |`true`   |v0.4.0   |If true, include an `id` attribute when emitting headings (h1, h2, h3, etc).|
 |headerPrefix|`string`  |`''`     |v0.3.0   |A string to prefix the `id` attribute when emitting headings (h1, h2, h3, etc).|
@@ -57,6 +57,8 @@ console.log(marked(markdownString));
 |silent      |`boolean` |`false`  |v0.2.7   |If true, the parser does not throw any exception.|
 |smartLists  |`boolean` |`false`  |v0.2.8   |If true, use smarter list behavior than those found in `markdown.pl`.|
 |smartypants |`boolean` |`false`  |v0.2.9   |If true, use "smart" typographic punctuation for things like quotes and dashes.|
+|tokenizer    |`object`  |`new Tokenizer()`|v1.0.0|An object containing functions to create tokens from markdown. See [extensibility](/#/USING_PRO.md) for more details.|
+|walkTokens   |`function`  |`null`|v1.1.0|A function which is called for every token. See [extensibility](/#/USING_PRO.md) for more details.|
 |xhtml       |`boolean` |`false`  |v0.3.2   |If true, emit self-closing HTML tags for void elements (&lt;br/&gt;, &lt;img/&gt;, etc.) with a "/" as required by XHTML.|
 
 <h2 id="highlight">Asynchronous highlighting</h2>

packages/markdown/marked/docs/USING_PRO.md

@@ -2,9 +2,21 @@
 
 To champion the single-responsibility and open/closed principles, we have tried to make it relatively painless to extend marked. If you are looking to add custom functionality, this is the place to start.
 
+<h2 id="use">marked.use()</h2>
+
+`marked.use(options)` is the recommended way to extend marked. The options object can contain any [option](#/USING_ADVANCED.md#options) available in marked.
+
+The `renderer` and `tokenizer` options can be an object with functions that will be merged into the `renderer` and `tokenizer` respectively.
+
+The `renderer` and `tokenizer` functions can return false to fallback to the previous function.
+
+The `walkTokens` option can be a function that will be called with every token before rendering. When calling `use` multiple times with different `walkTokens` functions each function will be called in the **reverse** order in which they were assigned.
+
+All other options will overwrite previously set options.
+
 <h2 id="renderer">The renderer</h2>
 
-The renderer is...
+The renderer defines the output of the parser.
 
 **Example:** Overriding default heading token by adding an embedded anchor tag like on GitHub.
 
@@ -12,24 +24,25 @@ The renderer is...
 // Create reference instance
 const marked = require('marked');
 
-// Get reference
-const renderer = new marked.Renderer();
-
 // Override function
-renderer.heading = function (text, level) {
-  const escapedText = text.toLowerCase().replace(/[^\w]+/g, '-');
-
-  return `
-          <h${level}>
-            <a name="${escapedText}" class="anchor" href="#${escapedText}">
-              <span class="header-link"></span>
-            </a>
-            ${text}
-          </h${level}>`;
+const renderer = {
+  heading(text, level) {
+    const escapedText = text.toLowerCase().replace(/[^\w]+/g, '-');
+
+    return `
+            <h${level}>
+              <a name="${escapedText}" class="anchor" href="#${escapedText}">
+                <span class="header-link"></span>
+              </a>
+              ${text}
+            </h${level}>`;
+  }
 };
 
+marked.use({ renderer });
+
 // Run marked
-console.log(marked('# heading+', { renderer: renderer }));
+console.log(marked('# heading+'));
 ```
 
 **Output:**
@@ -58,7 +71,7 @@ console.log(marked('# heading+', { renderer: renderer }));
 - tablerow(*string* content)
 - tablecell(*string* content, *object* flags)
 
-`slugger` has the `slug` method to create an unique id from value:
+`slugger` has the `slug` method to create a unique id from value:
 
 ```js
 slugger.slug('foo')   // foo
@@ -89,14 +102,130 @@ slugger.slug('foo-1') // foo-1-2
 - image(*string* href, *string* title, *string* text)
 - text(*string* text)
 
-<h2 id="lexer">The lexer</h2>
+<h2 id="tokenizer">The tokenizer</h2>
+
+The tokenizer defines how to turn markdown text into tokens.
+
+**Example:** Overriding default `codespan` tokenizer to include LaTeX.
+
+```js
+// Create reference instance
+const marked = require('marked');
 
-The lexer is...
+// Override function
+const tokenizer = {
+  codespan(src) {
+    const match = src.match(/\$+([^\$\n]+?)\$+/);
+    if (match) {
+      return {
+        type: 'codespan',
+        raw: match[0],
+        text: match[1].trim()
+      };
+    }
+
+    // return false to use original codespan tokenizer
+    return false;
+  }
+};
+
+marked.use({ tokenizer });
+
+// Run marked
+console.log(marked('$ latex code $\n\n` other code `'));
+```
+
+**Output:**
+
+```html
+<p><code>latex code</code></p>
+<p><code>other code</code></p>
+```
+
+### Block level tokenizer methods
+
+- space(*string* src)
+- code(*string* src, *array* tokens)
+- fences(*string* src)
+- heading(*string* src)
+- nptable(*string* src)
+- hr(*string* src)
+- blockquote(*string* src)
+- list(*string* src)
+- html(*string* src)
+- def(*string* src)
+- table(*string* src)
+- lheading(*string* src)
+- paragraph(*string* src)
+- text(*string* src, *array* tokens)
+
+### Inline level tokenizer methods
+
+- escape(*string* src)
+- tag(*string* src, *bool* inLink, *bool* inRawBlock)
+- link(*string* src)
+- reflink(*string* src, *object* links)
+- strong(*string* src)
+- em(*string* src)
+- codespan(*string* src)
+- br(*string* src)
+- del(*string* src)
+- autolink(*string* src, *function* mangle)
+- url(*string* src, *function* mangle)
+- inlineText(*string* src, *bool* inRawBlock, *function* smartypants)
+
+`mangle` is a method that changes text to HTML character references:
+
+```js
+mangle('test@example.com')
+// "&#x74;&#101;&#x73;&#116;&#x40;&#101;&#120;&#x61;&#x6d;&#112;&#108;&#101;&#46;&#x63;&#111;&#x6d;"
+```
 
+`smartypants` is a method that translates plain ASCII punctuation characters into “smart” typographic punctuation HTML entities:
+
+https://daringfireball.net/projects/smartypants/
+
+```js
+smartypants('"this ... string"')
+// "“this … string”"
+```
+
+<h2 id="walk-tokens">Walk Tokens</h2>
+
+The walkTokens function gets called with every token. Child tokens are called before moving on to sibling tokens. Each token is passed by reference so updates are persisted when passed to the parser. The return value of the function is ignored.
+
+**Example:** Overriding heading tokens to start at h2.
+
+```js
+const marked = require('marked');
+
+// Override function
+const walkTokens = (token) => {
+  if (token.type === 'heading') {
+    token.depth += 1;
+  }
+};
+
+marked.use({ walkTokens });
+
+// Run marked
+console.log(marked('# heading 2\n\n## heading 3'));
+```
+
+**Output:**
+
+```html
+<h2 id="heading-2">heading 2</h2>
+<h3 id="heading-3">heading 3</h3>
+```
+
+<h2 id="lexer">The lexer</h2>
+
+The lexer takes a markdown string and calls the tokenizer functions.
 
 <h2 id="parser">The parser</h2>
 
-The parser is...
+The parser takes tokens as input and calls the renderer functions.
 
 ***
 
@@ -105,30 +234,48 @@ The parser is...
 You also have direct access to the lexer and parser if you so desire.
 
 ``` js
-const tokens = marked.lexer(text, options);
+const tokens = marked.lexer(markdown, options);
 console.log(marked.parser(tokens, options));
 ```
 
 ``` js
 const lexer = new marked.Lexer(options);
-const tokens = lexer.lex(text);
+const tokens = lexer.lex(markdown);
 console.log(tokens);
-console.log(lexer.rules);
+console.log(lexer.tokenizer.rules.block); // block level rules used
+console.log(lexer.tokenizer.rules.inline); // inline level rules used
+console.log(marked.Lexer.rules.block); // all block level rules
+console.log(marked.Lexer.rules.inline); // all inline level rules
 ```
 
 ``` bash
 $ node
-> require('marked').lexer('> i am using marked.')
-[ { type: 'blockquote_start' },
-  { type: 'paragraph',
-    text: 'i am using marked.' },
-  { type: 'blockquote_end' },
-  links: {} ]
+> require('marked').lexer('> I am using marked.')
+[
+  {
+    type: "blockquote",
+    raw: "> I am using marked.",
+    tokens: [
+      {
+        type: "paragraph",
+        raw: "I am using marked.",
+        text: "I am using marked.",
+        tokens: [
+          {
+            type: "text",
+            raw: "I am using marked.",
+            text: "I am using marked."
+          }
+        ]
+      }
+    ]
+  },
+  links: {}
+]
 ```
 
-The Lexers build an array of tokens, which will be passed to their respective
-Parsers. The Parsers process each token in the token arrays,
-which are removed from the array of tokens:
+The Lexer builds an array of tokens, which will be passed to the Parser.
+The Parser processes each token in the token array:
 
 ``` js
 const marked = require('marked');
@@ -146,18 +293,60 @@ console.log(tokens);
 
 const html = marked.parser(tokens);
 console.log(html);
-
-console.log(tokens);
 ```
 
 ``` bash
-[ { type: 'heading', depth: 1, text: 'heading' },
-  { type: 'paragraph', text: '  [link][1]' },
-  { type: 'space' },
-  links: { '1': { href: '#heading', title: 'heading' } } ]
-
+[
+  {
+    type: "heading",
+    raw: "  # heading\n\n",
+    depth: 1,
+    text: "heading",
+    tokens: [
+      {
+        type: "text",
+        raw: "heading",
+        text: "heading"
+      }
+    ]
+  },
+  {
+    type: "paragraph",
+    raw: "  [link][1]",
+    text: "  [link][1]",
+    tokens: [
+      {
+        type: "text",
+        raw: "  ",
+        text: "  "
+      },
+      {
+        type: "link",
+        raw: "[link][1]",
+        text: "link",
+        href: "#heading",
+        title: "heading",
+        tokens: [
+          {
+            type: "text",
+            raw: "link",
+            text: "link"
+          }
+        ]
+      }
+    ]
+  },
+  {
+    type: "space",
+    raw: "\n\n"
+  },
+  links: {
+    "1": {
+      href: "#heading",
+      title: "heading"
+    }
+  }
+]
 <h1 id="heading">heading</h1>
 <p>  <a href="#heading" title="heading">link</a></p>
-
-[ links: { '1': { href: '#heading', title: 'heading' } } ]
 ```

packages/markdown/marked/docs/demo/demo.js

@@ -183,7 +183,7 @@ function handleIframeLoad() {
 
 function handleInput() {
   inputDirty = true;
-};
+}
 
 function handleVersionChange() {
   if ($markedVerElem.value === 'commit' || $markedVerElem.value === 'pr') {
@@ -256,7 +256,7 @@ function handleChange(panes, visiblePane) {
     }
   }
   return active;
-};
+}
 
 function addCommitVersion(value, text, commit) {
   if (markedVersions[value]) {
@@ -331,13 +331,13 @@ function jsonString(input) {
     .replace(/[\\"']/g, '\\$&')
     .replace(/\u0000/g, '\\0');
   return '"' + output + '"';
-};
+}
 
 function getScrollSize() {
   var e = $activeOutputElem;
 
   return e.scrollHeight - e.clientHeight;
-};
+}
 
 function getScrollPercent() {
   var size = getScrollSize();
@@ -347,11 +347,11 @@ function getScrollPercent() {
   }
 
   return $activeOutputElem.scrollTop / size;
-};
+}
 
 function setScrollPercent(percent) {
   $activeOutputElem.scrollTop = percent * getScrollSize();
-};
+}
 
 function updateLink() {
   var outputType = '';
@@ -446,7 +446,7 @@ function checkForChanges() {
     }
   }
   checkChangeTimeout = window.setTimeout(checkForChanges, delayTime);
-};
+}
 
 function setResponseTime(ms) {
   var amount = ms;

packages/markdown/marked/docs/demo/worker.js

@@ -1,4 +1,5 @@
 /* globals marked, unfetch, ES6Promise, Promise */ // eslint-disable-line no-redeclare
+
 if (!self.Promise) {
   self.importScripts('https://cdn.jsdelivr.net/npm/es6-promise/dist/es6-promise.js');
   self.Promise = ES6Promise;
@@ -48,38 +49,55 @@ function parse(e) {
     case 'parse':
       var startTime = new Date();
       var lexed = marked.lexer(e.data.markdown, e.data.options);
-      var lexedList = [];
-      for (var i = 0; i < lexed.length; i++) {
-        var lexedLine = [];
-        for (var j in lexed[i]) {
-          lexedLine.push(j + ':' + jsonString(lexed[i][j]));
-        }
-        lexedList.push('{' + lexedLine.join(', ') + '}');
-      }
+      var lexedList = jsonString(lexed);
       var parsed = marked.parser(lexed, e.data.options);
       var endTime = new Date();
-      // setTimeout(function () {
       postMessage({
         task: e.data.task,
-        lexed: lexedList.join('\n'),
+        lexed: lexedList,
         parsed: parsed,
         time: endTime - startTime
       });
-      // }, 10000);
       break;
   }
 }
 
-function jsonString(input) {
-  var output = (input + '')
-    .replace(/\n/g, '\\n')
-    .replace(/\r/g, '\\r')
-    .replace(/\t/g, '\\t')
-    .replace(/\f/g, '\\f')
-    .replace(/[\\"']/g, '\\$&')
-    .replace(/\u0000/g, '\\0');
-  return '"' + output + '"';
-};
+function stringRepeat(char, times) {
+  var s = '';
+  for (var i = 0; i < times; i++) {
+    s += char;
+  }
+  return s;
+}
+
+function jsonString(input, level) {
+  level = level || 0;
+  if (Array.isArray(input)) {
+    if (input.length === 0) {
+      return '[]';
+    }
+    var items = [],
+        i;
+    if (!Array.isArray(input[0]) && typeof input[0] === 'object' && input[0] !== null) {
+      for (i = 0; i < input.length; i++) {
+        items.push(stringRepeat(' ', 2 * level) + jsonString(input[i], level + 1));
+      }
+      return '[\n' + items.join('\n') + '\n]';
+    }
+    for (i = 0; i < input.length; i++) {
+      items.push(jsonString(input[i], level));
+    }
+    return '[' + items.join(', ') + ']';
+  } else if (typeof input === 'object' && input !== null) {
+    var props = [];
+    for (var prop in input) {
+      props.push(prop + ':' + jsonString(input[prop], level));
+    }
+    return '{' + props.join(', ') + '}';
+  } else {
+    return JSON.stringify(input);
+  }
+}
 
 function loadVersion(ver) {
   var promise;

packages/markdown/marked/docs/index.html

@@ -154,7 +154,10 @@
                     <li>
                         <a href="#/USING_PRO.md">Extensibility</a>
                         <ul>
+                            <li><a href="#/USING_PRO.md#use">marked.use()</a></li>
                             <li><a href="#/USING_PRO.md#renderer">Renderer</a></li>
+                            <li><a href="#/USING_PRO.md#tokenizer">Tokenizer</a></li>
+                            <li><a href="#/USING_PRO.md#walk-tokens">Walk Tokens</a></li>
                             <li><a href="#/USING_PRO.md#lexer">Lexer</a></li>
                             <li><a href="#/USING_PRO.md#parser">Parser</a></li>
                         </ul>

packages/markdown/marked/package.json

@@ -2,8 +2,9 @@
   "name": "marked",
   "description": "A markdown parser built for speed",
   "author": "Christopher Jeffrey",
-  "version": "0.8.0",
+  "version": "1.1.0",
   "main": "./src/marked.js",
+  "browser": "./lib/marked.js",
   "bin": "./bin/marked",
   "man": "./man/marked.1",
   "files": [
@@ -30,27 +31,27 @@
     "html"
   ],
   "devDependencies": {
-    "@babel/core": "^7.8.7",
-    "@babel/preset-env": "^7.8.7",
-    "@markedjs/html-differ": "^3.0.0",
-    "cheerio": "^0.22.0",
-    "commonmark": "^0.29.1",
-    "eslint": "^6.8.0",
-    "eslint-config-standard": "^14.1.0",
-    "eslint-plugin-import": "^2.20.1",
-    "eslint-plugin-node": "^11.0.0",
+    "@babel/core": "^7.9.6",
+    "@babel/preset-env": "^7.9.6",
+    "@markedjs/html-differ": "^3.0.2",
+    "cheerio": "^1.0.0-rc.3",
+    "commonmark": "0.29.x",
+    "eslint": "^7.0.0",
+    "eslint-config-standard": "^14.1.1",
+    "eslint-plugin-import": "^2.20.2",
+    "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^4.2.1",
     "eslint-plugin-standard": "^4.0.1",
-    "front-matter": "^3.1.0",
+    "front-matter": "^3.2.1",
     "jasmine": "^3.5.0",
-    "markdown": "^0.5.0",
-    "markdown-it": "^10.0.0",
+    "markdown": "0.5.x",
+    "markdown-it": "10.x",
     "node-fetch": "^2.6.0",
-    "rollup": "^2.0.6",
+    "rollup": "^2.10.2",
     "rollup-plugin-babel": "^4.4.0",
     "rollup-plugin-commonjs": "^10.1.0",
-    "rollup-plugin-license": "^0.13.0",
-    "uglify-js": "^3.8.0",
+    "rollup-plugin-license": "^2.0.1",
+    "uglify-js": "^3.9.3",
     "vuln-regex-detector": "^1.3.0"
   },
   "scripts": {
@@ -69,7 +70,7 @@
     "rollup:umd": "rollup -c rollup.config.js",
     "rollup:esm": "rollup -c rollup.config.esm.js",
     "minify": "uglifyjs lib/marked.js -cm  --comments /Copyright/ -o marked.min.js",
-    "preversion": "npm run build && (git diff --quiet || git commit -am 'build')"
+    "preversion": "npm run build && (git diff --quiet || git commit -am build)"
   },
   "engines": {
     "node": ">= 8.16.2"

packages/markdown/package.js

@@ -3,7 +3,7 @@
 Package.describe({
 	name: 'wekan-markdown',
 	summary: "GitHub flavored markdown parser for Meteor based on marked.js",
-	version: "1.0.7",
+	version: "1.0.8",
 	git: "https://github.com/wekan/markdown.git"
 });