Added CJK support + MathJax display

CHANGELOG.md

@@ -4,57 +4,60 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 ### Added
-- Configuration Wizard: Added Public Access option
-- Auth: Azure AD authentication provider is now available
-- Auth: Can now specify Read Access by default for all providers (except Local)
-- Navigation: All Pages section
+- **Auth**: Azure AD authentication provider is now available
+- **Auth**: Can now specify Read Access by default for all providers (except Local)
+- **View**: MathML and TeX math equations support
+- **Configuration Wizard**: Added Public Access option
+- **Navigation**: All Pages section
 
 ### Changed
-- Auth: Provider Strategies are now only loaded if enabled
+- **Auth**: Provider Strategies are now only loaded if enabled
 
 ### Fixed
-- UI: Scrollbar is no longer always shown in code blocks
-- Init: Malformed config file is now being reported correctly
+- **Configuration Wizard**: Git version detection no longer fails on MacOS
+- **Init**: Malformed config file is now being reported correctly
+- **UI**: Scrollbar is no longer always shown in code blocks
+- **Misc**: CJK (Chinese, Japanese & Korean) characters are now fully supported for pages, content and uploads
 
 ## [v1.0.0-beta.10] - 2017-04-08
 ### Added
-- Installation: Wiki.js can now install via local tarball
-- Installation: RAM check during install to prevent crashing due to low memory
+- **Installation**: Wiki.js can now install via local tarball
+- **Installation**: RAM check during install to prevent crashing due to low memory
 
 ### Changed
 - Updated dependencies + snyk policy
 
 ### Fixed
-- UI: Code blocks longer than page width are now displayed with scrollbars
-- Configuration Wizard: Git version check no longer fails if between 2.7.4 and 2.11.0
-- Init: Admin account is no longer attempted to be created during init
+- **UI**: Code blocks longer than page width are now displayed with scrollbars
+- **Configuration Wizard**: Git version check no longer fails if between 2.7.4 and 2.11.0
+- **Init**: Admin account is no longer attempted to be created during init
 
 ## [v1.0.0-beta.9] - 2017-04-05
 ### Added
 - Interactive setup
-- Auth: GitHub and Slack authentication providers are now available
-- Auth: LDAP authentication provider is now available
-- Logs: Support for the logging services: Bugsnag, Loggly, Papertrail, Rollbar and Sentry
-- Config: Can now use ENV variable to specify DB connection string ($VARNAME as db value in config.yml)
+- **Auth**: GitHub and Slack authentication providers are now available
+- **Auth**: LDAP authentication provider is now available
+- **Logs**: Support for the logging services: Bugsnag, Loggly, Papertrail, Rollbar and Sentry
+- **Config**: Can now use ENV variable to specify DB connection string ($VARNAME as db value in config.yml)
 
 ### Changed
-- Native Compilation Removal: Replaced farmhash with md5
-- Native Compilation Removal: Replaced leveldown with memdown
-- Native Compilation Removal: Replaced sharp with jimp
-- Sidebar: Contents is now Page Contents
-- Sidebar: Start is now Top of Page
-- UI: Content headers are now showing an anchor icon instead of a #
-- Dev: Replaced Gulp with Fuse-box
+- **Native Compilation Removal**: Replaced farmhash with md5
+- **Native Compilation Removal**: Replaced leveldown with memdown
+- **Native Compilation Removal**: Replaced sharp with jimp
+- **Sidebar**: Contents is now Page Contents
+- **Sidebar**: Start is now Top of Page
+- **UI**: Content headers are now showing an anchor icon instead of a #
+- **Dev**: Replaced Gulp with Fuse-box
 
 ### Fixed
-- Auth: Authentication would fail if email has uppercase chars and provider callback is in lowercase
-- Markdown: Fixed potential crash on markdown processing of video links
-- Search: Search index should now update upon article creation
-- Search: Search results are no longer duplicated upon article update
-- UI: Missing icons on login page
-- UI: Image alignement center and right should now behave correctly
-- Uploads: Error notification when upload is too large for server
-- Uploads: Fix uploads and temp-uploads folder permissions on unix-based systems
+- **Auth**: Authentication would fail if email has uppercase chars and provider callback is in lowercase
+- **Markdown**: Fixed potential crash on markdown processing of video links
+- **Search**: Search index should now update upon article creation
+- **Search**: Search results are no longer duplicated upon article update
+- **UI**: Missing icons on login page
+- **UI**: Image alignement center and right should now behave correctly
+- **Uploads**: Error notification when upload is too large for server
+- **Uploads**: Fix uploads and temp-uploads folder permissions on unix-based systems
 
 ## [v1.0.0-beta.8] - 2017-02-19
 ### Added

app/data.yml

@@ -53,6 +53,8 @@ defaults:
       signature:
         name: Wiki
         email: wiki@example.com
+    features:
+      mathjax: true
     externalLogging:
       bugsnap: false
       loggly: false

app/regex.js

+'use strict'
+
+module.exports = {
+  arabic: /([\u0600-\u06ff]|[\u0750-\u077f]|[\ufb50-\ufc3f]|[\ufe70-\ufefc])/,
+  cjk: /([\u4E00-\u9FBF]|[\u3040-\u309F\u30A0-\u30FF]|[ㄱ-ㅎ가-힣ㅏ-ㅣ])/,
+  youtube: /(?:(?:youtu\.be\/|v\/|vi\/|u\/\w\/|embed\/)|(?:(?:watch)?\?v(?:i)?=|&v(?:i)?=))([^#&?]*).*/,
+  vimeo: /vimeo.com\/(?:channels\/(?:\w+\/)?|groups\/(?:[^/]*)\/videos\/|album\/(?:\d+)\/video\/|)(\d+)(?:$|\/|\?)/,
+  dailymotion: /(?:dailymotion\.com(?:\/embed)?(?:\/video|\/hub)|dai\.ly)\/([0-9a-z]+)(?:[-_0-9a-zA-Z]+(?:#video=)?([a-z0-9]+)?)?/
+}

client/js/pages/view.js

 'use strict'
 
 import $ from 'jquery'
+import MathJax from 'mathjax'
 
 module.exports = (alerts) => {
   if ($('#page-type-view').length) {
     let currentBasePath = ($('#page-type-view').data('entrypath') !== 'home') ? $('#page-type-view').data('entrypath') : ''
 
+    // MathJax Render
+
+    MathJax.Hub.Config({
+      jax: ['input/TeX', 'input/MathML', 'output/SVG'],
+      extensions: ['tex2jax.js', 'mml2jax.js'],
+      TeX: {
+        extensions: ['AMSmath.js', 'AMSsymbols.js', 'noErrors.js', 'noUndefined.js']
+      },
+      SVG: {
+        scale: 120,
+        font: 'STIX-Web'
+      },
+      showMathMenu: false
+    })
+
     require('../modals/create.js')(currentBasePath)
     require('../modals/move.js')(currentBasePath, alerts)
   }

config.sample.yml

@@ -132,6 +132,14 @@ git:
     email: marty@example.com
 
 # ---------------------------------------------------------------------
+# Features
+# ---------------------------------------------------------------------
+# You can enable / disable specific features below
+
+features:
+  mathjax: true
+
+# ---------------------------------------------------------------------
 # External Logging
 # ---------------------------------------------------------------------
 

controllers/uploads.js

@@ -10,7 +10,7 @@ const fs = Promise.promisifyAll(require('fs-extra'))
 const path = require('path')
 const _ = require('lodash')
 
-const validPathRe = new RegExp('^([a-z0-9\\/-]+\\.[a-z0-9]+)$')
+const validPathRe = new RegExp('^(([a-z0-9/-]|' + appdata.regex.cjk.source + ')+\\.[a-z0-9]+)$')
 const validPathThumbsRe = new RegExp('^([a-z0-9]+\\.png)$')
 
 // ==========================================

fuse.js

@@ -59,6 +59,10 @@ const SHIMS = {
   jquery: {
     source: 'node_modules/jquery/dist/jquery.js',
     exports: '$'
+  },
+  mathjax: {
+    source: 'node_modules/mathjax/MathJax.js',
+    exports: 'MathJax'
   }
 }
 
@@ -69,6 +73,9 @@ const SHIMS = {
 console.info(colors.white('└── ') + colors.green('Running global tasks...'))
 
 let globalTasks = Promise.mapSeries([
+  /**
+   * ACE Modes
+   */
   () => {
     return fs.accessAsync('./assets/js/ace').then(() => {
       console.info(colors.white('  └── ') + colors.magenta('ACE modes directory already exists. Task aborted.'))
@@ -89,6 +96,54 @@ let globalTasks = Promise.mapSeries([
         throw err
       }
     })
+  },
+  /**
+   * MathJax
+   */
+  () => {
+    return fs.accessAsync('./assets/js/mathjax').then(() => {
+      console.info(colors.white('  └── ') + colors.magenta('MathJax directory already exists. Task aborted.'))
+      return true
+    }).catch(err => {
+      if (err.code === 'ENOENT') {
+        console.info(colors.white('  └── ') + colors.green('Copy MathJax dependencies to assets...'))
+        return fs.ensureDirAsync('./assets/js/mathjax').then(() => {
+          return fs.copyAsync('./node_modules/mathjax', './assets/js/mathjax', { filter: (src, dest) => {
+            let srcNormalized = src.replace(/\\/g, '/')
+            let shouldCopy = false
+            console.log(srcNormalized)
+            _.forEach([
+              '/node_modules/mathjax',
+              '/node_modules/mathjax/jax',
+              '/node_modules/mathjax/jax/input',
+              '/node_modules/mathjax/jax/output'
+            ], chk => {
+              if (srcNormalized.endsWith(chk)) {
+                shouldCopy = true
+              }
+            })
+            _.forEach([
+              '/node_modules/mathjax/extensions',
+              '/node_modules/mathjax/MathJax.js',
+              '/node_modules/mathjax/jax/element',
+              '/node_modules/mathjax/jax/input/MathML',
+              '/node_modules/mathjax/jax/input/TeX',
+              '/node_modules/mathjax/jax/output/SVG'
+            ], chk => {
+              if (srcNormalized.indexOf(chk) > 0) {
+                shouldCopy = true
+              }
+            })
+            if (shouldCopy && srcNormalized.indexOf('/fonts/') > 0 && srcNormalized.indexOf('/STIX-Web') <= 1) {
+              shouldCopy = false
+            }
+            return shouldCopy
+          }})
+        })
+      } else {
+        throw err
+      }
+    })
   }
 ], f => { return f() })
 

libs/config.js

@@ -13,7 +13,8 @@ const _ = require('lodash')
 module.exports = (confPaths) => {
   confPaths = _.defaults(confPaths, {
     config: './config.yml',
-    data: './app/data.yml'
+    data: './app/data.yml',
+    dataRegex: '../app/regex.js'
   })
 
   let appconfig = {}
@@ -22,6 +23,7 @@ module.exports = (confPaths) => {
   try {
     appconfig = yaml.safeLoad(fs.readFileSync(confPaths.config, 'utf8'))
     appdata = yaml.safeLoad(fs.readFileSync(confPaths.data, 'utf8'))
+    appdata.regex = require(confPaths.dataRegex)
   } catch (ex) {
     console.error(ex)
     process.exit(1)

libs/entries.js

@@ -5,6 +5,7 @@ const path = require('path')
 const fs = Promise.promisifyAll(require('fs-extra'))
 const _ = require('lodash')
 const crypto = require('crypto')
+const qs = require('querystring')
 
 /**
  * Entries Model
@@ -163,7 +164,8 @@ module.exports = {
    * @return     {String}  Safe entry path
    */
   parsePath (urlPath) {
-    let wlist = new RegExp('[^a-z0-9/-]', 'g')
+    urlPath = qs.unescape(urlPath)
+    let wlist = new RegExp('(?!([^a-z0-9]|' + appdata.regex.cjk.source + '|[/-]))', 'g')
 
     urlPath = _.toLower(urlPath).replace(wlist, '')
 

libs/local.js

@@ -152,7 +152,7 @@ module.exports = {
    */
   validateUploadsFilename (f, fld, isImage) {
     let fObj = path.parse(f)
-    let fname = _.chain(fObj.name).trim().toLower().kebabCase().value().replace(/[^a-z0-9-]+/g, '')
+    let fname = _.chain(fObj.name).trim().toLower().kebabCase().value().replace(new RegExp('(?!([^a-z0-9-]|' + appdata.regex.cjk.source + '))', 'g'), '')
     let fext = _.toLower(fObj.ext)
 
     if (isImage && !_.includes(['.jpg', '.jpeg', '.png', '.gif', '.webp'], fext)) {

libs/markdown.js

@@ -51,6 +51,11 @@ var mkdown = md({
   })
   .use(mdAttrs)
 
+if (appconfig) {
+  const mdMathjax = require('markdown-it-mathjax')
+  mkdown.use(mdMathjax())
+}
+
 // Rendering rules
 
 mkdown.renderer.rules.emoji = function (token, idx) {

package.json

@@ -82,6 +82,7 @@
     "markdown-it-expand-tabs": "^1.0.11",
     "markdown-it-external-links": "0.0.6",
     "markdown-it-footnote": "^3.0.1",
+    "markdown-it-mathjax": "^2.0.0",
     "markdown-it-task-lists": "^2.0.0",
     "memdown": "^1.2.4",
     "mime-types": "^2.1.15",
@@ -139,6 +140,7 @@
     "jquery-contextmenu": "^2.4.4",
     "jquery-simple-upload": "^1.0.0",
     "jquery-smooth-scroll": "^2.0.0",
+    "mathjax": "^2.7.0",
     "node-sass": "latest",
     "nodemon": "latest",
     "pug-lint": "latest",

views/layout.pug

@@ -18,6 +18,8 @@ html
     link(rel='manifest', href='/manifest.json')
 
     // JS / CSS
+    script(type='text/javascript').
+      window.MathJax = { root:"/js/mathjax" }
     script(type='text/javascript', src='/js/bundle.min.js')
 
     block head