feat: page cache + save/load logic + db fixes

client/components/admin/admin-utilities.vue

@@ -6,8 +6,9 @@
       .subheading.grey--text Maintenance and troubleshooting tools
     v-tabs(:color='$vuetify.dark ? "primary" : "grey lighten-4"', fixed-tabs, :slider-color='$vuetify.dark ? "white" : "primary"', show-arrows)
       v-tab(key='tools') Tools
+      v-tab(key='cache') Cache
       v-tab(key='telemetry') Telemetry
-      v-tab(key='telemetry') Support
+      v-tab(key='support') Support
 
       v-tab-item(key='tools', :transition='false', :reverse-transition='false')
         v-container(fluid, grid-list-lg, :class='$vuetify.dark ? "" : "grey lighten-5"')

server/controllers/common.js

@@ -34,7 +34,7 @@ router.get('/*', async (req, res, next) => {
     path: pageArgs.path,
     locale: pageArgs.locale,
     userId: req.user.id,
-    private: false
+    isPrivate: false
   })
   if (page) {
     res.render('page', { page })

server/db/migrations/2.0.0.js

@@ -5,6 +5,7 @@ exports.up = knex => {
     // =====================================
     // ASSETS ------------------------------
     .createTable('assets', table => {
+      table.charset('utf8mb4')
       table.increments('id').primary()
       table.string('filename').notNullable()
       table.string('basename').notNullable()
@@ -18,6 +19,7 @@ exports.up = knex => {
     })
     // ASSET FOLDERS -----------------------
     .createTable('assetFolders', table => {
+      table.charset('utf8mb4')
       table.increments('id').primary()
       table.string('name').notNullable()
       table.string('slug').notNullable()
@@ -25,6 +27,7 @@ exports.up = knex => {
     })
     // AUTHENTICATION ----------------------
     .createTable('authentication', table => {
+      table.charset('utf8mb4')
       table.string('key').notNullable().primary()
       table.boolean('isEnabled').notNullable().defaultTo(false)
       table.json('config').notNullable()
@@ -34,6 +37,7 @@ exports.up = knex => {
     })
     // COMMENTS ----------------------------
     .createTable('comments', table => {
+      table.charset('utf8mb4')
       table.increments('id').primary()
       table.text('content').notNullable()
       table.string('createdAt').notNullable()
@@ -41,12 +45,14 @@ exports.up = knex => {
     })
     // EDITORS -----------------------------
     .createTable('editors', table => {
+      table.charset('utf8mb4')
       table.string('key').notNullable().primary()
       table.boolean('isEnabled').notNullable().defaultTo(false)
       table.json('config').notNullable()
     })
     // GROUPS ------------------------------
     .createTable('groups', table => {
+      table.charset('utf8mb4')
       table.increments('id').primary()
       table.string('name').notNullable()
       table.string('createdAt').notNullable()
@@ -54,6 +60,7 @@ exports.up = knex => {
     })
     // LOCALES -----------------------------
     .createTable('locales', table => {
+      table.charset('utf8mb4')
       table.string('code', 2).notNullable().primary()
       table.json('strings')
       table.boolean('isRTL').notNullable().defaultTo(false)
@@ -64,6 +71,7 @@ exports.up = knex => {
     })
     // LOGGING ----------------------------
     .createTable('loggers', table => {
+      table.charset('utf8mb4')
       table.string('key').notNullable().primary()
       table.boolean('isEnabled').notNullable().defaultTo(false)
       table.string('level').notNullable().defaultTo('warn')
@@ -71,8 +79,10 @@ exports.up = knex => {
     })
     // PAGE HISTORY ------------------------
     .createTable('pageHistory', table => {
+      table.charset('utf8mb4')
       table.increments('id').primary()
       table.string('path').notNullable()
+      table.string('hash').notNullable()
       table.string('title').notNullable()
       table.string('description')
       table.boolean('isPrivate').notNullable().defaultTo(false)
@@ -85,8 +95,10 @@ exports.up = knex => {
     })
     // PAGES -------------------------------
     .createTable('pages', table => {
+      table.charset('utf8mb4')
       table.increments('id').primary()
       table.string('path').notNullable()
+      table.string('hash').notNullable()
       table.string('title').notNullable()
       table.string('description')
       table.boolean('isPrivate').notNullable().defaultTo(false)
@@ -102,24 +114,28 @@ exports.up = knex => {
     })
     // RENDERERS ---------------------------
     .createTable('renderers', table => {
+      table.charset('utf8mb4')
       table.string('key').notNullable().primary()
       table.boolean('isEnabled').notNullable().defaultTo(false)
       table.json('config')
     })
     // SEARCH ------------------------------
     .createTable('searchEngines', table => {
+      table.charset('utf8mb4')
       table.string('key').notNullable().primary()
       table.boolean('isEnabled').notNullable().defaultTo(false)
       table.json('config')
     })
     // SETTINGS ----------------------------
     .createTable('settings', table => {
+      table.charset('utf8mb4')
       table.string('key').notNullable().primary()
       table.json('value')
       table.string('updatedAt').notNullable()
     })
     // STORAGE -----------------------------
     .createTable('storage', table => {
+      table.charset('utf8mb4')
       table.string('key').notNullable().primary()
       table.boolean('isEnabled').notNullable().defaultTo(false)
       table.string('mode', ['sync', 'push', 'pull']).notNullable().defaultTo('push')
@@ -127,6 +143,7 @@ exports.up = knex => {
     })
     // TAGS --------------------------------
     .createTable('tags', table => {
+      table.charset('utf8mb4')
       table.increments('id').primary()
       table.string('tag').notNullable().unique()
       table.string('title')
@@ -135,6 +152,7 @@ exports.up = knex => {
     })
     // USERS -------------------------------
     .createTable('users', table => {
+      table.charset('utf8mb4')
       table.increments('id').primary()
       table.string('email').notNullable()
       table.string('name').notNullable()
@@ -155,18 +173,21 @@ exports.up = knex => {
     // =====================================
     // PAGE HISTORY TAGS ---------------------------
     .createTable('pageHistoryTags', table => {
+      table.charset('utf8mb4')
       table.increments('id').primary()
       table.integer('pageId').unsigned().references('id').inTable('pageHistory').onDelete('CASCADE')
       table.integer('tagId').unsigned().references('id').inTable('tags').onDelete('CASCADE')
     })
     // PAGE TAGS ---------------------------
     .createTable('pageTags', table => {
+      table.charset('utf8mb4')
       table.increments('id').primary()
       table.integer('pageId').unsigned().references('id').inTable('pages').onDelete('CASCADE')
       table.integer('tagId').unsigned().references('id').inTable('tags').onDelete('CASCADE')
     })
     // USER GROUPS -------------------------
     .createTable('userGroups', table => {
+      table.charset('utf8mb4')
       table.increments('id').primary()
       table.integer('userId').unsigned().references('id').inTable('users').onDelete('CASCADE')
       table.integer('groupId').unsigned().references('id').inTable('groups').onDelete('CASCADE')

server/helpers/page.js

 const qs = require('querystring')
 const _ = require('lodash')
+const crypto = require('crypto')
 
 module.exports = {
   /**
@@ -26,5 +27,11 @@ module.exports = {
     }
     pathObj.path = _.join(pathParts, '/')
     return pathObj
+  },
+  /**
+   * Generate unique hash from page
+   */
+  generateHash(opts) {
+    return crypto.createHash('sha1').update(`${opts.locale}|${opts.path}|${opts.privateNS}`).digest('hex')
   }
 }

server/jobs/render-page.js

@@ -20,7 +20,17 @@ module.exports = async (job) => {
         input: output
       })
     }
-    console.info(output)
+
+    // Save to DB
+    await WIKI.models.pages.query()
+      .patch({ render: output })
+      .where('id', job.data.page.id)
+
+    // Save to cache
+    await WIKI.models.pages.savePageToCache({
+      ...job.data.page,
+      render: output
+    })
 
     WIKI.logger.info(`Rendering page ${job.data.page.path}: [ COMPLETED ]`)
   } catch (err) {

server/models/pageHistory.js

@@ -16,6 +16,7 @@ module.exports = class PageHistory extends Model {
       properties: {
         id: {type: 'integer'},
         path: {type: 'string'},
+        hash: {type: 'string'},
         title: {type: 'string'},
         description: {type: 'string'},
         isPublished: {type: 'boolean'},
@@ -88,6 +89,7 @@ module.exports = class PageHistory extends Model {
       content: opts.content,
       description: opts.description,
       editorKey: opts.editorKey,
+      hash: opts.hash,
       isPrivate: opts.isPrivate,
       isPublished: opts.isPublished,
       localeCode: opts.localeCode,

server/models/pages.js

 const Model = require('objection').Model
 const _ = require('lodash')
+const JSBinType = require('js-binary').Type
+const pageHelper = require('../helpers/page')
+const path = require('path')
+const fs = require('fs-extra')
 
 /* global WIKI */
 
@@ -17,6 +21,7 @@ module.exports = class Page extends Model {
       properties: {
         id: {type: 'integer'},
         path: {type: 'string'},
+        hash: {type: 'string'},
         title: {type: 'string'},
         description: {type: 'string'},
         isPublished: {type: 'boolean'},
@@ -89,35 +94,33 @@ module.exports = class Page extends Model {
     this.updatedAt = new Date().toISOString()
   }
 
-  static async getPage(opts) {
-    const page = await WIKI.models.pages.query().where({
-      path: opts.path,
-      localeCode: opts.locale
-    }).andWhere(builder => {
-      builder.where({
-        isPublished: true
-      }).orWhere({
-        isPublished: false,
-        authorId: opts.userId
+  static get cacheSchema() {
+    return new JSBinType({
+      authorId: 'uint',
+      authorName: 'string',
+      createdAt: 'string',
+      creatorId: 'uint',
+      creatorName: 'string',
+      description: 'string',
+      isPrivate: 'boolean',
+      isPublished: 'boolean',
+      publishEndDate: 'string',
+      publishStartDate: 'string',
+      render: 'string',
+      title: 'string',
+      updatedAt: 'string'
     })
-    }).andWhere(builder => {
-      if (opts.private) {
-        builder.where({ isPrivate: true, privateNS: opts.privateNS })
-      } else {
-        builder.where({ isPrivate: false })
-      }
-    }).first()
-    return page
   }
 
   static async createPage(opts) {
-    const page = await WIKI.models.pages.query().insertAndFetch({
+    await WIKI.models.pages.query().insert({
       authorId: opts.authorId,
       content: opts.content,
       creatorId: opts.authorId,
       contentType: _.get(_.find(WIKI.data.editors, ['key', opts.editor]), `contentType`, 'text'),
       description: opts.description,
       editorKey: opts.editor,
+      hash: pageHelper.generateHash({ path: opts.path, locale: opts.locale, privateNS: opts.isPrivate ? 'TODO' : '' }),
       isPrivate: opts.isPrivate,
       isPublished: opts.isPublished,
       localeCode: opts.locale,
@@ -126,6 +129,7 @@ module.exports = class Page extends Model {
       publishStartDate: opts.publishStartDate,
       title: opts.title
     })
+    const page = await WIKI.models.pages.getPageFromDb(opts)
     await WIKI.models.pages.renderPage(page)
     await WIKI.models.storage.pageEvent({
       event: 'created',
@@ -140,7 +144,7 @@ module.exports = class Page extends Model {
       throw new Error('Invalid Page Id')
     }
     await WIKI.models.pageHistory.addVersion(ogPage)
-    const page = await WIKI.models.pages.query().patchAndFetchById(ogPage.id, {
+    await WIKI.models.pages.query().patch({
       authorId: opts.authorId,
       content: opts.content,
       description: opts.description,
@@ -148,7 +152,8 @@ module.exports = class Page extends Model {
       publishEndDate: opts.publishEndDate,
       publishStartDate: opts.publishStartDate,
       title: opts.title
-    })
+    }).where('id', ogPage.id)
+    const page = await WIKI.models.pages.getPageFromDb(opts)
     await WIKI.models.pages.renderPage(page)
     await WIKI.models.storage.pageEvent({
       event: 'updated',
@@ -167,4 +172,88 @@ module.exports = class Page extends Model {
       removeOnFail: true
     })
   }
+
+  static async getPage(opts) {
+    let page = await WIKI.models.pages.getPageFromCache(opts)
+    if (!page) {
+      page = await WIKI.models.pages.getPageFromDb(opts)
+      await WIKI.models.pages.savePageToCache(page)
+    }
+    return page
+  }
+
+  static async getPageFromDb(opts) {
+    const page = await WIKI.models.pages.query()
+      .column([
+        'pages.*',
+        {
+          authorName: 'author.name',
+          creatorName: 'creator.name'
+        }
+      ])
+      .joinRelation('author')
+      .joinRelation('creator')
+      .where({
+        'pages.path': opts.path,
+        'pages.localeCode': opts.locale
+      })
+      .andWhere(builder => {
+        builder.where({
+          'pages.isPublished': true
+        }).orWhere({
+          'pages.isPublished': false,
+          'pages.authorId': opts.userId
+        })
+      })
+      .andWhere(builder => {
+        if (opts.isPrivate) {
+          builder.where({ 'pages.isPrivate': true, 'pages.privateNS': opts.privateNS })
+        } else {
+          builder.where({ 'pages.isPrivate': false })
+        }
+      })
+      .first()
+    return page
+  }
+
+  static async savePageToCache(page) {
+    const cachePath = path.join(process.cwd(), `data/cache/${page.hash}.bin`)
+    await fs.outputFile(cachePath, WIKI.models.pages.cacheSchema.encode({
+      authorId: page.authorId,
+      authorName: page.authorName,
+      createdAt: page.createdAt,
+      creatorId: page.creatorId,
+      creatorName: page.creatorName,
+      description: page.description,
+      isPrivate: page.isPrivate === 1,
+      isPublished: page.isPublished === 1,
+      publishEndDate: page.publishEndDate,
+      publishStartDate: page.publishStartDate,
+      render: page.render,
+      title: page.title,
+      updatedAt: page.updatedAt
+    }))
+  }
+
+  static async getPageFromCache(opts) {
+    const pageHash = pageHelper.generateHash({ path: opts.path, locale: opts.locale, privateNS: opts.isPrivate ? 'TODO' : '' })
+    const cachePath = path.join(process.cwd(), `data/cache/${pageHash}.bin`)
+
+    try {
+      const pageBuffer = await fs.readFile(cachePath)
+      let page = WIKI.models.pages.cacheSchema.decode(pageBuffer)
+      return {
+        ...page,
+        path: opts.path,
+        localeCode: opts.locale,
+        isPrivate: opts.isPrivate
+      }
+    } catch (err) {
+      if (err.code === 'ENOENT') {
+        return false
+      }
+      WIKI.logger.error(err)
+      throw err
+    }
+  }
 }

server/setup.js

@@ -90,10 +90,10 @@ module.exports = () => {
       }
 
       // Create directory structure
-      const tmpPath = path.join(os.tmpdir(), 'wikijs')
-      await fs.ensureDir(tmpPath)
-      await fs.ensureDir(path.join(tmpPath, 'cache'))
-      await fs.ensureDir(path.join(tmpPath, 'uploads'))
+      const dataPath = path.join(process.cwd(), 'data')
+      await fs.ensureDir(dataPath)
+      await fs.ensureDir(path.join(dataPath, 'cache'))
+      await fs.ensureDir(path.join(dataPath, 'uploads'))
 
       // Set config
       _.set(WIKI.config, 'defaultEditor', 'markdown')