feat: local disk import all action + v1 import content (#1100)

server/core/auth.js

 const passport = require('passport')
 const passportJWT = require('passport-jwt')
 const _ = require('lodash')
-const path = require('path')
 const jwt = require('jsonwebtoken')
 const moment = require('moment')
 const Promise = require('bluebird')

server/helpers/page.js

@@ -4,6 +4,13 @@ const crypto = require('crypto')
 const path = require('path')
 
 const localeSegmentRegex = /^[A-Z]{2}(-[A-Z]{2})?$/i
+const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
+
+const contentToExt = {
+  markdown: 'md',
+  html: 'html'
+}
+const extToContent = _.invert(contentToExt)
 
 /* global WIKI */
 
@@ -94,13 +101,34 @@ module.exports = {
    * Get file extension from content type
    */
   getFileExtension(contentType) {
-    switch (contentType) {
-      case 'markdown':
-        return 'md'
-      case 'html':
-        return 'html'
-      default:
-        return 'txt'
+    _.get(contentToExt, contentType, 'txt')
+  },
+  /**
+   * Get content type from file extension
+   */
+  getContentType (filePath) {
+    const ext = _.last(filePath.split('.'))
+    return _.get(extToContent, ext, false)
+  },
+  /**
+   * Get Page Meta object from disk path
+   */
+  getPagePath (filePath) {
+    let fpath = filePath
+    if (process.platform === 'win32') {
+      fpath = filePath.replace(/\\/g, '/')
+    }
+    let meta = {
+      locale: WIKI.config.lang.code,
+      path: _.initial(fpath.split('.')).join('')
+    }
+    const result = localeFolderRegex.exec(meta.path)
+    if (result[1]) {
+      meta = {
+        locale: result[1],
+        path: result[2]
+      }
     }
+    return meta
   }
 }

server/models/editors.js

@@ -99,6 +99,8 @@ module.exports = class Editor extends Model {
     switch (contentType) {
       case 'markdown':
         return 'markdown'
+      case 'html':
+        return 'ckeditor'
       default:
         return 'code'
     }

server/models/pages.js

@@ -366,7 +366,7 @@ module.exports = class Page extends Model {
     }
 
     // -> Perform move?
-    if (opts.locale !== page.localeCode || opts.path !== page.path) {
+    if ((opts.locale && opts.locale !== page.localeCode) || (opts.path && opts.path !== page.path)) {
       await WIKI.models.pages.movePage({
         id: page.id,
         destinationLocale: opts.locale,

server/models/users.js

@@ -712,4 +712,14 @@ module.exports = class User extends Model {
     user.permissions = user.getGlobalPermissions()
     return user
   }
+
+  static async getRootUser () {
+    let user = await WIKI.models.users.query().findById(1)
+    if (!user) {
+      WIKI.logger.error('CRITICAL ERROR: Root Administrator user is missing!')
+      process.exit(1)
+    }
+    user.permissions = ['manage:system']
+    return user
+  }
 }

server/modules/storage/disk/definition.yml

@@ -29,3 +29,6 @@ actions:
   - handler: backup
     label: Create Backup
     hint: Will create a manual backup archive at this point in time, in a subfolder named _manual, from the contents currently on disk.
+  - handler: importAll
+    label: Import Everything
+    hint: Will import all content currently in the local disk folder.

server/modules/storage/disk/storage.js

@@ -3,8 +3,10 @@ const path = require('path')
 const tar = require('tar-fs')
 const zlib = require('zlib')
 const stream = require('stream')
+const _ = require('lodash')
 const Promise = require('bluebird')
 const pipeline = Promise.promisify(stream.pipeline)
+const klaw = require('klaw')
 const pageHelper = require('../../../helpers/page.js')
 const moment = require('moment')
 
@@ -113,5 +115,77 @@ module.exports = {
   },
   async backup() {
     return this.sync({ manual: true })
+  },
+  async importAll() {
+    WIKI.logger.info(`(STORAGE/DISK) Importing all content from local disk folder to the DB...`)
+
+    const rootUser = await WIKI.models.users.getRootUser()
+
+    await pipeline(
+      klaw(this.config.path, {
+        filter: (f) => {
+          return !_.includes(f, '.git')
+        }
+      }),
+      new stream.Transform({
+        objectMode: true,
+        transform: async (file, enc, cb) => {
+          const relPath = file.path.substr(this.config.path.length + 1)
+          if (relPath && relPath.length > 3) {
+            WIKI.logger.info(`(STORAGE/DISK) Processing ${relPath}...`)
+            const contentType = pageHelper.getContentType(relPath)
+            if (!contentType) {
+              return cb()
+            }
+            const contentPath = pageHelper.getPagePath(relPath)
+
+            let itemContents = ''
+            try {
+              itemContents = await fs.readFile(path.join(this.config.path, relPath), 'utf8')
+              const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType)
+              const currentPage = await WIKI.models.pages.query().findOne({
+                path: contentPath.path,
+                localeCode: contentPath.locale
+              })
+              if (currentPage) {
+                // Already in the DB, can mark as modified
+                WIKI.logger.info(`(STORAGE/DISK) Page marked as modified: ${relPath}`)
+                await WIKI.models.pages.updatePage({
+                  id: currentPage.id,
+                  title: _.get(pageData, 'title', currentPage.title),
+                  description: _.get(pageData, 'description', currentPage.description) || '',
+                  isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
+                  isPrivate: false,
+                  content: pageData.content,
+                  user: rootUser,
+                  skipStorage: true
+                })
+              } else {
+                // Not in the DB, can mark as new
+                WIKI.logger.info(`(STORAGE/DISK) Page marked as new: ${relPath}`)
+                const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType)
+                await WIKI.models.pages.createPage({
+                  path: contentPath.path,
+                  locale: contentPath.locale,
+                  title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))),
+                  description: _.get(pageData, 'description', '') || '',
+                  isPublished: _.get(pageData, 'isPublished', true),
+                  isPrivate: false,
+                  content: pageData.content,
+                  user: rootUser,
+                  editor: pageEditor,
+                  skipStorage: true
+                })
+              }
+            } catch (err) {
+              WIKI.logger.warn(`(STORAGE/DISK) Failed to process ${relPath}`)
+              WIKI.logger.warn(err)
+            }
+          }
+          cb()
+        }
+      })
+    )
+    WIKI.logger.info('(STORAGE/DISK) Import completed.')
   }
 }

server/modules/storage/git/storage.js

@@ -8,41 +8,8 @@ const pipeline = Promise.promisify(stream.pipeline)
 const klaw = require('klaw')
 const pageHelper = require('../../../helpers/page.js')
 
-const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
-
 /* global WIKI */
 
-const getContenType = (filePath) => {
-  const ext = _.last(filePath.split('.'))
-  switch (ext) {
-    case 'md':
-      return 'markdown'
-    case 'html':
-      return 'html'
-    default:
-      return false
-  }
-}
-
-const getPagePath = (filePath) => {
-  let fpath = filePath
-  if (process.platform === 'win32') {
-    fpath = filePath.replace(/\\/g, '/')
-  }
-  let meta = {
-    locale: 'en',
-    path: _.initial(fpath.split('.')).join('')
-  }
-  const result = localeFolderRegex.exec(meta.path)
-  if (result[1]) {
-    meta = {
-      locale: result[1],
-      path: result[2]
-    }
-  }
-  return meta
-}
-
 module.exports = {
   git: null,
   repoPath: path.join(process.cwd(), 'data/repo'),
@@ -145,6 +112,8 @@ module.exports = {
   async sync() {
     const currentCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {})
 
+    const rootUser = await WIKI.models.users.getRootUser()
+
     // Pull rebase
     if (_.includes(['sync', 'pull'], this.mode)) {
       WIKI.logger.info(`(STORAGE/GIT) Performing pull rebase from origin on branch ${this.config.branch}...`)
@@ -167,7 +136,7 @@ module.exports = {
 
       const diff = await this.git.diffSummary(['-M', currentCommitLog.hash, latestCommitLog.hash])
       if (_.get(diff, 'files', []).length > 0) {
-        await this.processFiles(diff.files)
+        await this.processFiles(diff.files, rootUser)
       }
     }
   },
@@ -176,13 +145,13 @@ module.exports = {
    *
    * @param {Array<String>} files Array of files to process
    */
-  async processFiles(files) {
+  async processFiles(files, user) {
     for (const item of files) {
-      const contentType = getContenType(item.file)
+      const contentType = pageHelper.getContentType(item.file)
       if (!contentType) {
         continue
       }
-      const contentPath = getPagePath(item.file)
+      const contentPath = pageHelper.getPagePath(item.file)
 
       let itemContents = ''
       try {
@@ -202,7 +171,7 @@ module.exports = {
             isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
             isPrivate: false,
             content: pageData.content,
-            authorId: 1,
+            user: user,
             skipStorage: true
           })
         } else {
@@ -217,7 +186,7 @@ module.exports = {
             isPublished: _.get(pageData, 'isPublished', true),
             isPrivate: false,
             content: pageData.content,
-            authorId: 1,
+            user: user,
             editor: pageEditor,
             skipStorage: true
           })
@@ -233,8 +202,7 @@ module.exports = {
             skipStorage: true
           })
         } else {
-          WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`)
-          console.error(err)
+          WIKI.logger.warn(`(STORAGE/GIT) Failed to process ${item.file}`)
           WIKI.logger.warn(err)
         }
       }
@@ -365,6 +333,9 @@ module.exports = {
    */
   async importAll() {
     WIKI.logger.info(`(STORAGE/GIT) Importing all content from local Git repo to the DB...`)
+
+    const rootUser = await WIKI.models.users.getRootUser()
+
     await pipeline(
       klaw(this.repoPath, {
         filter: (f) => {
@@ -378,10 +349,11 @@ module.exports = {
           if (relPath && relPath.length > 3) {
             WIKI.logger.info(`(STORAGE/GIT) Processing ${relPath}...`)
             await this.processFiles([{
+              user: rootUser,
               file: relPath,
               deletions: 0,
               insertions: 0
-            }])
+            }], rootUser)
           }
           cb()
         }
@@ -391,6 +363,7 @@ module.exports = {
   },
   async syncUntracked() {
     WIKI.logger.info(`(STORAGE/GIT) Adding all untracked content...`)
+
     await pipeline(
       WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
         isPrivate: false