feat: SFTP storage module + sensitive field option

client/graph/admin/storage/storage-query-targets.gql

 query {
   storage {
-    targets(orderBy: "title ASC") {
+    targets {
       isAvailable
       isEnabled
       key

package.json

@@ -156,6 +156,8 @@
     "simple-git": "1.129.0",
     "solr-node": "1.2.1",
     "sqlite3": "4.1.1",
+    "ssh2": "0.8.7",
+    "ssh2-promise": "0.1.6",
     "striptags": "3.1.1",
     "subscriptions-transport-ws": "0.9.16",
     "tar-fs": "2.0.0",

server/graph/resolvers/storage.js

@@ -13,7 +13,7 @@ module.exports = {
   StorageQuery: {
     async targets(obj, args, context, info) {
       let targets = await WIKI.models.storage.getTargets()
-      targets = targets.map(tgt => {
+      targets = _.sortBy(targets.map(tgt => {
         const targetInfo = _.find(WIKI.data.storage, ['key', tgt.key]) || {}
         return {
           ...targetInfo,
@@ -28,15 +28,13 @@ module.exports = {
                 key,
                 value: JSON.stringify({
                   ...configData,
-                  value
+                  value: (configData.sensitive && value.length > 0) ? '********' : value
                 })
               })
             }
           }, []), 'key')
         }
-      })
-      // if (args.filter) { targets = graphHelper.filter(targets, args.filter) }
-      if (args.orderBy) { targets = _.sortBy(targets, [args.orderBy]) }
+      }), ['title', 'key'])
       return targets
     },
     async status(obj, args, context, info) {
@@ -56,13 +54,22 @@ module.exports = {
   StorageMutation: {
     async updateTargets(obj, args, context) {
       try {
+        let dbTargets = await WIKI.models.storage.getTargets()
         for (let tgt of args.targets) {
+          const currentDbTarget = _.find(dbTargets, ['key', tgt.key])
+          if (!currentDbTarget) {
+            continue
+          }
           await WIKI.models.storage.query().patch({
             isEnabled: tgt.isEnabled,
             mode: tgt.mode,
             syncInterval: tgt.syncInterval,
             config: _.reduce(tgt.config, (result, value, key) => {
-              _.set(result, `${value.key}`, _.get(JSON.parse(value.value), 'v', null))
+              let configValue = _.get(JSON.parse(value.value), 'v', null)
+              if (configValue === '********') {
+                configValue = _.get(currentDbTarget.config, value.key, '')
+              }
+              _.set(result, `${value.key}`, configValue)
               return result
             }, {}),
             state: {

server/graph/schemas/storage.graphql

@@ -15,11 +15,7 @@ extend type Mutation {
 # -----------------------------------------------
 
 type StorageQuery {
-  targets(
-    filter: String
-    orderBy: String
-  ): [StorageTarget] @auth(requires: ["manage:system"])
-
+  targets: [StorageTarget] @auth(requires: ["manage:system"])
   status: [StorageStatus] @auth(requires: ["manage:system"])
 }
 

server/helpers/common.js

@@ -32,6 +32,7 @@ module.exports = {
         hint: value.hint || false,
         enum: value.enum || false,
         multiline: value.multiline || false,
+        sensitive: value.sensitive || false,
         order: value.order || 100
       })
       return result

server/models/storage.js

@@ -94,6 +94,14 @@ module.exports = class Storage extends Model {
       } else {
         WIKI.logger.info(`No new storage targets found: [ SKIPPED ]`)
       }
+
+      // -> Delete removed targets
+      for (const target of dbTargets) {
+        if (!_.some(WIKI.data.storage, ['key', target.key])) {
+          await WIKI.models.storage.query().where('key', target.key).del()
+          WIKI.logger.info(`Removed target ${target.key} because it is no longer present in the modules folder: [ OK ]`)
+        }
+      }
     } catch (err) {
       WIKI.logger.error(`Failed to scan or load new storage providers: [ FAILED ]`)
       WIKI.logger.error(err)

server/modules/storage/azure/definition.yml

@@ -21,6 +21,7 @@ props:
     title: Account Access Key
     default: ''
     hint: Either key 1 or key 2.
+    sensitive: true
     order: 2
   containerName:
     type: String
@@ -40,7 +41,4 @@ props:
 actions:
   - handler: exportAll
     label: Export All
-    hint: Output all content from the DB to Azure Blog Storage, overwriting any existing data. If you enabled Azure Blog Storage after content was created or you temporarily disabled Git, you'll want to execute this action to add the missing content.
-  - handler: importAll
-    label: Import Everything
-    hint: Will import all content currently in Azure Blog Storage. Useful for importing or restoring content from a previously backed up state.
+    hint: Output all content from the DB to Azure Blog Storage, overwriting any existing data. If you enabled Azure Blog Storage after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.

server/modules/storage/azure/storage.js

 const { BlobServiceClient, StorageSharedKeyCredential } = require('@azure/storage-blob')
+const stream = require('stream')
+const Promise = require('bluebird')
+const pipeline = Promise.promisify(stream.pipeline)
 const pageHelper = require('../../../helpers/page.js')
+const _ = require('lodash')
 
 /* global WIKI */
 
@@ -110,5 +114,48 @@ module.exports = {
     await sourceBlockBlobClient.delete({
       deleteSnapshots: 'include'
     })
+  },
+  /**
+   * HANDLERS
+   */
+  async exportAll() {
+    WIKI.logger.info(`(STORAGE/AZURE) Exporting all content to Azure Blob Storage...`)
+
+    // -> Pages
+    await pipeline(
+      WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
+        isPrivate: false
+      }).stream(),
+      new stream.Transform({
+        objectMode: true,
+        transform: async (page, enc, cb) => {
+          const filePath = getFilePath(page, 'path')
+          WIKI.logger.info(`(STORAGE/AZURE) Adding page ${filePath}...`)
+          const pageContent = pageHelper.injectPageMetadata(page)
+          const blockBlobClient = this.container.getBlockBlobClient(filePath)
+          await blockBlobClient.upload(pageContent, pageContent.length, { tier: this.config.storageTier })
+          cb()
+        }
+      })
+    )
+
+    // -> Assets
+    const assetFolders = await WIKI.models.assetFolders.getAllPaths()
+
+    await pipeline(
+      WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
+      new stream.Transform({
+        objectMode: true,
+        transform: async (asset, enc, cb) => {
+          const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
+          WIKI.logger.info(`(STORAGE/AZURE) Adding asset ${filename}...`)
+          const blockBlobClient = this.container.getBlockBlobClient(filename)
+          await blockBlobClient.upload(asset.data, asset.data.length, { tier: this.config.storageTier })
+          cb()
+        }
+      })
+    )
+
+    WIKI.logger.info('(STORAGE/AZURE) All content has been pushed to Azure Blob Storage.')
   }
 }

server/modules/storage/digitalocean/definition.yml

@@ -36,4 +36,10 @@ props:
     type: String
     title: Access Key Secret
     hint: The Access Key Secret for the Access Key ID you created above.
+    sensitive: true
     order: 4
+actions:
+  - handler: exportAll
+    label: Export All
+    hint: Output all content from the DB to DigitalOcean Spaces, overwriting any existing data. If you enabled DigitalOcean Spaces after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
+

server/modules/storage/git/definition.yml

@@ -50,6 +50,7 @@ props:
     title: B - SSH Private Key Contents
     hint: SSH Authentication Only - Paste the contents of the private key. The key must NOT be passphrase-protected. Mode must be set to contents to use this option.
     multiline: true
+    sensitive: true
     order: 13
   verifySSL:
     type: Boolean
@@ -66,6 +67,7 @@ props:
     type: String
     title: Password / PAT
     hint: Basic Authentication Only
+    sensitive: true
     order: 21
   defaultEmail:
     type: String

server/modules/storage/s3/common.js

 const S3 = require('aws-sdk/clients/s3')
+const stream = require('stream')
+const Promise = require('bluebird')
+const pipeline = Promise.promisify(stream.pipeline)
+const _ = require('lodash')
 const pageHelper = require('../../../helpers/page.js')
 
 /* global WIKI */
@@ -98,4 +102,44 @@ module.exports = class S3CompatibleStorage {
     await this.s3.copyObject({ CopySource: asset.path, Key: asset.destinationPath }).promise()
     await this.s3.deleteObject({ Key: asset.path }).promise()
   }
+  /**
+   * HANDLERS
+   */
+  async exportAll() {
+    WIKI.logger.info(`(STORAGE/${this.storageName}) Exporting all content to the cloud provider...`)
+
+    // -> Pages
+    await pipeline(
+      WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
+        isPrivate: false
+      }).stream(),
+      new stream.Transform({
+        objectMode: true,
+        transform: async (page, enc, cb) => {
+          const filePath = getFilePath(page, 'path')
+          WIKI.logger.info(`(STORAGE/${this.storageName}) Adding page ${filePath}...`)
+          await this.s3.putObject({ Key: filePath, Body: pageHelper.injectPageMetadata(page) }).promise()
+          cb()
+        }
+      })
+    )
+
+    // -> Assets
+    const assetFolders = await WIKI.models.assetFolders.getAllPaths()
+
+    await pipeline(
+      WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
+      new stream.Transform({
+        objectMode: true,
+        transform: async (asset, enc, cb) => {
+          const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
+          WIKI.logger.info(`(STORAGE/${this.storageName}) Adding asset ${filename}...`)
+          await this.s3.putObject({ Key: filename, Body: asset.data }).promise()
+          cb()
+        }
+      })
+    )
+
+    WIKI.logger.info(`(STORAGE/${this.storageName}) All content has been pushed to the cloud provider.`)
+  }
 }

server/modules/storage/s3/definition.yml

@@ -29,4 +29,9 @@ props:
     type: String
     title: Secret Access Key
     hint: The Secret Access Key for the Access Key ID you created above.
+    sensitive: true
     order: 4
+actions:
+  - handler: exportAll
+    label: Export All
+    hint: Output all content from the DB to S3, overwriting any existing data. If you enabled S3 after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.

server/modules/storage/scp/definition.yml

-key: scp
-title: SCP (SSH)
-description: SSH is a software package that enables secure system administration and file transfers over insecure networks.
-author: requarks.io
-logo: https://static.requarks.io/logo/ssh.svg
-website: https://www.ssh.com/ssh/
-props:
-  host: String
-  port:
-    type: Number
-    default: 22
-  username: String
-  privateKeyPath: String
-  basePath:
-    type: String
-    default: '~'

server/modules/storage/scp/storage.js

-module.exports = {
-  async activated() {
-
-  },
-  async deactivated() {
-
-  },
-  async init() {
-
-  },
-  async created() {
-
-  },
-  async updated() {
-
-  },
-  async deleted() {
-
-  },
-  async renamed() {
-
-  }
-}

server/modules/storage/sftp/definition.yml

+key: sftp
+title: SFTP
+description: SFTP (SSH File Transfer Protocol) is a secure file transfer protocol. It runs over the SSH protocol. It supports the full security and authentication functionality of SSH.
+author: requarks.io
+logo: https://static.requarks.io/logo/ssh.svg
+website: https://www.ssh.com/ssh/sftp
+isAvailable: true
+supportedModes:
+  - push
+defaultMode: push
+schedule: false
+props:
+  host:
+    type: String
+    title: Host
+    default: ''
+    hint: Hostname or IP of the remote SSH server.
+    order: 1
+  port:
+    type: Number
+    title: Port
+    default: 22
+    hint: SSH port of the remote server.
+    order: 2
+  authMode:
+    type: String
+    title: Authentication Method
+    default: 'privateKey'
+    hint: Whether to use Private Key or Password-based authentication. A private key is highly recommended for best security.
+    enum:
+      - privateKey
+      - password
+    order: 3
+  username:
+    type: String
+    title: Username
+    default: ''
+    hint: Username for authentication.
+    order: 4
+  privateKey:
+    type: String
+    title: Private Key Contents
+    default: ''
+    hint: (Private Key Authentication Only) - Contents of the private key
+    multiline: true
+    sensitive: true
+    order: 5
+  passphrase:
+    type: String
+    title: Private Key Passphrase
+    default: ''
+    hint: (Private Key Authentication Only) - Passphrase if the private key is encrypted, leave empty otherwise
+    sensitive: true
+    order: 6
+  password:
+    type: String
+    title: Password
+    default: ''
+    hint: (Password-based Authentication Only) - Password for authentication
+    sensitive: true
+    order: 6
+  basePath:
+    type: String
+    title: Base Directory Path
+    default: '/root/wiki'
+    hint: Base directory where files will be transferred to. The path must already exists and be writable by the user.
+actions:
+  - handler: exportAll
+    label: Export All
+    hint: Output all content from the DB to the remote SSH server, overwriting any existing data. If you enabled SFTP after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
+

server/modules/storage/sftp/storage.js

+const SSH2Promise = require('ssh2-promise')
+const _ = require('lodash')
+const path = require('path')
+const stream = require('stream')
+const Promise = require('bluebird')
+const pipeline = Promise.promisify(stream.pipeline)
+const pageHelper = require('../../../helpers/page.js')
+
+/* global WIKI */
+
+const getFilePath = (page, pathKey) => {
+  const fileName = `${page[pathKey]}.${pageHelper.getFileExtension(page.contentType)}`
+  const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode
+  return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName
+}
+
+module.exports = {
+  client: null,
+  sftp: null,
+  async activated() {
+
+  },
+  async deactivated() {
+
+  },
+  async init() {
+    WIKI.logger.info(`(STORAGE/SFTP) Initializing...`)
+    this.client = new SSH2Promise({
+      host: this.config.host,
+      port: this.config.port || 22,
+      username: this.config.username,
+      password: (this.config.authMode === 'password') ? this.config.password : null,
+      privateKey: (this.config.authMode === 'privateKey') ? this.config.privateKey : null,
+      passphrase: (this.config.authMode === 'privateKey') ? this.config.passphrase : null
+    })
+    await this.client.connect()
+    this.sftp = this.client.sftp()
+    try {
+      await this.sftp.readdir(this.config.basePath)
+    } catch (err) {
+      WIKI.logger.warn(`(STORAGE/SFTP) ${err.message}`)
+      throw new Error(`Unable to read specified base directory: ${err.message}`)
+    }
+    WIKI.logger.info(`(STORAGE/SFTP) Initialization completed.`)
+  },
+  async created(page) {
+    WIKI.logger.info(`(STORAGE/SFTP) Creating file ${page.path}...`)
+    const filePath = getFilePath(page, 'path')
+    await this.ensureDirectory(filePath)
+    await this.sftp.writeFile(path.posix.join(this.config.basePath, filePath), page.injectMetadata())
+  },
+  async updated(page) {
+    WIKI.logger.info(`(STORAGE/SFTP) Updating file ${page.path}...`)
+    const filePath = getFilePath(page, 'path')
+    await this.ensureDirectory(filePath)
+    await this.sftp.writeFile(path.posix.join(this.config.basePath, filePath), page.injectMetadata())
+  },
+  async deleted(page) {
+    WIKI.logger.info(`(STORAGE/SFTP) Deleting file ${page.path}...`)
+    const filePath = getFilePath(page, 'path')
+    await this.sftp.unlink(path.posix.join(this.config.basePath, filePath))
+  },
+  async renamed(page) {
+    WIKI.logger.info(`(STORAGE/SFTP) Renaming file ${page.path} to ${page.destinationPath}...`)
+    let sourceFilePath = getFilePath(page, 'path')
+    let destinationFilePath = getFilePath(page, 'destinationPath')
+    if (WIKI.config.lang.namespacing) {
+      if (WIKI.config.lang.code !== page.localeCode) {
+        sourceFilePath = `${page.localeCode}/${sourceFilePath}`
+      }
+      if (WIKI.config.lang.code !== page.destinationLocaleCode) {
+        destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}`
+      }
+    }
+    await this.ensureDirectory(destinationFilePath)
+    await this.sftp.rename(path.posix.join(this.config.basePath, sourceFilePath), path.posix.join(this.config.basePath, destinationFilePath))
+  },
+  /**
+   * ASSET UPLOAD
+   *
+   * @param {Object} asset Asset to upload
+   */
+  async assetUploaded (asset) {
+    WIKI.logger.info(`(STORAGE/SFTP) Creating new file ${asset.path}...`)
+    await this.ensureDirectory(asset.path)
+    await this.sftp.writeFile(path.posix.join(this.config.basePath, asset.path), asset.data)
+  },
+  /**
+   * ASSET DELETE
+   *
+   * @param {Object} asset Asset to delete
+   */
+  async assetDeleted (asset) {
+    WIKI.logger.info(`(STORAGE/SFTP) Deleting file ${asset.path}...`)
+    await this.sftp.unlink(path.posix.join(this.config.basePath, asset.path))
+  },
+  /**
+   * ASSET RENAME
+   *
+   * @param {Object} asset Asset to rename
+   */
+  async assetRenamed (asset) {
+    WIKI.logger.info(`(STORAGE/SFTP) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
+    await this.ensureDirectory(asset.destinationPath)
+    await this.sftp.rename(path.posix.join(this.config.basePath, asset.path), path.posix.join(this.config.basePath, asset.destinationPath))
+  },
+  /**
+   * HANDLERS
+   */
+  async exportAll() {
+    WIKI.logger.info(`(STORAGE/SFTP) Exporting all content to the remote server...`)
+
+    // -> Pages
+    await pipeline(
+      WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
+        isPrivate: false
+      }).stream(),
+      new stream.Transform({
+        objectMode: true,
+        transform: async (page, enc, cb) => {
+          const filePath = getFilePath(page, 'path')
+          WIKI.logger.info(`(STORAGE/SFTP) Adding page ${filePath}...`)
+          await this.ensureDirectory(filePath)
+          await this.sftp.writeFile(path.posix.join(this.config.basePath, filePath), pageHelper.injectPageMetadata(page))
+          cb()
+        }
+      })
+    )
+
+    // -> Assets
+    const assetFolders = await WIKI.models.assetFolders.getAllPaths()
+
+    await pipeline(
+      WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
+      new stream.Transform({
+        objectMode: true,
+        transform: async (asset, enc, cb) => {
+          const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
+          WIKI.logger.info(`(STORAGE/SFTP) Adding asset ${filename}...`)
+          await this.ensureDirectory(filename)
+          await this.sftp.writeFile(path.posix.join(this.config.basePath, filename), asset.data)
+          cb()
+        }
+      })
+    )
+
+    WIKI.logger.info('(STORAGE/SFTP) All content has been pushed to the remote server.')
+  },
+  async ensureDirectory(filePath) {
+    if (filePath.indexOf('/') >= 0) {
+      try {
+        const folderPaths = _.dropRight(filePath.split('/'))
+        for (let i = 1; i <= folderPaths.length; i++) {
+          const folderSection = _.take(folderPaths, i).join('/')
+          await this.sftp.mkdir(path.posix.join(this.config.basePath, folderSection))
+        }
+      } catch (err) {}
+    }
+  }
+}