Skip to content

W
wiki-js
Unverified Commit edb52937 authored by NGPixel's avatar NGPixel
Browse files
Options

refactor: fix models + storage modules

parent 027b1614
Hide whitespace changes
Inline Side-by-side
Showing with 1376 additions and 1072 deletions
dev.code-workspace
View file @ edb52937
{
"folders": [
{
"path": "ux"
"name": "server",
"path": "server"
},
{
"path": "server"
"name": "ux",
"path": "ux"
}
],
"settings": {
......@@ -13,4 +15,4 @@
"src/i18n/locales"
]
}
}
\ No newline at end of file
}
package.json
View file @ edb52937
......@@ -7,7 +7,8 @@
"dev": true,
"scripts": {
"start": "node server",
"dev": "node dev",
"dev": "nodemon server",
"dev-legacy": "node dev",
"test": "eslint --format codeframe --ext .js,.vue . && pug-lint server/views && jest",
"cypress:open": "cypress open"
},
......@@ -119,12 +120,12 @@
"moment-timezone": "0.5.31",
"ms": "2.1.3",
"multer": "1.4.4",
"nanoid": "3.2.0",
"nanoid": "3.3.2",
"node-2fa": "1.1.2",
"node-cache": "5.1.2",
"nodemailer": "6.7.3",
"objection": "3.0.1",
"passport": "0.4.1",
"passport": "0.5.2",
"passport-auth0": "1.4.2",
"passport-azure-ad": "4.3.1",
"passport-cas": "0.1.1",
......@@ -135,13 +136,13 @@
"passport-gitlab2": "5.0.0",
"passport-google-oauth20": "2.0.0",
"passport-jwt": "4.0.0",
"passport-ldapauth": "2.1.4",
"passport-ldapauth": "3.0.1",
"passport-local": "1.0.0",
"passport-microsoft": "0.1.0",
"passport-oauth2": "1.6.1",
"passport-okta-oauth": "0.0.1",
"passport-openidconnect": "0.0.2",
"passport-saml": "1.3.5",
"passport-openidconnect": "0.1.1",
"passport-saml": "3.2.1",
"passport-slack-oauth2": "1.1.1",
"passport-twitch-oauth": "1.0.0",
"pem-jwk": "2.0.0",
......@@ -241,7 +242,7 @@
"filepond-plugin-file-validate-type": "1.2.6",
"filesize.js": "2.0.0",
"graphql-persisted-document-loader": "2.0.0",
"graphql-tag": "2.11.0",
"graphql-tag": "2.12.6",
"hammerjs": "2.0.8",
"html-webpack-plugin": "4.5.0",
"html-webpack-pug-plugin": "2.0.0",
......@@ -256,6 +257,7 @@
"mini-css-extract-plugin": "0.11.3",
"moment-duration-format": "2.3.2",
"moment-timezone-data-webpack-plugin": "1.3.0",
"nodemon": "2.0.15",
"offline-plugin": "5.0.7",
"optimize-css-assets-webpack-plugin": "5.0.4",
"pako": "1.0.11",
......@@ -265,7 +267,7 @@
"postcss-import": "12.0.1",
"postcss-loader": "3.0.0",
"postcss-preset-env": "6.7.0",
"postcss-selector-parser": "6.0.9",
"postcss-selector-parser": "6.0.10",
"prismjs": "1.22.0",
"pug-lint": "2.6.0",
"pug-loader": "2.4.0",
......@@ -371,5 +373,14 @@
"type": "opencollective",
"url": "https://opencollective.com/wikijs",
"logo": "https://opencollective.com/opencollective/logo.txt"
},
"nodemonConfig": {
"env": {
"NODE_ENV": "development"
},
"ext": "js,json,graphql,gql",
"watch": [
"server/"
]
}
}
server/core/auth.js
View file @ edb52937
......@@ -66,7 +66,7 @@ module.exports = {
// Load JWT
passport.use('jwt', new passportJWT.Strategy({
jwtFromRequest: securityHelper.extractJWT,
secretOrKey: WIKI.config.certs.public,
secretOrKey: WIKI.config.auth.certs.public,
audience: WIKI.config.auth.audience,
issuer: 'urn:wiki.js',
algorithms: ['RS256']
......@@ -76,13 +76,13 @@ module.exports = {
// Load enabled strategies
const enabledStrategies = await WIKI.models.authentication.getStrategies()
for (let idx in enabledStrategies) {
for (const idx in enabledStrategies) {
const stg = enabledStrategies[idx]
try {
const strategy = require(`../modules/authentication/${stg.strategyKey}/authentication.js`)
const strategy = require(`../modules/authentication/${stg.module}/authentication.js`)
stg.config.callbackURL = `${WIKI.config.host}/login/${stg.key}/callback`
stg.config.key = stg.key;
stg.config.callbackURL = `${WIKI.config.host}/login/${stg.id}/callback`
stg.config.key = stg.id
strategy.init(passport, stg.config)
strategy.config = stg.config
......@@ -92,7 +92,7 @@ module.exports = {
}
WIKI.logger.info(`Authentication Strategy ${stg.displayName}: [ OK ]`)
} catch (err) {
WIKI.logger.error(`Authentication Strategy ${stg.displayName} (${stg.key}): [ FAILED ]`)
WIKI.logger.error(`Authentication Strategy ${stg.displayName} (${stg.id}): [ FAILED ]`)
WIKI.logger.error(err)
}
}
......
server/core/kernel.js
View file @ edb52937
......@@ -67,7 +67,6 @@ module.exports = {
await WIKI.models.analytics.refreshProvidersFromDisk()
await WIKI.models.authentication.refreshStrategiesFromDisk()
await WIKI.models.commentProviders.refreshProvidersFromDisk()
await WIKI.models.editors.refreshEditorsFromDisk()
await WIKI.models.renderers.refreshRenderersFromDisk()
await WIKI.models.storage.refreshTargetsFromDisk()
......@@ -76,7 +75,7 @@ module.exports = {
await WIKI.auth.activateStrategies()
await WIKI.models.commentProviders.initProvider()
await WIKI.models.storage.initTargets()
WIKI.scheduler.start()
// WIKI.scheduler.start()
await WIKI.models.subscribeToNotifications()
},
......
server/core/servers.js
View file @ edb52937
......@@ -4,6 +4,7 @@ const https = require('https')
const { ApolloServer } = require('apollo-server-express')
const Promise = require('bluebird')
const _ = require('lodash')
const { ApolloServerPluginLandingPageGraphQLPlayground, ApolloServerPluginLandingPageProductionDefault } = require('apollo-server-core')
/* global WIKI */
......@@ -123,6 +124,11 @@ module.exports = {
uploads: false,
context: ({ req, res }) => ({ req, res }),
plugins: [
process.env.NODE_ENV === 'development' ? ApolloServerPluginLandingPageGraphQLPlayground({
footer: false
}) : ApolloServerPluginLandingPageProductionDefault({
footer: false
})
// ApolloServerPluginDrainHttpServer({ httpServer: this.servers.http })
// ...(this.servers.https && ApolloServerPluginDrainHttpServer({ httpServer: this.servers.https }))
]
......
server/db/migrations/3.0.0.js
View file @ edb52937
......@@ -71,6 +71,12 @@ exports.up = async knex => {
table.jsonb('autoEnrollGroups').notNullable().defaultTo('[]')
table.jsonb('hideOnSites').notNullable().defaultTo('[]')
})
.createTable('commentProviders', table => {
table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()'))
table.string('module').notNullable()
table.boolean('isEnabled').notNullable().defaultTo(false)
table.json('config').notNullable()
})
// COMMENTS ----------------------------
.createTable('comments', table => {
table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()'))
......@@ -140,6 +146,7 @@ exports.up = async knex => {
table.timestamp('publishEndDate')
table.string('action').defaultTo('updated')
table.text('content')
table.string('editor').notNullable()
table.string('contentType').notNullable()
table.jsonb('extra').notNullable().defaultTo('{}')
table.jsonb('tags').defaultTo('[]')
......@@ -166,6 +173,7 @@ exports.up = async knex => {
table.text('content')
table.text('render')
table.jsonb('toc')
table.string('editor').notNullable()
table.string('contentType').notNullable()
table.jsonb('extra').notNullable().defaultTo('{}')
table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now())
......@@ -279,6 +287,9 @@ exports.up = async knex => {
.table('assetFolders', table => {
table.uuid('parentId').references('id').inTable('assetFolders').index()
})
.table('commentProviders', table => {
table.uuid('siteId').notNullable().references('id').inTable('sites')
})
.table('comments', table => {
table.uuid('pageId').notNullable().references('id').inTable('pages').index()
table.uuid('authorId').notNullable().references('id').inTable('users').index()
......@@ -306,6 +317,9 @@ exports.up = async knex => {
table.uuid('pageId').notNullable().references('id').inTable('pages').onDelete('CASCADE')
table.string('localeCode', 5).references('code').inTable('locales')
})
.table('renderers', table => {
table.uuid('siteId').notNullable().references('id').inTable('sites')
})
.table('storage', table => {
table.uuid('siteId').notNullable().references('id').inTable('sites')
})
......@@ -324,10 +338,51 @@ exports.up = async knex => {
// DEFAULT DATA
// =====================================
// -> GENERATE IDS
const groupAdminId = uuid()
const groupGuestId = '10000000-0000-4000-0000-000000000001'
const siteId = uuid()
const authModuleId = uuid()
const userAdminId = uuid()
const userGuestId = uuid()
// -> SYSTEM CONFIG
WIKI.logger.info('Generating certificates...')
const secret = crypto.randomBytes(32).toString('hex')
const certs = crypto.generateKeyPairSync('rsa', {
modulusLength: 2048,
publicKeyEncoding: {
type: 'pkcs1',
format: 'pem'
},
privateKeyEncoding: {
type: 'pkcs1',
format: 'pem',
cipher: 'aes-256-cbc',
passphrase: secret
}
})
await knex('settings').insert([
{
key: 'auth',
value: {
audience: 'urn:wiki.js',
tokenExpiration: '30m',
tokenRenewal: '14d',
certs: {
jwk: pem2jwk(certs.publicKey),
public: certs.publicKey,
private: certs.privateKey
},
secret,
rootAdminUserId: userAdminId,
guestUserId: userGuestId
}
},
{
key: 'mail',
value: {
senderName: '',
......@@ -368,12 +423,6 @@ exports.up = async knex => {
}
},
{
key: 'system',
value: {
sessionSecret: crypto.randomBytes(32).toString('hex')
}
},
{
key: 'update',
value: {
locales: true
......@@ -393,39 +442,11 @@ exports.up = async knex => {
// -> DEFAULT SITE
WIKI.logger.info('Generating certificates...')
const secret = crypto.randomBytes(32).toString('hex')
const certs = crypto.generateKeyPairSync('rsa', {
modulusLength: 2048,
publicKeyEncoding: {
type: 'pkcs1',
format: 'pem'
},
privateKeyEncoding: {
type: 'pkcs1',
format: 'pem',
cipher: 'aes-256-cbc',
passphrase: secret
}
})
const siteId = uuid()
await knex('sites').insert({
id: siteId,
hostname: '*',
isEnabled: true,
config: {
auth: {
audience: 'urn:wiki.js',
tokenExpiration: '30m',
tokenRenewal: '14d',
certs: {
jwk: pem2jwk(certs.publicKey),
public: certs.publicKey,
private: certs.privateKey
},
secret
},
title: 'My Wiki Site',
description: '',
company: '',
......@@ -471,8 +492,6 @@ exports.up = async knex => {
// -> DEFAULT GROUPS
const groupAdminId = uuid()
const groupGuestId = '10000000-0000-4000-0000-000000000001'
await knex('groups').insert([
{
id: groupAdminId,
......@@ -503,7 +522,6 @@ exports.up = async knex => {
// -> AUTHENTICATION MODULE
const authModuleId = uuid()
await knex('authentication').insert({
id: authModuleId,
module: 'local',
......@@ -513,8 +531,6 @@ exports.up = async knex => {
// -> USERS
const userAdminId = uuid()
const userGuestId = uuid()
await knex('users').insert([
{
id: userAdminId,
......
server/index.js
View file @ edb52937
......@@ -22,7 +22,11 @@ let WIKI = {
Error: require('./helpers/error'),
configSvc: require('./core/config'),
kernel: require('./core/kernel'),
startedAt: DateTime.utc()
startedAt: DateTime.utc(),
storage: {
defs: [],
modules: []
}
}
global.WIKI = WIKI
......
server/master.js
View file @ edb52937
......@@ -77,7 +77,7 @@ module.exports = async () => {
app.use(cookieParser())
app.use(session({
secret: WIKI.config.system.sessionSecret,
secret: WIKI.config.auth.secret,
resave: false,
saveUninitialized: false,
store: new KnexSessionStore({
......
server/models/analytics.js
View file @ edb52937
......@@ -12,7 +12,6 @@ const commonHelper = require('../helpers/common')
*/
module.exports = class Analytics extends Model {
static get tableName() { return 'analytics' }
static get idColumn() { return 'key' }
static get jsonSchema () {
return {
......@@ -52,7 +51,7 @@ module.exports = class Analytics extends Model {
WIKI.logger.info(`Loaded ${WIKI.data.analytics.length} analytics module definitions: [ OK ]`)
} catch (err) {
WIKI.logger.error(`Failed to scan or load new analytics providers: [ FAILED ]`)
WIKI.logger.error(`Failed to scan or load analytics providers: [ FAILED ]`)
WIKI.logger.error(err)
}
}
......
server/models/authentication.js
View file @ edb52937
......@@ -12,15 +12,15 @@ const commonHelper = require('../helpers/common')
*/
module.exports = class Authentication extends Model {
static get tableName() { return 'authentication' }
static get idColumn() { return 'key' }
static get jsonSchema () {
return {
type: 'object',
required: ['key'],
required: ['module'],
properties: {
key: {type: 'string'},
id: { type: 'string' },
module: { type: 'string' },
selfRegistration: {type: 'boolean'}
}
}
......@@ -43,79 +43,23 @@ module.exports = class Authentication extends Model {
}))
}
static async getStrategiesForLegacyClient() {
const strategies = await WIKI.models.authentication.query().select('key', 'selfRegistration')
let formStrategies = []
let socialStrategies = []
for (let stg of strategies) {
const stgInfo = _.find(WIKI.data.authentication, ['key', stg.key]) || {}
if (stgInfo.useForm) {
formStrategies.push({
key: stg.key,
title: stgInfo.title
})
} else {
socialStrategies.push({
...stgInfo,
...stg,
icon: await fs.readFile(path.join(WIKI.ROOTPATH, `assets/svg/auth-icon-${stg.key}.svg`), 'utf8').catch(err => {
if (err.code === 'ENOENT') {
return null
}
throw err
})
})
}
}
return {
formStrategies,
socialStrategies
}
}
static async refreshStrategiesFromDisk() {
try {
const dbStrategies = await WIKI.models.authentication.query()
// -> Fetch definitions from disk
const authDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/authentication'))
const authenticationDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/authentication'))
WIKI.data.authentication = []
for (let dir of authDirs) {
const defRaw = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/authentication', dir, 'definition.yml'), 'utf8')
const def = yaml.safeLoad(defRaw)
WIKI.data.authentication.push({
...def,
props: commonHelper.parseModuleProps(def.props)
})
}
for (const strategy of dbStrategies) {
const strategyDef = _.find(WIKI.data.authentication, ['key', strategy.strategyKey])
if (!strategyDef) {
await WIKI.models.authentication.query().delete().where('key', strategy.key)
WIKI.logger.info(`Authentication strategy ${strategy.strategyKey} was removed from disk: [ REMOVED ]`)
continue
}
strategy.config = _.transform(strategyDef.props, (result, value, key) => {
if (!_.has(result, key)) {
_.set(result, key, value.default)
}
return result
}, strategy.config)
// Fix pre-2.5 strategies displayName
if (!strategy.displayName) {
await WIKI.models.authentication.query().patch({
displayName: strategyDef.title
}).where('key', strategy.key)
}
for (const dir of authenticationDirs) {
const def = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/authentication', dir, 'definition.yml'), 'utf8')
const defParsed = yaml.load(def)
defParsed.key = dir
defParsed.props = commonHelper.parseModuleProps(defParsed.props)
WIKI.data.analytics.push(defParsed)
WIKI.logger.debug(`Loaded authentication module definition ${dir}: [ OK ]`)
}
WIKI.logger.info(`Loaded ${WIKI.data.authentication.length} authentication strategies: [ OK ]`)
WIKI.logger.info(`Loaded ${WIKI.data.analytics.length} authentication module definitions: [ OK ]`)
} catch (err) {
WIKI.logger.error(`Failed to scan or load new authentication providers: [ FAILED ]`)
WIKI.logger.error(`Failed to scan or load authentication providers: [ FAILED ]`)
WIKI.logger.error(err)
}
}
......
server/models/commentProviders.js
View file @ edb52937
......@@ -36,65 +36,27 @@ module.exports = class CommentProvider extends Model {
static async getProviders(isEnabled) {
const providers = await WIKI.models.commentProviders.query().where(_.isBoolean(isEnabled) ? { isEnabled } : {})
return _.sortBy(providers, ['key'])
return _.sortBy(providers, ['module'])
}
static async refreshProvidersFromDisk() {
let trx
try {
const dbProviders = await WIKI.models.commentProviders.query()
// -> Fetch definitions from disk
const commentDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/comments'))
let diskProviders = []
for (let dir of commentDirs) {
const commentsDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/comments'))
WIKI.data.commentProviders = []
for (const dir of commentsDirs) {
const def = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/comments', dir, 'definition.yml'), 'utf8')
diskProviders.push(yaml.safeLoad(def))
const defParsed = yaml.load(def)
defParsed.key = dir
defParsed.props = commonHelper.parseModuleProps(defParsed.props)
WIKI.data.commentProviders.push(defParsed)
WIKI.logger.debug(`Loaded comments provider module definition ${dir}: [ OK ]`)
}
WIKI.data.commentProviders = diskProviders.map(provider => ({
...provider,
props: commonHelper.parseModuleProps(provider.props)
}))
let newProviders = []
for (let provider of WIKI.data.commentProviders) {
if (!_.some(dbProviders, ['key', provider.key])) {
newProviders.push({
key: provider.key,
isEnabled: provider.key === 'default',
config: _.transform(provider.props, (result, value, key) => {
_.set(result, key, value.default)
return result
}, {})
})
} else {
const providerConfig = _.get(_.find(dbProviders, ['key', provider.key]), 'config', {})
await WIKI.models.commentProviders.query().patch({
config: _.transform(provider.props, (result, value, key) => {
if (!_.has(result, key)) {
_.set(result, key, value.default)
}
return result
}, providerConfig)
}).where('key', provider.key)
}
}
if (newProviders.length > 0) {
trx = await WIKI.models.Objection.transaction.start(WIKI.models.knex)
for (let provider of newProviders) {
await WIKI.models.commentProviders.query(trx).insert(provider)
}
await trx.commit()
WIKI.logger.info(`Loaded ${newProviders.length} new comment providers: [ OK ]`)
} else {
WIKI.logger.info(`No new comment providers found: [ SKIPPED ]`)
}
WIKI.logger.info(`Loaded ${WIKI.data.commentProviders.length} comments providers module definitions: [ OK ]`)
} catch (err) {
WIKI.logger.error(`Failed to scan or load new comment providers: [ FAILED ]`)
WIKI.logger.error(`Failed to scan or load comments providers: [ FAILED ]`)
WIKI.logger.error(err)
if (trx) {
trx.rollback()
}
}
}
......@@ -102,7 +64,7 @@ module.exports = class CommentProvider extends Model {
const commentProvider = await WIKI.models.commentProviders.query().findOne('isEnabled', true)
if (commentProvider) {
WIKI.data.commentProvider = {
..._.find(WIKI.data.commentProviders, ['key', commentProvider.key]),
..._.find(WIKI.data.commentProviders, ['key', commentProvider.module]),
head: '',
bodyStart: '',
bodyEnd: '',
......
server/models/editors.js
View file @ edb52937
const Model = require('objection').Model
const fs = require('fs-extra')
const path = require('path')
const _ = require('lodash')
const yaml = require('js-yaml')
const commonHelper = require('../helpers/common')
/* global WIKI */
......@@ -34,66 +29,6 @@ module.exports = class Editor extends Model {
return WIKI.models.editors.query()
}
static async refreshEditorsFromDisk() {
let trx
try {
const dbEditors = await WIKI.models.editors.query()
// -> Fetch definitions from disk
const editorDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/editor'))
let diskEditors = []
for (let dir of editorDirs) {
const def = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/editor', dir, 'definition.yml'), 'utf8')
diskEditors.push(yaml.safeLoad(def))
}
WIKI.data.editors = diskEditors.map(editor => ({
...editor,
props: commonHelper.parseModuleProps(editor.props)
}))
// -> Insert new editors
let newEditors = []
for (let editor of WIKI.data.editors) {
if (!_.some(dbEditors, ['key', editor.key])) {
newEditors.push({
key: editor.key,
isEnabled: false,
config: _.transform(editor.props, (result, value, key) => {
_.set(result, key, value.default)
return result
}, {})
})
} else {
const editorConfig = _.get(_.find(dbEditors, ['key', editor.key]), 'config', {})
await WIKI.models.editors.query().patch({
config: _.transform(editor.props, (result, value, key) => {
if (!_.has(result, key)) {
_.set(result, key, value.default)
}
return result
}, editorConfig)
}).where('key', editor.key)
}
}
if (newEditors.length > 0) {
trx = await WIKI.models.Objection.transaction.start(WIKI.models.knex)
for (let editor of newEditors) {
await WIKI.models.editors.query(trx).insert(editor)
}
await trx.commit()
WIKI.logger.info(`Loaded ${newEditors.length} new editors: [ OK ]`)
} else {
WIKI.logger.info(`No new editors found: [ SKIPPED ]`)
}
} catch (err) {
WIKI.logger.error(`Failed to scan or load new editors: [ FAILED ]`)
WIKI.logger.error(err)
if (trx) {
trx.rollback()
}
}
}
static async getDefaultEditor(contentType) {
// TODO - hardcoded for now
switch (contentType) {
......
server/models/pages.js
View file @ edb52937
......@@ -41,7 +41,7 @@ module.exports = class Page extends Model {
hash: {type: 'string'},
title: {type: 'string'},
description: {type: 'string'},
isPublished: {type: 'boolean'},
publishState: {type: 'string'},
privateNS: {type: 'string'},
publishStartDate: {type: 'string'},
publishEndDate: {type: 'string'},
......@@ -96,14 +96,6 @@ module.exports = class Page extends Model {
to: 'users.id'
}
},
editor: {
relation: Model.BelongsToOneRelation,
modelClass: require('./editors'),
join: {
from: 'pages.editorKey',
to: 'editors.key'
}
},
locale: {
relation: Model.BelongsToOneRelation,
modelClass: require('./locales'),
......@@ -143,16 +135,14 @@ module.exports = class Page extends Model {
creatorId: 'uint',
creatorName: 'string',
description: 'string',
editorKey: 'string',
isPrivate: 'boolean',
isPublished: 'boolean',
editor: 'string',
publishState: 'string',
publishEndDate: 'string',
publishStartDate: 'string',
render: 'string',
tags: [
{
tag: 'string',
title: 'string'
tag: 'string'
}
],
extra: {
......@@ -301,10 +291,9 @@ module.exports = class Page extends Model {
creatorId: opts.user.id,
contentType: _.get(_.find(WIKI.data.editors, ['key', opts.editor]), `contentType`, 'text'),
description: opts.description,
editorKey: opts.editor,
hash: pageHelper.generateHash({ path: opts.path, locale: opts.locale, privateNS: opts.isPrivate ? 'TODO' : '' }),
isPrivate: opts.isPrivate,
isPublished: opts.isPublished,
editor: opts.editor,
hash: pageHelper.generateHash({ path: opts.path, locale: opts.locale }),
publishState: opts.publishState,
localeCode: opts.locale,
path: opts.path,
publishEndDate: opts.publishEndDate || '',
......@@ -319,8 +308,7 @@ module.exports = class Page extends Model {
const page = await WIKI.models.pages.getPageFromDb({
path: opts.path,
locale: opts.locale,
userId: opts.user.id,
isPrivate: opts.isPrivate
userId: opts.user.id
})
// -> Save Tags
......@@ -389,7 +377,6 @@ module.exports = class Page extends Model {
// -> Create version snapshot
await WIKI.models.pageHistory.addVersion({
...ogPage,
isPublished: ogPage.isPublished === true || ogPage.isPublished === 1,
action: opts.action ? opts.action : 'updated',
versionDate: ogPage.updatedAt
})
......@@ -426,7 +413,7 @@ module.exports = class Page extends Model {
authorId: opts.user.id,
content: opts.content,
description: opts.description,
isPublished: opts.isPublished === true || opts.isPublished === 1,
publishState: opts.publishState,
publishEndDate: opts.publishEndDate || '',
publishStartDate: opts.publishStartDate || '',
title: opts.title,
......@@ -500,7 +487,7 @@ module.exports = class Page extends Model {
throw new Error('Invalid Page Id')
}
if (ogPage.editorKey === opts.editor) {
if (ogPage.editor === opts.editor) {
throw new Error('Page is already using this editor. Nothing to convert.')
}
......@@ -631,7 +618,6 @@ module.exports = class Page extends Model {
if (shouldConvert) {
await WIKI.models.pageHistory.addVersion({
...ogPage,
isPublished: ogPage.isPublished === true || ogPage.isPublished === 1,
action: 'updated',
versionDate: ogPage.updatedAt
})
......@@ -640,7 +626,7 @@ module.exports = class Page extends Model {
// -> Update page
await WIKI.models.pages.query().patch({
contentType: targetContentType,
editorKey: opts.editor,
editor: opts.editor,
...(convertedContent ? { content: convertedContent } : {})
}).where('id', ogPage.id)
const page = await WIKI.models.pages.getPageFromDb(ogPage.id)
......@@ -721,7 +707,7 @@ module.exports = class Page extends Model {
versionDate: page.updatedAt
})
const destinationHash = pageHelper.generateHash({ path: opts.destinationPath, locale: opts.destinationLocale, privateNS: opts.isPrivate ? 'TODO' : '' })
const destinationHash = pageHelper.generateHash({ path: opts.destinationPath, locale: opts.destinationLocale })
// -> Move page
const destinationTitle = (page.title === page.path ? opts.destinationPath : page.title)
......@@ -991,9 +977,7 @@ module.exports = class Page extends Model {
'pages.hash',
'pages.title',
'pages.description',
'pages.isPrivate',
'pages.isPublished',
'pages.privateNS',
'pages.publishState',
'pages.publishStartDate',
'pages.publishEndDate',
'pages.content',
......@@ -1002,7 +986,7 @@ module.exports = class Page extends Model {
'pages.contentType',
'pages.createdAt',
'pages.updatedAt',
'pages.editorKey',
'pages.editor',
'pages.localeCode',
'pages.authorId',
'pages.creatorId',
......@@ -1018,7 +1002,7 @@ module.exports = class Page extends Model {
.joinRelated('creator')
.withGraphJoined('tags')
.modifyGraph('tags', builder => {
builder.select('tag', 'title')
builder.select('tag')
})
.where(queryModeID ? {
'pages.id': opts
......@@ -1066,17 +1050,16 @@ module.exports = class Page extends Model {
creatorId: page.creatorId,
creatorName: page.creatorName,
description: page.description,
editorKey: page.editorKey,
editor: page.editor,
extra: {
css: _.get(page, 'extra.css', ''),
js: _.get(page, 'extra.js', '')
},
isPrivate: page.isPrivate === 1 || page.isPrivate === true,
isPublished: page.isPublished === 1 || page.isPublished === true,
publishState: page.publishState,
publishEndDate: page.publishEndDate,
publishStartDate: page.publishStartDate,
render: page.render,
tags: page.tags.map(t => _.pick(t, ['tag', 'title'])),
tags: page.tags.map(t => _.pick(t, ['tag'])),
title: page.title,
toc: _.isString(page.toc) ? page.toc : JSON.stringify(page.toc),
updatedAt: page.updatedAt
......@@ -1090,7 +1073,7 @@ module.exports = class Page extends Model {
* @returns {Promise} Promise of the Page Model Instance
*/
static async getPageFromCache(opts) {
const pageHash = pageHelper.generateHash({ path: opts.path, locale: opts.locale, privateNS: opts.isPrivate ? 'TODO' : '' })
const pageHash = pageHelper.generateHash({ path: opts.path, locale: opts.locale })
const cachePath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, `cache/${pageHash}.bin`)
try {
......
server/models/renderers.js
View file @ edb52937
......@@ -13,15 +13,15 @@ const commonHelper = require('../helpers/common')
*/
module.exports = class Renderer extends Model {
static get tableName() { return 'renderers' }
static get idColumn() { return 'key' }
static get jsonSchema () {
return {
type: 'object',
required: ['key', 'isEnabled'],
required: ['module', 'isEnabled'],
properties: {
key: {type: 'string'},
id: {type: 'string'},
module: {type: 'string'},
isEnabled: {type: 'boolean'}
}
}
......@@ -36,77 +36,35 @@ module.exports = class Renderer extends Model {
}
static async fetchDefinitions() {
const rendererDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/rendering'))
let diskRenderers = []
for (let dir of rendererDirs) {
const def = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/rendering', dir, 'definition.yml'), 'utf8')
diskRenderers.push(yaml.safeLoad(def))
}
WIKI.data.renderers = diskRenderers.map(renderer => ({
...renderer,
props: commonHelper.parseModuleProps(renderer.props)
}))
}
static async refreshRenderersFromDisk() {
let trx
try {
const dbRenderers = await WIKI.models.renderers.query()
// -> Fetch definitions from disk
await WIKI.models.renderers.fetchDefinitions()
// -> Insert new Renderers
let newRenderers = []
for (let renderer of WIKI.data.renderers) {
if (!_.some(dbRenderers, ['key', renderer.key])) {
newRenderers.push({
key: renderer.key,
isEnabled: _.get(renderer, 'enabledDefault', true),
config: _.transform(renderer.props, (result, value, key) => {
_.set(result, key, value.default)
return result
}, {})
})
} else {
const rendererConfig = _.get(_.find(dbRenderers, ['key', renderer.key]), 'config', {})
await WIKI.models.renderers.query().patch({
config: _.transform(renderer.props, (result, value, key) => {
if (!_.has(result, key)) {
_.set(result, key, value.default)
}
return result
}, rendererConfig)
}).where('key', renderer.key)
}
}
if (newRenderers.length > 0) {
trx = await WIKI.models.Objection.transaction.start(WIKI.models.knex)
for (let renderer of newRenderers) {
await WIKI.models.renderers.query(trx).insert(renderer)
}
await trx.commit()
WIKI.logger.info(`Loaded ${newRenderers.length} new renderers: [ OK ]`)
} else {
WIKI.logger.info(`No new renderers found: [ SKIPPED ]`)
const renderersDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/rendering'))
WIKI.data.renderers = []
for (const dir of renderersDirs) {
const def = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/rendering', dir, 'definition.yml'), 'utf8')
const defParsed = yaml.load(def)
defParsed.key = dir
defParsed.props = commonHelper.parseModuleProps(defParsed.props)
WIKI.data.renderers.push(defParsed)
WIKI.logger.debug(`Loaded renderers module definition ${dir}: [ OK ]`)
}
// -> Delete removed Renderers
for (const renderer of dbRenderers) {
if (!_.some(WIKI.data.renderers, ['key', renderer.key])) {
await WIKI.models.renderers.query().where('key', renderer.key).del()
WIKI.logger.info(`Removed renderer ${renderer.key} because it is no longer present in the modules folder: [ OK ]`)
}
}
WIKI.logger.info(`Loaded ${WIKI.data.renderers.length} renderers module definitions: [ OK ]`)
} catch (err) {
WIKI.logger.error(`Failed to scan or load new renderers: [ FAILED ]`)
WIKI.logger.error(`Failed to scan or load renderers providers: [ FAILED ]`)
WIKI.logger.error(err)
if (trx) {
trx.rollback()
}
}
}
static async refreshRenderersFromDisk() {
// const dbRenderers = await WIKI.models.renderers.query()
// -> Fetch definitions from disk
await WIKI.models.renderers.fetchDefinitions()
// TODO: Merge existing configs with updated modules
}
static async getRenderingPipeline(contentType) {
const renderersDb = await WIKI.models.renderers.query().where('isEnabled', true)
if (renderersDb && renderersDb.length > 0) {
......
server/models/sites.js 0 → 100644
View file @ edb52937
const Model = require('objection').Model
const crypto = require('crypto')
const pem2jwk = require('pem-jwk').pem2jwk
const _ = require('lodash')
/* global WIKI */
/**
* Site model
*/
module.exports = class Site extends Model {
static get tableName () { return 'sites' }
static get jsonSchema () {
return {
type: 'object',
required: ['hostname'],
properties: {
id: { type: 'string' },
hostname: { type: 'string' },
isEnabled: { type: 'boolean', default: false }
}
}
}
static get jsonAttributes () {
return ['config']
}
static async createSite (hostname, config) {
const newSite = await WIKI.models.sites.query().insertAndFetch({
hostname,
isEnabled: true,
config: _.defaultsDeep(config, {
title: 'My Wiki Site',
description: '',
company: '',
contentLicense: '',
defaults: {
timezone: 'America/New_York',
dateFormat: 'YYYY-MM-DD',
timeFormat: '12h'
},
features: {
ratings: false,
ratingsMode: 'off',
comments: false,
contributions: false,
profile: true,
search: true
},
logoUrl: '',
logoText: true,
robots: {
index: true,
follow: true
},
locale: 'en',
localeNamespacing: false,
localeNamespaces: [],
theme: {
dark: false,
colorPrimary: '#1976d2',
colorSecondary: '#02c39a',
colorAccent: '#f03a47',
colorHeader: '#000000',
colorSidebar: '#1976d2',
injectCSS: '',
injectHead: '',
injectBody: '',
sidebarPosition: 'left',
tocPosition: 'right',
showSharingMenu: true,
showPrintBtn: true
}
})
})
await WIKI.models.storage.query().insert({
module: 'db',
siteId: newSite.id,
isEnabled: true,
contentTypes: {
activeTypes: ['pages', 'images', 'documents', 'others', 'large'],
largeThreshold: '5MB'
},
assetDelivery: {
streaming: true,
directAccess: false
},
state: {
current: 'ok'
}
})
return newSite
}
static async updateSite (id, patch) {
return WIKI.models.sites.query().findById(id).patch(patch)
}
static async deleteSite (id) {
await WIKI.models.storage.query().delete().where('siteId', id)
return WIKI.models.sites.query().deleteById(id)
}
}
server/models/storage.js
View file @ edb52937
......@@ -17,93 +17,45 @@ module.exports = class Storage extends Model {
static get jsonSchema () {
return {
type: 'object',
required: ['key', 'isEnabled'],
required: ['module', 'isEnabled', 'siteId'],
properties: {
key: {type: 'string'},
module: {type: 'string'},
isEnabled: {type: 'boolean'},
mode: {type: 'string'}
SVGAnimatedInteger: {type: 'string'}
}
}
}
static get jsonAttributes() {
return ['config', 'state']
return ['contentTypes', 'assetDelivery', 'versioning', 'schedule', 'config', 'state']
}
static async getTargets() {
return WIKI.models.storage.query()
static async getTargets ({ siteId }) {
return WIKI.models.storage.query().where(builder => {
if (siteId) {
builder.where('siteId', siteId)
}
})
}
static async refreshTargetsFromDisk() {
static async refreshTargetsFromDisk () {
let trx
try {
const dbTargets = await WIKI.models.storage.query()
// -> Fetch definitions from disk
const storageDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/storage'))
let diskTargets = []
for (let dir of storageDirs) {
WIKI.storage.defs = []
for (const dir of storageDirs) {
const def = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/storage', dir, 'definition.yml'), 'utf8')
diskTargets.push(yaml.safeLoad(def))
}
WIKI.data.storage = diskTargets.map(target => ({
...target,
isAvailable: _.get(target, 'isAvailable', false),
props: commonHelper.parseModuleProps(target.props)
}))
// -> Insert new targets
let newTargets = []
for (let target of WIKI.data.storage) {
if (!_.some(dbTargets, ['key', target.key])) {
newTargets.push({
key: target.key,
isEnabled: false,
mode: target.defaultMode || 'push',
syncInterval: target.schedule || 'P0D',
config: _.transform(target.props, (result, value, key) => {
_.set(result, key, value.default)
return result
}, {}),
state: {
status: 'pending',
message: '',
lastAttempt: null
}
})
} else {
const targetConfig = _.get(_.find(dbTargets, ['key', target.key]), 'config', {})
await WIKI.models.storage.query().patch({
config: _.transform(target.props, (result, value, key) => {
if (!_.has(result, key)) {
_.set(result, key, value.default)
}
return result
}, targetConfig)
}).where('key', target.key)
}
}
if (newTargets.length > 0) {
trx = await WIKI.models.Objection.transaction.start(WIKI.models.knex)
for (let target of newTargets) {
await WIKI.models.storage.query(trx).insert(target)
}
await trx.commit()
WIKI.logger.info(`Loaded ${newTargets.length} new storage targets: [ OK ]`)
} else {
WIKI.logger.info(`No new storage targets found: [ SKIPPED ]`)
}
// -> Delete removed targets
for (const target of dbTargets) {
if (!_.some(WIKI.data.storage, ['key', target.key])) {
await WIKI.models.storage.query().where('key', target.key).del()
WIKI.logger.info(`Removed target ${target.key} because it is no longer present in the modules folder: [ OK ]`)
}
const defParsed = yaml.load(def)
defParsed.key = dir
defParsed.isLoaded = false
WIKI.storage.defs.push(defParsed)
WIKI.logger.debug(`Loaded storage module definition ${dir}: [ OK ]`)
}
WIKI.logger.info(`Loaded ${WIKI.storage.defs.length} storage module definitions: [ OK ]`)
} catch (err) {
WIKI.logger.error(`Failed to scan or load new storage providers: [ FAILED ]`)
WIKI.logger.error('Failed to scan or load new storage providers: [ FAILED ]')
WIKI.logger.error(err)
if (trx) {
trx.rollback()
......@@ -112,65 +64,90 @@ module.exports = class Storage extends Model {
}
/**
* Ensure a storage module is loaded
*/
static async ensureModule (moduleName) {
if (!_.has(WIKI.storage.modules, moduleName)) {
try {
WIKI.storage.modules[moduleName] = require(`../modules/storage/${moduleName}/storage`)
WIKI.logger.debug(`Activated storage module ${moduleName}: [ OK ]`)
return true
} catch (err) {
WIKI.logger.warn(`Failed to load storage module ${moduleName}: [ FAILED ]`)
WIKI.logger.warn(err)
return false
}
} else {
return true
}
}
/**
* Initialize active storage targets
*/
static async initTargets() {
this.targets = await WIKI.models.storage.query().where('isEnabled', true).orderBy('key')
static async initTargets () {
const dbTargets = await WIKI.models.storage.query().where('isEnabled', true)
const activeModules = _.uniq(dbTargets.map(t => t.module))
try {
// -> Stop and delete existing jobs
const prevjobs = _.remove(WIKI.scheduler.jobs, job => job.name === `sync-storage`)
if (prevjobs.length > 0) {
prevjobs.forEach(job => job.stop())
// const prevjobs = _.remove(WIKI.scheduler.jobs, job => job.name === 'sync-storage')
// if (prevjobs.length > 0) {
// prevjobs.forEach(job => job.stop())
// }
// -> Load active modules
for (const md of activeModules) {
this.ensureModule(md)
}
// -> Initialize targets
for (let target of this.targets) {
const targetDef = _.find(WIKI.data.storage, ['key', target.key])
target.fn = require(`../modules/storage/${target.key}/storage`)
target.fn.config = target.config
target.fn.mode = target.mode
try {
await target.fn.init()
// -> Save succeeded init state
await WIKI.models.storage.query().patch({
state: {
status: 'operational',
message: '',
lastAttempt: new Date().toISOString()
}
}).where('key', target.key)
// -> Set recurring sync job
if (targetDef.schedule && target.syncInterval !== `P0D`) {
WIKI.scheduler.registerJob({
name: `sync-storage`,
immediate: false,
schedule: target.syncInterval,
repeat: true
}, target.key)
}
// -> Set internal recurring sync job
if (targetDef.internalSchedule && targetDef.internalSchedule !== `P0D`) {
WIKI.scheduler.registerJob({
name: `sync-storage`,
immediate: false,
schedule: target.internalSchedule,
repeat: true
}, target.key)
}
} catch (err) {
// -> Save initialization error
await WIKI.models.storage.query().patch({
state: {
status: 'error',
message: err.message,
lastAttempt: new Date().toISOString()
}
}).where('key', target.key)
}
}
// for (const target of this.targets) {
// const targetDef = _.find(WIKI.data.storage, ['key', target.key])
// target.fn = require(`../modules/storage/${target.key}/storage`)
// target.fn.config = target.config
// target.fn.mode = target.mode
// try {
// await target.fn.init()
// // -> Save succeeded init state
// await WIKI.models.storage.query().patch({
// state: {
// status: 'operational',
// message: '',
// lastAttempt: new Date().toISOString()
// }
// }).where('key', target.key)
// // -> Set recurring sync job
// if (targetDef.schedule && target.syncInterval !== 'P0D') {
// WIKI.scheduler.registerJob({
// name: 'sync-storage',
// immediate: false,
// schedule: target.syncInterval,
// repeat: true
// }, target.key)
// }
// // -> Set internal recurring sync job
// if (targetDef.internalSchedule && targetDef.internalSchedule !== 'P0D') {
// WIKI.scheduler.registerJob({
// name: 'sync-storage',
// immediate: false,
// schedule: target.internalSchedule,
// repeat: true
// }, target.key)
// }
// } catch (err) {
// // -> Save initialization error
// await WIKI.models.storage.query().patch({
// state: {
// status: 'error',
// message: err.message,
// lastAttempt: new Date().toISOString()
// }
// }).where('key', target.key)
// }
// }
} catch (err) {
WIKI.logger.warn(err)
throw err
......
server/models/tags.js
View file @ edb52937
......@@ -17,7 +17,6 @@ module.exports = class Tag extends Model {
properties: {
id: {type: 'integer'},
tag: {type: 'string'},
title: {type: 'string'},
createdAt: {type: 'string'},
updatedAt: {type: 'string'}
......@@ -59,10 +58,7 @@ module.exports = class Tag extends Model {
// Create missing tags
const newTags = _.filter(tags, t => !_.some(existingTags, ['tag', t])).map(t => ({
tag: t,
title: t
}))
const newTags = _.filter(tags, t => !_.some(existingTags, ['tag', t])).map(t => ({ tag: t }))
if (newTags.length > 0) {
if (WIKI.config.db.type === 'postgres') {
const createdTags = await WIKI.models.tags.query().insert(newTags)
......
server/models/users.js
View file @ edb52937
......@@ -861,7 +861,7 @@ module.exports = class User extends Model {
* Logout the current user
*/
static async logout (context) {
if (!context.req.user || context.req.user.id === 2) {
if (!context.req.user || context.req.user.id === WIKI.config.auth.guestUserId) {
return '/'
}
const usr = await WIKI.models.users.query().findById(context.req.user.id).select('providerKey')
......@@ -870,7 +870,7 @@ module.exports = class User extends Model {
}
static async getGuestUser () {
const user = await WIKI.models.users.query().findById(2).withGraphJoined('groups').modifyGraph('groups', builder => {
const user = await WIKI.models.users.query().findById(WIKI.config.auth.guestUserId).withGraphJoined('groups').modifyGraph('groups', builder => {
builder.select('groups.id', 'permissions')
})
if (!user) {
......@@ -882,7 +882,7 @@ module.exports = class User extends Model {
}
static async getRootUser () {
let user = await WIKI.models.users.query().findById(1)
let user = await WIKI.models.users.query().findById(WIKI.config.auth.rootAdminUserId)
if (!user) {
WIKI.logger.error('CRITICAL ERROR: Root Administrator user is missing!')
process.exit(1)
......
server/modules/editor/api/definition.yml deleted 100644 → 0
View file @ 027b1614
key: api
title: API Docs
description: REST / GraphQL Editor
contentType: yml
author: requarks.io
props: {}
server/modules/editor/ckeditor/definition.yml deleted 100644 → 0
View file @ 027b1614
key: ckeditor
title: Visual Editor
description: Rich-text WYSIWYG Editor
contentType: html
author: requarks.io
props: {}
server/modules/editor/code/definition.yml deleted 100644 → 0
View file @ 027b1614
key: code
title: Code
description: Raw HTML editor
contentType: html
author: requarks.io
props: {}
server/modules/editor/markdown/definition.yml deleted 100644 → 0
View file @ 027b1614
key: markdown
title: Markdown
description: Basic Markdown editor
contentType: markdown
author: requarks.io
props: {}
server/modules/editor/redirect/definition.yml deleted 100644 → 0
View file @ 027b1614
key: redirect
title: Redirection
description: Redirect the user
contentType: redirect
author: requarks.io
props: {}
server/modules/editor/wysiwyg/definition.yml deleted 100644 → 0
View file @ 027b1614
key: wysiwyg
title: WYSIWYG
description: Advanced Visual HTML Builder
contentType: html
author: requarks.io
props: {}
server/modules/storage/azure/definition.yml
View file @ edb52937
key: azure
title: Azure Blob Storage
description: Azure Blob Storage by Microsoft provides massively scalable object storage for unstructured data.
author: requarks.io
logo: https://static.requarks.io/logo/azure.svg
website: https://azure.microsoft.com/services/storage/blobs/
isAvailable: true
supportedModes:
- push
defaultMode: push
schedule: false
icon: '/_assets/icons/ultraviolet-azure.svg'
banner: '/_assets/storage/azure.jpg'
description: Azure Blob Storage is Microsoft's object storage solution for the cloud. Blob storage is optimized for storing massive amounts of unstructured data.
vendor: Microsoft Corporation
website: 'https://azure.microsoft.com'
assetDelivery:
isStreamingSupported: true
isDirectAccessSupported: true
defaultStreamingEnabled: true
defaultDirectAccessEnabled: true
contentTypes:
defaultTypesEnabled: ['images', 'documents', 'others', 'large']
defaultLargeThreshold: '5MB'
versioning:
isSupported: false
defaultEnabled: false
sync: false
props:
accountName:
type: String
title: Account Name
default: ''
hint: Your unique account name.
icon: 3d-touch
order: 1
accountKey:
type: String
title: Account Access Key
default: ''
hint: Either key 1 or key 2.
icon: key
sensitive: true
order: 2
containerName:
type: String
title: Container Name
default: 'wiki'
default: wiki
hint: Will automatically be created if it doesn't exist yet.
icon: shipping-container
order: 3
storageTier:
type: String
title: Storage Tier
hint: Represents the access tier on a blob. Use Cool for lower storage costs but at higher retrieval costs.
icon: scan-stock
order: 4
default: 'Cool'
default: cool
enum:
- 'Hot'
- 'Cool'
- hot|Hot
- cool|Cool
actions:
- handler: exportAll
label: Export All
label: Export All DB Assets to Azure
hint: Output all content from the DB to Azure Blog Storage, overwriting any existing data. If you enabled Azure Blog Storage after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
icon: this-way-up
server/modules/storage/box/definition.yml deleted 100644 → 0
View file @ 027b1614
key: box
title: Box
description: Box is a cloud content management and file sharing service for businesses.
author: requarks.io
logo: https://static.requarks.io/logo/box.svg
website: https://www.box.com/platform
props:
clientId: String
clientSecret: String
rootFolder: String
server/modules/storage/box/storage.js deleted 100644 → 0
View file @ 027b1614
module.exports = {
async activated() {
},
async deactivated() {
},
async init() {
},
async created() {
},
async updated() {
},
async deleted() {
},
async renamed() {
},
async getLocalLocation () {
}
}
server/modules/storage/db/definition.yml 0 → 100644
View file @ edb52937
title: 'Database'
icon: '/_assets/icons/ultraviolet-database.svg'
banner: '/_assets/storage/database.jpg'
description: 'The local PostgreSQL database can store any assets. It is however not recommended to store large files directly in the database as this can cause performance issues.'
vendor: 'Wiki.js'
website: 'https://js.wiki'
assetDelivery:
isStreamingSupported: true
isDirectAccessSupported: false
defaultStreamingEnabled: true
defaultDirectAccessEnabled: false
contentTypes:
defaultTypesEnabled: ['pages', 'images', 'documents', 'others', 'large']
defaultLargeThreshold: '5MB'
versioning:
isSupported: true
defaultEnabled: false
sync: false
props: {}
actions:
- handler: purge
label: Purge All Assets
hint: Delete all asset data from the database (not the metadata). Useful if you moved assets to another storage target and want to reduce the size of the database.
warn: This is a destructive action! Make sure all asset files are properly stored on another storage module! This action cannot be undone!
icon: explosion
server/modules/storage/db/storage.js 0 → 100644
View file @ edb52937
module.exports = {
async activated () { },
async deactivated () { },
async init () { },
async created (page) { },
async updated (page) { },
async deleted (page) { },
async renamed (page) { },
async assetUploaded (asset) { },
async assetDeleted (asset) { },
async assetRenamed (asset) { },
async getLocalLocation () { },
async exportAll () { }
}
server/modules/storage/digitalocean/definition.yml deleted 100644 → 0
View file @ 027b1614
key: digitalocean
title: DigitalOcean Spaces
description: DigitalOcean provides developers and businesses a reliable, easy-to-use cloud computing platform of virtual servers (Droplets), object storage (Spaces) and more.
author: andrewsim
logo: https://static.requarks.io/logo/digitalocean.svg
website: https://www.digitalocean.com/products/spaces/
isAvailable: true
supportedModes:
- push
defaultMode: push
schedule: false
props:
endpoint:
type: String
title: Endpoint
hint: The DigitalOcean spaces endpoint that has the form ${REGION}.digitaloceanspaces.com
default: nyc3.digitaloceanspaces.com
enum:
- ams3.digitaloceanspaces.com
- fra1.digitaloceanspaces.com
- nyc3.digitaloceanspaces.com
- sfo2.digitaloceanspaces.com
- sgp1.digitaloceanspaces.com
order: 1
bucket:
type: String
title: Space Unique Name
hint: The unique space name to create (e.g. wiki-johndoe)
order: 2
accessKeyId:
type: String
title: Access Key ID
hint: The Access Key (Generated in API > Tokens/Keys > Spaces access keys).
order: 3
secretAccessKey :
type: String
title: Access Key Secret
hint: The Access Key Secret for the Access Key ID you created above.
sensitive: true
order: 4
actions:
- handler: exportAll
label: Export All
hint: Output all content from the DB to DigitalOcean Spaces, overwriting any existing data. If you enabled DigitalOcean Spaces after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
server/modules/storage/digitalocean/storage.js deleted 100644 → 0
View file @ 027b1614
const S3CompatibleStorage = require('../s3/common')
module.exports = new S3CompatibleStorage('Digitalocean')
server/modules/storage/disk/definition.yml
View file @ edb52937
key: disk
title: Local File System
description: Local storage on disk or network shares.
author: requarks.io
logo: https://static.requarks.io/logo/local-fs.svg
website: https://wiki.js.org
isAvailable: true
supportedModes:
- push
defaultMode: push
schedule: false
icon: '/_assets/icons/ultraviolet-hdd.svg'
banner: '/_assets/storage/disk.jpg'
description: Store files on the local file system or over network attached storage. Note that you must use replicated storage if using high-availability instances.
vendor: Wiki.js
website: 'https://js.wiki'
assetDelivery:
isStreamingSupported: true
isDirectAccessSupported: false
defaultStreamingEnabled: true
defaultDirectAccessEnabled: false
contentTypes:
defaultTypesEnabled: ['images', 'documents', 'others', 'large']
defaultLargeThreshold: '5MB'
versioning:
isSupported: false
defaultEnabled: false
sync: false
internalSchedule: P1D
props:
path:
type: String
title: Path
hint: Absolute path without a trailing slash (e.g. /home/wiki/backup, C:\wiki\backup)
icon: symlink-directory
order: 1
createDailyBackups:
type: Boolean
default: false
title: Create Daily Backups
hint: A tar.gz archive containing all content will be created daily in subfolder named _daily. Archives are kept for a month.
icon: archive-folder
order: 2
actions:
- handler: dump
label: Dump all content to disk
hint: Output all content from the DB to the local disk. If you enabled this module after content was created or you temporarily disabled this module, you'll want to execute this action to add the missing files.
icon: downloads
- handler: backup
label: Create Backup
hint: Will create a manual backup archive at this point in time, in a subfolder named _manual, from the contents currently on disk.
icon: archive-folder
- handler: importAll
label: Import Everything
hint: Will import all content currently in the local disk folder.
icon: database-daily-import
server/modules/storage/disk/storage.js
View file @ edb52937
......@@ -127,15 +127,12 @@ module.exports = {
// -> Pages
await pipeline(
WIKI.models.knex.column('id', 'path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt', 'createdAt', 'editorKey').select().from('pages').where({
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt', 'createdAt').select().from('pages').where({
isPrivate: false
}).stream(),
new stream.Transform({
objectMode: true,
transform: async (page, enc, cb) => {
const pageObject = await WIKI.models.pages.query().findById(page.id)
page.tags = await pageObject.$relatedQuery('tags')
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
......
server/modules/storage/dropbox/definition.yml deleted 100644 → 0
View file @ 027b1614
key: dropbox
title: Dropbox
description: Dropbox is a file hosting service that offers cloud storage, file synchronization, personal cloud, and client software.
author: requarks.io
logo: https://static.requarks.io/logo/dropbox.svg
website: https://dropbox.com
props:
appKey: String
appSecret: String
server/modules/storage/dropbox/storage.js deleted 100644 → 0
View file @ 027b1614
module.exports = {
async activated() {
},
async deactivated() {
},
async init() {
},
async created() {
},
async updated() {
},
async deleted() {
},
async renamed() {
},
async getLocalLocation () {
}
}
server/modules/storage/gcs/definition.yml 0 → 100644
View file @ edb52937
title: Google Cloud Storage
icon: '/_assets/icons/ultraviolet-google.svg'
banner: '/_assets/storage/gcs.jpg'
description: Google Cloud Storage is an online file storage web service for storing and accessing data on Google Cloud Platform infrastructure.
vendor: Alphabet Inc.
website: 'https://cloud.google.com'
assetDelivery:
isStreamingSupported: true
isDirectAccessSupported: true
defaultStreamingEnabled: true
defaultDirectAccessEnabled: true
contentTypes:
defaultTypesEnabled: ['images', 'documents', 'others', 'large']
defaultLargeThreshold: '5MB'
versioning:
isSupported: false
defaultEnabled: false
sync: false
props:
accountName:
type: String
title: Project ID
hint: The project ID from the Google Developer's Console (e.g. grape-spaceship-123).
icon: 3d-touch
default: ''
order: 1
credentialsJSON:
type: String
title: JSON Credentials
hint: Contents of the JSON credentials file for the service account having Cloud Storage permissions.
icon: key
default: ''
multiline: true
sensitive: true
order: 2
bucket:
type: String
title: Unique bucket name
hint: The unique bucket name to create (e.g. wiki-johndoe).
icon: open-box
order: 3
storageTier:
type: String
title: Storage Tier
hint: Select the storage class to use when uploading new assets.
icon: scan-stock
order: 4
default: STANDARD
enum:
- STANDARD|Standard
- NEARLINE|Nearline
- COLDLINE|Coldline
- ARCHIVE|Archive
apiEndpoint:
type: String
title: API Endpoint
hint: The API endpoint of the service used to make requests.
icon: api
default: storage.google.com
order: 5
actions:
- handler: exportAll
label: Export All DB Assets to GCS
hint: Output all content from the DB to Google Cloud Storage, overwriting any existing data. If you enabled Google Cloud Storage after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
icon: this-way-up
server/modules/storage/gcs/storage.js 0 → 100644
View file @ edb52937
const { BlobServiceClient, StorageSharedKeyCredential } = require('@azure/storage-blob')
const stream = require('stream')
const Promise = require('bluebird')
const pipeline = Promise.promisify(stream.pipeline)
const pageHelper = require('../../../helpers/page.js')
const _ = require('lodash')
/* global WIKI */
const getFilePath = (page, pathKey) => {
const fileName = `${page[pathKey]}.${pageHelper.getFileExtension(page.contentType)}`
const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode
return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName
}
module.exports = {
async activated() {
},
async deactivated() {
},
async init() {
WIKI.logger.info(`(STORAGE/AZURE) Initializing...`)
const { accountName, accountKey, containerName } = this.config
this.client = new BlobServiceClient(
`https://${accountName}.blob.core.windows.net`,
new StorageSharedKeyCredential(accountName, accountKey)
)
this.container = this.client.getContainerClient(containerName)
try {
await this.container.create()
} catch (err) {
if (err.statusCode !== 409) {
WIKI.logger.warn(err)
throw err
}
}
WIKI.logger.info(`(STORAGE/AZURE) Initialization completed.`)
},
async created (page) {
WIKI.logger.info(`(STORAGE/AZURE) Creating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
const pageContent = page.injectMetadata()
const blockBlobClient = this.container.getBlockBlobClient(filePath)
await blockBlobClient.upload(pageContent, pageContent.length, { tier: this.config.storageTier })
},
async updated (page) {
WIKI.logger.info(`(STORAGE/AZURE) Updating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
const pageContent = page.injectMetadata()
const blockBlobClient = this.container.getBlockBlobClient(filePath)
await blockBlobClient.upload(pageContent, pageContent.length, { tier: this.config.storageTier })
},
async deleted (page) {
WIKI.logger.info(`(STORAGE/AZURE) Deleting file ${page.path}...`)
const filePath = getFilePath(page, 'path')
const blockBlobClient = this.container.getBlockBlobClient(filePath)
await blockBlobClient.delete({
deleteSnapshots: 'include'
})
},
async renamed(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.path} to ${page.destinationPath}...`)
let sourceFilePath = getFilePath(page, 'path')
let destinationFilePath = getFilePath(page, 'destinationPath')
if (WIKI.config.lang.namespacing) {
if (WIKI.config.lang.code !== page.localeCode) {
sourceFilePath = `${page.localeCode}/${sourceFilePath}`
}
if (WIKI.config.lang.code !== page.destinationLocaleCode) {
destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}`
}
}
const sourceBlockBlobClient = this.container.getBlockBlobClient(sourceFilePath)
const destBlockBlobClient = this.container.getBlockBlobClient(destinationFilePath)
await destBlockBlobClient.syncCopyFromURL(sourceBlockBlobClient.url)
await sourceBlockBlobClient.delete({
deleteSnapshots: 'include'
})
},
/**
* ASSET UPLOAD
*
* @param {Object} asset Asset to upload
*/
async assetUploaded (asset) {
WIKI.logger.info(`(STORAGE/AZURE) Creating new file ${asset.path}...`)
const blockBlobClient = this.container.getBlockBlobClient(asset.path)
await blockBlobClient.upload(asset.data, asset.data.length, { tier: this.config.storageTier })
},
/**
* ASSET DELETE
*
* @param {Object} asset Asset to delete
*/
async assetDeleted (asset) {
WIKI.logger.info(`(STORAGE/AZURE) Deleting file ${asset.path}...`)
const blockBlobClient = this.container.getBlockBlobClient(asset.path)
await blockBlobClient.delete({
deleteSnapshots: 'include'
})
},
/**
* ASSET RENAME
*
* @param {Object} asset Asset to rename
*/
async assetRenamed (asset) {
WIKI.logger.info(`(STORAGE/AZURE) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
const sourceBlockBlobClient = this.container.getBlockBlobClient(asset.path)
const destBlockBlobClient = this.container.getBlockBlobClient(asset.destinationPath)
await destBlockBlobClient.syncCopyFromURL(sourceBlockBlobClient.url)
await sourceBlockBlobClient.delete({
deleteSnapshots: 'include'
})
},
async getLocalLocation () {
},
/**
* HANDLERS
*/
async exportAll() {
WIKI.logger.info(`(STORAGE/AZURE) Exporting all content to Azure Blob Storage...`)
// -> Pages
await pipeline(
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt', 'createdAt').select().from('pages').where({
isPrivate: false
}).stream(),
new stream.Transform({
objectMode: true,
transform: async (page, enc, cb) => {
const filePath = getFilePath(page, 'path')
WIKI.logger.info(`(STORAGE/AZURE) Adding page ${filePath}...`)
const pageContent = pageHelper.injectPageMetadata(page)
const blockBlobClient = this.container.getBlockBlobClient(filePath)
await blockBlobClient.upload(pageContent, pageContent.length, { tier: this.config.storageTier })
cb()
}
})
)
// -> Assets
const assetFolders = await WIKI.models.assetFolders.getAllPaths()
await pipeline(
WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
new stream.Transform({
objectMode: true,
transform: async (asset, enc, cb) => {
const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
WIKI.logger.info(`(STORAGE/AZURE) Adding asset ${filename}...`)
const blockBlobClient = this.container.getBlockBlobClient(filename)
await blockBlobClient.upload(asset.data, asset.data.length, { tier: this.config.storageTier })
cb()
}
})
)
WIKI.logger.info('(STORAGE/AZURE) All content has been pushed to Azure Blob Storage.')
}
}
server/modules/storage/gdrive/definition.yml deleted 100644 → 0
View file @ 027b1614
key: gdrive
title: Google Drive
description: Google Drive is a file storage and synchronization service developed by Google.
author: requarks.io
logo: https://static.requarks.io/logo/google-drive.svg
website: https://www.google.com/drive/
props:
clientId: String
clientSecret: String
server/modules/storage/gdrive/storage.js deleted 100644 → 0
View file @ 027b1614
module.exports = {
async activated() {
},
async deactivated() {
},
async init() {
},
async created() {
},
async updated() {
},
async deleted() {
},
async renamed() {
},
async getLocalLocation () {
}
}
server/modules/storage/git/definition.yml
View file @ edb52937
key: git
title: Git
description: Git is a version control system for tracking changes in computer files and coordinating work on those files among multiple people.
author: requarks.io
logo: https://static.requarks.io/logo/git-alt.svg
website: https://git-scm.com/
isAvailable: true
supportedModes:
- sync
- push
- pull
defaultMode: sync
schedule: PT5M
icon: '/_assets/icons/ultraviolet-git.svg'
banner: '/_assets/storage/git.jpg'
description: Git is a version control system for tracking changes in computer files and coordinating work on those files among multiple people. If using GitHub, use the GitHub module instead!
vendor: Software Freedom Conservancy, Inc.
website: 'https://git-scm.com'
assetDelivery:
isStreamingSupported: true
isDirectAccessSupported: false
defaultStreamingEnabled: true
defaultDirectAccessEnabled: false
contentTypes:
defaultTypesEnabled: ['pages', 'images', 'documents', 'others', 'large']
defaultLargeThreshold: '5MB'
versioning:
isSupported: true
defaultEnabled: true
isForceEnabled: true
sync:
supportedModes:
- sync
- push
- pull
defaultMode: sync
schedule: PT5M
props:
authType:
type: String
default: 'ssh'
title: Authentication Type
hint: Use SSH for maximum security.
icon: security-configuration
enum:
- 'basic'
- 'ssh'
- basic|Basic
- ssh|SSH
enumDisplay: buttons
order: 1
repoUrl:
type: String
title: Repository URI
hint: Git-compliant URI (e.g. git@github.com:org/repo.git for ssh, https://github.com/org/repo.git for basic)
hint: Git-compliant URI (e.g. git@server.com:org/repo.git for ssh, https://server.com/org/repo.git for basic)
icon: dns
order: 2
branch:
type: String
default: 'master'
default: 'main'
title: Branch
hint: The branch to use during pull / push
icon: code-fork
order: 3
sshPrivateKeyMode:
type: String
title: SSH Private Key Mode
hint: SSH Authentication Only - The mode to use to load the private key. Fill in the corresponding field below.
hint: The mode to use to load the private key. Fill in the corresponding field below.
icon: grand-master-key
order: 11
default: 'path'
default: inline
enum:
- 'path'
- 'contents'
- path|File Path
- inline|Inline Contents
enumDisplay: buttons
if:
- { key: 'authType', eq: 'ssh' }
sshPrivateKeyPath:
type: String
title: A - SSH Private Key Path
hint: SSH Authentication Only - Absolute path to the key. The key must NOT be passphrase-protected. Mode must be set to path to use this option.
title: SSH Private Key Path
hint: Absolute path to the key. The key must NOT be passphrase-protected.
icon: key
order: 12
if:
- { key: 'authType', eq: 'ssh' }
- { key: 'sshPrivateKeyMode', eq: 'path' }
sshPrivateKeyContent:
type: String
title: B - SSH Private Key Contents
hint: SSH Authentication Only - Paste the contents of the private key. The key must NOT be passphrase-protected. Mode must be set to contents to use this option.
title: SSH Private Key Contents
hint: Paste the contents of the private key. The key must NOT be passphrase-protected.
icon: key
multiline: true
sensitive: true
order: 13
if:
- { key: 'sshPrivateKeyMode', eq: 'inline' }
verifySSL:
type: Boolean
default: true
title: Verify SSL Certificate
hint: Some hosts requires SSL certificate checking to be disabled. Leave enabled for proper security.
icon: security-ssl
order: 14
basicUsername:
type: String
title: Username
hint: Basic Authentication Only
icon: test-account
order: 20
if:
- { key: 'authType', eq: 'basic' }
basicPassword:
type: String
title: Password / PAT
hint: Basic Authentication Only
icon: password
sensitive: true
order: 21
if:
- { key: 'authType', eq: 'basic' }
defaultEmail:
type: String
title: Default Author Email
default: 'name@company.com'
hint: 'Used as fallback in case the author of the change is not present.'
order: 22
icon: email
order: 30
defaultName:
type: String
title: Default Author Name
default: 'John Smith'
hint: 'Used as fallback in case the author of the change is not present.'
order: 23
icon: customer
order: 31
localRepoPath:
type: String
title: Local Repository Path
default: './data/repo'
hint: 'Path where the local git repository will be created.'
order: 30
icon: symlink-directory
order: 32
gitBinaryPath:
type: String
title: Git Binary Path
default: ''
hint: Optional - Absolute path to the Git binary, when not available in PATH. Leave empty to use the default PATH location (recommended).
icon: run-command
order: 50
actions:
- handler: syncUntracked
label: Add Untracked Changes
hint: Output all content from the DB to the local Git repository to ensure all untracked content is saved. If you enabled Git after content was created or you temporarily disabled Git, you'll want to execute this action to add the missing untracked changes.
icon: database-daily-export
- handler: sync
label: Force Sync
hint: Will trigger an immediate sync operation, regardless of the current sync schedule. The sync direction is respected.
icon: synchronize
- handler: importAll
label: Import Everything
hint: Will import all content currently in the local Git repository, regardless of the latest commit state. Useful for importing content from the remote repository created before git was enabled.
icon: database-daily-import
- handler: purge
label: Purge Local Repository
hint: If you have unrelated merge histories, clearing the local repository can resolve this issue. This will not affect the remote repository or perform any commit.
icon: trash
server/modules/storage/git/storage.js
View file @ edb52937
......@@ -73,7 +73,7 @@ module.exports = {
mode: 0o600
})
} catch (err) {
WIKI.logger.error(err)
console.error(err)
throw err
}
}
......@@ -142,9 +142,7 @@ module.exports = {
if (_.get(diff, 'files', []).length > 0) {
let filesToProcess = []
for (const f of diff.files) {
const fMoved = f.file.split(' => ')
const fName = fMoved.length === 2 ? fMoved[1] : fMoved[0]
const fPath = path.join(this.repoPath, fName)
const fPath = path.join(this.repoPath, f.file)
let fStats = { size: 0 }
try {
fStats = await fs.stat(fPath)
......@@ -161,8 +159,7 @@ module.exports = {
path: fPath,
stats: fStats
},
oldPath: fMoved[0],
relPath: fName
relPath: f.file
})
}
await this.processFiles(filesToProcess, rootUser)
......@@ -177,25 +174,11 @@ module.exports = {
async processFiles(files, user) {
for (const item of files) {
const contentType = pageHelper.getContentType(item.relPath)
const fileExists = await fs.pathExists(item.file.path)
const fileExists = await fs.pathExists(item.file)
if (!item.binary && contentType) {
// -> Page
if (fileExists && !item.importAll && item.relPath !== item.oldPath) {
// Page was renamed by git, so rename in DB
WIKI.logger.info(`(STORAGE/GIT) Page marked as renamed: from ${item.oldPath} to ${item.relPath}`)
const contentPath = pageHelper.getPagePath(item.oldPath)
const contentDestinationPath = pageHelper.getPagePath(item.relPath)
await WIKI.models.pages.movePage({
user: user,
path: contentPath.path,
destinationPath: contentDestinationPath.path,
locale: contentPath.locale,
destinationLocale: contentPath.locale,
skipStorage: true
})
} else if (!fileExists && !item.importAll && item.deletions > 0 && item.insertions === 0) {
if (!fileExists && item.deletions > 0 && item.insertions === 0) {
// Page was deleted by git, can safely mark as deleted in DB
WIKI.logger.info(`(STORAGE/GIT) Page marked as deleted: ${item.relPath}`)
......@@ -224,23 +207,7 @@ module.exports = {
} else {
// -> Asset
if (fileExists && !item.importAll && ((item.before === item.after) || (item.deletions === 0 && item.insertions === 0))) {
// Asset was renamed by git, so rename in DB
WIKI.logger.info(`(STORAGE/GIT) Asset marked as renamed: from ${item.oldPath} to ${item.relPath}`)
const fileHash = assetHelper.generateHash(item.relPath)
const assetToRename = await WIKI.models.assets.query().findOne({ hash: fileHash })
if (assetToRename) {
await WIKI.models.assets.query().patch({
filename: item.relPath,
hash: fileHash
}).findById(assetToRename.id)
await assetToRename.deleteAssetCache()
} else {
WIKI.logger.info(`(STORAGE/GIT) Asset was not found in the DB, nothing to rename: ${item.relPath}`)
}
continue
} else if (!fileExists && !item.importAll && ((item.before > 0 && item.after === 0) || (item.deletions > 0 && item.insertions === 0))) {
if (!fileExists && ((item.before > 0 && item.after === 0) || (item.deletions > 0 && item.insertions === 0))) {
// Asset was deleted by git, can safely mark as deleted in DB
WIKI.logger.info(`(STORAGE/GIT) Asset marked as deleted: ${item.relPath}`)
......@@ -427,8 +394,7 @@ module.exports = {
relPath,
file,
deletions: 0,
insertions: 0,
importAll: true
insertions: 0
}], rootUser)
}
cb()
......@@ -445,15 +411,12 @@ module.exports = {
// -> Pages
await pipeline(
WIKI.models.knex.column('id', 'path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt', 'createdAt', 'editorKey').select().from('pages').where({
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt', 'createdAt').select().from('pages').where({
isPrivate: false
}).stream(),
new stream.Transform({
objectMode: true,
transform: async (page, enc, cb) => {
const pageObject = await WIKI.models.pages.query().findById(page.id)
page.tags = await pageObject.$relatedQuery('tags')
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
......
server/modules/storage/github/definition.yml 0 → 100644
View file @ edb52937
title: GitHub
icon: '/_assets/icons/ultraviolet-github.svg'
banner: '/_assets/storage/github.jpg'
description: Millions of developers and companies build, ship, and maintain their software on GitHub - the largest and most advanced development platform in the world.
vendor: GitHub, Inc.
website: 'https://github.com'
assetDelivery:
isStreamingSupported: false
isDirectAccessSupported: false
defaultStreamingEnabled: false
defaultDirectAccessEnabled: false
contentTypes:
defaultTypesEnabled: ['pages', 'images', 'documents', 'others', 'large']
defaultLargeThreshold: '5MB'
versioning:
isSupported: true
defaultEnabled: true
isForceEnabled: true
sync: false
setup:
handler: github
defaultValues:
accountType: org
org: ''
publicUrl: https://
props:
appName:
readOnly: true
type: String
title: App Name
hint: Name of the generated app in GitHub.
icon: 3d-touch
repoFullName:
readOnly: true
type: String
title: GitHub Repository
hint: The GitHub repository used for content synchronization.
icon: github
repoDefaultBranch:
readOnly: true
type: String
title: Default Branch
hint: The repository default branch.
icon: code-fork
actions:
- handler: exportAll
label: Export All DB Assets to GitHub
hint: Output all content from the DB to GitHub, overwriting any existing data. If you enabled GitHub after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
icon: this-way-up
server/modules/storage/github/storage.js 0 → 100644
View file @ edb52937
const { Octokit, App } = require('octokit')
/* global WIKI */
module.exports = {
async activated () { },
async deactivated () { },
async init () { },
/**
* SETUP FUNCTIONS
*/
async setup (id, state) {
try {
switch (state.step) {
// --------------------------------------------
// -> VALIDATE CALLBACK CODE AFTER APP CREATION
// --------------------------------------------
case 'connect': {
const gh = new Octokit({
userAgent: 'wikijs'
})
const resp = await gh.request('POST /app-manifests/{code}/conversions', {
code: state.code
})
if (resp.status > 200 && resp.status < 300) {
await WIKI.models.storage.query().patch({
config: {
appId: resp.data.id,
appName: resp.data.name,
appSlug: resp.data.slug,
appClientId: resp.data.client_id,
appClientSecret: resp.data.client_secret,
appWebhookSecret: resp.data.webhook_secret,
appPem: resp.data.pem,
appPermissions: resp.data.permissions,
appEvents: resp.data.events,
ownerLogin: resp.data.owner?.login,
ownerId: resp.data.owner?.id
},
state: {
current: 'ok',
setup: 'pendinginstall'
}
}).where('id', id)
return {
nextStep: 'installApp',
url: `https://github.com/apps/${resp.data.slug}/installations/new/permissions?target_id=${resp.data.owner?.id}`
}
} else {
throw new Error('GitHub refused the code or could not be reached.')
}
}
// -----------------------
// VERIFY APP INSTALLATION
// -----------------------
case 'verify': {
const tgt = await WIKI.models.storage.query().findById(id)
if (!tgt) {
throw new Error('Invalid Target ID')
}
const ghApp = new App({
appId: tgt.config.appId,
privateKey: tgt.config.appPem,
Octokit: Octokit.defaults({
userAgent: 'wikijs'
}),
oauth: {
clientId: tgt.config.appClientId,
clientSecret: tgt.config.appClientSecret
},
webhooks: {
secret: tgt.config.appWebhookSecret
}
})
// -> Find Installation ID
let installId = null
let installTotal = 0
for await (const { installation } of ghApp.eachInstallation.iterator()) {
if (installTotal < 1) {
installId = installation.id
WIKI.logger.debug(`Using GitHub App installation ID ${installId}`)
}
installTotal++
}
if (installTotal < 1) {
throw new Error('App is not installed on any GitHub account!')
} else if (installTotal > 1) {
WIKI.logger.warn(`GitHub App ${tgt.config.appName} is installed on more than 1 account. Only the first one ${installId} will be used.`)
}
// -> Fetch Repository Info
let repo = null
let repoTotal = 0
for await (const { repository } of ghApp.eachRepository.iterator({ installationId: installId })) {
if (repository.archived || repository.disabled) {
WIKI.logger.debug(`Skipping GitHub Repository ${repo.id} because of it is archived or disabled.`)
continue
}
if (repoTotal < 1) {
repo = repository
WIKI.logger.debug(`Using GitHub Repository ${repo.id}`)
}
repoTotal++
}
if (repoTotal < 1) {
throw new Error('App is not installed on any GitHub repository!')
} else if (repoTotal > 1) {
WIKI.logger.warn(`GitHub App ${tgt.config.appName} is installed on more than 1 repository. Only the first one (${repo.full_name}) will be used.`)
}
// -> Save install/repo info
await WIKI.models.storage.query().patch({
isEnabled: true,
config: {
...tgt.config,
installId,
repoId: repo.id,
repoName: repo.name,
repoOwner: repo.owner?.login,
repoDefaultBranch: repo.default_branch,
repoFullName: repo.full_name
},
state: {
current: 'ok',
setup: 'configured'
}
}).where('id', id)
return {
nextStep: 'completed'
}
}
default: {
throw new Error('Invalid Setup Step')
}
}
} catch (err) {
WIKI.logger.warn('GitHub Storage Module Setup Failed:')
WIKI.logger.warn(err)
throw err
}
},
async setupDestroy (id) {
try {
const tgt = await WIKI.models.storage.query().findById(id)
if (!tgt) {
throw new Error('Invalid Target ID')
}
WIKI.logger.info('Resetting GitHub storage configuration...')
const ghApp = new App({
appId: tgt.config.appId,
privateKey: tgt.config.appPem,
Octokit: Octokit.defaults({
userAgent: 'wikijs'
}),
oauth: {
clientId: tgt.config.appClientId,
clientSecret: tgt.config.appClientSecret
},
webhooks: {
secret: tgt.config.appWebhookSecret
}
})
// -> Reset storage module config
await WIKI.models.storage.query().patch({
isEnabled: false,
config: {},
state: {
current: 'ok',
setup: 'notconfigured'
}
}).where('id', id)
// -> Try to delete installation on GitHub
if (tgt.config.installId) {
try {
await ghApp.octokit.request('DELETE /app/installations/{installation_id}', {
installation_id: tgt.config.installId
})
WIKI.logger.info('Deleted GitHub installation successfully.')
} catch (err) {
WIKI.logger.warn('Could not delete GitHub installation automatically. Please remove the installation on GitHub.')
}
}
} catch (err) {
WIKI.logger.warn('GitHub Storage Module Destroy Failed:')
WIKI.logger.warn(err)
throw err
}
},
async created (page) { },
async updated (page) { },
async deleted (page) { },
async renamed (page) { },
async assetUploaded (asset) { },
async assetDeleted (asset) { },
async assetRenamed (asset) { },
async getLocalLocation () { },
async exportAll () { }
}
server/modules/storage/onedrive/definition.yml deleted 100644 → 0
View file @ 027b1614
key: onedrive
title: OneDrive
description: OneDrive is a file hosting service operated by Microsoft as part of its suite of Office Online services.
author: requarks.io
logo: https://static.requarks.io/logo/onedrive.svg
website: https://onedrive.live.com/about/
props:
clientId: String
clientSecret: String
server/modules/storage/onedrive/storage.js deleted 100644 → 0
View file @ 027b1614
module.exports = {
async activated() {
},
async deactivated() {
},
async init() {
},
async created() {
},
async updated() {
},
async deleted() {
},
async renamed() {
},
async getLocalLocation () {
}
}
server/modules/storage/s3/common.js deleted 100644 → 0
View file @ 027b1614
const S3 = require('aws-sdk/clients/s3')
const stream = require('stream')
const Promise = require('bluebird')
const pipeline = Promise.promisify(stream.pipeline)
const _ = require('lodash')
const pageHelper = require('../../../helpers/page.js')
/* global WIKI */
/**
* Deduce the file path given the `page` object and the object's key to the page's path.
*/
const getFilePath = (page, pathKey) => {
const fileName = `${page[pathKey]}.${pageHelper.getFileExtension(page.contentType)}`
const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode
return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName
}
/**
* Can be used with S3 compatible storage.
*/
module.exports = class S3CompatibleStorage {
constructor(storageName) {
this.storageName = storageName
this.bucketName = ""
}
async activated() {
// not used
}
async deactivated() {
// not used
}
async init() {
WIKI.logger.info(`(STORAGE/${this.storageName}) Initializing...`)
const { accessKeyId, secretAccessKey, bucket } = this.config
const s3Config = {
accessKeyId,
secretAccessKey,
params: { Bucket: bucket },
apiVersions: '2006-03-01'
}
if (!_.isNil(this.config.region)) {
s3Config.region = this.config.region
}
if (!_.isNil(this.config.endpoint)) {
s3Config.endpoint = this.config.endpoint
}
if (!_.isNil(this.config.sslEnabled)) {
s3Config.sslEnabled = this.config.sslEnabled
}
if (!_.isNil(this.config.s3ForcePathStyle)) {
s3Config.s3ForcePathStyle = this.config.s3ForcePathStyle
}
if (!_.isNil(this.config.s3BucketEndpoint)) {
s3Config.s3BucketEndpoint = this.config.s3BucketEndpoint
}
this.s3 = new S3(s3Config)
this.bucketName = bucket
// determine if a bucket exists and you have permission to access it
await this.s3.headBucket().promise()
WIKI.logger.info(`(STORAGE/${this.storageName}) Initialization completed.`)
}
async created(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Creating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
}
async updated(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Updating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
}
async deleted(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.deleteObject({ Key: filePath }).promise()
}
async renamed(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.path} to ${page.destinationPath}...`)
let sourceFilePath = getFilePath(page, 'path')
let destinationFilePath = getFilePath(page, 'destinationPath')
if (WIKI.config.lang.namespacing) {
if (WIKI.config.lang.code !== page.localeCode) {
sourceFilePath = `${page.localeCode}/${sourceFilePath}`
}
if (WIKI.config.lang.code !== page.destinationLocaleCode) {
destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}`
}
}
await this.s3.copyObject({ CopySource: `${this.bucketName}/${sourceFilePath}`, Key: destinationFilePath }).promise()
await this.s3.deleteObject({ Key: sourceFilePath }).promise()
}
/**
* ASSET UPLOAD
*
* @param {Object} asset Asset to upload
*/
async assetUploaded (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Creating new file ${asset.path}...`)
await this.s3.putObject({ Key: asset.path, Body: asset.data }).promise()
}
/**
* ASSET DELETE
*
* @param {Object} asset Asset to delete
*/
async assetDeleted (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${asset.path}...`)
await this.s3.deleteObject({ Key: asset.path }).promise()
}
/**
* ASSET RENAME
*
* @param {Object} asset Asset to rename
*/
async assetRenamed (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
await this.s3.copyObject({ CopySource: `${this.bucketName}/${asset.path}`, Key: asset.destinationPath }).promise()
await this.s3.deleteObject({ Key: asset.path }).promise()
}
async getLocalLocation () {
}
/**
* HANDLERS
*/
async exportAll() {
WIKI.logger.info(`(STORAGE/${this.storageName}) Exporting all content to the cloud provider...`)
// -> Pages
await pipeline(
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt', 'createdAt').select().from('pages').where({
isPrivate: false
}).stream(),
new stream.Transform({
objectMode: true,
transform: async (page, enc, cb) => {
const filePath = getFilePath(page, 'path')
WIKI.logger.info(`(STORAGE/${this.storageName}) Adding page ${filePath}...`)
await this.s3.putObject({ Key: filePath, Body: pageHelper.injectPageMetadata(page) }).promise()
cb()
}
})
)
// -> Assets
const assetFolders = await WIKI.models.assetFolders.getAllPaths()
await pipeline(
WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
new stream.Transform({
objectMode: true,
transform: async (asset, enc, cb) => {
const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
WIKI.logger.info(`(STORAGE/${this.storageName}) Adding asset ${filename}...`)
await this.s3.putObject({ Key: filename, Body: asset.data }).promise()
cb()
}
})
)
WIKI.logger.info(`(STORAGE/${this.storageName}) All content has been pushed to the cloud provider.`)
}
}
server/modules/storage/s3/definition.yml
View file @ edb52937
key: s3
title: Amazon S3
description: Amazon S3 is a cloud computing web service offered by Amazon Web Services which provides object storage.
author: andrewsim
logo: https://static.requarks.io/logo/aws-s3.svg
website: https://aws.amazon.com/s3/
isAvailable: true
supportedModes:
- push
defaultMode: push
schedule: false
title: AWS S3 / DigitalOcean Spaces
icon: '/_assets/icons/ultraviolet-amazon-web-services.svg'
banner: '/_assets/storage/s3.jpg'
description: Amazon Simple Storage Service (Amazon S3) is an object storage service offering industry-leading scalability, data availability, security, and performance.
vendor: Amazon.com, Inc.
website: 'https://aws.amazon.com'
assetDelivery:
isStreamingSupported: true
isDirectAccessSupported: true
defaultStreamingEnabled: true
defaultDirectAccessEnabled: true
contentTypes:
defaultTypesEnabled: ['images', 'documents', 'others', 'large']
defaultLargeThreshold: '5MB'
versioning:
isSupported: false
defaultEnabled: false
sync: false
props:
region:
mode:
type: String
title: Mode
hint: Select a preset configuration mode or define a custom one.
icon: tune
default: aws
order: 1
enum:
- aws|AWS S3
- do|DigitalOcean Spaces
- custom|Custom
awsRegion:
type: String
title: Region
hint: The AWS datacenter region where the bucket will be created.
order: 1
icon: geography
default: us-east-1
enum:
- af-south-1|af-south-1 - Africa (Cape Town)
- ap-east-1|ap-east-1 - Asia Pacific (Hong Kong)
- ap-southeast-3|ap-southeast-3 - Asia Pacific (Jakarta)
- ap-south-1|ap-south-1 - Asia Pacific (Mumbai)
- ap-northeast-3|ap-northeast-3 - Asia Pacific (Osaka)
- ap-northeast-2|ap-northeast-2 - Asia Pacific (Seoul)
- ap-southeast-1|ap-southeast-1 - Asia Pacific (Singapore)
- ap-southeast-2|ap-southeast-2 - Asia Pacific (Sydney)
- ap-northeast-1|ap-northeast-1 - Asia Pacific (Tokyo)
- ca-central-1|ca-central-1 - Canada (Central)
- cn-north-1|cn-north-1 - China (Beijing)
- cn-northwest-1|cn-northwest-1 - China (Ningxia)
- eu-central-1|eu-central-1 - Europe (Frankfurt)
- eu-west-1|eu-west-1 - Europe (Ireland)
- eu-west-2|eu-west-2 - Europe (London)
- eu-south-1|eu-south-1 - Europe (Milan)
- eu-west-3|eu-west-3 - Europe (Paris)
- eu-north-1|eu-north-1 - Europe (Stockholm)
- me-south-1|me-south-1 - Middle East (Bahrain)
- sa-east-1|sa-east-1 - South America (São Paulo)
- us-east-1|us-east-1 - US East (N. Virginia)
- us-east-2|us-east-2 - US East (Ohio)
- us-west-1|us-west-1 - US West (N. California)
- us-west-2|us-west-2 - US West (Oregon)
order: 2
if:
- { key: 'mode', eq: 'aws' }
doRegion:
type: String
title: Region
hint: The DigitalOcean Spaces region
icon: geography
default: nyc3
enum:
- ams3|Amsterdam
- fra1|Frankfurt
- nyc3|New York
- sfo2|San Francisco 2
- sfo3|San Francisco 3
- sgp1|Singapore
order: 2
if:
- { key: 'mode', eq: 'do' }
endpoint:
type: String
title: Endpoint URI
hint: The full S3-compliant endpoint URI.
icon: dns
default: https://service.region.example.com
order: 2
if:
- { key: 'mode', eq: 'custom' }
bucket:
type: String
title: Unique bucket name
hint: The unique bucket name to create (e.g. wiki-johndoe).
order: 2
icon: open-box
order: 3
accessKeyId:
type: String
title: Access Key ID
hint: The Access Key.
order: 3
icon: 3d-touch
order: 4
secretAccessKey:
type: String
title: Secret Access Key
hint: The Secret Access Key for the Access Key ID you created above.
icon: key
sensitive: true
order: 4
order: 5
storageTier:
type: String
title: Storage Tier
hint: The storage tier to use when adding files.
icon: scan-stock
order: 6
default: STANDARD
enum:
- STANDARD|Standard
- STANDARD_IA|Standard Infrequent Access
- INTELLIGENT_TIERING|Intelligent Tiering
- ONEZONE_IA|One Zone Infrequent Access
- REDUCED_REDUNDANCY|Reduced Redundancy
- GLACIER_IR|Glacier Instant Retrieval
- GLACIER|Glacier Flexible Retrieval
- DEEP_ARCHIVE|Glacier Deep Archive
- OUTPOSTS|Outposts
if:
- { key: 'mode', eq: 'aws' }
sslEnabled:
type: Boolean
title: Use SSL
hint: Whether to enable SSL for requests
icon: secure
default: true
order: 10
if:
- { key: 'mode', eq: 'custom' }
s3ForcePathStyle:
type: Boolean
title: Force Path Style for S3 objects
hint: Whether to force path style URLs for S3 objects.
icon: filtration
default: false
order: 11
if:
- { key: 'mode', eq: 'custom' }
s3BucketEndpoint:
type: Boolean
title: Single Bucket Endpoint
hint: Whether the provided endpoint addresses an individual bucket.
icon: swipe-right
default: false
order: 12
if:
- { key: 'mode', eq: 'custom' }
actions:
- handler: exportAll
label: Export All
label: Export All DB Assets to S3
hint: Output all content from the DB to S3, overwriting any existing data. If you enabled S3 after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
icon: this-way-up
server/modules/storage/s3/storage.js
View file @ edb52937
const S3CompatibleStorage = require('./common')
const S3 = require('aws-sdk/clients/s3')
const stream = require('stream')
const Promise = require('bluebird')
const pipeline = Promise.promisify(stream.pipeline)
const _ = require('lodash')
const pageHelper = require('../../../helpers/page.js')
module.exports = new S3CompatibleStorage('S3')
/* global WIKI */
/**
* Deduce the file path given the `page` object and the object's key to the page's path.
*/
const getFilePath = (page, pathKey) => {
const fileName = `${page[pathKey]}.${pageHelper.getFileExtension(page.contentType)}`
const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode
return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName
}
/**
* Can be used with S3 compatible storage.
*/
module.exports = class S3CompatibleStorage {
constructor(storageName) {
this.storageName = storageName
}
async activated() {
// not used
}
async deactivated() {
// not used
}
async init() {
WIKI.logger.info(`(STORAGE/${this.storageName}) Initializing...`)
const { accessKeyId, secretAccessKey, bucket } = this.config
const s3Config = {
accessKeyId,
secretAccessKey,
params: { Bucket: bucket },
apiVersions: '2006-03-01'
}
if (!_.isNil(this.config.region)) {
s3Config.region = this.config.region
}
if (!_.isNil(this.config.endpoint)) {
s3Config.endpoint = this.config.endpoint
}
if (!_.isNil(this.config.sslEnabled)) {
s3Config.sslEnabled = this.config.sslEnabled
}
if (!_.isNil(this.config.s3ForcePathStyle)) {
s3Config.s3ForcePathStyle = this.config.s3ForcePathStyle
}
if (!_.isNil(this.config.s3BucketEndpoint)) {
s3Config.s3BucketEndpoint = this.config.s3BucketEndpoint
}
this.s3 = new S3(s3Config)
// determine if a bucket exists and you have permission to access it
await this.s3.headBucket().promise()
WIKI.logger.info(`(STORAGE/${this.storageName}) Initialization completed.`)
}
async created(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Creating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
}
async updated(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Updating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
}
async deleted(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.deleteObject({ Key: filePath }).promise()
}
async renamed(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.path} to ${page.destinationPath}...`)
let sourceFilePath = getFilePath(page, 'path')
let destinationFilePath = getFilePath(page, 'destinationPath')
if (WIKI.config.lang.namespacing) {
if (WIKI.config.lang.code !== page.localeCode) {
sourceFilePath = `${page.localeCode}/${sourceFilePath}`
}
if (WIKI.config.lang.code !== page.destinationLocaleCode) {
destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}`
}
}
await this.s3.copyObject({ CopySource: sourceFilePath, Key: destinationFilePath }).promise()
await this.s3.deleteObject({ Key: sourceFilePath }).promise()
}
/**
* ASSET UPLOAD
*
* @param {Object} asset Asset to upload
*/
async assetUploaded (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Creating new file ${asset.path}...`)
await this.s3.putObject({ Key: asset.path, Body: asset.data }).promise()
}
/**
* ASSET DELETE
*
* @param {Object} asset Asset to delete
*/
async assetDeleted (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${asset.path}...`)
await this.s3.deleteObject({ Key: asset.path }).promise()
}
/**
* ASSET RENAME
*
* @param {Object} asset Asset to rename
*/
async assetRenamed (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
await this.s3.copyObject({ CopySource: asset.path, Key: asset.destinationPath }).promise()
await this.s3.deleteObject({ Key: asset.path }).promise()
}
async getLocalLocation () {
}
/**
* HANDLERS
*/
async exportAll() {
WIKI.logger.info(`(STORAGE/${this.storageName}) Exporting all content to the cloud provider...`)
// -> Pages
await pipeline(
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt', 'createdAt').select().from('pages').where({
isPrivate: false
}).stream(),
new stream.Transform({
objectMode: true,
transform: async (page, enc, cb) => {
const filePath = getFilePath(page, 'path')
WIKI.logger.info(`(STORAGE/${this.storageName}) Adding page ${filePath}...`)
await this.s3.putObject({ Key: filePath, Body: pageHelper.injectPageMetadata(page) }).promise()
cb()
}
})
)
// -> Assets
const assetFolders = await WIKI.models.assetFolders.getAllPaths()
await pipeline(
WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
new stream.Transform({
objectMode: true,
transform: async (asset, enc, cb) => {
const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
WIKI.logger.info(`(STORAGE/${this.storageName}) Adding asset ${filename}...`)
await this.s3.putObject({ Key: filename, Body: asset.data }).promise()
cb()
}
})
)
WIKI.logger.info(`(STORAGE/${this.storageName}) All content has been pushed to the cloud provider.`)
}
}
server/modules/storage/s3generic/definition.yml deleted 100644 → 0
View file @ 027b1614
key: s3generic
title: S3 Generic
description: Generic storage module for S3-compatible services.
author: requarks.io
logo: https://static.requarks.io/logo/aws-s3-alt.svg
website: https://wiki.js.org
isAvailable: true
supportedModes:
- push
defaultMode: push
schedule: false
props:
endpoint:
type: String
title: Endpoint URI
hint: The full S3-compliant endpoint URI.
default: https://service.region.example.com
order: 1
bucket:
type: String
title: Unique bucket name
hint: The unique bucket name to create (e.g. wiki-johndoe)
order: 2
accessKeyId:
type: String
title: Access Key ID
hint: The Access Key ID.
order: 3
secretAccessKey:
type: String
title: Access Key Secret
hint: The Access Key Secret for the Access Key ID above.
sensitive: true
order: 4
sslEnabled:
type: Boolean
title: Use SSL
hint: Whether to enable SSL for requests
default: true
order: 5
s3ForcePathStyle:
type: Boolean
title: Force Path Style for S3 objects
hint: Whether to force path style URLs for S3 objects.
default: false
order: 6
s3BucketEndpoint:
type: Boolean
title: Single Bucket Endpoint
hint: Whether the provided endpoint addresses an individual bucket.
default: false
order: 7
actions:
- handler: exportAll
label: Export All
hint: Output all content from the DB to the external service, overwriting any existing data. If you enabled this module after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
server/modules/storage/s3generic/storage.js deleted 100644 → 0
View file @ 027b1614
const S3CompatibleStorage = require('../s3/common')
module.exports = new S3CompatibleStorage('S3Generic')
server/modules/storage/sftp/definition.yml
View file @ edb52937
key: sftp
title: SFTP
description: SFTP (SSH File Transfer Protocol) is a secure file transfer protocol. It runs over the SSH protocol. It supports the full security and authentication functionality of SSH.
author: requarks.io
logo: https://static.requarks.io/logo/ssh.svg
website: https://www.ssh.com/ssh/sftp
isAvailable: true
supportedModes:
- push
defaultMode: push
schedule: false
title: 'SFTP'
icon: '/_assets/icons/ultraviolet-nas.svg'
banner: '/_assets/storage/ssh.jpg'
description: 'Store files over a remote connection using the SSH File Transfer Protocol.'
vendor: 'Wiki.js'
website: 'https://js.wiki'
assetDelivery:
isStreamingSupported: false
isDirectAccessSupported: false
defaultStreamingEnabled: false
defaultDirectAccessEnabled: false
contentTypes:
defaultTypesEnabled: ['pages', 'images', 'documents', 'others', 'large']
defaultLargeThreshold: '5MB'
versioning:
isSupported: false
defaultEnabled: false
sync: false
props:
host:
type: String
title: Host
default: ''
hint: Hostname or IP of the remote SSH server.
icon: dns
order: 1
port:
type: Number
title: Port
default: 22
hint: SSH port of the remote server.
icon: ethernet-off
order: 2
authMode:
type: String
title: Authentication Method
default: 'privateKey'
hint: Whether to use Private Key or Password-based authentication. A private key is highly recommended for best security.
icon: grand-master-key
enum:
- privateKey
- password
- privateKey|Private Key
- password|Password
enumDisplay: buttons
order: 3
username:
type: String
title: Username
default: ''
hint: Username for authentication.
icon: test-account
order: 4
privateKey:
type: String
title: Private Key Contents
default: ''
hint: (Private Key Authentication Only) - Contents of the private key
hint: Contents of the private key
icon: key
multiline: true
sensitive: true
order: 5
if:
- { key: 'authMode', eq: 'privateKey' }
passphrase:
type: String
title: Private Key Passphrase
default: ''
hint: (Private Key Authentication Only) - Passphrase if the private key is encrypted, leave empty otherwise
hint: Passphrase if the private key is encrypted, leave empty otherwise
icon: password
sensitive: true
order: 6
if:
- { key: 'authMode', eq: 'privateKey' }
password:
type: String
title: Password
default: ''
hint: (Password-based Authentication Only) - Password for authentication
hint: Password for authentication
icon: password
sensitive: true
order: 6
if:
- { key: 'authMode', eq: 'password' }
basePath:
type: String
title: Base Directory Path
default: '/root/wiki'
hint: Base directory where files will be transferred to. The path must already exists and be writable by the user.
icon: symlink-directory
actions:
- handler: exportAll
label: Export All
label: Export All DB Assets to Remote
hint: Output all content from the DB to the remote SSH server, overwriting any existing data. If you enabled SFTP after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
icon: this-way-up
server/modules/storage/sftp/storage.js
View file @ edb52937
......@@ -155,12 +155,7 @@ module.exports = {
const folderPaths = _.dropRight(filePath.split('/'))
for (let i = 1; i <= folderPaths.length; i++) {
const folderSection = _.take(folderPaths, i).join('/')
const folderDir = path.posix.join(this.config.basePath, folderSection)
try {
await this.sftp.readdir(folderDir)
} catch (err) {
await this.sftp.mkdir(folderDir)
}
await this.sftp.mkdir(path.posix.join(this.config.basePath, folderSection))
}
} catch (err) {}
}
......
yarn.lock
View file @ edb52937
This diff was suppressed by a .gitattributes entry.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment