Skip to content

W
wiki-js
Unverified Commit 24ddab73 authored by Nicolas Giard's avatar Nicolas Giard
Browse files
Options

feat: scheduler history + retries + admin scheduler page (wip)

parent 39b273b2
Hide whitespace changes
Inline Side-by-side
Showing with 322 additions and 90 deletions
server/app/data.yml
View file @ 24ddab73
......@@ -33,6 +33,8 @@ defaults:
workers: 3
pollingCheck: 5
scheduledCheck: 300
maxRetries: 5
retryBackoff: 60
# DB defaults
api:
isEnabled: false
......
server/core/scheduler.js
View file @ 24ddab73
......@@ -55,12 +55,13 @@ module.exports = {
WIKI.logger.info('Scheduler: [ STARTED ]')
},
async addJob ({ task, payload, waitUntil, isScheduled = false, notify = true }) {
async addJob ({ task, payload, waitUntil, maxRetries, isScheduled = false, notify = true }) {
try {
await WIKI.db.knex('jobs').insert({
task,
useWorker: !(typeof this.tasks[task] === 'function'),
payload,
maxRetries: maxRetries ?? WIKI.config.scheduler.maxRetries,
isScheduled,
waitUntil,
createdBy: WIKI.INSTANCE_ID
......@@ -76,6 +77,7 @@ module.exports = {
}
},
async processJob () {
let jobId = null
try {
await WIKI.db.knex.transaction(async trx => {
const jobs = await trx('jobs')
......@@ -85,19 +87,74 @@ module.exports = {
if (jobs && jobs.length === 1) {
const job = jobs[0]
WIKI.logger.info(`Processing new job ${job.id}: ${job.task}...`)
if (job.useWorker) {
await this.workerPool.execute({
id: job.id,
name: job.task,
data: job.payload
jobId = job.id
// -> Add to Job History
await WIKI.db.knex('jobHistory').insert({
id: job.id,
task: job.task,
state: 'active',
useWorker: job.useWorker,
wasScheduled: job.isScheduled,
payload: job.payload,
attempt: job.retries + 1,
maxRetries: job.maxRetries,
createdAt: job.createdAt
}).onConflict('id').merge({
startedAt: new Date()
})
// -> Start working on it
try {
if (job.useWorker) {
await this.workerPool.execute({
id: job.id,
name: job.task,
data: job.payload
})
} else {
await this.tasks[job.task](job.payload)
}
// -> Update job history (success)
await WIKI.db.knex('jobHistory').where({
id: job.id
}).update({
state: 'completed',
completedAt: new Date()
})
WIKI.logger.info(`Completed job ${job.id}: ${job.task} [ SUCCESS ]`)
} catch (err) {
WIKI.logger.warn(`Failed to complete job ${job.id}: ${job.task} [ FAILED ]`)
WIKI.logger.warn(err)
// -> Update job history (fail)
await WIKI.db.knex('jobHistory').where({
id: job.id
}).update({
state: 'failed',
lastErrorMessage: err.message
})
} else {
await this.tasks[job.task](job.payload)
// -> Reschedule for retry
if (job.retries < job.maxRetries) {
const backoffDelay = (2 ** job.retries) * WIKI.config.scheduler.retryBackoff
await trx('jobs').insert({
...job,
retries: job.retries + 1,
waitUntil: DateTime.utc().plus({ seconds: backoffDelay }).toJSDate(),
updatedAt: new Date()
})
WIKI.logger.warn(`Rescheduling new attempt for job ${job.id}: ${job.task}...`)
}
}
}
})
} catch (err) {
WIKI.logger.warn(err)
if (jobId) {
WIKI.db.knex('jobHistory').where({
id: jobId
}).update({
state: 'interrupted',
lastErrorMessage: err.message
})
}
}
},
async addScheduled () {
......
server/db/migrations/3.0.0.js
View file @ 24ddab73
......@@ -125,12 +125,16 @@ exports.up = async knex => {
.createTable('jobHistory', table => {
table.uuid('id').notNullable().primary()
table.string('task').notNullable()
table.string('state').notNullable()
table.enum('state', ['active', 'completed', 'failed', 'interrupted']).notNullable()
table.boolean('useWorker').notNullable().defaultTo(false)
table.boolean('wasScheduled').notNullable().defaultTo(false)
table.jsonb('payload')
table.string('lastErrorMessage')
table.integer('attempt').notNullable().defaultTo(1)
table.integer('maxRetries').notNullable().defaultTo(0)
table.text('lastErrorMessage')
table.timestamp('createdAt').notNullable()
table.timestamp('startedAt').notNullable()
table.timestamp('completedAt').notNullable().defaultTo(knex.fn.now())
table.timestamp('startedAt').notNullable().defaultTo(knex.fn.now())
table.timestamp('completedAt')
})
// JOB SCHEDULE ------------------------
.createTable('jobSchedule', table => {
......@@ -154,6 +158,8 @@ exports.up = async knex => {
table.string('task').notNullable()
table.boolean('useWorker').notNullable().defaultTo(false)
table.jsonb('payload')
table.integer('retries').notNullable().defaultTo(0)
table.integer('maxRetries').notNullable().defaultTo(0)
table.timestamp('waitUntil')
table.boolean('isScheduled').notNullable().defaultTo(false)
table.string('createdBy')
......
server/graph/resolvers/system.js
View file @ 24ddab73
......@@ -7,7 +7,6 @@ const path = require('path')
const fs = require('fs-extra')
const { DateTime } = require('luxon')
const graphHelper = require('../../helpers/graph')
const cronParser = require('cron-parser')
module.exports = {
Query: {
......@@ -28,33 +27,34 @@ module.exports = {
return WIKI.config.security
},
async systemJobs (obj, args) {
switch (args.type) {
switch (args.state) {
case 'ACTIVE': {
// const result = await WIKI.scheduler.boss.fetch('*', 25, { includeMeta: true })
return []
}
case 'COMPLETED': {
const result = await WIKI.scheduler.boss.fetchCompleted('*', 25, { includeMeta: true })
console.info(result)
return result ?? []
return []
}
case 'FAILED': {
return []
}
case 'INTERRUPTED': {
return []
}
default: {
WIKI.logger.warn('Invalid Job Type requested.')
WIKI.logger.warn('Invalid Job State requested.')
return []
}
}
},
async systemScheduledJobs (obj, args) {
const jobs = await WIKI.scheduler.boss.getSchedules()
return jobs.map(job => ({
id: job.name,
name: job.name,
cron: job.cron,
timezone: job.timezone,
nextExecution: cronParser.parseExpression(job.cron, { tz: job.timezone }).next(),
createdAt: job.created_on,
updatedAt: job.updated_on
}))
async systemJobsScheduled (obj, args) {
return WIKI.db.knex('jobSchedule').orderBy('task')
},
async systemJobsUpcoming (obj, args) {
return WIKI.db.knex('jobs').orderBy([
{ column: 'waitUntil', order: 'asc', nulls: 'first' },
{ column: 'createdAt', order: 'asc' }
])
}
},
Mutation: {
......@@ -123,15 +123,19 @@ module.exports = {
httpsPort () {
return WIKI.servers.servers.https ? _.get(WIKI.servers.servers.https.address(), 'port', 0) : 0
},
isMailConfigured () {
return WIKI.config?.mail?.host?.length > 2
},
async isSchedulerHealthy () {
const results = await WIKI.db.knex('jobHistory').count('* as total').whereIn('state', ['failed', 'interrupted']).andWhere('startedAt', '>=', DateTime.utc().minus({ days: 1 }).toISO()).first()
return _.toSafeInteger(results?.total) === 0
},
latestVersion () {
return WIKI.system.updates.version
},
latestVersionReleaseDate () {
return DateTime.fromISO(WIKI.system.updates.releaseDate).toJSDate()
},
mailConfigured () {
return WIKI.config?.mail?.host?.length > 2
},
nodeVersion () {
return process.version.substr(1)
},
......@@ -168,12 +172,6 @@ module.exports = {
sslSubscriberEmail () {
return WIKI.config.ssl.enabled && WIKI.config.ssl.provider === 'letsencrypt' ? WIKI.config.ssl.subscriberEmail : null
},
telemetry () {
return WIKI.telemetry.enabled
},
telemetryClientId () {
return WIKI.config.telemetry.clientId
},
async upgradeCapable () {
return !_.isNil(process.env.UPGRADE_COMPANION)
},
......
server/graph/schemas/system.graphql
View file @ 24ddab73
......@@ -8,9 +8,10 @@ extend type Query {
systemInfo: SystemInfo
systemSecurity: SystemSecurity
systemJobs(
type: SystemJobType!
state: SystemJobState
): [SystemJob]
systemScheduledJobs: [SystemScheduledJob]
systemJobsScheduled: [SystemJobScheduled]
systemJobsUpcoming: [SystemJobUpcoming]
}
extend type Mutation {
......@@ -72,9 +73,10 @@ type SystemInfo {
httpPort: Int
httpRedirection: Boolean
httpsPort: Int
isMailConfigured: Boolean
isSchedulerHealthy: Boolean
latestVersion: String
latestVersionReleaseDate: Date
mailConfigured: Boolean
nodeVersion: String
operatingSystem: String
pagesTotal: Int
......@@ -86,8 +88,6 @@ type SystemInfo {
sslStatus: String
sslSubscriberEmail: String
tagsTotal: Int
telemetry: Boolean
telemetryClientId: String
upgradeCapable: Boolean
usersTotal: Int
workingDirectory: String
......@@ -151,22 +151,46 @@ enum SystemSecurityCorsMode {
type SystemJob {
id: UUID
name: String
priority: Int
state: String
task: String
state: SystemJobState
useWorker: Boolean
wasScheduled: Boolean
payload: JSON
attempt: Int
maxRetries: Int
lastErrorMessage: String
createdAt: Date
startedAt: Date
completedAt: Date
}
type SystemScheduledJob {
id: String
name: String
type SystemJobScheduled {
id: UUID
task: String
cron: String
timezone: String
nextExecution: Date
type: String
payload: JSON
createdAt: Date
updatedAt: Date
}
type SystemJobUpcoming {
id: UUID
task: String
useWorker: Boolean
payload: JSON
retries: Int
maxRetries: Int
waitUntil: Date
isScheduled: Boolean
createdBy: String
createdAt: Date
updatedAt: Date
}
enum SystemJobType {
enum SystemJobState {
ACTIVE
COMPLETED
FAILED
INTERRUPTED
}
ux/jsconfig.json
View file @ 24ddab73
......@@ -36,5 +36,11 @@
"dist",
".quasar",
"node_modules"
]
],
"vueCompilerOptions": {
"target": 3,
"plugins": [
"@volar/vue-language-plugin-pug"
]
}
}
ux/package.json
View file @ 24ddab73
......@@ -89,6 +89,7 @@
"@intlify/vite-plugin-vue-i18n": "6.0.1",
"@quasar/app-vite": "1.0.6",
"@types/lodash": "4.14.184",
"@volar/vue-language-plugin-pug": "1.0.1",
"browserlist": "latest",
"eslint": "8.22.0",
"eslint-config-standard": "17.0.0",
......
ux/src/i18n/locales/en.json
View file @ 24ddab73
......@@ -1515,7 +1515,15 @@
"admin.scheduler.completedNone": "There are no recently completed job to display.",
"admin.scheduler.scheduledNone": "There are no scheduled jobs at the moment.",
"admin.scheduler.cron": "Cron",
"admin.scheduler.nextExecutionIn": "Next run {date}",
"admin.scheduler.nextExecution": "Next Run",
"admin.scheduler.timezone": "Timezone"
"admin.scheduler.createdBy": "by instance {instance}",
"admin.scheduler.upcoming": "Upcoming",
"admin.scheduler.failed": "Failed",
"admin.scheduler.type": "Type",
"admin.scheduler.createdAt": "Created",
"admin.scheduler.updatedAt": "Last Updated",
"common.field.task": "Task",
"admin.scheduler.upcomingNone": "There are no upcoming job for the moment.",
"admin.scheduler.waitUntil": "Start",
"admin.scheduler.attempt": "Attempt",
"admin.scheduler.useWorker": "Execution Mode"
}
ux/src/layouts/AdminLayout.vue
View file @ 24ddab73
......@@ -156,6 +156,8 @@ q-layout.admin(view='hHh Lpr lff')
q-item-section(avatar)
q-icon(name='img:/_assets/icons/fluent-bot.svg')
q-item-section {{ t('admin.scheduler.title') }}
q-item-section(side)
status-light(:color='adminStore.info.isSchedulerHealthy ? `positive` : `warning`')
q-item(to='/_admin/security', v-ripple, active-class='bg-primary text-white')
q-item-section(avatar)
q-icon(name='img:/_assets/icons/fluent-protect.svg')
......
ux/src/pages/AdminScheduler.vue
View file @ 24ddab73
......@@ -18,7 +18,9 @@ q-page.admin-terminal
:color='$q.dark.isActive ? `dark-1` : `white`'
:options=`[
{ label: t('admin.scheduler.scheduled'), value: 'scheduled' },
{ label: t('admin.scheduler.completed'), value: 'completed' }
{ label: t('admin.scheduler.upcoming'), value: 'upcoming' },
{ label: t('admin.scheduler.completed'), value: 'completed' },
{ label: t('admin.scheduler.failed'), value: 'failed' },
]`
)
q-separator.q-mr-md(vertical)
......@@ -66,18 +68,62 @@ q-page.admin-terminal
size='xs'
)
//- q-icon(name='las la-stopwatch', color='primary', size='sm')
template(v-slot:body-cell-name='props')
template(v-slot:body-cell-task='props')
q-td(:props='props')
strong {{props.value}}
div: small.text-grey {{props.row.id}}
template(v-slot:body-cell-cron='props')
q-td(:props='props')
span {{ props.value }}
template(v-else-if='state.displayMode === `upcoming`')
q-card.rounded-borders(
v-if='state.upcomingJobs.length < 1'
flat
:class='$q.dark.isActive ? `bg-dark-5 text-white` : `bg-grey-3 text-dark`'
)
q-card-section.items-center(horizontal)
q-card-section.col-auto.q-pr-none
q-icon(name='las la-info-circle', size='sm')
q-card-section.text-caption {{ t('admin.scheduler.upcomingNone') }}
q-card.shadow-1(v-else)
q-table(
:rows='state.upcomingJobs'
:columns='upcomingJobsHeaders'
row-key='name'
flat
hide-bottom
:rows-per-page-options='[0]'
:loading='state.loading > 0'
)
template(v-slot:body-cell-id='props')
q-td(:props='props')
q-icon(name='las la-chess-knight', color='primary', size='sm')
template(v-slot:body-cell-task='props')
q-td(:props='props')
strong {{props.value}}
div: small.text-grey {{props.row.id}}
template(v-slot:body-cell-waituntil='props')
q-td(:props='props')
span {{ props.value }}
div: small.text-grey {{humanizeDate(props.row.waitUntil)}}
template(v-slot:body-cell-retries='props')
q-td(:props='props')
span #[strong {{props.value + 1}}] #[span.text-grey / {{props.row.maxRetries}}]
template(v-slot:body-cell-useworker='props')
q-td(:props='props')
template(v-if='props.value')
q-icon(name='las la-microchip', color='brown', size='sm')
small.q-ml-xs.text-brown Worker
template(v-else)
q-icon(name='las la-leaf', color='teal', size='sm')
small.q-ml-xs.text-teal In-Process
template(v-slot:body-cell-date='props')
q-td(:props='props')
i18n-t.text-caption(keypath='admin.scheduler.nextExecutionIn', tag='div')
template(#date)
strong {{ humanizeDate(props.value) }}
small {{props.value}}
span {{props.value}}
div
i18n-t.text-grey(keypath='admin.scheduler.createdBy', tag='small')
template(#instance)
strong {{props.row.createdBy}}
template(v-else)
q-card.rounded-borders(
v-if='state.jobs.length < 1'
......@@ -122,8 +168,9 @@ useMeta({
// DATA
const state = reactive({
displayMode: 'scheduled',
displayMode: 'upcoming',
scheduledJobs: [],
upcomingJobs: [],
jobs: [],
loading: 0
})
......@@ -137,10 +184,10 @@ const scheduledJobsHeaders = [
style: 'width: 15px; padding-right: 0;'
},
{
label: t('common.field.name'),
label: t('common.field.task'),
align: 'left',
field: 'name',
name: 'name',
field: 'task',
name: 'task',
sortable: true
},
{
......@@ -148,21 +195,77 @@ const scheduledJobsHeaders = [
align: 'left',
field: 'cron',
name: 'cron',
sortable: false
sortable: true
},
{
label: t('admin.scheduler.type'),
align: 'left',
field: 'type',
name: 'type',
sortable: true
},
{
label: t('admin.scheduler.createdAt'),
align: 'left',
field: 'createdAt',
name: 'created',
sortable: true,
format: v => DateTime.fromISO(v).toRelative()
},
{
label: t('admin.scheduler.updatedAt'),
align: 'left',
field: 'updatedAt',
name: 'updated',
sortable: true,
format: v => DateTime.fromISO(v).toRelative()
}
]
const upcomingJobsHeaders = [
{
align: 'center',
field: 'id',
name: 'id',
sortable: false,
style: 'width: 15px; padding-right: 0;'
},
{
label: t('common.field.task'),
align: 'left',
field: 'task',
name: 'task',
sortable: true
},
{
label: t('admin.scheduler.timezone'),
label: t('admin.scheduler.waitUntil'),
align: 'left',
field: 'timezone',
name: 'timezone',
sortable: false
field: 'waitUntil',
name: 'waituntil',
sortable: true,
format: v => DateTime.fromISO(v).toRelative()
},
{
label: t('admin.scheduler.nextExecution'),
label: t('admin.scheduler.attempt'),
align: 'left',
field: 'nextExecution',
field: 'retries',
name: 'retries',
sortable: true
},
{
label: t('admin.scheduler.useWorker'),
align: 'left',
field: 'useWorker',
name: 'useworker',
sortable: true
},
{
label: t('admin.scheduler.createdAt'),
align: 'left',
field: 'createdAt',
name: 'date',
sortable: false
sortable: true,
format: v => DateTime.fromISO(v).toRelative()
}
]
......@@ -174,33 +277,59 @@ watch(() => state.displayMode, (newValue) => {
// METHODS
function humanizeDate (val) {
return DateTime.fromISO(val).toFormat('fff')
}
async function load () {
state.loading++
try {
if (state.displayMode === 'scheduled') {
const resp = await APOLLO_CLIENT.query({
query: gql`
query getSystemScheduledJobs {
systemScheduledJobs {
query getSystemJobsScheduled {
systemJobsScheduled {
id
name
task
cron
timezone
nextExecution
type
createdAt
updatedAt
}
}
`,
fetchPolicy: 'network-only'
})
state.scheduledJobs = resp?.data?.systemScheduledJobs
state.scheduledJobs = resp?.data?.systemJobsScheduled
} else if (state.displayMode === 'upcoming') {
const resp = await APOLLO_CLIENT.query({
query: gql`
query getSystemJobsUpcoming {
systemJobsUpcoming {
id
task
useWorker
retries
maxRetries
waitUntil
isScheduled
createdBy
createdAt
updatedAt
}
}
`,
fetchPolicy: 'network-only'
})
state.upcomingJobs = resp?.data?.systemJobsUpcoming
} else {
const resp = await APOLLO_CLIENT.query({
query: gql`
query getSystemJobs (
$type: SystemJobType!
$state: SystemJobState!
) {
systemJobs (
type: $type
state: $state
) {
id
name
......@@ -210,7 +339,7 @@ async function load () {
}
`,
variables: {
type: state.displayMode.toUpperCase()
state: state.displayMode.toUpperCase()
},
fetchPolicy: 'network-only'
})
......@@ -226,10 +355,6 @@ async function load () {
state.loading--
}
function humanizeDate (val) {
return DateTime.fromISO(val).toRelative()
}
// MOUNTED
onMounted(() => {
......
ux/src/stores/admin.js
View file @ 24ddab73
......@@ -16,7 +16,8 @@ export const useAdminStore = defineStore('admin', {
usersTotal: 0,
loginsPastDay: 0,
isApiEnabled: false,
isMailConfigured: false
isMailConfigured: false,
isSchedulerHealthy: false
},
overlay: null,
overlayOpts: {},
......@@ -63,7 +64,8 @@ export const useAdminStore = defineStore('admin', {
usersTotal
currentVersion
latestVersion
mailConfigured
isMailConfigured
isSchedulerHealthy
}
}
`,
......@@ -74,7 +76,8 @@ export const useAdminStore = defineStore('admin', {
this.info.currentVersion = clone(resp?.data?.systemInfo?.currentVersion ?? 'n/a')
this.info.latestVersion = clone(resp?.data?.systemInfo?.latestVersion ?? 'n/a')
this.info.isApiEnabled = clone(resp?.data?.apiState ?? false)
this.info.isMailConfigured = clone(resp?.data?.systemInfo?.mailConfigured ?? false)
this.info.isMailConfigured = clone(resp?.data?.systemInfo?.isMailConfigured ?? false)
this.info.isSchedulerHealthy = clone(resp?.data?.systemInfo?.isSchedulerHealthy ?? false)
}
}
})
ux/yarn.lock
View file @ 24ddab73
This diff was suppressed by a .gitattributes entry.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment