use apprise for better notification & fix callback bug & improve log

README.md

@@ -34,11 +34,13 @@ Please refer to [Quickstart](https://github.com/lyc8503/UptimeFlare/wiki/Quickst
 
 - [x] Specify region for monitors
 - [x] TCP `opened` promise
-- [x] Telegram example
+- [x] Use apprise to support different notification channels
-- [x] [Bark](https://bark.day.app) example
+- [x] ~~Telegram example~~
+- [x] ~~[Bark](https://bark.day.app) example~~
+- [x] ~~Email notification via Cloudflare Email Workers~~
 - [ ] Improve docs by providing simple examples
 - [ ] Notification grace period
 - [ ] SSL certificate checks
 - [ ] Self-host Dockerfile
-- [ ] Email notification via Cloudflare Email Workers
 - [ ] Incident timeline
+- [ ] Remove old incidents

uptime.config.ts

@@ -58,6 +58,20 @@ const workerConfig = {
       timeout: 5000,
     },
   ],
+  notification: {
+    // [Optional] apprise API server URL
+    // if not specified, no notification will be sent
+    appriseApiServer: "https://apprise.example.com/notify",
+    // [Optional] recipient URL for apprise, refer to https://github.com/caronc/apprise
+    // if not specified, no notification will be sent
+    recipientUrl: "tgram://bottoken/ChatID",
+    // [Optional] timezone used in notification messages, default to "Etc/GMT"
+    timeZone: "Asia/Shanghai",
+    // [Optional] grace period in minutes before sending a notification
+    // notification will be sent only if the monitor is down for N continuous checks after the initial failure
+    // if not specified, notification will be sent immediately
+    gracePeriod: 5,
+  },
   callbacks: {
     onStatusChange: async (
       env: any,
@@ -70,8 +84,8 @@ const workerConfig = {
       // This callback will be called when there's a status change for any monitor
       // Write any Typescript code here
 
-      // By default, this sends Bark and Telegram notification on every status change if you setup Cloudflare env variables correctly.
+      // This will not follow the grace period settings and will be called immediately when the status changes
-      await notify(env, monitor, isUp, timeIncidentStart, timeNow, reason)
+      // You need to handle the grace period manually if you want to implement it
     },
     onIncident: async (
       env: any,
@@ -86,117 +100,5 @@ const workerConfig = {
   },
 }
 
-// Below is code for sending Telegram & Bark notification
-// You can safely ignore them
-const escapeMarkdown = (text: string) => {
-  return text.replace(/[_*[\](){}~`>#+\-=|.!\\]/g, '\\$&');
-};
-
-async function notify(
-  env: any,
-  monitor: any,
-  isUp: boolean,
-  timeIncidentStart: number,
-  timeNow: number,
-  reason: string,
-) {
-  const dateFormatter = new Intl.DateTimeFormat('en-US', {
-    month: 'numeric',
-    day: '2-digit',
-    hour: '2-digit',
-    minute: '2-digit',
-    hour12: false,
-    timeZone: 'Asia/Shanghai',
-  });
-
-  let downtimeDuration = Math.round((timeNow - timeIncidentStart) / 60);
-  const timeIncidentStartFormatted = dateFormatter.format(new Date(timeIncidentStart * 1000));
-  let statusText = isUp
-    ? `The service is up again after being down for ${downtimeDuration} minutes.`
-    : `Service became unavailable at ${timeIncidentStartFormatted}. Issue: ${reason || 'unspecified'}`;
-
-  console.log('Notifying: ', monitor.name, statusText);
-
-  if (env.BARK_SERVER && env.BARK_DEVICE_KEY) {
-    try {
-      let title = isUp ? `✅ ${monitor.name} is up again!` : `🔴 ${monitor.name} is currently down.`;
-      await sendBarkNotification(env, monitor, title, statusText);
-    } catch (error) {
-      console.error('Error sending Bark notification:', error);
-    }
-  }
-
-  if (env.SECRET_TELEGRAM_CHAT_ID && env.SECRET_TELEGRAM_API_TOKEN) {
-    try {
-      let operationalLabel = isUp ? 'Up' : 'Down';
-      let statusEmoji = isUp ? '✅' : '🔴';
-      let telegramText = `*${escapeMarkdown(
-        monitor.name,
-      )}* is currently *${operationalLabel}*\n${statusEmoji} ${escapeMarkdown(statusText)}`;
-      await notifyTelegram(env, monitor, isUp, telegramText);
-    } catch (error) {
-      console.error('Error sending Telegram notification:', error);
-    }
-  }
-}
-
-export async function notifyTelegram(env: any, monitor: any, operational: boolean, text: string) {
-  const chatId = env.SECRET_TELEGRAM_CHAT_ID;
-  const apiToken = env.SECRET_TELEGRAM_API_TOKEN;
-
-  const payload = new URLSearchParams({
-    chat_id: chatId,
-    parse_mode: 'MarkdownV2',
-    text: text,
-  });
-
-  try {
-    const response = await fetch(`https://api.telegram.org/bot${apiToken}/sendMessage`, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/x-www-form-urlencoded',
-      },
-      body: payload.toString(),
-    });
-
-    if (!response.ok) {
-      console.error(
-        `Failed to send Telegram notification "${text}",  ${response.status} ${response.statusText
-        } ${await response.text()}`,
-      );
-    }
-  } catch (error) {
-    console.error('Error sending Telegram notification:', error);
-  }
-}
-
-async function sendBarkNotification(env: any, monitor: any, title: string, body: string, group: string = '') {
-  const barkServer = env.BARK_SERVER;
-  const barkDeviceKey = env.BARK_DEVICE_KEY;
-  const barkUrl = `${barkServer}/push`;
-  const data = {
-    title: title,
-    body: body,
-    group: group,
-    url: monitor.url,
-    device_key: barkDeviceKey,
-  };
-
-  const response = await fetch(barkUrl, {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify(data),
-  });
-
-  if (response.ok) {
-    console.log('Bark notification sent successfully.');
-  } else {
-    const respText = await response.text();
-    console.error('Failed to send Bark notification:', response.status, response.statusText, respText);
-  }
-}
-
 // Don't forget this, otherwise compilation fails.
 export { pageConfig, workerConfig }

worker/src/index.ts

@@ -1,5 +1,5 @@
 import { workerConfig } from '../../uptime.config'
-import { getWorkerLocation } from './util'
+import { formatStatusChangeNotification, getWorkerLocation, notifyWithApprise } from './util'
 import { MonitorState } from '../../uptime.types'
 import { getStatus } from './monitor'
 
@@ -42,6 +42,34 @@ export default {
     const workerLocation = (await getWorkerLocation()) || 'ERROR'
     console.log(`Running scheduled event on ${workerLocation}...`)
 
+    // Auxiliary function to format notification and send it via apprise
+    let formatAndNotify = async (
+      monitor: any,
+      isUp: boolean,
+      timeIncidentStart: number,
+      timeNow: number,
+      reason: string
+    ) => {
+      if (workerConfig.notification?.appriseApiServer && workerConfig.notification?.recipientUrl) {
+        const notification = formatStatusChangeNotification(
+          monitor,
+          isUp,
+          timeIncidentStart,
+          timeNow,
+          reason,
+          workerConfig.notification?.timeZone ?? 'Etc/GMT'
+        )
+        await notifyWithApprise(
+          workerConfig.notification.appriseApiServer,
+          workerConfig.notification.recipientUrl,
+          notification.title,
+          notification.body
+        )
+      } else {
+        console.log(`Apprise API server or recipient URL not set, skipping apprise notification for ${monitor.name}`)
+      }
+    }
+
     // Read state, set init state if it doesn't exist
     let state =
       ((await env.UPTIMEFLARE_STATE.get('state', {
@@ -63,10 +91,10 @@ export default {
 
     // Check each monitor
     // TODO: concurrent status check
-
     for (const monitor of workerConfig.monitors) {
       console.log(`[${workerLocation}] Checking ${monitor.name}...`)
 
+      let monitorStatusChanged = false
       let checkLocation = workerLocation
       let status
 
@@ -109,16 +137,33 @@ export default {
         },
       ]
       // Then lastIncident here must not be undefined
-      const lastIncident = state.incident[monitor.id].slice(-1)[0]
+      let lastIncident = state.incident[monitor.id].slice(-1)[0]
 
       if (status.up) {
         // Current status is up
         // close existing incident if any
         if (lastIncident.end === undefined) {
           lastIncident.end = currentTimeSecond
-          statusChanged = true
+          monitorStatusChanged = true
-
           try {
+            if (
+              // grace period not set OR ...
+              workerConfig.notification?.gracePeriod === undefined ||
+              // only when we have sent a notification for DOWN status, we will send a notification for UP status (within 30 seconds of possible drift)
+              currentTimeSecond - lastIncident.start[0] >= (workerConfig.notification.gracePeriod + 1) * 60 - 30
+            ) {
+              await formatAndNotify(
+                monitor,
+                true,
+                lastIncident.start[0],
+                currentTimeSecond,
+                'OK'
+              )
+            } else {
+              console.log(`grace period (${workerConfig.notification?.gracePeriod}m) not met, skipping apprise UP notification for ${monitor.name}`)
+            }
+
+            console.log('Calling config onStatusChange callback...')
             await workerConfig.callbacks.onStatusChange(
               env,
               monitor,
@@ -141,21 +186,7 @@ export default {
             end: undefined,
             error: [status.err],
           })
-          statusChanged = true
+          monitorStatusChanged = true
-
-          try {
-            await workerConfig.callbacks.onStatusChange(
-              env,
-              monitor,
-              false,
-              currentTimeSecond,
-              currentTimeSecond,
-              status.err
-            )
-          } catch (e) {
-            console.log('Error calling callback: ')
-            console.log(e)
-          }
         } else if (
           lastIncident.end === undefined &&
           lastIncident.error.slice(-1)[0] !== status.err
@@ -163,28 +194,62 @@ export default {
           // append if the error message changes
           lastIncident.start.push(currentTimeSecond)
           lastIncident.error.push(status.err)
-          statusChanged = true
+          monitorStatusChanged = true
+        }
 
-          try {
+        const currentIncident = state.incident[monitor.id].slice(-1)[0]
+        try {
+          if (
+            // monitor status changed AND...
+            (monitorStatusChanged && (
+              // grace period not set OR ...
+              workerConfig.notification?.gracePeriod === undefined ||
+              // have sent a notification for DOWN status
+              currentTimeSecond - currentIncident.start[0] >= (workerConfig.notification.gracePeriod + 1) * 60 - 30
+            ))
+            ||
+            (
+              // grace period is set AND...
+              workerConfig.notification?.gracePeriod !== undefined &&
+              (
+                // grace period is met
+                currentTimeSecond - currentIncident.start[0] >= workerConfig.notification.gracePeriod * 60 - 30 &&
+                currentTimeSecond - currentIncident.start[0] < workerConfig.notification.gracePeriod * 60 + 30
+              )
+            )) {
+            await formatAndNotify(
+              monitor,
+              false,
+              currentIncident.start[0],
+              currentTimeSecond,
+              status.err
+            )
+          } else {
+            console.log(`Grace period (${workerConfig.notification?.gracePeriod}m) not met (currently down for ${currentTimeSecond - currentIncident.start[0]}s, changed ${monitorStatusChanged}), skipping apprise DOWN notification for ${monitor.name}`)
+          }
+
+          if (monitorStatusChanged) {
+            console.log('Calling config onStatusChange callback...')
             await workerConfig.callbacks.onStatusChange(
               env,
               monitor,
               false,
-              lastIncident.start[0],
+              currentIncident.start[0],
               currentTimeSecond,
               status.err
             )
-          } catch (e) {
-            console.log('Error calling callback: ')
-            console.log(e)
           }
+        } catch (e) {
+          console.log('Error calling callback: ')
+          console.log(e)
         }
 
         try {
+          console.log('Calling config onIncident callback...')
           await workerConfig.callbacks.onIncident(
             env,
             monitor,
-            lastIncident.start[0],
+            currentIncident.start[0],
             currentTimeSecond,
             status.err
           )
@@ -218,6 +283,9 @@ export default {
         latencyLists.all.shift()
       }
       state.latency[monitor.id] = latencyLists
+      // TODO: discard old incidents
+
+      statusChanged ||= monitorStatusChanged
     }
 
     console.log(`statusChanged: ${statusChanged}, lastUpdate: ${state.lastUpdate}, currentTime: ${currentTimeSecond}`)

worker/src/monitor.ts

@@ -57,6 +57,7 @@ export async function getStatus(
 
       if (monitor.expectedCodes) {
         if (!monitor.expectedCodes.includes(response.status)) {
+          console.log(`${monitor.name} expected ${monitor.expectedCodes}, got ${response.status}`)
           status.up = false
           status.err = `Expected codes: ${JSON.stringify(monitor.expectedCodes)}, Got: ${response.status
             }`
@@ -64,6 +65,7 @@ export async function getStatus(
         }
       } else {
         if (response.status < 200 || response.status > 299) {
+          console.log(`${monitor.name} expected 2xx, got ${response.status}`)
           status.up = false
           status.err = `Expected codes: 2xx, Got: ${response.status}`
           return status
@@ -73,6 +75,7 @@ export async function getStatus(
       if (monitor.responseKeyword) {
         const responseBody = await response.text()
         if (!responseBody.includes(monitor.responseKeyword)) {
+          console.log(`${monitor.name} expected keyword ${monitor.responseKeyword}, not found in response (truncated to 100 chars): ${responseBody.slice(0, 100)}`)
           status.up = false
           status.err = "HTTP response doesn't contain the configured keyword"
           return status

worker/src/util.ts

@@ -1,29 +1,100 @@
 async function getWorkerLocation() {
-	const res = await fetch('https://cloudflare.com/cdn-cgi/trace')
+  const res = await fetch('https://cloudflare.com/cdn-cgi/trace')
-	const text = await res.text()
+  const text = await res.text()
 
-	const colo = /^colo=(.*)$/m.exec(text)?.[1]
+  const colo = /^colo=(.*)$/m.exec(text)?.[1]
-	return colo
+  return colo
 }
 
 const fetchTimeout = (
-	url: string,
+  url: string,
-	ms: number,
+  ms: number,
-	{ signal, ...options }: RequestInit<RequestInitCfProperties> | undefined = {}
+  { signal, ...options }: RequestInit<RequestInitCfProperties> | undefined = {}
 ): Promise<Response> => {
-	const controller = new AbortController()
+  const controller = new AbortController()
-	const promise = fetch(url, { signal: controller.signal, ...options })
+  const promise = fetch(url, { signal: controller.signal, ...options })
-	if (signal) signal.addEventListener('abort', () => controller.abort())
+  if (signal) signal.addEventListener('abort', () => controller.abort())
-	const timeout = setTimeout(() => controller.abort(), ms)
+  const timeout = setTimeout(() => controller.abort(), ms)
-	return promise.finally(() => clearTimeout(timeout))
+  return promise.finally(() => clearTimeout(timeout))
 }
 
 function withTimeout<T>(millis: number, promise: Promise<T>): Promise<T> {
-	const timeout = new Promise<T>((resolve, reject) =>
+  const timeout = new Promise<T>((resolve, reject) =>
-		setTimeout(() => reject(new Error(`Promise timed out after ${millis}ms`)), millis)
+    setTimeout(() => reject(new Error(`Promise timed out after ${millis}ms`)), millis)
-	)
+  )
 
-	return Promise.race([promise, timeout])
+  return Promise.race([promise, timeout])
 }
 
-export { getWorkerLocation, fetchTimeout, withTimeout }
+function formatStatusChangeNotification(
+  monitor: any,
+  isUp: boolean,
+  timeIncidentStart: number,
+  timeNow: number,
+  reason: string,
+  timeZone: string
+) {
+  const dateFormatter = new Intl.DateTimeFormat('en-US', {
+    month: 'numeric',
+    day: '2-digit',
+    hour: '2-digit',
+    minute: '2-digit',
+    hour12: false,
+    timeZone: timeZone,
+  })
+
+  let downtimeDuration = Math.round((timeNow - timeIncidentStart) / 60);
+  const timeNowFormatted = dateFormatter.format(new Date(timeNow * 1000))
+  const timeIncidentStartFormatted = dateFormatter.format(new Date(timeIncidentStart * 1000))
+
+  if (isUp) {
+    return {
+      title: `✅ ${monitor.name} is up!`,
+      body: `The service is up again after being down for ${downtimeDuration} minutes.`,
+    }
+  } else if (timeNow == timeIncidentStart) {
+    return {
+      title: `🔴 ${monitor.name} is currently down.`,
+      body: `Service is unavailable at ${timeNowFormatted}. Issue: ${reason || 'unspecified'}`,
+    }
+  } else {
+    return {
+      title: `🔴 ${monitor.name} is still down.`,
+      body: `Service is unavailable since ${timeIncidentStartFormatted} (${downtimeDuration} minutes). Issue: ${reason || 'unspecified'}`,
+    }
+  }
+}
+
+async function notifyWithApprise(
+  appriseApiServer: string,
+  recipientUrl: string,
+  title: string,
+  body: string
+) {
+  console.log('Sending Apprise notification: ' + title + '-' + body + ' to ' + recipientUrl + ' via ' + appriseApiServer)
+  try {
+    const resp = await fetchTimeout(appriseApiServer, 5000, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        urls: recipientUrl,
+        title,
+        body,
+        type: 'warning',
+        format: 'text'
+      }),
+    })
+
+    if (!resp.ok) {
+      console.log('Error calling apprise server, code: ' + resp.status + ', response: ' + await resp.text())
+    } else {
+      console.log('Apprise notification sent successfully, code: ' + resp.status)
+    }
+  } catch (e) {
+    console.log('Error calling apprise server: ' + e)
+  }
+}
+
+export { getWorkerLocation, fetchTimeout, withTimeout, notifyWithApprise, formatStatusChangeNotification }