feat: D1 upgrade: deployment and migration process DONE!

2026-01-03 20:47:43 +08:00 · 2026-01-03 20:47:43 +08:00 · 3de3640d17
parent 0258ae1ddc
commit 3de3640d17
12 changed files with 278 additions and 76 deletions
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@ -42,8 +42,6 @@ jobs:
        env:
          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}

-      - nname: TODO migrate and init D1
-
      - name: Install packages
        run: |
          npm install
@ -59,26 +57,48 @@ jobs:
        run: |
          npx @cloudflare/next-on-pages

+      - name: Create D1 database and tables
+        run: |
+          python3 deploy/init_d1.py  # This sets D1_ID in GITHUB_ENV
+
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ steps.fetch_account_id.outputs.account_id }}
+
+      - name: Migrate state from KV (if needed)
+        run: |
+          python3 deploy/migrate_kv.py
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ steps.fetch_account_id.outputs.account_id }}
+          D1_ID: ${{ env.D1_ID }}
+
      - name: Deploy using Terraform
        # As we don't save terraform state somewhere, we need to import the existing resources
        run: |
          terraform init

-          KV_ID=$(curl https://api.cloudflare.com/client/v4/accounts/$TF_VAR_CLOUDFLARE_ACCOUNT_ID/storage/kv/namespaces\?per_page\=100 --header 'Authorization: Bearer '$CLOUDFLARE_API_TOKEN | jq -r '.result[] | select(.title == "uptimeflare_kv") | .id')
-          if [ -n "$KV_ID" ]; then
-            echo "Importing existing resources..."
-            terraform import cloudflare_workers_kv_namespace.uptimeflare_kv "$TF_VAR_CLOUDFLARE_ACCOUNT_ID/$KV_ID"
-            terraform import cloudflare_workers_script.uptimeflare_worker "$TF_VAR_CLOUDFLARE_ACCOUNT_ID/uptimeflare_worker" || echo "WARNING: Worker script import failed, continuing..."
-            terraform import cloudflare_workers_cron_trigger.uptimeflare_worker_cron "$TF_VAR_CLOUDFLARE_ACCOUNT_ID/uptimeflare_worker" || echo "WARNING: Cron trigger import failed, continuing..."
-            terraform import cloudflare_pages_project.uptimeflare "$TF_VAR_CLOUDFLARE_ACCOUNT_ID/uptimeflare" || echo "WARNING: Pages project import failed, continuing..."
+          DO_RESP=$(curl "https://api.cloudflare.com/client/v4/accounts/$TF_VAR_CLOUDFLARE_ACCOUNT_ID/workers/durable_objects/namespaces?per_page=1000" \
+            -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN")
+
+          if echo "$DO_RESP" | jq -e '.result[] | select(.script == "uptimeflare_worker" and .class == "RemoteChecker")' > /dev/null; then
+            echo "Existing Durable Object namespace found for uptimeflare_worker RemoteChecker. No migration needed."
          else
-            echo "KV namespace not found, first-time setup."
+            echo "No existing Durable Object namespace found for uptimeflare_worker RemoteChecker. Need migration."
+            export TF_VAR_enable_do_migration=true
          fi

+          echo "Try importing existing resources..."
+          terraform import cloudflare_d1_database.uptimeflare_d1 "$TF_VAR_CLOUDFLARE_ACCOUNT_ID/$D1_ID"
+          terraform import cloudflare_workers_script.uptimeflare_worker "$TF_VAR_CLOUDFLARE_ACCOUNT_ID/uptimeflare_worker" || echo "WARNING: Worker script import failed, continuing..."
+          terraform import cloudflare_workers_cron_trigger.uptimeflare_worker_cron "$TF_VAR_CLOUDFLARE_ACCOUNT_ID/uptimeflare_worker" || echo "WARNING: Cron trigger import failed, continuing..."
+          terraform import cloudflare_pages_project.uptimeflare "$TF_VAR_CLOUDFLARE_ACCOUNT_ID/uptimeflare" || echo "WARNING: Pages project import failed, continuing..."
+
          terraform apply -auto-approve -input=false
        env:
          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
          TF_VAR_CLOUDFLARE_ACCOUNT_ID: ${{ steps.fetch_account_id.outputs.account_id }}
+          D1_ID: ${{ env.D1_ID }}

      # Currently Terraform Cloudflare provider doesn't support direct upload, use wrangler to upload instead.
      - name: Upload pages
--- a/README.md
+++ b/README.md
@ -7,6 +7,10 @@

 A more advanced, serverless, and free uptime monitoring & status page solution, powered by Cloudflare Workers, complete with a user-friendly interface.

+🎉 **[UPDATE 2026/01/03]** I have just migrated UptimeFlare from KV to D1 Database. I also updated the Terraform Cloudflare provider to v5 and improved the deployment process. The data structure has been optimized to resolve long-standing performance issues.
+
+New users can deploy directly, while existing users can have a simple auto migration process (upgrade docs below)! Feel free to open an issue if you run into any trouble deploying.
+
 ## ⭐Features

 - Open-source, easy to deploy (in under 10 minutes, no local tools required), and free
@ -69,8 +73,8 @@ To contribute new features or customize your deployment furthermore, see [here](
 - [x] Compatibility date update
 - [x] Scheduled Maintenance
 - [x] Add docs for dev
- [ ] Migration to Terraform Cloudflare provider version 5.x
- [ ] Cloudflare D1 database
+- [x] Migration to Terraform Cloudflare provider version 5.x
+- [x] Cloudflare D1 database
 - [x] Scheduled maintenances (via IIFE)
 - [x] Simpler config example
 - [x] Upcoming maintenances
@ -80,5 +84,5 @@ To contribute new features or customize your deployment furthermore, see [here](
 - [x] Add default UA
 - [x] Customizable footer
 - [x] New header logo
- [ ] Improve CPU time usage
- [ ] Local deployment
+- [x] Improve CPU time usage
+- [x] Local deployment (docs WIP)
--- a/deploy.tf
+++ b/deploy.tf
@ -16,49 +16,49 @@ variable "CLOUDFLARE_ACCOUNT_ID" {
  type = string
 }

-resource "cloudflare_workers_kv_namespace" "uptimeflare_kv" {
-  account_id = var.CLOUDFLARE_ACCOUNT_ID
-  title      = "uptimeflare_kv"
+variable "enable_do_migration" {
+  type    = bool
+  default = false
 }

 resource "cloudflare_d1_database" "uptimeflare_d1" {
-  account_id = var.CLOUDFLARE_ACCOUNT_ID
-  name = "uptimeflare_d1"
-  primary_location_hint = "wnam"
+  account_id            = var.CLOUDFLARE_ACCOUNT_ID
+  name                  = "uptimeflare_d1"
  read_replication = {
    mode = "auto"
  }
 }

 resource "cloudflare_workers_script" "uptimeflare_worker" {
-  account_id         = var.CLOUDFLARE_ACCOUNT_ID
-  script_name        = "uptimeflare_worker"
-  main_module = "worker/dist/index.js"
-  content_file       = "worker/dist/index.js"
-  content_sha256     = filesha256("worker/dist/index.js")
-  compatibility_date = "2025-04-02"
+  account_id          = var.CLOUDFLARE_ACCOUNT_ID
+  script_name         = "uptimeflare_worker"
+  main_module         = "worker/dist/index.js"
+  content_file        = "worker/dist/index.js"
+  content_sha256      = filesha256("worker/dist/index.js")
+  compatibility_date  = "2025-04-02"
  compatibility_flags = ["nodejs_compat"]

  observability = {
    enabled = true
    logs = {
-      enabled = true
+      enabled         = true
      invocation_logs = true
    }
  }

+  migrations = var.enable_do_migration ? {
+    new_tag            = "v1"
+    new_sqlite_classes = ["RemoteChecker"]
+  } : null
+
  bindings = [{
    name       = "REMOTE_CHECKER_DO"
    class_name = "RemoteChecker"
    type       = "durable_object_namespace"
-  }, {
-    name = "UPTIMEFLARE_STATE"
-    type = "kv_namespace"
-    namespace_id = cloudflare_workers_kv_namespace.uptimeflare_kv.id
-  }, {
+    }, {
    name = "UPTIMEFLARE_D1"
    type = "d1"
-    id = cloudflare_d1_database.uptimeflare_d1.id
+    id   = cloudflare_d1_database.uptimeflare_d1.id
  }]
 }

@ -66,7 +66,7 @@ resource "cloudflare_workers_cron_trigger" "uptimeflare_worker_cron" {
  account_id  = var.CLOUDFLARE_ACCOUNT_ID
  script_name = cloudflare_workers_script.uptimeflare_worker.script_name
  schedules = [{
-    cron = "* * * * *" # every 1 minute, you can reduce the KV write by increase the worker settings of `kvWriteCooldownMinutes`
+    cron = "* * * * *" # every 1 minute, you can reduce the write counts by increase the worker settings of `kvWriteCooldownMinutes`
  }]
 }

@ -81,11 +81,6 @@ resource "cloudflare_pages_project" "uptimeflare" {
      fail_open = false
    }
    production = {
-      kv_namespaces = {
-        UPTIMEFLARE_STATE = {
-          namespace_id = cloudflare_workers_kv_namespace.uptimeflare_kv.id
-        }
-      }
      d1_databases = {
        UPTIMEFLARE_D1 = {
          id = cloudflare_d1_database.uptimeflare_d1.id
@ -93,7 +88,7 @@ resource "cloudflare_pages_project" "uptimeflare" {
      }
      compatibility_date  = "2025-04-02"
      compatibility_flags = ["nodejs_compat"]
-      fail_open = false
+      fail_open           = false
    }
  }

--- a/deploy/init_d1.py
+++ b/deploy/init_d1.py
@ -0,0 +1,76 @@
+# This is a script to create D1 database, initialize tables, and get D1_ID for later use.
+import requests
+import os
+
+api_endpoint = f"https://api.cloudflare.com/client/v4/accounts/{os.environ['CLOUDFLARE_ACCOUNT_ID']}"
+headers = {
+    "Authorization": f"Bearer {os.environ['CLOUDFLARE_API_TOKEN']}",
+}
+d1_name = "uptimeflare_d1"
+with open('init.sql', 'r') as f:
+    init_sql = f.read()
+
+# Try to create D1 database (if it doesn't exist)
+r = requests.post(
+    api_endpoint + "/d1/database",
+    headers=headers,
+    json={
+        "name": d1_name,
+        "primary_location_hint": "wnam"
+    }
+).json()
+
+if not r['success']:
+    print("Error creating D1 database: ", r)
+    if r['errors'][0]['code'] == 7502:
+        print("D1 database already exists, skipping creation.")
+    elif r['errors'][0]['code'] == 10000:
+        print("Authentication error when creating D1 database. Please make sure your CLOUDFLARE_API_TOKEN has the necessary permissions for D1 Database. This is required for versions after 2026/01/02.")
+        exit(1)
+    else:
+        print("Unknown error creating D1 database: ", r)
+        print("Please report this issue at https://github.com/lyc8503/UptimeFlare/issues.")
+        exit(1)
+else:
+    print("D1 database created successfully: ", r)
+
+# Fetch D1 database ID
+r = requests.get(
+    api_endpoint + "/d1/database?per_page=1000",
+    headers=headers
+).json()
+
+if not r['success']:
+    print("Error fetching D1 database info: ", r)
+    exit(1)
+
+d1_id = ''
+for db in r['result']:
+    if db['name'] == d1_name:
+        d1_id = db['uuid']
+        break
+
+if d1_id == '':
+    print("D1 database not found after creation. Please report this issue at https://github.com/lyc8503/UptimeFlare/issues.")
+    print("Full response: ", r)
+    exit(1)
+print(f"Got D1 database ID: {d1_id}")
+
+# Create initial table in D1
+r = requests.post(
+    api_endpoint + f"/d1/database/{d1_id}/query",
+    headers=headers,
+    json={
+        "sql": init_sql,
+        "params": []
+    }
+).json()
+
+print("Initialize D1 database response: ", r)
+if not r['success']:
+    print("Error initializing D1 database.")
+    exit(1)
+print("D1 database initialized successfully.")
+
+with open(os.environ['GITHUB_ENV'], "a") as f:
+    f.write(f"D1_ID={d1_id}\n")
--- a/deploy/migrate_kv.py
+++ b/deploy/migrate_kv.py
@ -0,0 +1,94 @@
+# This is a script to migrate state from KV to D1 database.
+# It reads the state from KV namespace, compacts it, and writes it to D1 database.
+# It also deletes the KV namespace after a successful migration.
+import requests
+import os
+import json
+
+api_endpoint = f"https://api.cloudflare.com/client/v4/accounts/{os.environ['CLOUDFLARE_ACCOUNT_ID']}"
+headers = {
+    "Authorization": f"Bearer {os.environ['CLOUDFLARE_API_TOKEN']}",
+}
+d1_id = os.environ['D1_ID']
+kv_name = "uptimeflare_kv"
+
+# Fetch KV namespace ID
+r = requests.get(
+    api_endpoint + "/storage/kv/namespaces?per_page=1000",
+    headers=headers
+).json()
+
+if not r['success']:
+    print("Error fetching KV namespace info: ", r)
+    exit(1)
+
+kv_id = ''
+for ns in r['result']:
+    if ns['title'] == kv_name:
+        kv_id = ns['id']
+        break
+
+if kv_id == '':
+    print("KV namespace not found. Skipping migration.")
+    exit(0)
+
+print(f"Got KV namespace ID: {kv_id}")
+
+# Fetch state from KV
+r = requests.post(
+    api_endpoint + f"/storage/kv/namespaces/{kv_id}/bulk/get",
+    headers=headers,
+    json={
+        "keys": ["state"]
+    }
+).json()
+
+if not r['success']:
+    print("Error fetching state from KV: ", r)
+    exit(1)
+
+# Compact it
+original_state = r['result']['values']['state']
+state = json.loads(original_state)
+compacted_state = {
+    'lastUpdate': state['lastUpdate'],
+    'overallUp': state['overallUp'],
+    'overallDown': state['overallDown'],
+    'incident': {
+        k: {
+            'start': [x['start'] for x in v],
+            'end': [x.get('end') for x in v],
+            'error': [x['error'] for x in v]
+        } for k, v in state['incident'].items()
+    },
+    'latency': {}
+}
+compacted_state_str = json.dumps(compacted_state)
+
+print("Original state: ", original_state[:256] + "..." if len(original_state) > 256 else original_state)
+print("Compacted state: ", compacted_state_str[:256] + "..." if len(compacted_state_str) > 256 else compacted_state_str)
+
+# Write compacted state to D1
+r = requests.post(
+    api_endpoint + f"/d1/database/{d1_id}/query",
+    headers=headers,
+    json={
+        "sql": "INSERT INTO uptimeflare (key, value) VALUES (?, ?)",
+        "params": ["state", compacted_state_str]
+    }
+).json()
+
+if r['success'] :
+    print("State migrated to D1 successfully. Trying to delete unused KV namespace...")
+    r = requests.delete(
+        api_endpoint + f"/storage/kv/namespaces/{kv_id}",
+        headers=headers
+    ).json()
+    if r['success']:
+        print("KV namespace deleted successfully.")
+    else:
+        print("Error deleting KV namespace: ", r)
+else:
+    print("Error writing state to D1: ", r)
+    print("Migration failed. Please report this issue at https://github.com/lyc8503/UptimeFlare/issues.")
+    exit(1)
--- a/pages/api/badge.ts
+++ b/pages/api/badge.ts
@ -49,7 +49,10 @@ export default async function handler(req: NextRequest): Promise<Response> {
      await getFromStore(process.env as any, 'state')
    )

-    const lastIncident = compactedState.getIncident(monitorId, compactedState.incidentLen(monitorId) - 1)
+    const lastIncident = compactedState.getIncident(
+      monitorId,
+      compactedState.incidentLen(monitorId) - 1
+    )
    const isUp = lastIncident?.end !== null

    const badge: BadgePayload = {
--- a/pages/api/data.ts
+++ b/pages/api/data.ts
@ -12,19 +12,24 @@ const headers = {
 }

 export default async function handler(req: NextRequest): Promise<Response> {
-  const compactedState = new CompactedMonitorStateWrapper(await getFromStore(process.env as any, 'state'))
+  const compactedState = new CompactedMonitorStateWrapper(
+    await getFromStore(process.env as any, 'state')
+  )

  if (compactedState.data.lastUpdate === 0) {
    return new Response(JSON.stringify({ error: 'No data available' }), {
      status: 500,
-      headers
+      headers,
    })
  }

  let monitors: any = {}

  for (let monitor of workerConfig.monitors) {
-    const lastIncident = compactedState.getIncident(monitor.id, compactedState.incidentLen(monitor.id) - 1)
+    const lastIncident = compactedState.getIncident(
+      monitor.id,
+      compactedState.incidentLen(monitor.id) - 1
+    )

    const isUp = lastIncident?.end !== null
    const latency = compactedState.getLastLatency(monitor.id)
@ -44,6 +49,6 @@ export default async function handler(req: NextRequest): Promise<Response> {
  }

  return new Response(JSON.stringify(ret), {
-    headers
+    headers,
  })
 }
--- a/types/config.ts
+++ b/types/config.ts
@ -94,7 +94,7 @@ export type Callbacks<TEnv = Env> = {

 export type IncidentRecord = {
  start: number[]
-  end: number | null  // null if it's still open
+  end: number | null // null if it's still open
  error: string[]
 }

@ -109,15 +109,18 @@ export type MonitorState = {
  overallUp: number
  overallDown: number
  incident: Record<string, IncidentRecord[]>
-  latency: Record<string, LatencyRecord[]>  // recent 12 hour data, N min interval
+  latency: Record<string, LatencyRecord[]> // recent 12 hour data, N min interval
 }

-// This is now the actual stored format (after TODO D1 migration) to improve (de)serialization performance
-// This gives a ~3.5x speedup in computing and a 58% reduction in size
-// The CPULimitExceeded issue with 10+ monitors on free tier should be largely mitigated by this change
+// This is now the actual stored format (after 2026/01/01 D1 migration) to improve (de)serialization performance
+// This gives a ~3.5x speedup in computing and a 40-60% reduction in size
+// The CPULimitExceeded issue with 10+ monitors on free tier should be mitigated by this change
 // local profiling result (1 op = parse + stringify):
 // MonitorState (original): 277 ops/s, ±0.51%   | slowest, 71.09% slower
 // MonitorStateCompacted:   958 ops/s, ±1.17%   | fastest
+// Real world test with 8 monitors and a few hundred incidents and full latency data (status.lyc8503.net):
+// original: 433KB size, 11.24ms P50 cpu time, 18.11ms P99 cpu time
+// compacted: 181KB size (59% smaller), 6.36ms P50 cpu time (43% faster), 8.86ms P99 cpu time (51% faster)
 export type MonitorStateCompacted = {
  lastUpdate: number
  overallUp: number
--- a/worker/src/index.ts
+++ b/worker/src/index.ts
@ -17,9 +17,7 @@ const Worker = {
    console.log(`Running scheduled event on ${workerLocation}...`)

    // Create a wrapped MonitorState from stored compacted state
-    const state = new CompactedMonitorStateWrapper(
-      await getFromStore(env, 'state')
-    )
+    const state = new CompactedMonitorStateWrapper(await getFromStore(env, 'state'))
    state.data.overallDown = 0
    state.data.overallUp = 0

@ -147,10 +145,7 @@ const Worker = {
            error: [status.err],
          })
          monitorStatusChanged = true
-        } else if (
-          lastIncident.end === null &&
-          lastIncident.error.slice(-1)[0] !== status.err
-        ) {
+        } else if (lastIncident.end === null && lastIncident.error.slice(-1)[0] !== status.err) {
          // append if the error message changes
          lastIncident.start.push(currentTimeSecond)
          lastIncident.error.push(status.err)
@ -160,10 +155,7 @@ const Worker = {
          monitorStatusChanged = true
        }

-        const currentIncident = state.getIncident(
-          monitor.id,
-          state.incidentLen(monitor.id) - 1
-        )
+        const currentIncident = state.getIncident(monitor.id, state.incidentLen(monitor.id) - 1)
        try {
          if (
            // monitor status changed AND...
@ -240,7 +232,7 @@ const Worker = {
      }

      // append to latency data
-      state.appendLatency(monitor.id,{
+      state.appendLatency(monitor.id, {
        loc: checkLocation,
        ping: status.ping,
        time: currentTimeSecond,
@ -283,7 +275,8 @@ const Worker = {
    // Allow for a cooldown period before writing to storage
    if (
      statusChanged ||
-      currentTimeSecond - state.data.lastUpdate >= (workerConfig.kvWriteCooldownMinutes ?? 3) * 60 - 10 // Allow for 10 seconds of clock drift
+      currentTimeSecond - state.data.lastUpdate >=
+        (workerConfig.kvWriteCooldownMinutes ?? 3) * 60 - 10 // Allow for 10 seconds of clock drift
    ) {
      console.log('Updating state...')
      state.data.lastUpdate = currentTimeSecond
--- a/worker/src/store.ts
+++ b/worker/src/store.ts
@ -1,5 +1,10 @@
 import { Env } from '.'
-import { IncidentRecord, LatencyRecord, MonitorState, MonitorStateCompacted } from '../../types/config'
+import {
+  IncidentRecord,
+  LatencyRecord,
+  MonitorState,
+  MonitorStateCompacted,
+} from '../../types/config'

 export async function getFromStore(env: Env, key: string): Promise<string | null> {
  const stmt = env.UPTIMEFLARE_D1.prepare('SELECT value FROM uptimeflare WHERE key = ?')
@ -50,8 +55,13 @@ export class CompactedMonitorStateWrapper {
      state.incident[monitorId] = []
      const incidents = this.data.incident[monitorId]

-      if (incidents.start.length !== incidents.end.length || incidents.start.length !== incidents.error.length) {
-        throw new Error('Inconsistent incident data lengths, please report an issue at https://github.com/lyc8503/UptimeFlare')
+      if (
+        incidents.start.length !== incidents.end.length ||
+        incidents.start.length !== incidents.error.length
+      ) {
+        throw new Error(
+          'Inconsistent incident data lengths, please report an issue at https://github.com/lyc8503/UptimeFlare'
+        )
      }

      for (let i = 0; i < incidents.start.length; i++) {
@ -79,7 +89,9 @@ export class CompactedMonitorStateWrapper {
      const pingArr = new Uint16Array(Uint8Array.fromHex(latencies.ping).buffer)

      if (timeArr.length !== pingArr.length || timeArr.length !== locUncompacted.length) {
-        throw new Error('Inconsistent latency data lengths, please report an issue at https://github.com/lyc8503/UptimeFlare.')
+        throw new Error(
+          'Inconsistent latency data lengths, please report an issue at https://github.com/lyc8503/UptimeFlare.'
+        )
      }

      for (let i = 0; i < timeArr.length; i++) {
@ -155,7 +167,7 @@ export class CompactedMonitorStateWrapper {
  latencyLen(monitorId: string): number {
    const latencies = this.data.latency[monitorId]
    if (!latencies) return 0
-    return latencies.ping.length / 4  // Uint16Array, 4 characters per entry in hex
+    return latencies.ping.length / 4 // Uint16Array, 4 characters per entry in hex
  }

  appendLatency(monitorId: string, record: LatencyRecord) {
--- a/worker/src/util.ts
+++ b/worker/src/util.ts
@ -157,9 +157,7 @@ const formatAndNotify = async (
  // Skip notification if monitor is in the skip list
  const skipList = workerConfig.notification?.skipNotificationIds
  if (skipList && skipList.includes(monitor.id)) {
-    console.log(
-      `Skipping notification for ${monitor.name} (${monitor.id} in skipNotificationIds)`
-    )
+    console.log(`Skipping notification for ${monitor.name} (${monitor.id} in skipNotificationIds)`)
    return
  }

@ -193,7 +191,6 @@ const formatAndNotify = async (
  }
 }

-
 export {
  getWorkerLocation,
  fetchTimeout,