content,dns: resolve proxy fake IP addresses safely

Use DNS over HTTPS only when a host has only proxy fake IP addresses. Keep private address checks and pinned article connections. Test DNS failures, redirects, and interruption.
2026-10-11 16:21:39 +08:00
1 parent ce87e56053
commit 3751fd00d3
@@ -26,6 +26,7 @@
    "@hugeicons/core-free-icons": "^4.3.5",
    "@hugeicons/react": "^1.1.10",
    "@mozilla/readability": "^0.6.0",
+    "@multiformats/dns": "^1.0.15",
    "@tailwindcss/vite": "^4",
    "@tanstack/react-router": "latest",
    "@tanstack/react-start": "latest",
@@ -1,6 +1,7 @@
 import assert from "node:assert/strict"
-import test from "node:test"
-import { Effect } from "effect"
+import dns from "node:dns/promises"
+import test, { type TestContext } from "node:test"
+import { ConfigProvider, Deferred, Effect, Fiber } from "effect"
 import { Agent, MockAgent } from "undici"
 import {
  canonicalUrl,
@@ -10,6 +11,42 @@ import {
  validatePublicUrl,
 } from "./content"

+const testUrl = "https://www.eflx.top/blog/ui-composition-first-principle"
+const dnsEndpoint = "https://cloudflare-dns.com/dns-query"
+
+function checkUrl(value = testUrl, endpoint = dnsEndpoint) {
+  return validatePublicUrl(value).pipe(
+    Effect.provideService(
+      ConfigProvider.ConfigProvider,
+      ConfigProvider.fromUnknown({ DNS_OVER_HTTPS_URL: endpoint }),
+    ),
+  )
+}
+
+function fakeDns(context: TestContext) {
+  return context.mock.method(dns, "lookup", async () => [
+    { address: "198.18.0.109", family: 4 },
+    { address: "198.19.255.254", family: 4 },
+  ])
+}
+
+function dnsAnswer(type: "A" | "AAAA", addresses: string[] = []) {
+  return Response.json({
+    Status: 0,
+    TC: false,
+    Question: [{ name: "www.eflx.top", type: type === "A" ? 1 : 28 }],
+    Answer: [
+      { name: "www.eflx.top", type: 5, TTL: 300, data: "cname.example.com" },
+      ...addresses.map((address) => ({
+        name: "cname.example.com",
+        type: type === "A" ? 1 : 28,
+        TTL: 300,
+        data: address,
+      })),
+    ],
+  })
+}
+
 test("article URLs reject local, private, mapped, and reserved addresses", async () => {
  for (const address of [
    "0.0.0.0",
@@ -21,6 +58,8 @@ test("article URLs reject local, private, mapped, and reserved addresses", async
    "100.64.0.1",
    "224.0.0.1",
    "192.0.2.1",
+    "198.18.0.109",
+    "198.19.255.254",
    "198.51.100.1",
    "203.0.113.1",
    "::",
@@ -68,6 +107,260 @@ test("article URLs accept public IPv4 and IPv6 addresses", async () => {
  }
 })

+test("public DNS answers retain the native route without an HTTPS lookup", async (context) => {
+  const addresses = [
+    { address: "66.33.60.129", family: 4 },
+    { address: "2606:4700:4700::1111", family: 6 },
+  ]
+  context.mock.method(dns, "lookup", async () => addresses)
+  const remote = context.mock.method(globalThis, "fetch", async () => {
+    throw new Error("A public native answer must not use the fallback.")
+  })
+  const result = await Effect.runPromise(checkUrl(testUrl, "invalid URL"))
+  assert.deepEqual(result.addresses, addresses)
+  assert.equal(remote.mock.callCount(), 0)
+})
+
+test("fake DNS uses public HTTPS answers when one address family has no records", async (context) => {
+  fakeDns(context)
+  const questions: string[] = []
+  context.mock.method(
+    globalThis,
+    "fetch",
+    async (...[input, options]: Parameters<typeof globalThis.fetch>) => {
+      const url = new URL(String(input))
+      assert.equal(`${url.origin}${url.pathname}`, dnsEndpoint)
+      assert.equal(url.searchParams.get("name"), "www.eflx.top")
+      assert.ok(options?.signal instanceof AbortSignal)
+      assert.equal(
+        new Headers(options.headers).get("accept"),
+        "application/dns-json",
+      )
+      const type = url.searchParams.get("type")
+      assert.ok(type === "A" || type === "AAAA")
+      questions.push(type)
+      return dnsAnswer(
+        type,
+        type === "A" ? ["66.33.60.129", "66.33.60.130"] : [],
+      )
+    },
+  )
+  const result = await Effect.runPromise(checkUrl())
+  assert.equal(result.url.toString(), testUrl)
+  assert.deepEqual(questions.sort(), ["A", "AAAA"])
+  assert.deepEqual(result.addresses, [
+    { address: "66.33.60.129", family: 4 },
+    { address: "66.33.60.130", family: 4 },
+  ])
+})
+
+test("fake DNS retains both public address families from the HTTPS resolver", async (context) => {
+  fakeDns(context)
+  context.mock.method(
+    globalThis,
+    "fetch",
+    async (...[input]: Parameters<typeof globalThis.fetch>) => {
+      const type = new URL(String(input)).searchParams.get("type")
+      return type === "A"
+        ? dnsAnswer("A", ["66.33.60.129"])
+        : dnsAnswer("AAAA", ["2606:4700:4700::1111"])
+    },
+  )
+  const result = await Effect.runPromise(checkUrl())
+  assert.deepEqual(result.addresses, [
+    { address: "66.33.60.129", family: 4 },
+    { address: "2606:4700:4700::1111", family: 6 },
+  ])
+})
+
+test("private hosts, IP literals, and mixed DNS answers cannot use the fallback", async (context) => {
+  let addresses = [{ address: "198.18.0.109", family: 4 }]
+  context.mock.method(dns, "lookup", async () => addresses)
+  const remote = context.mock.method(globalThis, "fetch", async () => {
+    throw new Error("An unsafe host must not use the fallback.")
+  })
+  for (const host of [
+    "198.18.0.109",
+    "198.19.255.254",
+    "[::ffff:198.18.0.109]",
+    "localhost",
+    "feed.local",
+    "feed.internal",
+  ]) {
+    await assert.rejects(Effect.runPromise(checkUrl(`https://${host}/`)))
+  }
+  for (const values of [
+    ["127.0.0.1"],
+    ["10.1.2.3"],
+    ["198.18.0.109", "10.1.2.3"],
+    ["198.18.0.109", "66.33.60.129"],
+    ["198.18.0.109", "198.51.100.1"],
+    ["66.33.60.129", "127.0.0.1"],
+  ]) {
+    addresses = values.map((address) => ({ address, family: 4 }))
+    await assert.rejects(Effect.runPromise(checkUrl()))
+  }
+  assert.equal(remote.mock.callCount(), 0)
+})
+
+test("native lookup failures cannot bypass the DNS policy", async (context) => {
+  context.mock.method(dns, "lookup", async () => {
+    throw Object.assign(new Error("No DNS entry"), { code: "ENOTFOUND" })
+  })
+  const remote = context.mock.method(globalThis, "fetch", async () => {
+    throw new Error("A lookup failure must not use the fallback.")
+  })
+  await assert.rejects(Effect.runPromise(checkUrl()), /host could not be found/)
+  assert.equal(remote.mock.callCount(), 0)
+})
+
+test("HTTPS resolution rejects failures, incomplete answers, and private addresses", async (context) => {
+  fakeDns(context)
+  const remote = context.mock.method(globalThis, "fetch", async () =>
+    dnsAnswer("A", ["66.33.60.129"]),
+  )
+  const invalidResponses = [
+    () => new Response("Unavailable", { status: 503 }),
+    () => Response.json({ Status: 3, TC: false, Answer: [] }),
+    () => Response.json({ Status: 0, TC: true, Answer: [] }),
+    () => dnsAnswer("A"),
+    () => dnsAnswer("A", ["66.33.60.129", "127.0.0.1"]),
+    () => dnsAnswer("A", ["198.18.0.109"]),
+    () => dnsAnswer("AAAA", ["::ffff:127.0.0.1"]),
+    () => dnsAnswer("A", ["invalid address"]),
+    () => new Response("invalid JSON", { status: 200 }),
+  ]
+  for (const response of invalidResponses) {
+    remote.mock.mockImplementation(async () => response())
+    await assert.rejects(Effect.runPromise(checkUrl()))
+  }
+  remote.mock.mockImplementation(async (input) => {
+    const type = new URL(String(input)).searchParams.get("type")
+    return type === "A"
+      ? dnsAnswer("A", ["66.33.60.129"])
+      : dnsAnswer("AAAA", ["::1"])
+  })
+  await assert.rejects(Effect.runPromise(checkUrl()))
+  remote.mock.mockImplementation(async () => {
+    throw new Error("The resolver connection failed.")
+  })
+  await assert.rejects(Effect.runPromise(checkUrl()))
+})
+
+test("HTTPS resolution can be disabled and requires a valid trusted endpoint", async (context) => {
+  fakeDns(context)
+  const remote = context.mock.method(globalThis, "fetch", async () => {
+    throw new Error("An invalid endpoint must not receive a query.")
+  })
+  for (const endpoint of [
+    "off",
+    "invalid URL",
+    "http://cloudflare-dns.com/dns-query",
+    "https://user:secret@cloudflare-dns.com/dns-query",
+    "https://cloudflare-dns.com/dns-query?name=other.example",
+    "https://cloudflare-dns.com/dns-query#fragment",
+  ]) {
+    await assert.rejects(Effect.runPromise(checkUrl(testUrl, endpoint)))
+  }
+  assert.equal(remote.mock.callCount(), 0)
+})
+
+test("interrupting HTTPS resolution cancels both address queries", {
+  timeout: 3_000,
+}, async (context) => {
+  fakeDns(context)
+  const started = Deferred.makeUnsafe<void>()
+  let starts = 0
+  let cancellations = 0
+  context.mock.method(
+    globalThis,
+    "fetch",
+    (...[_input, options]: Parameters<typeof globalThis.fetch>) => {
+      const signal = options?.signal
+      assert.ok(signal instanceof AbortSignal)
+      return new Promise<Response>((_resolve, reject) => {
+        signal.addEventListener(
+          "abort",
+          () => {
+            cancellations++
+            reject(signal.reason)
+          },
+          { once: true },
+        )
+        if (++starts === 2) Effect.runSync(Deferred.succeed(started, undefined))
+      })
+    },
+  )
+  const fiber = Effect.runFork(checkUrl())
+  context.after(() => Effect.runPromise(Fiber.interrupt(fiber)))
+  await Effect.runPromise(Deferred.await(started))
+  await Effect.runPromise(Fiber.interrupt(fiber))
+  assert.equal(cancellations, 2)
+})
+
+test("an HTTPS resolver failure cancels the other address query", {
+  timeout: 3_000,
+}, async (context) => {
+  fakeDns(context)
+  let starts = 0
+  let failQuery: (() => void) | undefined
+  let cancelled = false
+  context.mock.method(
+    globalThis,
+    "fetch",
+    (...[input, options]: Parameters<typeof globalThis.fetch>) => {
+      const type = new URL(String(input)).searchParams.get("type")
+      const signal = options?.signal
+      assert.ok(signal instanceof AbortSignal)
+      return new Promise<Response>((resolve, reject) => {
+        if (type === "A")
+          failQuery = () =>
+            resolve(new Response("Unavailable", { status: 503 }))
+        signal.addEventListener(
+          "abort",
+          () => {
+            if (type === "AAAA") cancelled = true
+            reject(signal.reason)
+          },
+          { once: true },
+        )
+        if (++starts === 2) failQuery?.()
+      })
+    },
+  )
+  await assert.rejects(Effect.runPromise(checkUrl()))
+  assert.equal(cancelled, true)
+})
+
+test("a public HTTPS DNS answer cannot permit a redirect to a private host", async (context) => {
+  fakeDns(context)
+  const resolver = context.mock.method(
+    globalThis,
+    "fetch",
+    async (...[input]: Parameters<typeof globalThis.fetch>) => {
+      const type = new URL(String(input)).searchParams.get("type")
+      return type === "A" ? dnsAnswer("A", ["66.33.60.129"]) : dnsAnswer("AAAA")
+    },
+  )
+  const mock = new MockAgent()
+  mock.disableNetConnect()
+  context.mock.method(
+    Agent.prototype,
+    "dispatch",
+    (...[options, handler]: Parameters<Agent["dispatch"]>) =>
+      mock.get(String(options.origin)).dispatch(options, handler),
+  )
+  context.after(() => mock.close())
+  mock
+    .get("https://www.eflx.top")
+    .intercept({ path: "/blog/ui-composition-first-principle", method: "GET" })
+    .reply(302, "", {
+      headers: { location: "http://169.254.169.254/latest/meta-data/" },
+    })
+  await assert.rejects(Effect.runPromise(fetchDocument(testUrl)), /public host/)
+  assert.equal(resolver.mock.callCount(), 2)
+})
+
 test("URL normalization retains article parameters and removes tracking", () => {
  assert.equal(
    canonicalUrl(
@@ -1,7 +1,9 @@
-import { lookup } from "node:dns/promises"
+import dns from "node:dns/promises"
 import { isIP } from "node:net"
 import { Readability } from "@mozilla/readability"
-import { Effect } from "effect"
+import { RecordType } from "@multiformats/dns"
+import { dnsJsonOverHttps } from "@multiformats/dns/resolvers"
+import { Config, Effect } from "effect"
 import ipaddr from "ipaddr.js"
 import { parseHTML } from "linkedom"
 import { Agent, fetch } from "undici"
@@ -10,6 +12,7 @@ import { AppError } from "./errors"
 const MAX_BYTES = 2_000_000
 const MAX_TEXT = 60_000
 const NETWORK_TIMEOUT = 20_000
+const FAKE_IP_RANGE = ipaddr.IPv4.parseCIDR("198.18.0.0/15")
 const redirects = new Set([301, 302, 303, 307, 308])

 export function isPublicAddress(address: string): boolean {
@@ -51,6 +54,81 @@ export function canonicalUrl(value: string): string {
  return url.toString()
 }

+const resolveProxyHostname = Effect.fn("content.resolveProxyHostname")(
+  function* (hostname: string) {
+    const endpoint = yield* Config.String("DNS_OVER_HTTPS_URL").pipe(
+      Config.withDefault("https://cloudflare-dns.com/dns-query"),
+    )
+    if (endpoint === "off") return []
+    const url = yield* Effect.try({
+      try: () => {
+        const url = new URL(endpoint)
+        if (
+          url.protocol !== "https:" ||
+          url.username ||
+          url.password ||
+          url.search ||
+          url.hash
+        )
+          throw new Error(
+            "Use an HTTPS DNS endpoint without credentials or parameters.",
+          )
+        return url
+      },
+      catch: (cause) =>
+        new AppError({
+          message: "Set DNS_OVER_HTTPS_URL to a valid HTTPS DNS endpoint.",
+          status: 500,
+          cause,
+        }),
+    })
+    const resolve = dnsJsonOverHttps(url.toString())
+    const responses = yield* Effect.forEach(
+      [RecordType.A, RecordType.AAAA],
+      (type) =>
+        Effect.tryPromise({
+          try: (signal) => resolve(hostname, { types: type, signal }),
+          catch: (cause) =>
+            new AppError({
+              message: "The public DNS lookup failed. Try again.",
+              status: 502,
+              cause,
+            }),
+        }).pipe(
+          Effect.filterOrFail(
+            (response) => response.Status === 0 && !response.TC,
+            () =>
+              new AppError({
+                message: "The public DNS lookup failed. Try again.",
+                status: 502,
+              }),
+          ),
+        ),
+      { concurrency: 2 },
+    ).pipe(
+      Effect.timeout("10 seconds"),
+      Effect.mapError((cause) =>
+        cause instanceof AppError
+          ? cause
+          : new AppError({
+              message: "The public DNS lookup timed out. Try again.",
+              status: 502,
+              cause,
+            }),
+      ),
+    )
+    return responses.flatMap((response) =>
+      response.Answer.flatMap((answer) => {
+        // The resolver normalizes numeric DNS record types to their names.
+        const type = String(answer.type)
+        return ["A", "AAAA", "1", "28"].includes(type)
+          ? [{ address: answer.data, family: isIP(answer.data) }]
+          : []
+      }),
+    )
+  },
+)
+
 export const validatePublicUrl = Effect.fn("content.validatePublicUrl")(
  function* (value: string) {
    const url = yield* Effect.try({
@@ -73,11 +151,11 @@ export const validatePublicUrl = Effect.fn("content.validatePublicUrl")(
        status: 400,
      })
    }
-    const addresses = yield* Effect.tryPromise({
+    let addresses = yield* Effect.tryPromise({
      try: () =>
        isIP(hostname)
          ? Promise.resolve([{ address: hostname, family: isIP(hostname) }])
-          : lookup(hostname, { all: true, verbatim: true }),
+          : dns.lookup(hostname, { all: true, verbatim: true }),
      catch: (cause) =>
        new AppError({
          message: "The URL host could not be found.",
@@ -96,6 +174,17 @@ export const validatePublicUrl = Effect.fn("content.validatePublicUrl")(
            }),
      ),
    )
+    if (
+      !isIP(hostname) &&
+      addresses.length > 0 &&
+      addresses.every(
+        ({ address }) =>
+          isIP(address) === 4 &&
+          ipaddr.IPv4.parse(address).match(FAKE_IP_RANGE),
+      )
+    ) {
+      addresses = yield* resolveProxyHostname(hostname)
+    }
    if (
      !addresses.length ||
      addresses.some(({ address }) => !isPublicAddress(address))