auth,feeds,ai,ui,deploy: build Stellar
Use Effect services for requests, AI tasks, and the RSS worker. Add admin accounts, feed sharing, and stored reading profiles. Add Biome, Effect tools, and Podman deployment with Caddy.
This commit is contained in:
EFLKumo committed
1 parent
ba8f925eed
commit
ce87e56053
137 files changed
+14652
-2457
No files matched your search
@@ -0,0 +1,273 @@
|
||||
import { lookup } from "node:dns/promises"
|
||||
import { isIP } from "node:net"
|
||||
import { Readability } from "@mozilla/readability"
|
||||
import { Effect } from "effect"
|
||||
import ipaddr from "ipaddr.js"
|
||||
import { parseHTML } from "linkedom"
|
||||
import { Agent, fetch } from "undici"
|
||||
import { AppError } from "./errors"
|
||||
|
||||
const MAX_BYTES = 2_000_000
|
||||
const MAX_TEXT = 60_000
|
||||
const NETWORK_TIMEOUT = 20_000
|
||||
const redirects = new Set([301, 302, 303, 307, 308])
|
||||
|
||||
export function isPublicAddress(address: string): boolean {
|
||||
try {
|
||||
return ipaddr.process(address).range() === "unicast"
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export function canonicalUrl(value: string): string {
|
||||
let url: URL
|
||||
try {
|
||||
url = new URL(value)
|
||||
} catch {
|
||||
throw new AppError({ message: "Enter a valid URL.", status: 400 })
|
||||
}
|
||||
if (
|
||||
!["http:", "https:"].includes(url.protocol) ||
|
||||
url.username ||
|
||||
url.password ||
|
||||
(url.port && !["80", "443"].includes(url.port))
|
||||
) {
|
||||
throw new AppError({
|
||||
message: "Use a public HTTP or HTTPS URL.",
|
||||
status: 400,
|
||||
})
|
||||
}
|
||||
url.hash = ""
|
||||
for (const key of [...url.searchParams.keys()]) {
|
||||
if (
|
||||
/^utm_/i.test(key) ||
|
||||
["fbclid", "gclid", "mc_cid", "mc_eid"].includes(key)
|
||||
) {
|
||||
url.searchParams.delete(key)
|
||||
}
|
||||
}
|
||||
url.searchParams.sort()
|
||||
return url.toString()
|
||||
}
|
||||
|
||||
export const validatePublicUrl = Effect.fn("content.validatePublicUrl")(
|
||||
function* (value: string) {
|
||||
const url = yield* Effect.try({
|
||||
try: () => new URL(canonicalUrl(value)),
|
||||
catch: (cause) =>
|
||||
cause instanceof AppError
|
||||
? cause
|
||||
: new AppError({ message: "Enter a valid URL.", status: 400, cause }),
|
||||
})
|
||||
const hostname = url.hostname.replace(/^\[|\]$/g, "")
|
||||
if (
|
||||
hostname === "localhost" ||
|
||||
hostname.endsWith(".localhost") ||
|
||||
hostname.endsWith(".local") ||
|
||||
hostname.endsWith(".internal") ||
|
||||
(!hostname.includes(".") && !isIP(hostname))
|
||||
) {
|
||||
return yield* new AppError({
|
||||
message: "The URL must use a public host.",
|
||||
status: 400,
|
||||
})
|
||||
}
|
||||
const addresses = yield* Effect.tryPromise({
|
||||
try: () =>
|
||||
isIP(hostname)
|
||||
? Promise.resolve([{ address: hostname, family: isIP(hostname) }])
|
||||
: lookup(hostname, { all: true, verbatim: true }),
|
||||
catch: (cause) =>
|
||||
new AppError({
|
||||
message: "The URL host could not be found.",
|
||||
status: 400,
|
||||
cause,
|
||||
}),
|
||||
}).pipe(
|
||||
Effect.timeout("10 seconds"),
|
||||
Effect.mapError((cause) =>
|
||||
cause instanceof AppError
|
||||
? cause
|
||||
: new AppError({
|
||||
message: "The URL host lookup timed out.",
|
||||
status: 502,
|
||||
cause,
|
||||
}),
|
||||
),
|
||||
)
|
||||
if (
|
||||
!addresses.length ||
|
||||
addresses.some(({ address }) => !isPublicAddress(address))
|
||||
) {
|
||||
return yield* new AppError({
|
||||
message: "The URL must use a public host.",
|
||||
status: 400,
|
||||
})
|
||||
}
|
||||
return { url, addresses }
|
||||
},
|
||||
)
|
||||
|
||||
export const fetchDocument = Effect.fn("content.fetchDocument")(function* (
|
||||
value: string,
|
||||
accept = "text/html,text/plain,application/xhtml+xml",
|
||||
) {
|
||||
let current = value
|
||||
for (let count = 0; count <= 5; count++) {
|
||||
const { url, addresses } = yield* validatePublicUrl(current)
|
||||
const result = yield* Effect.acquireUseRelease(
|
||||
Effect.sync(
|
||||
() =>
|
||||
new Agent({
|
||||
connect: {
|
||||
timeout: 10_000,
|
||||
lookup: (_hostname, options, callback) => {
|
||||
// The connection uses the addresses that passed the check.
|
||||
const selected =
|
||||
addresses.find(
|
||||
({ family }) =>
|
||||
!options.family || family === options.family,
|
||||
) ?? addresses[0]
|
||||
if (!selected) {
|
||||
callback(new Error("No public address is available."), "", 4)
|
||||
} else if (options.all) {
|
||||
callback(null, addresses)
|
||||
} else {
|
||||
callback(null, selected.address, selected.family)
|
||||
}
|
||||
},
|
||||
},
|
||||
}),
|
||||
),
|
||||
(dispatcher) =>
|
||||
Effect.tryPromise({
|
||||
try: async (signal) => {
|
||||
const response = await fetch(url, {
|
||||
dispatcher,
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.any([
|
||||
signal,
|
||||
AbortSignal.timeout(NETWORK_TIMEOUT),
|
||||
]),
|
||||
headers: {
|
||||
Accept: accept,
|
||||
"User-Agent": "Stellar/1.0 (feed reader)",
|
||||
},
|
||||
})
|
||||
if (redirects.has(response.status)) {
|
||||
await response.body?.cancel()
|
||||
const location = response.headers.get("location")
|
||||
if (!location) throw new Error("The redirect has no URL.")
|
||||
return {
|
||||
kind: "redirect" as const,
|
||||
url: new URL(location, url).toString(),
|
||||
}
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel()
|
||||
throw new Error(
|
||||
`The remote server returned HTTP ${response.status}.`,
|
||||
)
|
||||
}
|
||||
if (Number(response.headers.get("content-length")) > MAX_BYTES) {
|
||||
await response.body?.cancel()
|
||||
throw new Error("The document exceeds the size limit.")
|
||||
}
|
||||
const contentType = response.headers.get("content-type") ?? ""
|
||||
if (!/text\/|xml|json/i.test(contentType)) {
|
||||
await response.body?.cancel()
|
||||
throw new Error("The URL must return a text document.")
|
||||
}
|
||||
const reader = response.body?.getReader()
|
||||
if (!reader) throw new Error("The document is empty.")
|
||||
const decoder = new TextDecoder()
|
||||
let bytes = 0
|
||||
let text = ""
|
||||
try {
|
||||
while (true) {
|
||||
const chunk = await reader.read()
|
||||
if (chunk.done) break
|
||||
bytes += chunk.value.byteLength
|
||||
if (bytes > MAX_BYTES)
|
||||
throw new Error("The document exceeds the size limit.")
|
||||
text += decoder.decode(chunk.value, { stream: true })
|
||||
}
|
||||
text += decoder.decode()
|
||||
} finally {
|
||||
await reader.cancel()
|
||||
}
|
||||
return {
|
||||
kind: "document" as const,
|
||||
url: url.toString(),
|
||||
text,
|
||||
contentType,
|
||||
}
|
||||
},
|
||||
catch: (cause) =>
|
||||
new AppError({
|
||||
message: "The document could not be read.",
|
||||
status: 502,
|
||||
cause,
|
||||
}),
|
||||
}),
|
||||
(dispatcher) => Effect.promise(() => dispatcher.destroy()),
|
||||
)
|
||||
if (result.kind === "redirect") {
|
||||
current = result.url
|
||||
} else {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return yield* new AppError({
|
||||
message: "The URL has too many redirects.",
|
||||
status: 400,
|
||||
})
|
||||
})
|
||||
|
||||
export const extractArticle = Effect.fn("content.extractArticle")(function* (
|
||||
url: string,
|
||||
) {
|
||||
const source = yield* fetchDocument(url)
|
||||
return yield* Effect.try({
|
||||
try: () => {
|
||||
if (/^text\/plain/i.test(source.contentType)) {
|
||||
return {
|
||||
url: source.url,
|
||||
title: new URL(source.url).hostname,
|
||||
content: source.text.slice(0, MAX_TEXT),
|
||||
}
|
||||
}
|
||||
const { document } = parseHTML(source.text)
|
||||
Object.defineProperty(document, "documentURI", { value: source.url })
|
||||
Object.defineProperty(document, "baseURI", { value: source.url })
|
||||
const article = new Readability(document as unknown as Document).parse()
|
||||
for (const element of document.querySelectorAll(
|
||||
"script,style,nav,footer,header",
|
||||
)) {
|
||||
element.remove()
|
||||
}
|
||||
const content = (article?.textContent || document.body?.textContent || "")
|
||||
.replace(/\n[\t ]+/g, "\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim()
|
||||
.slice(0, MAX_TEXT)
|
||||
if (!content) throw new Error("The document has no article text.")
|
||||
return {
|
||||
url: source.url,
|
||||
title: (
|
||||
article?.title ||
|
||||
document.title ||
|
||||
new URL(source.url).hostname
|
||||
).slice(0, 300),
|
||||
content,
|
||||
}
|
||||
},
|
||||
catch: (cause) =>
|
||||
new AppError({
|
||||
message: "The article text could not be read.",
|
||||
status: 422,
|
||||
cause,
|
||||
}),
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user