Linkvorschau für den ersten Link eines Beitrags
Der Server holt zum ersten Link Titel, Seiten- bzw. Kanalname und ein kleines Vorschaubild (320 px) und speichert das Bild lokal; der Browser lädt nur von uns, Besucher-IPs gehen nie an die verlinkte Seite. YouTube über oEmbed (alle Linkformen kanonisch als watch?v=<id>), alle übrigen Seiten über OpenGraph bzw. <title>. Abruf asynchron in einem Worker, eine Zeile je Link in link_preview, Auffrischen nach 30 Tagen. Der Abruf-Client lässt nur öffentliche IPs auf Port 80/443 zu, geprüft nach DNS-Auflösung und bei jeder Weiterleitung (SSRF). Bestehende Beiträge: kver link-previews. Neue Abhängigkeit golang.org/x/net, damit go 1.26. DEV bekommt ein beschreibbares Volume für die Vorschaubilder. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TiXsPUqw7oeomZ8wZrQW5q
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a8f392edba
commit
26b0182391
+56
-1
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
@@ -16,8 +17,13 @@ func runCommand(args []string) error {
|
|||||||
return errors.New("Verwendung: kver reset-link <nutzername>")
|
return errors.New("Verwendung: kver reset-link <nutzername>")
|
||||||
}
|
}
|
||||||
return cmdResetLink(args[1])
|
return cmdResetLink(args[1])
|
||||||
|
case "link-previews":
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("Verwendung: kver link-previews")
|
||||||
|
}
|
||||||
|
return cmdLinkPreviews()
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unbekannter Befehl %q (bekannt: reset-link)", args[0])
|
return fmt.Errorf("unbekannter Befehl %q (bekannt: reset-link, link-previews)", args[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -39,3 +45,52 @@ func cmdResetLink(username string) error {
|
|||||||
fmt.Println("Ältere Reset-Links dieses Nutzers sind damit ungültig.")
|
fmt.Println("Ältere Reset-Links dieses Nutzers sind damit ungültig.")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// cmdLinkPreviews trägt bei bestehenden Beiträgen den ersten Link nach
|
||||||
|
// (entry.link_url) und holt alle offenen Vorschauen nacheinander. Läuft der
|
||||||
|
// Server parallel, holt er höchstens dieselben Links noch einmal -- harmlos.
|
||||||
|
func cmdLinkPreviews() error {
|
||||||
|
rows, err := db.Query(`SELECT pid, content FROM entry WHERE deleted = 0 AND link_url = '' AND content LIKE '%http%'`)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
links := map[int64]string{}
|
||||||
|
for rows.Next() {
|
||||||
|
var pid int64
|
||||||
|
var content string
|
||||||
|
if err := rows.Scan(&pid, &content); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if key := previewKey(content); key != "" {
|
||||||
|
links[pid] = key
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for pid, key := range links {
|
||||||
|
if _, err := db.Exec(`UPDATE entry SET link_url = $1 WHERE pid = $2`, key, pid); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := db.Exec(`INSERT INTO link_preview (url) VALUES ($1) ON CONFLICT DO NOTHING`, key); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("%d Beiträge mit Link nachgetragen.\n", len(links))
|
||||||
|
|
||||||
|
pending := pendingPreviews(100000)
|
||||||
|
for i, key := range pending {
|
||||||
|
fmt.Printf("[%d/%d] %s\n", i+1, len(pending), key)
|
||||||
|
fetchPreview(context.Background(), key)
|
||||||
|
}
|
||||||
|
var ok, none int
|
||||||
|
if err := db.QueryRow(
|
||||||
|
`SELECT COUNT(*) FILTER (WHERE status = 'ok'), COUNT(*) FILTER (WHERE status = 'none') FROM link_preview`).
|
||||||
|
Scan(&ok, &none); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("Vorschauen: %d fertig, %d ohne Vorschau.\n", ok, none)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -47,6 +47,8 @@ CREATE TABLE IF NOT EXISTS entry (
|
|||||||
bump_count BIGINT NOT NULL DEFAULT 0,
|
bump_count BIGINT NOT NULL DEFAULT 0,
|
||||||
last_bump BIGINT NOT NULL DEFAULT 0
|
last_bump BIGINT NOT NULL DEFAULT 0
|
||||||
);
|
);
|
||||||
|
-- Kanonische URL des ersten Links im Inhalt (previewKey), '' ohne Link.
|
||||||
|
ALTER TABLE entry ADD COLUMN IF NOT EXISTS link_url TEXT NOT NULL DEFAULT '';
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS vote (
|
CREATE TABLE IF NOT EXISTS vote (
|
||||||
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||||
@@ -93,6 +95,19 @@ CREATE TABLE IF NOT EXISTS password_reset (
|
|||||||
expires BIGINT NOT NULL
|
expires BIGINT NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Linkvorschauen, eine Zeile je kanonischer URL (siehe preview.go), geteilt
|
||||||
|
-- von allen Beiträgen mit diesem Link. status: pending (wird geholt), ok,
|
||||||
|
-- none (keine Vorschau möglich). thumb ist das lokal gespeicherte Bild.
|
||||||
|
CREATE TABLE IF NOT EXISTS link_preview (
|
||||||
|
url TEXT PRIMARY KEY,
|
||||||
|
kind TEXT NOT NULL DEFAULT '',
|
||||||
|
title TEXT NOT NULL DEFAULT '',
|
||||||
|
site TEXT NOT NULL DEFAULT '',
|
||||||
|
thumb TEXT NOT NULL DEFAULT '',
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
fetched_at BIGINT NOT NULL DEFAULT 0
|
||||||
|
);
|
||||||
|
|
||||||
-- Indizes für die häufigen Zugriffspfade; ohne sie werden Feed (ORDER BY
|
-- Indizes für die häufigen Zugriffspfade; ohne sie werden Feed (ORDER BY
|
||||||
-- last_activity), Thread (WHERE reply_to) und Vote-Zähler (WHERE pid) mit
|
-- last_activity), Thread (WHERE reply_to) und Vote-Zähler (WHERE pid) mit
|
||||||
-- wachsender Tabelle zu Full-Table-Scans. username/session.value/vote(uid,pid)
|
-- wachsender Tabelle zu Full-Table-Scans. username/session.value/vote(uid,pid)
|
||||||
|
|||||||
@@ -41,7 +41,14 @@ podman build -q -t "$IMAGE" . 9>&-
|
|||||||
|
|
||||||
mounts=()
|
mounts=()
|
||||||
[[ -d "$DATA/data" ]] && mounts+=(-v "$DATA/data:/app/data:ro" -e KVER_GEOIP_ASN=/app/data/GeoLite2-ASN.mmdb)
|
[[ -d "$DATA/data" ]] && mounts+=(-v "$DATA/data:/app/data:ro" -e KVER_GEOIP_ASN=/app/data/GeoLite2-ASN.mmdb)
|
||||||
[[ -d "$DATA/media" ]] && mounts+=(-v "$DATA/media:/app/static/media:ro")
|
if [[ -d "$DATA/media" ]]; then
|
||||||
|
mounts+=(-v "$DATA/media:/app/static/media:ro")
|
||||||
|
# Linkvorschau-Bilder schreibt der Server selbst: beschreibbares Volume über
|
||||||
|
# dem read-only Media-Mount. Der Mountpunkt muss im Quellverzeichnis
|
||||||
|
# existieren, weil der Container ihn im read-only Mount nicht anlegen kann.
|
||||||
|
mkdir -p "$DATA/media/preview"
|
||||||
|
mounts+=(-v kver-dev-preview:/app/static/media/preview:U)
|
||||||
|
fi
|
||||||
|
|
||||||
podman rm -f "$NAME" >/dev/null 2>&1 9>&- || true
|
podman rm -f "$NAME" >/dev/null 2>&1 9>&- || true
|
||||||
podman run -d --name "$NAME" -p "127.0.0.1:$PORT:8080" \
|
podman run -d --name "$NAME" -p "127.0.0.1:$PORT:8080" \
|
||||||
|
|||||||
@@ -34,6 +34,8 @@ type Entry struct {
|
|||||||
LastBump int64 `json:"last_bump"`
|
LastBump int64 `json:"last_bump"`
|
||||||
Username string `json:"username"`
|
Username string `json:"username"`
|
||||||
Avatar string `json:"avatar"`
|
Avatar string `json:"avatar"`
|
||||||
|
// Preview ist die fertige Vorschau des ersten Links, siehe preview.go.
|
||||||
|
Preview *Preview `json:"preview,omitempty"`
|
||||||
// Latest ist nur in Feed und Antwortliste gesetzt, siehe attachLatest.
|
// Latest ist nur in Feed und Antwortliste gesetzt, siehe attachLatest.
|
||||||
Latest *Latest `json:"latest,omitempty"`
|
Latest *Latest `json:"latest,omitempty"`
|
||||||
}
|
}
|
||||||
@@ -47,20 +49,29 @@ type Latest struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// LEFT JOIN + COALESCE, damit soft-gelöschte Beiträge (uid auf 0 gesetzt) als
|
// LEFT JOIN + COALESCE, damit soft-gelöschte Beiträge (uid auf 0 gesetzt) als
|
||||||
// [deleted]-Platzhalter erhalten bleiben und Threads nicht verwaisen.
|
// [deleted]-Platzhalter erhalten bleiben und Threads nicht verwaisen. Die
|
||||||
|
// Linkvorschau hängt nur dran, wenn sie fertig ist (status ok).
|
||||||
const entrySelect = `
|
const entrySelect = `
|
||||||
SELECT e.pid, e.uid, e.created_at, e.content, e.filepath,
|
SELECT e.pid, e.uid, e.created_at, e.content, e.filepath,
|
||||||
e.reply_to, e.reply_count, e.last_activity, e.deleted,
|
e.reply_to, e.reply_count, e.last_activity, e.deleted,
|
||||||
e.bump_count, e.last_bump,
|
e.bump_count, e.last_bump,
|
||||||
COALESCE(u.username, ''), COALESCE(u.avatar, '')
|
COALESCE(u.username, ''), COALESCE(u.avatar, ''),
|
||||||
FROM entry e LEFT JOIN account u ON u.uid = e.uid`
|
COALESCE(lp.url, ''), COALESCE(lp.kind, ''), COALESCE(lp.title, ''),
|
||||||
|
COALESCE(lp.site, ''), COALESCE(lp.thumb, '')
|
||||||
|
FROM entry e LEFT JOIN account u ON u.uid = e.uid
|
||||||
|
LEFT JOIN link_preview lp ON lp.url = e.link_url AND lp.status = 'ok'`
|
||||||
|
|
||||||
// scanEntry liest eine Zeile mit entrySelect-Spalten (aus *sql.Row oder *sql.Rows).
|
// scanEntry liest eine Zeile mit entrySelect-Spalten (aus *sql.Row oder *sql.Rows).
|
||||||
func scanEntry(row interface{ Scan(...any) error }) (Entry, error) {
|
func scanEntry(row interface{ Scan(...any) error }) (Entry, error) {
|
||||||
var e Entry
|
var e Entry
|
||||||
|
var p Preview
|
||||||
err := row.Scan(&e.PID, &e.UID, &e.CreatedAt, &e.Content, &e.Filepath,
|
err := row.Scan(&e.PID, &e.UID, &e.CreatedAt, &e.Content, &e.Filepath,
|
||||||
&e.ReplyTo, &e.ReplyCount, &e.LastActivity, &e.Deleted,
|
&e.ReplyTo, &e.ReplyCount, &e.LastActivity, &e.Deleted,
|
||||||
&e.BumpCount, &e.LastBump, &e.Username, &e.Avatar)
|
&e.BumpCount, &e.LastBump, &e.Username, &e.Avatar,
|
||||||
|
&p.URL, &p.Kind, &p.Title, &p.Site, &p.Thumb)
|
||||||
|
if p.URL != "" {
|
||||||
|
e.Preview = &p
|
||||||
|
}
|
||||||
return e, err
|
return e, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -308,6 +319,7 @@ func handleCreateEntry(w http.ResponseWriter, r *http.Request) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now().Unix()
|
now := time.Now().Unix()
|
||||||
|
linkURL := previewKey(content)
|
||||||
|
|
||||||
tx, err := db.Begin()
|
tx, err := db.Begin()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -320,10 +332,10 @@ func handleCreateEntry(w http.ResponseWriter, r *http.Request) error {
|
|||||||
for try := 0; ; try++ {
|
for try := 0; ; try++ {
|
||||||
pid = int64(rand.IntN(999999999999)) + 1
|
pid = int64(rand.IntN(999999999999)) + 1
|
||||||
res, err := tx.Exec(
|
res, err := tx.Exec(
|
||||||
`INSERT INTO entry (pid, uid, created_at, content, filepath, reply_to, reply_count, last_activity)
|
`INSERT INTO entry (pid, uid, created_at, content, filepath, reply_to, reply_count, last_activity, link_url)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, 0, $7)
|
VALUES ($1, $2, $3, $4, $5, $6, 0, $7, $8)
|
||||||
ON CONFLICT (pid) DO NOTHING`,
|
ON CONFLICT (pid) DO NOTHING`,
|
||||||
pid, uid, now, content, filepath, replyTo, now,
|
pid, uid, now, content, filepath, replyTo, now, linkURL,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
tx.Rollback()
|
tx.Rollback()
|
||||||
@@ -362,6 +374,7 @@ func handleCreateEntry(w http.ResponseWriter, r *http.Request) error {
|
|||||||
if err := tx.Commit(); err != nil {
|
if err := tx.Commit(); err != nil {
|
||||||
return Internal(err)
|
return Internal(err)
|
||||||
}
|
}
|
||||||
|
requestPreview(linkURL)
|
||||||
|
|
||||||
writeJSON(w, http.StatusCreated, map[string]any{
|
writeJSON(w, http.StatusCreated, map[string]any{
|
||||||
"pid": pid,
|
"pid": pid,
|
||||||
@@ -389,9 +402,11 @@ func handleEditEntry(w http.ResponseWriter, r *http.Request) error {
|
|||||||
return Invalid("entry.empty", "Inhalt darf nicht leer sein.").At("content")
|
return Invalid("entry.empty", "Inhalt darf nicht leer sein.").At("content")
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := db.Exec(`UPDATE entry SET content = $1 WHERE pid = $2`, content, pid); err != nil {
|
linkURL := previewKey(content)
|
||||||
|
if _, err := db.Exec(`UPDATE entry SET content = $1, link_url = $2 WHERE pid = $3`, content, linkURL, pid); err != nil {
|
||||||
return Internal(err)
|
return Internal(err)
|
||||||
}
|
}
|
||||||
|
requestPreview(linkURL)
|
||||||
|
|
||||||
writeJSON(w, http.StatusOK, map[string]any{"pid": pid, "content": content})
|
writeJSON(w, http.StatusOK, map[string]any{"pid": pid, "content": content})
|
||||||
return nil
|
return nil
|
||||||
@@ -429,7 +444,7 @@ func handleDeleteEntry(w http.ResponseWriter, r *http.Request) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if _, err := db.Exec(
|
if _, err := db.Exec(
|
||||||
`UPDATE entry SET deleted = 1, content = '', filepath = '', uid = 0 WHERE pid = $1`, pid,
|
`UPDATE entry SET deleted = 1, content = '', filepath = '', link_url = '', uid = 0 WHERE pid = $1`, pid,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return Internal(err)
|
return Internal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -204,6 +204,55 @@ a.button:hover, button:hover {
|
|||||||
max-width: 100%;
|
max-width: 100%;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Linkvorschau: abgesetzte Karte unter dem Text, kleines Vorschaubild links,
|
||||||
|
Titel und Seiten- bzw. Kanalname daneben. Rahmen wie bei den Buttons,
|
||||||
|
weiß auch beim Card-Hover. */
|
||||||
|
.link-preview {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
border: 1px solid black;
|
||||||
|
background-color: white;
|
||||||
|
margin: 0 0 0.5rem;
|
||||||
|
text-decoration: none;
|
||||||
|
color: black;
|
||||||
|
}
|
||||||
|
|
||||||
|
.link-preview:hover .link-preview-title {
|
||||||
|
text-decoration: underline;
|
||||||
|
}
|
||||||
|
|
||||||
|
.card .link-preview img {
|
||||||
|
flex: 0 0 auto;
|
||||||
|
align-self: stretch;
|
||||||
|
width: 160px;
|
||||||
|
max-width: 40%;
|
||||||
|
aspect-ratio: 16 / 9;
|
||||||
|
object-fit: cover;
|
||||||
|
margin: 0;
|
||||||
|
border-right: 1px solid black;
|
||||||
|
}
|
||||||
|
|
||||||
|
.link-preview-text {
|
||||||
|
min-width: 0;
|
||||||
|
padding: 0.3rem 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.link-preview-title,
|
||||||
|
.link-preview-site {
|
||||||
|
display: block;
|
||||||
|
overflow-wrap: break-word;
|
||||||
|
}
|
||||||
|
|
||||||
|
.link-preview-title {
|
||||||
|
font-size: 1.1rem;
|
||||||
|
line-height: 1.3;
|
||||||
|
}
|
||||||
|
|
||||||
|
.link-preview-site {
|
||||||
|
font-size: 0.9rem;
|
||||||
|
color: gray;
|
||||||
|
}
|
||||||
|
|
||||||
/* --- Avatare --- */
|
/* --- Avatare --- */
|
||||||
img.avatar {
|
img.avatar {
|
||||||
display: inline-block;
|
display: inline-block;
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { MouseEvent } from "react";
|
import type { MouseEvent } from "react";
|
||||||
import { Link, useNavigate } from "react-router";
|
import { Link, useNavigate } from "react-router";
|
||||||
import type { Entry } from "../types";
|
import type { Entry, LinkPreview } from "../types";
|
||||||
import { avatarUrl, formatTime, linkify } from "../format";
|
import { avatarUrl, formatTime, linkify } from "../format";
|
||||||
import { Votes } from "./Votes";
|
import { Votes } from "./Votes";
|
||||||
import { useBump } from "./Bump";
|
import { useBump } from "./Bump";
|
||||||
@@ -36,6 +36,21 @@ function DeletedCard({ entry: e }: { entry: Entry }) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LinkPreviewCard zeigt die Vorschau des ersten Links als abgesetzte Karte:
|
||||||
|
// kleines Bild links, Titel und Seite daneben.
|
||||||
|
// Alles darin kommt von unserem Server; erst der Klick führt zur fremden Seite.
|
||||||
|
function LinkPreviewCard({ preview: p }: { preview: LinkPreview }) {
|
||||||
|
return (
|
||||||
|
<a className="link-preview" href={p.url} target="_blank" rel="noopener noreferrer">
|
||||||
|
{p.thumb && <img src={`/${p.thumb}`} alt="" loading="lazy" />}
|
||||||
|
<span className="link-preview-text">
|
||||||
|
<span className="link-preview-title">{p.title}</span>
|
||||||
|
{p.site && <span className="link-preview-site">{p.site}</span>}
|
||||||
|
</span>
|
||||||
|
</a>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function LiveCard({ entry: e }: { entry: Entry }) {
|
function LiveCard({ entry: e }: { entry: Entry }) {
|
||||||
const open = useOpenOnClick(e.pid);
|
const open = useOpenOnClick(e.pid);
|
||||||
const bump = useBump(e);
|
const bump = useBump(e);
|
||||||
@@ -60,6 +75,7 @@ function LiveCard({ entry: e }: { entry: Entry }) {
|
|||||||
<div className="body">
|
<div className="body">
|
||||||
{report.form}
|
{report.form}
|
||||||
<div className="content">{linkify(e.content)}</div>
|
<div className="content">{linkify(e.content)}</div>
|
||||||
|
{e.preview && <LinkPreviewCard preview={e.preview} />}
|
||||||
{e.filepath && <img src={`/${e.filepath}`} alt="" loading="lazy" />}
|
{e.filepath && <img src={`/${e.filepath}`} alt="" loading="lazy" />}
|
||||||
<div className="interactions">
|
<div className="interactions">
|
||||||
<Link className="button reply" to={`/e/${e.pid}`}>Antworten ({e.reply_count})</Link>
|
<Link className="button reply" to={`/e/${e.pid}`}>Antworten ({e.reply_count})</Link>
|
||||||
|
|||||||
@@ -15,11 +15,22 @@ export interface Entry {
|
|||||||
last_bump: number;
|
last_bump: number;
|
||||||
username: string;
|
username: string;
|
||||||
avatar: string;
|
avatar: string;
|
||||||
|
// Vorschau des ersten Links, sobald der Server sie geholt hat.
|
||||||
|
preview?: LinkPreview;
|
||||||
// Nur in Feed und Antwortliste: jüngster Beitrag im Teilbaum, skipped =
|
// Nur in Feed und Antwortliste: jüngster Beitrag im Teilbaum, skipped =
|
||||||
// Beiträge auf dem Pfad dazwischen (0 = direkte Antwort).
|
// Beiträge auf dem Pfad dazwischen (0 = direkte Antwort).
|
||||||
latest?: { entry: Entry; skipped: number };
|
latest?: { entry: Entry; skipped: number };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Vom Server geholte Linkvorschau; thumb liegt lokal (leer = kein Bild).
|
||||||
|
export interface LinkPreview {
|
||||||
|
url: string;
|
||||||
|
kind: "youtube" | "page";
|
||||||
|
title: string;
|
||||||
|
site: string;
|
||||||
|
thumb: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface Thread {
|
export interface Thread {
|
||||||
entry: Entry;
|
entry: Entry;
|
||||||
ancestors: Entry[];
|
ancestors: Entry[];
|
||||||
|
|||||||
@@ -119,6 +119,7 @@ E-Mail: <a>stress@bnd.wtf</a>
|
|||||||
<li>Optional: Hochgeladenes Bild (JPG, PNG, GIF; max. 1024x1024 Pixel)</li>
|
<li>Optional: Hochgeladenes Bild (JPG, PNG, GIF; max. 1024x1024 Pixel)</li>
|
||||||
</ul>
|
</ul>
|
||||||
<em>Sichtbarkeit:</em> Beiträge und Benutzername sind öffentlich einsehbar<br>
|
<em>Sichtbarkeit:</em> Beiträge und Benutzername sind öffentlich einsehbar<br>
|
||||||
|
<em>Linkvorschau:</em> Enthält ein Beitrag einen Link, ruft unser Server von der verlinkten Seite Titel, Seiten- bzw. Kanalname und Vorschaubild ab (bei YouTube über den öffentlichen oEmbed-Dienst von YouTube) und speichert sie auf unserem Server. Die Vorschau wird Ihnen ausschließlich von unserem Server ausgeliefert; beim Anzeigen werden keine Daten an die verlinkte Seite oder an YouTube übermittelt. Erst wenn Sie die Vorschau oder den Link anklicken, rufen Sie die fremde Seite selbst auf.<br>
|
||||||
<em>Zweck:</em> Bereitstellung der Forumsfunktion<br>
|
<em>Zweck:</em> Bereitstellung der Forumsfunktion<br>
|
||||||
<em>Rechtsgrundlage:</em> Vertragserfüllung (Art. 6 Abs. 1 lit. b DSGVO)<br>
|
<em>Rechtsgrundlage:</em> Vertragserfüllung (Art. 6 Abs. 1 lit. b DSGVO)<br>
|
||||||
<em>Speicherdauer:</em> Bis zur Löschung durch den Nutzer
|
<em>Speicherdauer:</em> Bis zur Löschung durch den Nutzer
|
||||||
|
|||||||
@@ -1,14 +1,15 @@
|
|||||||
module kver
|
module kver
|
||||||
|
|
||||||
go 1.25.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/go-chi/chi/v5 v5.3.0
|
github.com/go-chi/chi/v5 v5.3.0
|
||||||
github.com/go-chi/httprate v0.15.0
|
github.com/go-chi/httprate v0.15.0
|
||||||
github.com/jackc/pgx/v5 v5.11.0
|
github.com/jackc/pgx/v5 v5.11.0
|
||||||
github.com/oschwald/maxminddb-golang v1.13.1
|
github.com/oschwald/maxminddb-golang v1.13.1
|
||||||
golang.org/x/crypto v0.52.0
|
golang.org/x/crypto v0.57.0
|
||||||
golang.org/x/image v0.41.0
|
golang.org/x/image v0.41.0
|
||||||
|
golang.org/x/net v0.59.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@@ -17,7 +18,7 @@ require (
|
|||||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||||
golang.org/x/sync v0.22.0 // indirect
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
golang.org/x/sys v0.47.0 // indirect
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
golang.org/x/text v0.37.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -28,16 +28,18 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
|||||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo=
|
golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo=
|
||||||
golang.org/x/image v0.41.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA=
|
golang.org/x/image v0.41.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA=
|
||||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
|
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||||
|
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
|||||||
@@ -57,6 +57,8 @@ func main() {
|
|||||||
// die geschlossene DB oder den unmap-ten GeoIP-Reader zugreifen.
|
// die geschlossene DB oder den unmap-ten GeoIP-Reader zugreifen.
|
||||||
startImpressionWorker()
|
startImpressionWorker()
|
||||||
defer stopImpressionWorker()
|
defer stopImpressionWorker()
|
||||||
|
startPreviewWorker()
|
||||||
|
defer stopPreviewWorker()
|
||||||
|
|
||||||
srv := &http.Server{
|
srv := &http.Server{
|
||||||
Addr: envOr("KVER_ADDR", ":8080"),
|
Addr: envOr("KVER_ADDR", ":8080"),
|
||||||
|
|||||||
@@ -78,6 +78,18 @@ Ein Beitrag (`Entry`):
|
|||||||
"last_bump": 1700000400, "username": "max", "avatar": "static/media/…" }
|
"last_bump": 1700000400, "username": "max", "avatar": "static/media/…" }
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Enthält der Text einen Link, trägt der Beitrag `preview`, sobald der Server
|
||||||
|
die Vorschau des **ersten** Links geholt hat (asynchron nach dem Anlegen bzw.
|
||||||
|
Bearbeiten; vorher und ohne verwertbare Vorschau fehlt das Feld). `url` ist
|
||||||
|
die kanonische Adresse (YouTube-Links aller Formen als `watch?v=<id>`),
|
||||||
|
`kind` ist `youtube` oder `page`, `site` Kanal- bzw. Seitenname, `thumb` ein
|
||||||
|
lokales Bild (leer, wenn es keins gibt). Siehe `preview.go`.
|
||||||
|
|
||||||
|
```json
|
||||||
|
"preview": { "url": "https://www.youtube.com/watch?v=…", "kind": "youtube",
|
||||||
|
"title": "…", "site": "…", "thumb": "static/media/preview/….jpg" }
|
||||||
|
```
|
||||||
|
|
||||||
In Hauptfeed und `replies` des Threads trägt jeder Beitrag mit Antworten
|
In Hauptfeed und `replies` des Threads trägt jeder Beitrag mit Antworten
|
||||||
zusätzlich `latest`: den jüngsten nicht gelöschten Beitrag aus seinem ganzen
|
zusätzlich `latest`: den jüngsten nicht gelöschten Beitrag aus seinem ganzen
|
||||||
Teilbaum (nach `created_at`, höchstens 50 Ebenen tief). `skipped` zählt die
|
Teilbaum (nach `created_at`, höchstens 50 Ebenen tief). `skipped` zählt die
|
||||||
|
|||||||
+567
@@ -0,0 +1,567 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"image"
|
||||||
|
"image/jpeg"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"mime"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/netip"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/image/webp"
|
||||||
|
"golang.org/x/net/html"
|
||||||
|
"golang.org/x/net/html/charset"
|
||||||
|
|
||||||
|
xdraw "golang.org/x/image/draw"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Linkvorschau: Zum ersten Link eines Beitrags holt der Server Titel, Seiten-
|
||||||
|
// bzw. Kanalname und Vorschaubild und speichert das Bild lokal. Der Browser
|
||||||
|
// lädt die Vorschau damit ausschließlich von uns -- Besucher-IPs gehen nie an
|
||||||
|
// die verlinkte Seite (kein Einwilligungsbedarf, CSP bleibt 'self').
|
||||||
|
//
|
||||||
|
// Ablauf: Beim Anlegen/Bearbeiten wird der erste Link kanonisiert (previewKey)
|
||||||
|
// und als entry.link_url gespeichert. requestPreview legt dazu eine Zeile in
|
||||||
|
// link_preview an und reicht sie an den Worker; der holt die Daten im
|
||||||
|
// Hintergrund (das Posten wartet nie). Mehrere Beiträge mit demselben Link
|
||||||
|
// teilen sich eine Zeile und ein Bild. entrySelect joint die fertige Vorschau
|
||||||
|
// an jeden Beitrag.
|
||||||
|
//
|
||||||
|
// Provider: YouTube (oEmbed, liefert auch ohne Cookie-Banner Titel und Kanal)
|
||||||
|
// und allgemein OpenGraph/<title> für alle übrigen Seiten. Weitere Spezialfälle
|
||||||
|
// kommen als eigener Zweig in previewKey/fetchPreview dazu.
|
||||||
|
|
||||||
|
// Preview ist die fertige Vorschau eines Links, wie die API sie ausliefert.
|
||||||
|
type Preview struct {
|
||||||
|
URL string `json:"url"` // Linkziel der Vorschau (kanonisch)
|
||||||
|
Kind string `json:"kind"` // "youtube" | "page"
|
||||||
|
Title string `json:"title"` // Video- bzw. Seitentitel
|
||||||
|
Site string `json:"site"` // Kanal- bzw. Seitenname
|
||||||
|
Thumb string `json:"thumb"` // lokales Bild unter static/media/preview, ggf. ""
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
previewRefreshOK = 30 * 24 * 60 * 60 // erfolgreiche Vorschau nach 30 Tagen auffrischen
|
||||||
|
previewRetryFailed = 24 * 60 * 60 // fehlgeschlagene frühestens nach einem Tag erneut
|
||||||
|
maxPreviewURL = 2048
|
||||||
|
maxPageBytes = 1 << 20 // gelesener HTML-Anfang
|
||||||
|
maxThumbBytes = 8 << 20
|
||||||
|
thumbSize = 320 // Zielkante des gespeicherten Vorschaubilds (klein, steht neben dem Text)
|
||||||
|
previewDir = "static/media/preview"
|
||||||
|
)
|
||||||
|
|
||||||
|
// linkPattern findet Links wie linkify im Frontend (format.tsx), damit die
|
||||||
|
// Vorschau exakt zum ersten dort verlinkten Text gehört. \p{Z} und U+FEFF
|
||||||
|
// gleichen Gos ASCII-\s an das Unicode-\s von JavaScript an.
|
||||||
|
var linkPattern = regexp.MustCompile(`https?://[^\s\p{Z}\x{feff}<]+`)
|
||||||
|
|
||||||
|
var youtubeIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{11}$`)
|
||||||
|
|
||||||
|
// previewKey liefert die kanonische URL des ersten Links in content oder "".
|
||||||
|
// YouTube-Links aller Formen werden auf watch?v=<id> gebracht, damit dasselbe
|
||||||
|
// Video nur einmal geholt wird.
|
||||||
|
func previewKey(content string) string {
|
||||||
|
raw := linkPattern.FindString(content)
|
||||||
|
if raw == "" || len(raw) > maxPreviewURL {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
u, err := url.Parse(raw)
|
||||||
|
if err != nil || u.Hostname() == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if id := youtubeID(u); id != "" {
|
||||||
|
return "https://www.youtube.com/watch?v=" + id
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// youtubeID erkennt die Video-ID in watch?v=, youtu.be/, /shorts/, /live/,
|
||||||
|
// /embed/ und /v/ auf youtube.com, m./music./www.-Varianten und
|
||||||
|
// youtube-nocookie.com. "" für alles andere (Kanäle, Playlists, ...).
|
||||||
|
func youtubeID(u *url.URL) string {
|
||||||
|
host := strings.ToLower(u.Hostname())
|
||||||
|
for _, p := range []string{"www.", "m.", "music."} {
|
||||||
|
host = strings.TrimPrefix(host, p)
|
||||||
|
}
|
||||||
|
seg := strings.Split(strings.Trim(u.Path, "/"), "/")
|
||||||
|
var id string
|
||||||
|
switch host {
|
||||||
|
case "youtu.be":
|
||||||
|
id = seg[0]
|
||||||
|
case "youtube.com", "youtube-nocookie.com":
|
||||||
|
switch {
|
||||||
|
case u.Path == "/watch":
|
||||||
|
id = u.Query().Get("v")
|
||||||
|
case len(seg) >= 2 && (seg[0] == "shorts" || seg[0] == "live" || seg[0] == "embed" || seg[0] == "v"):
|
||||||
|
id = seg[1]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !youtubeIDPattern.MatchString(id) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
// requestPreview sorgt dafür, dass zu key eine Vorschau existiert: neue Links
|
||||||
|
// werden angelegt, veraltete oder vor über einem Tag gescheiterte erneut
|
||||||
|
// angestoßen. Nur dann geht der Key an den Worker. Ist dessen Puffer voll,
|
||||||
|
// bleibt die Zeile auf pending und wird beim nächsten Start nachgeholt.
|
||||||
|
func requestPreview(key string) {
|
||||||
|
if key == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now().Unix()
|
||||||
|
res, err := db.Exec(
|
||||||
|
`INSERT INTO link_preview (url) VALUES ($1)
|
||||||
|
ON CONFLICT (url) DO UPDATE SET status = 'pending'
|
||||||
|
WHERE (link_preview.status = 'ok' AND link_preview.fetched_at < $2)
|
||||||
|
OR (link_preview.status = 'none' AND link_preview.fetched_at < $3)`,
|
||||||
|
key, now-previewRefreshOK, now-previewRetryFailed)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("linkvorschau %s: %v", key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case previewCh <- key:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ein einzelner Worker holt die Vorschauen nacheinander, wie beim Impression-
|
||||||
|
// Worker deckelt das die ausgehenden Requests (Link-Spam erzeugt keine Flut).
|
||||||
|
// Beim Shutdown bricht previewCtx einen laufenden Abruf ab; was noch im Puffer
|
||||||
|
// liegt, bleibt pending und wird beim nächsten Start nachgeholt.
|
||||||
|
var (
|
||||||
|
previewCh = make(chan string, 64)
|
||||||
|
previewDone = make(chan struct{})
|
||||||
|
previewCtx, previewShutdown = context.WithCancel(context.Background())
|
||||||
|
)
|
||||||
|
|
||||||
|
// startPreviewWorker startet den Worker (nach initDB) und holt zuerst die
|
||||||
|
// liegengebliebenen pending-Zeilen nach.
|
||||||
|
func startPreviewWorker() {
|
||||||
|
go func() {
|
||||||
|
defer close(previewDone)
|
||||||
|
for _, key := range pendingPreviews(64) {
|
||||||
|
fetchPreview(previewCtx, key)
|
||||||
|
}
|
||||||
|
for key := range previewCh {
|
||||||
|
fetchPreview(previewCtx, key)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopPreviewWorker bricht laufende Abrufe ab und wartet auf den Worker (vor
|
||||||
|
// db.Close rufen).
|
||||||
|
func stopPreviewWorker() {
|
||||||
|
previewShutdown()
|
||||||
|
close(previewCh)
|
||||||
|
<-previewDone
|
||||||
|
}
|
||||||
|
|
||||||
|
func pendingPreviews(limit int) []string {
|
||||||
|
rows, err := db.Query(`SELECT url FROM link_preview WHERE status = 'pending' LIMIT $1`, limit)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("linkvorschau: %v", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var keys []string
|
||||||
|
for rows.Next() {
|
||||||
|
var k string
|
||||||
|
if rows.Scan(&k) == nil {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchPreview holt die Vorschau zu key und speichert das Ergebnis: status ok
|
||||||
|
// mit Daten oder none (Seite ohne verwertbaren Titel, Video privat/gelöscht,
|
||||||
|
// Fehler). Bei abgebrochenem ctx (Shutdown) bleibt die Zeile pending.
|
||||||
|
func fetchPreview(ctx context.Context, key string) {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
var p Preview
|
||||||
|
var imgs []string // Bildkandidaten, der erste ladbare gewinnt
|
||||||
|
var err error
|
||||||
|
if u, _ := url.Parse(key); u != nil && youtubeID(u) != "" {
|
||||||
|
p, imgs, err = fetchYouTube(ctx, youtubeID(u))
|
||||||
|
} else {
|
||||||
|
p, imgs, err = fetchPage(ctx, key)
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.Canceled) && previewCtx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now().Unix()
|
||||||
|
thumb := thumbPath(key)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("linkvorschau %s: %v", key, err)
|
||||||
|
os.Remove(thumb)
|
||||||
|
db.Exec(`UPDATE link_preview SET status = 'none', title = '', site = '', thumb = '', fetched_at = $2 WHERE url = $1`, key, now)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
stored := false
|
||||||
|
for _, img := range imgs {
|
||||||
|
if err := storeThumb(ctx, img, thumb); err != nil {
|
||||||
|
log.Printf("linkvorschau %s: bild %s: %v", key, img, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
stored = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if !stored {
|
||||||
|
os.Remove(thumb)
|
||||||
|
thumb = ""
|
||||||
|
}
|
||||||
|
if previewCtx.Err() != nil {
|
||||||
|
return // Shutdown mitten im Bildabruf: pending lassen, nicht ohne Bild speichern
|
||||||
|
}
|
||||||
|
db.Exec(
|
||||||
|
`UPDATE link_preview SET status = 'ok', kind = $2, title = $3, site = $4, thumb = $5, fetched_at = $6 WHERE url = $1`,
|
||||||
|
key, p.Kind, truncateRunes(p.Title, 300), truncateRunes(p.Site, 100), thumb, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// thumbPath leitet den Bildnamen aus dem Key ab: deterministisch (Auffrischen
|
||||||
|
// überschreibt dasselbe Bild) und ohne Nutzereingabe im Dateinamen.
|
||||||
|
func thumbPath(key string) string {
|
||||||
|
sum := sha256.Sum256([]byte(key))
|
||||||
|
return previewDir + "/" + hex.EncodeToString(sum[:10]) + ".jpg"
|
||||||
|
}
|
||||||
|
|
||||||
|
// YouTube-Endpunkte; Variablen, damit Tests sie auf einen lokalen Server lenken.
|
||||||
|
// mqdefault (320x180) gibt es für jedes Video, in 16:9 ohne schwarze Balken
|
||||||
|
// und genau in der Größe, in der wir es zeigen.
|
||||||
|
var (
|
||||||
|
youtubeOEmbed = "https://www.youtube.com/oembed"
|
||||||
|
youtubeThumbs = []string{"https://i.ytimg.com/vi/%s/mqdefault.jpg"}
|
||||||
|
)
|
||||||
|
|
||||||
|
// fetchYouTube fragt den öffentlichen oEmbed-Endpunkt (ohne API-Key). Das
|
||||||
|
// Bild kommt von der festen Thumbnail-Adresse statt aus thumbnail_url, so
|
||||||
|
// bestimmt allein die geprüfte ID, welche Hosts wir ansprechen.
|
||||||
|
func fetchYouTube(ctx context.Context, id string) (Preview, []string, error) {
|
||||||
|
watch := "https://www.youtube.com/watch?v=" + id
|
||||||
|
resp, err := previewGet(ctx, youtubeOEmbed+"?format=json&url="+url.QueryEscape(watch), "application/json")
|
||||||
|
if err != nil {
|
||||||
|
return Preview{}, nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
// 401/403: privat oder nicht einbettbar, 404: gelöscht.
|
||||||
|
return Preview{}, nil, fmt.Errorf("oembed: status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
var o struct {
|
||||||
|
Title string `json:"title"`
|
||||||
|
AuthorName string `json:"author_name"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&o); err != nil {
|
||||||
|
return Preview{}, nil, fmt.Errorf("oembed: %w", err)
|
||||||
|
}
|
||||||
|
if o.Title == "" {
|
||||||
|
return Preview{}, nil, errors.New("oembed: kein Titel")
|
||||||
|
}
|
||||||
|
imgs := make([]string, len(youtubeThumbs))
|
||||||
|
for i, f := range youtubeThumbs {
|
||||||
|
imgs[i] = fmt.Sprintf(f, id)
|
||||||
|
}
|
||||||
|
return Preview{URL: watch, Kind: "youtube", Title: o.Title, Site: o.AuthorName}, imgs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchPage liest den Kopf einer HTML-Seite und nimmt OpenGraph- bzw.
|
||||||
|
// Twitter-Angaben, ersatzweise <title>. Ohne Titel gibt es keine Vorschau.
|
||||||
|
func fetchPage(ctx context.Context, pageURL string) (Preview, []string, error) {
|
||||||
|
resp, err := previewGet(ctx, pageURL, "text/html,application/xhtml+xml")
|
||||||
|
if err != nil {
|
||||||
|
return Preview{}, nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return Preview{}, nil, fmt.Errorf("status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
ct := resp.Header.Get("Content-Type")
|
||||||
|
if mt, _, _ := mime.ParseMediaType(ct); mt != "text/html" && mt != "application/xhtml+xml" {
|
||||||
|
return Preview{}, nil, fmt.Errorf("kein HTML: %q", ct)
|
||||||
|
}
|
||||||
|
body, err := charset.NewReader(io.LimitReader(resp.Body, maxPageBytes), ct)
|
||||||
|
if err != nil {
|
||||||
|
return Preview{}, nil, err
|
||||||
|
}
|
||||||
|
m := parsePageMeta(body)
|
||||||
|
|
||||||
|
title := firstNonEmpty(m["og:title"], m["twitter:title"], m["title"])
|
||||||
|
if title == "" {
|
||||||
|
return Preview{}, nil, errors.New("kein Titel")
|
||||||
|
}
|
||||||
|
// Nach Redirects gilt die endgültige Adresse als Basis für relative Bild-URLs.
|
||||||
|
base := resp.Request.URL
|
||||||
|
site := firstNonEmpty(m["og:site_name"], strings.TrimPrefix(base.Hostname(), "www."))
|
||||||
|
var imgs []string
|
||||||
|
if ref := firstNonEmpty(m["og:image:secure_url"], m["og:image"], m["og:image:url"], m["twitter:image"], m["twitter:image:src"]); ref != "" {
|
||||||
|
if u, err := base.Parse(ref); err == nil && (u.Scheme == "http" || u.Scheme == "https") {
|
||||||
|
imgs = []string{u.String()}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Preview{URL: pageURL, Kind: "page", Title: title, Site: site}, imgs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parsePageMeta sammelt <meta property|name=... content=...> und den
|
||||||
|
// <title>-Text aus dem Dokumentkopf. Kleingeschriebene Schlüssel, jeweils der
|
||||||
|
// erste Treffer zählt; Schluss beim <body> bzw. </head>.
|
||||||
|
func parsePageMeta(r io.Reader) map[string]string {
|
||||||
|
m := map[string]string{}
|
||||||
|
set := func(k, v string) {
|
||||||
|
v = strings.Join(strings.Fields(strings.ToValidUTF8(v, "")), " ")
|
||||||
|
if _, ok := m[k]; !ok && v != "" {
|
||||||
|
m[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
z := html.NewTokenizer(r)
|
||||||
|
inTitle := false
|
||||||
|
for {
|
||||||
|
switch z.Next() {
|
||||||
|
case html.ErrorToken:
|
||||||
|
return m
|
||||||
|
case html.StartTagToken, html.SelfClosingTagToken:
|
||||||
|
t := z.Token()
|
||||||
|
switch t.Data {
|
||||||
|
case "body":
|
||||||
|
return m
|
||||||
|
case "title":
|
||||||
|
inTitle = true
|
||||||
|
case "meta":
|
||||||
|
var key, content string
|
||||||
|
for _, a := range t.Attr {
|
||||||
|
switch a.Key {
|
||||||
|
case "property", "name":
|
||||||
|
if key == "" {
|
||||||
|
key = strings.ToLower(a.Val)
|
||||||
|
}
|
||||||
|
case "content":
|
||||||
|
content = a.Val
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if key != "" && key != "title" {
|
||||||
|
set(key, content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case html.TextToken:
|
||||||
|
if inTitle {
|
||||||
|
set("title", string(z.Text()))
|
||||||
|
}
|
||||||
|
case html.EndTagToken:
|
||||||
|
switch z.Token().Data {
|
||||||
|
case "title":
|
||||||
|
inTitle = false
|
||||||
|
case "head":
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstNonEmpty(vals ...string) string {
|
||||||
|
for _, v := range vals {
|
||||||
|
if v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// storeThumb lädt ein Bild, verkleinert es auf thumbSize und speichert es als
|
||||||
|
// JPEG unter name. Das Re-Encode verwirft Metadaten und alles, was kein Bild
|
||||||
|
// ist; animierte GIFs werden zum ersten Frame.
|
||||||
|
func storeThumb(ctx context.Context, imgURL, name string) error {
|
||||||
|
resp, err := previewGet(ctx, imgURL, "image/*")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return fmt.Errorf("status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(io.LimitReader(resp.Body, maxThumbBytes+1))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(data) > maxThumbBytes {
|
||||||
|
return errors.New("bild zu groß")
|
||||||
|
}
|
||||||
|
|
||||||
|
// WebP gezielt dekodieren statt global zu registrieren -- sonst nähme
|
||||||
|
// auch der Upload (storeImage) plötzlich WebP an.
|
||||||
|
decodeConfig, decode := image.DecodeConfig, func(r io.Reader) (image.Image, error) {
|
||||||
|
img, _, err := image.Decode(r)
|
||||||
|
return img, err
|
||||||
|
}
|
||||||
|
if len(data) >= 12 && string(data[:4]) == "RIFF" && string(data[8:12]) == "WEBP" {
|
||||||
|
decodeConfig = func(r io.Reader) (image.Config, string, error) {
|
||||||
|
c, err := webp.DecodeConfig(r)
|
||||||
|
return c, "webp", err
|
||||||
|
}
|
||||||
|
decode = webp.Decode
|
||||||
|
}
|
||||||
|
cfg, _, err := decodeConfig(bytes.NewReader(data))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if cfg.Width <= 0 || cfg.Height <= 0 || cfg.Width*cfg.Height > maxImagePixels {
|
||||||
|
return fmt.Errorf("bild zu groß: %dx%d", cfg.Width, cfg.Height)
|
||||||
|
}
|
||||||
|
img, err := decode(bytes.NewReader(data))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
b := img.Bounds()
|
||||||
|
nw, nh := b.Dx(), b.Dy()
|
||||||
|
if nw > thumbSize || nh > thumbSize {
|
||||||
|
if nw > nh {
|
||||||
|
nw, nh = thumbSize, max(nh*thumbSize/nw, 1)
|
||||||
|
} else {
|
||||||
|
nw, nh = max(nw*thumbSize/nh, 1), thumbSize
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dst := image.NewRGBA(image.Rect(0, 0, nw, nh))
|
||||||
|
// Weißer Grund für transparente PNG/WebP, JPEG kennt keinen Alphakanal.
|
||||||
|
xdraw.Draw(dst, dst.Bounds(), image.White, image.Point{}, xdraw.Src)
|
||||||
|
xdraw.CatmullRom.Scale(dst, dst.Bounds(), img, b, xdraw.Over, nil)
|
||||||
|
|
||||||
|
if err := os.MkdirAll(previewDir, 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Erst in eine Temp-Datei, dann umbenennen: ein Auffrischen ersetzt das
|
||||||
|
// alte Bild atomar, Besucher sehen nie ein halb geschriebenes.
|
||||||
|
tmp, err := os.CreateTemp(previewDir, ".tmp-*")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.Remove(tmp.Name())
|
||||||
|
if err := jpeg.Encode(tmp, dst, &jpeg.Options{Quality: 80}); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Chmod(tmp.Name(), 0o644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(tmp.Name(), name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// previewGet ist ein GET über previewClient mit ehrlichem User-Agent.
|
||||||
|
func previewGet(ctx context.Context, target, accept string) (*http.Response, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "kver-linkvorschau/1.0")
|
||||||
|
req.Header.Set("Accept", accept)
|
||||||
|
req.Header.Set("Accept-Language", "de,en;q=0.5")
|
||||||
|
return previewClient.Do(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// previewClient ruft beliebige, von Nutzern gepostete URLs ab und muss daher
|
||||||
|
// gegen SSRF gehärtet sein: Wer "http://postgres:5432/" oder eine Adresse im
|
||||||
|
// internen Netz postet, darf unseren Server nicht dorthin schicken. Geprüft
|
||||||
|
// wird in Dialer.Control, also NACH der DNS-Auflösung auf der tatsächlich
|
||||||
|
// angewählten IP -- das deckt auch DNS-Rebinding und jeden Redirect-Schritt ab.
|
||||||
|
// Kein Proxy aus der Umgebung (der würde die Prüfung umgehen).
|
||||||
|
var previewClient = &http.Client{
|
||||||
|
Timeout: 20 * time.Second,
|
||||||
|
Transport: &http.Transport{
|
||||||
|
Proxy: nil,
|
||||||
|
DialContext: (&net.Dialer{
|
||||||
|
Timeout: 5 * time.Second,
|
||||||
|
Control: func(network, address string, _ syscall.RawConn) error {
|
||||||
|
return previewDialGuard(address)
|
||||||
|
},
|
||||||
|
}).DialContext,
|
||||||
|
TLSHandshakeTimeout: 5 * time.Second,
|
||||||
|
ResponseHeaderTimeout: 10 * time.Second,
|
||||||
|
MaxIdleConns: 4,
|
||||||
|
IdleConnTimeout: 30 * time.Second,
|
||||||
|
},
|
||||||
|
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||||
|
if len(via) >= 5 {
|
||||||
|
return errors.New("zu viele Weiterleitungen")
|
||||||
|
}
|
||||||
|
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
|
||||||
|
return fmt.Errorf("weiterleitung auf %q", req.URL.Scheme)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// previewDialGuard lässt nur öffentliche Adressen auf Port 80/443 zu.
|
||||||
|
// Variable, damit Tests gegen einen lokalen httptest-Server laufen können.
|
||||||
|
var previewDialGuard = func(address string) error {
|
||||||
|
host, port, err := net.SplitHostPort(address)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if port != "80" && port != "443" {
|
||||||
|
return fmt.Errorf("port %s nicht erlaubt", port)
|
||||||
|
}
|
||||||
|
ip, err := netip.ParseAddr(host)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !isPublicIP(ip) {
|
||||||
|
return fmt.Errorf("adresse %s nicht öffentlich", ip)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nicht-öffentliche Bereiche, die IsGlobalUnicast/IsPrivate nicht abdecken.
|
||||||
|
var reservedNets = []netip.Prefix{
|
||||||
|
netip.MustParsePrefix("0.0.0.0/8"), // "dieses Netz"
|
||||||
|
netip.MustParsePrefix("100.64.0.0/10"), // Carrier-Grade-NAT
|
||||||
|
netip.MustParsePrefix("192.0.0.0/24"), // IETF-Protokollzuweisungen
|
||||||
|
netip.MustParsePrefix("192.0.2.0/24"), // Dokumentation
|
||||||
|
netip.MustParsePrefix("198.18.0.0/15"), // Benchmarking
|
||||||
|
netip.MustParsePrefix("198.51.100.0/24"), // Dokumentation
|
||||||
|
netip.MustParsePrefix("203.0.113.0/24"), // Dokumentation
|
||||||
|
netip.MustParsePrefix("240.0.0.0/4"), // reserviert, inkl. Broadcast
|
||||||
|
netip.MustParsePrefix("64:ff9b::/96"), // NAT64: bildet IPv4 inkl. interner Netze ab
|
||||||
|
netip.MustParsePrefix("64:ff9b:1::/48"), // lokales NAT64
|
||||||
|
netip.MustParsePrefix("2001:db8::/32"), // Dokumentation
|
||||||
|
netip.MustParsePrefix("2002::/16"), // 6to4: kann interne IPv4 einbetten
|
||||||
|
}
|
||||||
|
|
||||||
|
func isPublicIP(ip netip.Addr) bool {
|
||||||
|
ip = ip.Unmap()
|
||||||
|
if !ip.IsGlobalUnicast() || ip.IsPrivate() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, p := range reservedNets {
|
||||||
|
if p.Contains(ip) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
+273
@@ -0,0 +1,273 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"image"
|
||||||
|
"image/png"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/netip"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPreviewKey(t *testing.T) {
|
||||||
|
yt := "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
|
||||||
|
cases := map[string]string{
|
||||||
|
"guck mal https://youtu.be/dQw4w9WgXcQ?t=42": yt,
|
||||||
|
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=x": yt,
|
||||||
|
"https://m.youtube.com/watch?feature=share&v=dQw4w9WgXcQ": yt,
|
||||||
|
"https://youtube.com/shorts/dQw4w9WgXcQ": yt,
|
||||||
|
"https://music.youtube.com/watch?v=dQw4w9WgXcQ": yt,
|
||||||
|
"https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ": yt,
|
||||||
|
"https://www.youtube.com/@kanal": "https://www.youtube.com/@kanal",
|
||||||
|
"https://www.youtube.com/watch?v=kurz": "https://www.youtube.com/watch?v=kurz",
|
||||||
|
"erst https://example.org/a dann https://youtu.be/dQw4w9WgXcQ": "https://example.org/a",
|
||||||
|
"ohne Link": "",
|
||||||
|
"ftp://example.org": "",
|
||||||
|
"https://example.org/x danach": "https://example.org/x",
|
||||||
|
"https://example.org/<b>": "https://example.org/",
|
||||||
|
}
|
||||||
|
for in, want := range cases {
|
||||||
|
if got := previewKey(in); got != want {
|
||||||
|
t.Errorf("previewKey(%q) = %q, erwartet %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := previewKey("https://example.org/" + strings.Repeat("a", maxPreviewURL)); got != "" {
|
||||||
|
t.Errorf("überlange URL ergab %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsPublicIP(t *testing.T) {
|
||||||
|
cases := map[string]bool{
|
||||||
|
"93.184.216.34": true,
|
||||||
|
"2a00:1450:4001::200e": true,
|
||||||
|
"127.0.0.1": false,
|
||||||
|
"10.1.2.3": false,
|
||||||
|
"172.16.0.1": false,
|
||||||
|
"192.168.161.4": false,
|
||||||
|
"169.254.169.254": false, // Cloud-Metadaten
|
||||||
|
"100.64.0.1": false,
|
||||||
|
"0.1.2.3": false,
|
||||||
|
"0.0.0.0": false,
|
||||||
|
"255.255.255.255": false,
|
||||||
|
"::1": false,
|
||||||
|
"fd00::1": false,
|
||||||
|
"fe80::1": false,
|
||||||
|
"::ffff:127.0.0.1": false,
|
||||||
|
"::ffff:93.184.216.34": true,
|
||||||
|
"64:ff9b::a00:1": false,
|
||||||
|
"2002:c0a8:101::1": false,
|
||||||
|
"224.0.0.1": false,
|
||||||
|
}
|
||||||
|
for in, want := range cases {
|
||||||
|
if got := isPublicIP(netip.MustParseAddr(in)); got != want {
|
||||||
|
t.Errorf("isPublicIP(%s) = %v, erwartet %v", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Der echte Wächter muss interne Ziele abweisen, ohne dass ein Request das
|
||||||
|
// Netz verlässt -- auch nach einer Weiterleitung.
|
||||||
|
func TestPreviewClientBlocksInternal(t *testing.T) {
|
||||||
|
internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
t.Error("interner Server wurde erreicht")
|
||||||
|
}))
|
||||||
|
defer internal.Close()
|
||||||
|
if _, err := previewGet(context.Background(), internal.URL, "text/html"); err == nil {
|
||||||
|
t.Fatal("Abruf von 127.0.0.1 wurde nicht blockiert")
|
||||||
|
}
|
||||||
|
if err := previewDialGuard("8.8.8.8:5432"); err == nil {
|
||||||
|
t.Error("Port 5432 wurde nicht blockiert")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParsePageMeta(t *testing.T) {
|
||||||
|
doc := `<!doctype html><html><head>
|
||||||
|
<title> Seiten
|
||||||
|
titel </title>
|
||||||
|
<meta property="og:title" content="OG-Titel">
|
||||||
|
<meta property="og:title" content="zweiter OG-Titel">
|
||||||
|
<meta name="twitter:image" content="/bild.png">
|
||||||
|
<meta property="OG:Site_Name" content="Beispiel">
|
||||||
|
</head><body><meta property="og:image" content="/zu-spaet.png"></body></html>`
|
||||||
|
m := parsePageMeta(strings.NewReader(doc))
|
||||||
|
want := map[string]string{
|
||||||
|
"title": "Seiten titel",
|
||||||
|
"og:title": "OG-Titel",
|
||||||
|
"twitter:image": "/bild.png",
|
||||||
|
"og:site_name": "Beispiel",
|
||||||
|
}
|
||||||
|
for k, v := range want {
|
||||||
|
if m[k] != v {
|
||||||
|
t.Errorf("%s = %q, erwartet %q", k, m[k], v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, ok := m["og:image"]; ok {
|
||||||
|
t.Error("Meta aus dem body wurde übernommen")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeRemote stellt eine Seite mit og-Tags, ein Bild und einen oEmbed-
|
||||||
|
// Endpunkt bereit und lenkt die Vorschau-Abrufe für die Testdauer dorthin.
|
||||||
|
func fakeRemote(t *testing.T) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
var buf strings.Builder
|
||||||
|
img := image.NewRGBA(image.Rect(0, 0, 1280, 720))
|
||||||
|
png.Encode(&buf, img)
|
||||||
|
pngData := buf.String()
|
||||||
|
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.HandleFunc("/seite", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
w.Write([]byte(`<html><head><meta property="og:title" content="Ein Artikel">
|
||||||
|
<meta property="og:image" content="/bild.png"></head><body></body></html>`))
|
||||||
|
})
|
||||||
|
mux.HandleFunc("/umleitung", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.Redirect(w, r, "/seite", http.StatusFound)
|
||||||
|
})
|
||||||
|
mux.HandleFunc("/ohne-titel", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "text/html")
|
||||||
|
w.Write([]byte(`<html><head></head><body>nix</body></html>`))
|
||||||
|
})
|
||||||
|
mux.HandleFunc("/bild.png", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "image/png")
|
||||||
|
w.Write([]byte(pngData))
|
||||||
|
})
|
||||||
|
mux.HandleFunc("/oembed", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
u, _ := url.Parse(r.URL.Query().Get("url"))
|
||||||
|
if u == nil || u.Query().Get("v") != "dQw4w9WgXcQ" {
|
||||||
|
http.Error(w, "Not Found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
json.NewEncoder(w).Encode(map[string]string{"title": "Ein Video", "author_name": "Ein Kanal"})
|
||||||
|
})
|
||||||
|
mux.HandleFunc("/vi/", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "image/png")
|
||||||
|
w.Write([]byte(pngData))
|
||||||
|
})
|
||||||
|
srv := httptest.NewServer(mux)
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
|
||||||
|
oldGuard, oldOEmbed, oldThumbs := previewDialGuard, youtubeOEmbed, youtubeThumbs
|
||||||
|
previewDialGuard = func(string) error { return nil }
|
||||||
|
youtubeOEmbed = srv.URL + "/oembed"
|
||||||
|
// Erster Kandidat fehlt -> der nächste greift.
|
||||||
|
youtubeThumbs = []string{srv.URL + "/fehlt/%s.jpg", srv.URL + "/vi/%s/hqdefault.jpg"}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
previewDialGuard, youtubeOEmbed, youtubeThumbs = oldGuard, oldOEmbed, oldThumbs
|
||||||
|
})
|
||||||
|
|
||||||
|
// Bilder landen relativ zum CWD unter static/media/preview.
|
||||||
|
dir := t.TempDir()
|
||||||
|
wd, _ := os.Getwd()
|
||||||
|
if err := os.Chdir(dir); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { os.Chdir(wd) })
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFetchPage(t *testing.T) {
|
||||||
|
remote := fakeRemote(t)
|
||||||
|
p, img, err := fetchPage(context.Background(), remote.URL+"/umleitung")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.Title != "Ein Artikel" || p.Site != "127.0.0.1" || p.URL != remote.URL+"/umleitung" {
|
||||||
|
t.Errorf("unerwartete Vorschau %+v", p)
|
||||||
|
}
|
||||||
|
// Relatives Bild gegen die Adresse NACH der Weiterleitung aufgelöst.
|
||||||
|
if len(img) != 1 || img[0] != remote.URL+"/bild.png" {
|
||||||
|
t.Errorf("bild = %q", img)
|
||||||
|
}
|
||||||
|
if _, _, err := fetchPage(context.Background(), remote.URL+"/ohne-titel"); err == nil {
|
||||||
|
t.Error("Seite ohne Titel ergab eine Vorschau")
|
||||||
|
}
|
||||||
|
if _, _, err := fetchPage(context.Background(), remote.URL+"/bild.png"); err == nil {
|
||||||
|
t.Error("Bild als Seite ergab eine Vorschau")
|
||||||
|
}
|
||||||
|
|
||||||
|
name := thumbPath("x")
|
||||||
|
if err := storeThumb(context.Background(), img[0], name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f, err := os.Open(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
cfg, format, err := image.DecodeConfig(f)
|
||||||
|
if err != nil || format != "jpeg" || cfg.Width != thumbSize || cfg.Height != 180 {
|
||||||
|
t.Errorf("thumb: %s %dx%d %v", format, cfg.Width, cfg.Height, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ende-zu-Ende: Beitrag mit YouTube-Link anlegen, Vorschau holen, im Feed
|
||||||
|
// sehen; Bearbeiten ohne Link nimmt sie wieder weg.
|
||||||
|
func TestLinkPreviewInFeed(t *testing.T) {
|
||||||
|
srv := newTestServer(t)
|
||||||
|
fakeRemote(t)
|
||||||
|
c := registerAndLogin(t, srv, "vorschau")
|
||||||
|
|
||||||
|
pid := createEntry(t, c, srv, "schau https://youtu.be/dQw4w9WgXcQ und https://example.org")
|
||||||
|
key := "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
|
||||||
|
feedPreview := func() *Preview {
|
||||||
|
t.Helper()
|
||||||
|
resp, err := http.Get(srv.URL + "/api/entry/feed/0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
var entries []Entry
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&entries); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.PID == pid {
|
||||||
|
return e.Preview
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("Beitrag %d fehlt im Feed", pid)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if p := feedPreview(); p != nil {
|
||||||
|
t.Errorf("Vorschau vor dem Abruf: %+v", p)
|
||||||
|
}
|
||||||
|
fetchPreview(context.Background(), key)
|
||||||
|
p := feedPreview()
|
||||||
|
if p == nil {
|
||||||
|
t.Fatal("keine Vorschau nach dem Abruf")
|
||||||
|
}
|
||||||
|
if p.URL != key || p.Kind != "youtube" || p.Title != "Ein Video" || p.Site != "Ein Kanal" || p.Thumb != thumbPath(key) {
|
||||||
|
t.Errorf("unerwartete Vorschau %+v", p)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(p.Thumb); err != nil {
|
||||||
|
t.Errorf("thumb fehlt: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ein privates/gelöschtes Video ergibt keine Vorschau.
|
||||||
|
other := "https://www.youtube.com/watch?v=aaaaaaaaaaa"
|
||||||
|
requestPreview(other)
|
||||||
|
fetchPreview(context.Background(), other)
|
||||||
|
var status string
|
||||||
|
db.QueryRow(`SELECT status FROM link_preview WHERE url = $1`, other).Scan(&status)
|
||||||
|
if status != "none" {
|
||||||
|
t.Errorf("status = %q, erwartet none", status)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp := postForm(t, c, srv.URL+"/api/entry/"+itoa(pid)+"/edit", url.Values{"content": {"jetzt ohne Link"}})
|
||||||
|
resp.Body.Close()
|
||||||
|
if p := feedPreview(); p != nil {
|
||||||
|
t.Errorf("Vorschau nach Bearbeiten ohne Link: %+v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func itoa(n int64) string {
|
||||||
|
b, _ := json.Marshal(n)
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
@@ -208,7 +208,7 @@ func handleUserDelete(w http.ResponseWriter, r *http.Request) error {
|
|||||||
`DELETE FROM session WHERE uid = $1`,
|
`DELETE FROM session WHERE uid = $1`,
|
||||||
`DELETE FROM password_reset WHERE uid = $1`,
|
`DELETE FROM password_reset WHERE uid = $1`,
|
||||||
`DELETE FROM vote WHERE uid = $1`,
|
`DELETE FROM vote WHERE uid = $1`,
|
||||||
`UPDATE entry SET deleted = 1, content = '', filepath = '', uid = 0 WHERE uid = $1`,
|
`UPDATE entry SET deleted = 1, content = '', filepath = '', link_url = '', uid = 0 WHERE uid = $1`,
|
||||||
`DELETE FROM account WHERE uid = $1`,
|
`DELETE FROM account WHERE uid = $1`,
|
||||||
} {
|
} {
|
||||||
if _, err := tx.Exec(q, uid); err != nil {
|
if _, err := tx.Exec(q, uid); err != nil {
|
||||||
|
|||||||
Reference in New Issue
Block a user