Add YouTube feed support via yt-dlp, MIME-based extensions, robust state
- ytdlp: thin wrapper around yt-dlp CLI for channel/playlist discovery (--flat-playlist) and per-video metadata (--print-json); defaults to player_client=web_embedded to avoid SABR-only format restrictions. - downloader: YouTube path extracts audio via yt-dlp -x, probes duration with ffprobe and rejects clips far shorter than the expected length. - main: YouTube channel/playlist feeds, incremental discovery with pre-start-date boundary for newest-first channels, two-phase record-then-enqueue so discovered jobs survive crashes. - poller: propagate enclosure MIME type; computeDestPath appends the matching file extension. - state: jobs carry expected duration; AddJobWithStatus for skipped items. - cmd/probe: dry-run discovery/date-resolution diagnostics.
This commit is contained in:
+189
@@ -0,0 +1,189 @@
|
||||
// Package ytdlp provides a thin wrapper around the yt-dlp CLI for
|
||||
// YouTube channel/playlist discovery and per-video metadata resolution.
|
||||
package ytdlp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// maxJSONLineBytes bounds one line of --print-json output. A full video's
|
||||
// JSON (including the formats list) routinely exceeds 1 MB for long videos,
|
||||
// so the limit must be generous.
|
||||
const maxJSONLineBytes = 16 * 1024 * 1024
|
||||
|
||||
// Settings controls how yt-dlp is invoked.
|
||||
type Settings struct {
|
||||
// BinPath is the path to the yt-dlp executable (or a name on $PATH).
|
||||
BinPath string
|
||||
// PlayerClient is passed via --extractor-args "youtube:player_client=..."
|
||||
// to work around YouTube client restrictions (see DefaultPlayerClient).
|
||||
PlayerClient string
|
||||
}
|
||||
|
||||
// DefaultPlayerClient works around YouTube's SABR-only streaming experiment,
|
||||
// which withholds plain HTTPS URLs from the default web/android clients.
|
||||
const DefaultPlayerClient = "web_embedded"
|
||||
|
||||
// FlatVideo is one entry from a --flat-playlist enumeration.
|
||||
type FlatVideo struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Duration *float64 `json:"duration"` // seconds; may be nil (e.g. some shorts)
|
||||
Channel string `json:"channel"`
|
||||
}
|
||||
|
||||
// Discovery enumerates all videos in a YouTube channel or playlist using
|
||||
// `yt-dlp --flat-playlist` (fast; does not resolve each video).
|
||||
// Returns videos in the order yt-dlp emits them:
|
||||
//
|
||||
// - channel: newest first
|
||||
// - playlist: playlist order (not necessarily chronological)
|
||||
func Discovery(ctx context.Context, s Settings, sourceURL string) ([]FlatVideo, error) {
|
||||
args := []string{"--flat-playlist", "--print-json", sourceURL}
|
||||
stdout, stderr, err := run(ctx, s, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("yt-dlp discovery %s: %v: %s", sourceURL, err, LastLines(string(stderr), 3))
|
||||
}
|
||||
|
||||
var vids []FlatVideo
|
||||
sc := bufio.NewScanner(bytes.NewReader(stdout))
|
||||
sc.Buffer(make([]byte, 0, 64*1024), maxJSONLineBytes)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var v FlatVideo
|
||||
if err := json.Unmarshal([]byte(line), &v); err != nil {
|
||||
continue // skip malformed lines
|
||||
}
|
||||
if v.ID == "" {
|
||||
continue
|
||||
}
|
||||
vids = append(vids, v)
|
||||
}
|
||||
if len(vids) == 0 {
|
||||
return nil, fmt.Errorf("yt-dlp discovery %s: no videos found", sourceURL)
|
||||
}
|
||||
return vids, nil
|
||||
}
|
||||
|
||||
// VideoInfo carries resolved metadata for a single video.
|
||||
type VideoInfo struct {
|
||||
ID string
|
||||
Title string
|
||||
Duration float64 // seconds, 0 if unknown
|
||||
UploadDate time.Time
|
||||
}
|
||||
|
||||
// Info resolves full metadata for a single video by ID.
|
||||
func Info(ctx context.Context, s Settings, videoID string) (VideoInfo, error) {
|
||||
url := "https://www.youtube.com/watch?v=" + videoID
|
||||
stdout, stderr, err := run(ctx, s,
|
||||
"--no-download", "--no-playlist", "--print-json", url)
|
||||
if err != nil {
|
||||
return VideoInfo{}, fmt.Errorf("yt-dlp info %s: %v: %s", videoID, err, LastLines(string(stderr), 3))
|
||||
}
|
||||
|
||||
sc := bufio.NewScanner(bytes.NewReader(stdout))
|
||||
sc.Buffer(make([]byte, 0, 64*1024), maxJSONLineBytes)
|
||||
var data map[string]any
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal([]byte(line), &data); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if data == nil {
|
||||
if err := sc.Err(); err != nil {
|
||||
return VideoInfo{}, fmt.Errorf("yt-dlp info %s: %v", videoID, err)
|
||||
}
|
||||
return VideoInfo{}, fmt.Errorf("yt-dlp info %s: empty or unparseable output", videoID)
|
||||
}
|
||||
|
||||
info := VideoInfo{ID: videoID}
|
||||
if t, _ := data["title"].(string); t != "" {
|
||||
info.Title = t
|
||||
}
|
||||
if d, ok := num(data["duration"]); ok && d > 0 {
|
||||
info.Duration = d
|
||||
}
|
||||
// upload_date is "YYYYMMDD" or "NA" (or null).
|
||||
if ud, ok := data["upload_date"].(string); ok && ud != "" && ud != "NA" {
|
||||
if t, err := time.Parse("20060102", ud); err == nil {
|
||||
info.UploadDate = t
|
||||
}
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// CommandFor builds an *exec.Cmd running yt-dlp with the standard
|
||||
// arguments for the given settings (binary + YouTube extractor args).
|
||||
// Additional arguments may be appended to cmd.Args by the caller.
|
||||
func CommandFor(ctx context.Context, s Settings) *exec.Cmd {
|
||||
bin := s.BinPath
|
||||
if bin == "" {
|
||||
bin = "yt-dlp"
|
||||
}
|
||||
pc := s.PlayerClient
|
||||
if pc == "" {
|
||||
pc = DefaultPlayerClient
|
||||
}
|
||||
return exec.CommandContext(ctx, bin,
|
||||
"--no-warnings", "--extractor-args", "youtube:player_client="+pc)
|
||||
}
|
||||
|
||||
// run executes yt-dlp with the given arguments and returns stdout and stderr.
|
||||
func run(ctx context.Context, s Settings, args ...string) ([]byte, []byte, error) {
|
||||
cmd := CommandFor(ctx, s)
|
||||
cmd.Args = append(cmd.Args, args...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return stdout.Bytes(), stderr.Bytes(), err
|
||||
}
|
||||
return stdout.Bytes(), stderr.Bytes(), nil
|
||||
}
|
||||
|
||||
// LastLines returns the last n non-empty lines of s, joined with " | ".
|
||||
// Useful for compact error messages from yt-dlp output.
|
||||
func LastLines(s string, n int) string {
|
||||
var lines []string
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if t := strings.TrimSpace(l); t != "" {
|
||||
lines = append(lines, t)
|
||||
}
|
||||
}
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, " | ")
|
||||
}
|
||||
|
||||
func num(v any) (float64, bool) {
|
||||
switch x := v.(type) {
|
||||
case float64:
|
||||
return x, true
|
||||
case float32:
|
||||
return float64(x), true
|
||||
case int:
|
||||
return float64(x), true
|
||||
case int64:
|
||||
return float64(x), true
|
||||
case json.Number:
|
||||
f, err := x.Float64()
|
||||
return f, err == nil
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
Reference in New Issue
Block a user