Add YouTube feed support via yt-dlp, MIME-based extensions, robust state

- ytdlp: thin wrapper around yt-dlp CLI for channel/playlist discovery
  (--flat-playlist) and per-video metadata (--print-json); defaults to
  player_client=web_embedded to avoid SABR-only format restrictions.
- downloader: YouTube path extracts audio via yt-dlp -x, probes duration
  with ffprobe and rejects clips far shorter than the expected length.
- main: YouTube channel/playlist feeds, incremental discovery with
  pre-start-date boundary for newest-first channels, two-phase
  record-then-enqueue so discovered jobs survive crashes.
- poller: propagate enclosure MIME type; computeDestPath appends the
  matching file extension.
- state: jobs carry expected duration; AddJobWithStatus for skipped items.
- cmd/probe: dry-run discovery/date-resolution diagnostics.
This commit is contained in:
gmp
2026-09-23 09:06:36 -04:00
parent 6f188d1939
commit bc26e73a90
7 changed files with 768 additions and 49 deletions
+13 -2
View File
@@ -12,7 +12,8 @@ import (
// EnclosureInfo carries an enclosure URL and its associated item's metadata.
type EnclosureInfo struct {
URL string
Title string // episode title, if available
Title string // episode title, if available
MimeType string // MIME type from the enclosure (e.g. "audio/mpeg")
PublishedAt time.Time // zero value means no date available
}
@@ -26,6 +27,11 @@ type FeedResult struct {
Error error
}
// UserAgent is sent on all outbound HTTP requests (feed fetches and
// enclosure downloads). Some hosts (e.g. Buzzsprout) reject Go's default
// "Go-http-client" UA with 403.
const UserAgent = "Mozilla/5.0 (compatible; rssd/1.0; +https://github.com/rssd)"
// Fetcher abstracts the HTTP fetch + parse cycle for a single feed.
type Fetcher struct {
httpClient *http.Client
@@ -50,6 +56,7 @@ func (f *Fetcher) Poll(url, lastModified, etag string) (*FeedResult, error) {
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("User-Agent", UserAgent)
if etag != "" {
req.Header.Set("If-None-Match", etag)
@@ -91,7 +98,11 @@ func (f *Fetcher) Poll(url, lastModified, etag string) (*FeedResult, error) {
for _, item := range feed.Items {
for _, enc := range item.Enclosures {
if enc.URL != "" {
encInfo := EnclosureInfo{URL: enc.URL, Title: item.Title}
encInfo := EnclosureInfo{
URL: enc.URL,
Title: item.Title,
MimeType: enc.Type,
}
if item.PublishedParsed != nil {
encInfo.PublishedAt = *item.PublishedParsed
}