Add YouTube feed support via yt-dlp, MIME-based extensions, robust state

- ytdlp: thin wrapper around yt-dlp CLI for channel/playlist discovery
  (--flat-playlist) and per-video metadata (--print-json); defaults to
  player_client=web_embedded to avoid SABR-only format restrictions.
- downloader: YouTube path extracts audio via yt-dlp -x, probes duration
  with ffprobe and rejects clips far shorter than the expected length.
- main: YouTube channel/playlist feeds, incremental discovery with
  pre-start-date boundary for newest-first channels, two-phase
  record-then-enqueue so discovered jobs survive crashes.
- poller: propagate enclosure MIME type; computeDestPath appends the
  matching file extension.
- state: jobs carry expected duration; AddJobWithStatus for skipped items.
- cmd/probe: dry-run discovery/date-resolution diagnostics.
This commit is contained in:
gmp
2026-09-23 09:06:36 -04:00
parent 6f188d1939
commit bc26e73a90
7 changed files with 768 additions and 49 deletions
+24 -7
View File
@@ -32,15 +32,17 @@ type Job struct {
MaxRetries int `json:"max_retries"`
NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"` // nil means ready now
Error string `json:"error,omitempty"`
Duration float64 `json:"duration,omitempty"` // expected media duration in seconds (YouTube)
}
// JobStatus values.
const (
StatusPending = "pending"
StatusPending = "pending"
StatusDownloading = "downloading"
StatusRetrying = "retrying"
StatusDownloaded = "downloaded"
StatusFailed = "failed"
StatusRetrying = "retrying"
StatusDownloaded = "downloaded"
StatusFailed = "failed"
StatusSkipped = "skipped" // resolved but excluded by start_date; never downloaded
)
// Manager provides thread-safe access to persisted state.
@@ -132,6 +134,7 @@ type JobSnapshot struct {
MaxRetries int
NextAttemptAt *time.Time // nil means ready now
Error string
Duration float64
}
// ForEachJob iterates over all jobs and calls fn for each one.
@@ -153,6 +156,7 @@ func (m *Manager) ForEachJob(fn func(*JobSnapshot)) {
MaxRetries: j.MaxRetries,
NextAttemptAt: j.NextAttemptAt,
Error: j.Error,
Duration: j.Duration,
}
}
m.mu.Unlock()
@@ -170,8 +174,20 @@ func (m *Manager) State() *State {
}
// AddJob adds a new download job if one with this enclosure URL doesn't already exist.
// Returns true if the job was added, false if it already existed.
func (m *Manager) AddJob(feedURL, enclosureURL, destPath string, maxRetries int) bool {
// duration is the expected media duration in seconds (0 = unknown, used for
// YouTube duration sanity checks). Returns true if the job was added.
func (m *Manager) AddJob(feedURL, enclosureURL, destPath string, maxRetries int, duration float64) bool {
return m.addJob(feedURL, enclosureURL, destPath, StatusPending, maxRetries, duration)
}
// AddJobWithStatus adds a new job with an explicit initial status (used to
// record videos resolved but excluded by the start_date filter, so they are
// not re-resolved on every poll). Returns true if the job was added.
func (m *Manager) AddJobWithStatus(feedURL, enclosureURL, destPath, status string, duration float64) bool {
return m.addJob(feedURL, enclosureURL, destPath, status, 0, duration)
}
func (m *Manager) addJob(feedURL, enclosureURL, destPath, status string, maxRetries int, duration float64) bool {
m.mu.Lock()
defer m.mu.Unlock()
@@ -186,11 +202,12 @@ func (m *Manager) AddJob(feedURL, enclosureURL, destPath string, maxRetries int)
FeedURL: feedURL,
EnclosureURL: enclosureURL,
DestPath: destPath,
Status: StatusPending,
Status: status,
AttemptCount: 0,
MaxRetries: maxRetries,
NextAttemptAt: nil,
Error: "",
Duration: duration,
}
m.state.Jobs = append(m.state.Jobs, job)