Add YouTube feed support via yt-dlp, MIME-based extensions, robust state
- ytdlp: thin wrapper around yt-dlp CLI for channel/playlist discovery (--flat-playlist) and per-video metadata (--print-json); defaults to player_client=web_embedded to avoid SABR-only format restrictions. - downloader: YouTube path extracts audio via yt-dlp -x, probes duration with ffprobe and rejects clips far shorter than the expected length. - main: YouTube channel/playlist feeds, incremental discovery with pre-start-date boundary for newest-first channels, two-phase record-then-enqueue so discovered jobs survive crashes. - poller: propagate enclosure MIME type; computeDestPath appends the matching file extension. - state: jobs carry expected duration; AddJobWithStatus for skipped items. - cmd/probe: dry-run discovery/date-resolution diagnostics.
This commit is contained in:
+24
-7
@@ -32,15 +32,17 @@ type Job struct {
|
||||
MaxRetries int `json:"max_retries"`
|
||||
NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"` // nil means ready now
|
||||
Error string `json:"error,omitempty"`
|
||||
Duration float64 `json:"duration,omitempty"` // expected media duration in seconds (YouTube)
|
||||
}
|
||||
|
||||
// JobStatus values.
|
||||
const (
|
||||
StatusPending = "pending"
|
||||
StatusPending = "pending"
|
||||
StatusDownloading = "downloading"
|
||||
StatusRetrying = "retrying"
|
||||
StatusDownloaded = "downloaded"
|
||||
StatusFailed = "failed"
|
||||
StatusRetrying = "retrying"
|
||||
StatusDownloaded = "downloaded"
|
||||
StatusFailed = "failed"
|
||||
StatusSkipped = "skipped" // resolved but excluded by start_date; never downloaded
|
||||
)
|
||||
|
||||
// Manager provides thread-safe access to persisted state.
|
||||
@@ -132,6 +134,7 @@ type JobSnapshot struct {
|
||||
MaxRetries int
|
||||
NextAttemptAt *time.Time // nil means ready now
|
||||
Error string
|
||||
Duration float64
|
||||
}
|
||||
|
||||
// ForEachJob iterates over all jobs and calls fn for each one.
|
||||
@@ -153,6 +156,7 @@ func (m *Manager) ForEachJob(fn func(*JobSnapshot)) {
|
||||
MaxRetries: j.MaxRetries,
|
||||
NextAttemptAt: j.NextAttemptAt,
|
||||
Error: j.Error,
|
||||
Duration: j.Duration,
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
@@ -170,8 +174,20 @@ func (m *Manager) State() *State {
|
||||
}
|
||||
|
||||
// AddJob adds a new download job if one with this enclosure URL doesn't already exist.
|
||||
// Returns true if the job was added, false if it already existed.
|
||||
func (m *Manager) AddJob(feedURL, enclosureURL, destPath string, maxRetries int) bool {
|
||||
// duration is the expected media duration in seconds (0 = unknown, used for
|
||||
// YouTube duration sanity checks). Returns true if the job was added.
|
||||
func (m *Manager) AddJob(feedURL, enclosureURL, destPath string, maxRetries int, duration float64) bool {
|
||||
return m.addJob(feedURL, enclosureURL, destPath, StatusPending, maxRetries, duration)
|
||||
}
|
||||
|
||||
// AddJobWithStatus adds a new job with an explicit initial status (used to
|
||||
// record videos resolved but excluded by the start_date filter, so they are
|
||||
// not re-resolved on every poll). Returns true if the job was added.
|
||||
func (m *Manager) AddJobWithStatus(feedURL, enclosureURL, destPath, status string, duration float64) bool {
|
||||
return m.addJob(feedURL, enclosureURL, destPath, status, 0, duration)
|
||||
}
|
||||
|
||||
func (m *Manager) addJob(feedURL, enclosureURL, destPath, status string, maxRetries int, duration float64) bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
@@ -186,11 +202,12 @@ func (m *Manager) AddJob(feedURL, enclosureURL, destPath string, maxRetries int)
|
||||
FeedURL: feedURL,
|
||||
EnclosureURL: enclosureURL,
|
||||
DestPath: destPath,
|
||||
Status: StatusPending,
|
||||
Status: status,
|
||||
AttemptCount: 0,
|
||||
MaxRetries: maxRetries,
|
||||
NextAttemptAt: nil,
|
||||
Error: "",
|
||||
Duration: duration,
|
||||
}
|
||||
|
||||
m.state.Jobs = append(m.state.Jobs, job)
|
||||
|
||||
Reference in New Issue
Block a user