Initial commit.
This commit is contained in:
@@ -0,0 +1,140 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"rssd/state"
|
||||
)
|
||||
|
||||
// DownloadJob carries all information needed for a single download.
|
||||
type DownloadJob struct {
|
||||
FeedURL string
|
||||
EnclosureURL string
|
||||
DestPath string
|
||||
}
|
||||
|
||||
// Pool manages a fixed-size worker pool for downloading files.
|
||||
type Pool struct {
|
||||
poolSize int
|
||||
backoffBase time.Duration
|
||||
state *state.Manager
|
||||
}
|
||||
|
||||
// NewPool creates a download pool with the given parameters.
|
||||
func NewPool(poolSize int, backoffBase time.Duration, sm *state.Manager) *Pool {
|
||||
return &Pool{poolSize: poolSize, backoffBase: backoffBase, state: sm}
|
||||
}
|
||||
|
||||
// Run starts the worker goroutines. It returns immediately after spawning the workers.
|
||||
func (p *Pool) Run(jobs <-chan DownloadJob) {
|
||||
for i := 0; i < p.poolSize; i++ {
|
||||
go func() {
|
||||
for job := range jobs {
|
||||
p.worker(job)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Pool) worker(job DownloadJob) {
|
||||
fmt.Fprintf(os.Stderr, "rssd: worker started download %s\n", job.EnclosureURL)
|
||||
if err := p.download(job); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "ERROR: download %s failed: %v\n", job.EnclosureURL, err)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "OK: downloaded %s -> %s\n", job.EnclosureURL, job.DestPath)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "rssd: worker finished download %s\n", job.EnclosureURL)
|
||||
}
|
||||
|
||||
func (p *Pool) download(job DownloadJob) error {
|
||||
// Update state to downloading.
|
||||
if !p.state.UpdateJob(job.EnclosureURL, func(j *state.Job) {
|
||||
j.Status = state.StatusDownloading
|
||||
j.AttemptCount++
|
||||
}) {
|
||||
return fmt.Errorf("job not found for %s", job.EnclosureURL)
|
||||
}
|
||||
if err := p.state.Save(); err != nil {
|
||||
return fmt.Errorf("save state (downloading): %w", err)
|
||||
}
|
||||
|
||||
// Ensure output directory exists.
|
||||
if err := os.MkdirAll(filepath.Dir(job.DestPath), 0755); err != nil {
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("create dest dir: %w", err))
|
||||
return err
|
||||
}
|
||||
|
||||
// Create destination file.
|
||||
dstFile, err := os.Create(job.DestPath)
|
||||
if err != nil {
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("create dest file: %w", err))
|
||||
return err
|
||||
}
|
||||
|
||||
// Download with timeout.
|
||||
client := &http.Client{Timeout: 30 * time.Minute}
|
||||
req, err := http.NewRequest("GET", job.EnclosureURL, nil)
|
||||
if err != nil {
|
||||
dstFile.Close()
|
||||
os.Remove(job.DestPath)
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("create request: %w", err))
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
dstFile.Close()
|
||||
os.Remove(job.DestPath)
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("fetch: %w", err))
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
dstFile.Close()
|
||||
os.Remove(job.DestPath)
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("HTTP %d", resp.StatusCode))
|
||||
return fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Copy body to file.
|
||||
if _, err := io.Copy(dstFile, resp.Body); err != nil {
|
||||
dstFile.Close()
|
||||
os.Remove(job.DestPath)
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("write file: %w", err))
|
||||
return err
|
||||
}
|
||||
if err := dstFile.Close(); err != nil {
|
||||
os.Remove(job.DestPath)
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("close file: %w", err))
|
||||
return err
|
||||
}
|
||||
|
||||
// Mark as downloaded.
|
||||
p.state.UpdateJob(job.EnclosureURL, func(j *state.Job) {
|
||||
j.Status = state.StatusDownloaded
|
||||
})
|
||||
if err := p.state.Save(); err != nil {
|
||||
return fmt.Errorf("save state (downloaded): %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// markRetry updates a job to retrying status with the given error and computes backoff delay.
|
||||
func (p *Pool) markRetry(enclosureURL string, err error) {
|
||||
p.state.UpdateJob(enclosureURL, func(j *state.Job) {
|
||||
j.Status = state.StatusRetrying
|
||||
j.Error = err.Error()
|
||||
// Exponential backoff: base * 2^attemptCount (attempt already incremented).
|
||||
delay := p.backoffBase * time.Duration(1<<uint(j.AttemptCount-1))
|
||||
t := time.Now().Add(delay)
|
||||
j.NextAttemptAt = &t
|
||||
})
|
||||
p.state.Save()
|
||||
}
|
||||
Reference in New Issue
Block a user