package main import ( "git.wow.st/gmp/ohlc" "fmt" "io" "net/http" "os" "sync" "time" ) var ( retries = 5 concurrency = 1 ch chan string clk chan bool wg sync.WaitGroup freq = time.Second * 15 throttle bool errs map[string]int mu sync.Mutex ) func get_(symbol string) (error) { url := fmt.Sprintf("https://query2.finance.yahoo.com/v8/finance/chart/%s?period1=1&period2=3122064000&interval=1d&includeAdjustedClose=true&events=div,splits,capitalGains", symbol) req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { fmt.Println("Request error (%s): ", symbol, err) return err } req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") resp, err := http.DefaultClient.Do(req) if err != nil { fmt.Println("Get error (%s): ", symbol, err) return err } switch resp.StatusCode { case 401, 429, 500: throttle = true return fmt.Errorf("%s", resp.Status) err_(symbol,0) // do not limit retries return fmt.Errorf("%s", resp.Status) case 404: err_(symbol,5) // do not retry return fmt.Errorf("%s", resp.Status) case 200: default: err_(symbol,1) return fmt.Errorf("%s", resp.Status) } filename := fmt.Sprintf("%s.json", symbol) out, err := os.Create(filename) if err != nil { fmt.Println("Error opening output file.") os.Exit(-1) } defer out.Close() _, err = io.Copy(out, resp.Body) resp.Body.Close() if err != nil { fmt.Println("Error copying to file.") os.Exit(-1) } out.WriteString("\n") out.Close() ohlc.Conv(symbol) return nil } func err_(symbol string, x int) { mu.Lock() defer mu.Unlock() errs[symbol] = errs[symbol]+x str := fmt.Sprintf("Error fetching %s. Retries: %d", symbol, errs[symbol]) if errs[symbol] < retries { go func() { wg.Add(1) ch <- symbol }() } else { str = fmt.Sprintf("%s: aborting", str) } fmt.Println(str) } func get() { <-clk symbol := <-ch defer wg.Done() fmt.Println("Symbol = ", symbol) err := get_(symbol) if err != nil { fmt.Printf("Get error (%s): %s\n", symbol, err) return } } func clock() { for { if (throttle) { time.Sleep(20 * time.Second) throttle = false } else { clk <- true } time.Sleep(freq) } } func main() { fmt.Println("goget\n") ch = make(chan string) clk = make(chan bool) errs = make(map[string]int) if len(os.Args) < 2 { fmt.Println("Usage: goget symbol1 [symbol2...]") os.Exit(-1) } go clock() for i := 0; i < concurrency; i++ { go func() { for { get() } }() } for _, x := range(os.Args[1:]) { wg.Add(1) ch <- x } wg.Wait() var inc []string for k,v := range errs { if v >= 5 { inc = append(inc, k) } } if len(inc) > 0 { fmt.Println("Failed downloads: ", inc) } os.Exit(0) }