155 lines
2.7 KiB
Go
155 lines
2.7 KiB
Go
package main
|
|
|
|
import (
|
|
"git.wow.st/gmp/ohlc"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
retries = 5
|
|
concurrency = 1
|
|
ch chan string
|
|
clk chan bool
|
|
wg sync.WaitGroup
|
|
|
|
freq = time.Second * 15
|
|
throttle bool
|
|
errs map[string]int
|
|
mu sync.Mutex
|
|
)
|
|
|
|
func get_(symbol string) (error) {
|
|
url := fmt.Sprintf("https://query2.finance.yahoo.com/v8/finance/chart/%s?period1=1&period2=3122064000&interval=1d&includeAdjustedClose=true&events=div,splits,capitalGains", symbol)
|
|
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
if err != nil {
|
|
fmt.Println("Request error (%s): ", symbol, err)
|
|
return err
|
|
}
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
fmt.Println("Get error (%s): ", symbol, err)
|
|
return err
|
|
}
|
|
switch resp.StatusCode {
|
|
case 401, 429, 500:
|
|
throttle = true
|
|
return fmt.Errorf("%s", resp.Status)
|
|
err_(symbol,0) // do not limit retries
|
|
return fmt.Errorf("%s", resp.Status)
|
|
case 404:
|
|
err_(symbol,5) // do not retry
|
|
return fmt.Errorf("%s", resp.Status)
|
|
case 200:
|
|
default:
|
|
err_(symbol,1)
|
|
return fmt.Errorf("%s", resp.Status)
|
|
}
|
|
|
|
filename := fmt.Sprintf("%s.json", symbol)
|
|
out, err := os.Create(filename)
|
|
if err != nil {
|
|
fmt.Println("Error opening output file.")
|
|
os.Exit(-1)
|
|
}
|
|
defer out.Close()
|
|
_, err = io.Copy(out, resp.Body)
|
|
resp.Body.Close()
|
|
if err != nil {
|
|
fmt.Println("Error copying to file.")
|
|
os.Exit(-1)
|
|
}
|
|
out.WriteString("\n")
|
|
out.Close()
|
|
|
|
ohlc.Conv(symbol)
|
|
|
|
return nil
|
|
}
|
|
|
|
func err_(symbol string, x int) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
errs[symbol] = errs[symbol]+x
|
|
str := fmt.Sprintf("Error fetching %s. Retries: %d", symbol, errs[symbol])
|
|
if errs[symbol] < retries {
|
|
go func() {
|
|
wg.Add(1)
|
|
ch <- symbol
|
|
}()
|
|
} else {
|
|
str = fmt.Sprintf("%s: aborting", str)
|
|
}
|
|
fmt.Println(str)
|
|
}
|
|
|
|
func get() {
|
|
<-clk
|
|
symbol := <-ch
|
|
defer wg.Done()
|
|
fmt.Println("Symbol = ", symbol)
|
|
|
|
err := get_(symbol)
|
|
if err != nil {
|
|
fmt.Printf("Get error (%s): %s\n", symbol, err)
|
|
return
|
|
}
|
|
}
|
|
|
|
func clock() {
|
|
for {
|
|
if (throttle) {
|
|
time.Sleep(20 * time.Second)
|
|
throttle = false
|
|
} else {
|
|
clk <- true
|
|
}
|
|
time.Sleep(freq)
|
|
}
|
|
}
|
|
|
|
func main() {
|
|
fmt.Println("goget\n")
|
|
ch = make(chan string)
|
|
clk = make(chan bool)
|
|
errs = make(map[string]int)
|
|
|
|
if len(os.Args) < 2 {
|
|
fmt.Println("Usage: goget symbol1 [symbol2...]")
|
|
os.Exit(-1)
|
|
}
|
|
|
|
go clock()
|
|
for i := 0; i < concurrency; i++ {
|
|
go func() {
|
|
for {
|
|
get()
|
|
}
|
|
}()
|
|
}
|
|
|
|
for _, x := range(os.Args[1:]) {
|
|
wg.Add(1)
|
|
ch <- x
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
var inc []string
|
|
|
|
for k,v := range errs {
|
|
if v >= 5 {
|
|
inc = append(inc, k)
|
|
}
|
|
}
|
|
if len(inc) > 0 {
|
|
fmt.Println("Failed downloads: ", inc)
|
|
}
|
|
os.Exit(0)
|
|
}
|