goget/main.go

package main

import (
	"git.wow.st/gmp/ohlc"
	"fmt"
	"io"
	"net/http"
	"os"
	"sync"
	"time"
)

var (
	retries = 5
	concurrency = 1
	ch chan string
	clk chan bool
	wg sync.WaitGroup

	freq = time.Second * 15
	throttle bool
	errs map[string]int
	mu sync.Mutex
)

func get_(symbol string) (error) {
	url := fmt.Sprintf("https://query2.finance.yahoo.com/v8/finance/chart/%s?period1=1&period2=3122064000&interval=1d&includeAdjustedClose=true&events=div,splits,capitalGains", symbol)
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		fmt.Println("Request error (%s): ", symbol, err)
		return err
	}
	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Println("Get error (%s): ", symbol, err)
		return err
	}
	switch resp.StatusCode {
	case 401, 429, 500:
		throttle = true
		return fmt.Errorf("%s", resp.Status)
		err_(symbol,0) // do not limit retries
		return fmt.Errorf("%s", resp.Status)
	case 404:
		err_(symbol,5) // do not retry
		return fmt.Errorf("%s", resp.Status)
	case 200:
	default:
		err_(symbol,1)
		return fmt.Errorf("%s", resp.Status)
	}

	filename := fmt.Sprintf("%s.json", symbol)
	out, err := os.Create(filename)
	if err != nil {
		fmt.Println("Error opening output file.")
		os.Exit(-1)
	}
	defer out.Close()
	_, err = io.Copy(out, resp.Body)
	resp.Body.Close()
	if err != nil {
		fmt.Println("Error copying to file.")
		os.Exit(-1)
	}
	out.WriteString("\n")
	out.Close()

	ohlc.Conv(symbol)

	return nil
}

func err_(symbol string, x int) {
	mu.Lock()
	defer mu.Unlock()
	errs[symbol] = errs[symbol]+x
	str := fmt.Sprintf("Error fetching %s. Retries: %d", symbol, errs[symbol])
	if errs[symbol] < retries {
		go func() {
			wg.Add(1)
			ch <- symbol
		}()
	} else {
		str = fmt.Sprintf("%s: aborting", str)
	}
	fmt.Println(str)
}

func get() {
	<-clk
	symbol := <-ch
	defer wg.Done()
	fmt.Println("Symbol = ", symbol)

	err := get_(symbol)
	if err != nil {
		fmt.Printf("Get error (%s): %s\n", symbol, err)
		return
	}
}

func clock() {
	for {
		if (throttle) {
			time.Sleep(20 * time.Second)
			throttle = false
		} else {
			clk <- true
		}
		time.Sleep(freq)
	}
}

func main() {
	fmt.Println("goget\n")
	ch = make(chan string)
	clk = make(chan bool)
	errs = make(map[string]int)

	if len(os.Args) < 2 {
		fmt.Println("Usage: goget symbol1 [symbol2...]")
		os.Exit(-1)
	}

	go clock()
	for i := 0; i < concurrency; i++ {
		go func() {
			for {
				get()
			}
		}()
	}

	for _, x := range(os.Args[1:]) {
		wg.Add(1)
		ch <- x
	}

	wg.Wait()

	var inc []string

	for k,v := range errs {
		if v >= 5 {
			inc = append(inc, k)
		}
	}
	if len(inc) > 0 {
		fmt.Println("Failed downloads: ", inc)
	}
	os.Exit(0)
}