first commit

go.mod

@@ -0,0 +1,5 @@
+module git.wow.st/gmp/goget
+
+go 1.23.0
+
+require git.wow.st/gmp/ohlc v0.0.0-20241115023606-83f41a06596c // indirect

go.sum

@@ -0,0 +1,2 @@
+git.wow.st/gmp/ohlc v0.0.0-20241115023606-83f41a06596c h1:etBdVfh2E4k00IzRBMuHsGm5HZg7S96JBZQpCGWshNE=
+git.wow.st/gmp/ohlc v0.0.0-20241115023606-83f41a06596c/go.mod h1:QSSqMCu9AFoYtfqA9Hbfpxy3mp5MT/d4fk0AzYnuoo8=

main.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"git.wow.st/gmp/ohlc"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+)
+
+var (
+	retries = 5
+	concurrency = 30
+	ch chan string
+	clk chan bool
+	wg sync.WaitGroup
+
+	freq = time.Second / 30
+	throttle bool
+	errs map[string]int
+	mu sync.Mutex
+)
+
+func get_(symbol string) (error) {
+	url := fmt.Sprintf("https://query2.finance.yahoo.com/v8/finance/chart/%s?period1=1&period2=3122064000&interval=1d&includeAdjustedClose=true&events=div,splits,capitalGains", symbol)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		fmt.Println("Request error (%s): ", symbol, err)
+		return err
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		fmt.Println("Get error (%s): ", symbol, err)
+		return err
+	}
+	switch resp.StatusCode {
+	case 401, 429:
+		throttle = true
+		return fmt.Errorf("%s", resp.Status)
+		err_(symbol,0) // do not limit retries
+		return fmt.Errorf("%s", resp.Status)
+	case 404:
+		err_(symbol,5) // do not retry
+		return fmt.Errorf("%s", resp.Status)
+	case 200:
+	default:
+		err_(symbol,1)
+		return fmt.Errorf("%s", resp.Status)
+	}
+
+	filename := fmt.Sprintf("%s.json", symbol)
+	out, err := os.Create(filename)
+	if err != nil {
+		fmt.Println("Error opening output file.")
+		os.Exit(-1)
+	}
+	defer out.Close()
+	_, err = io.Copy(out, resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		fmt.Println("Error copying to file.")
+		os.Exit(-1)
+	}
+	out.WriteString("\n")
+	out.Close()
+
+	ohlc.Conv(symbol)
+
+	return nil
+}
+
+func err_(symbol string, x int) {
+	mu.Lock()
+	defer mu.Unlock()
+	errs[symbol] = errs[symbol]+x
+	str := fmt.Sprintf("Error fetching %s. Retries: %d", symbol, errs[symbol])
+	if errs[symbol] < retries {
+		go func() {
+			wg.Add(1)
+			ch <- symbol
+		}()
+	} else {
+		str = fmt.Sprintf("%s: aborting", str)
+	}
+	fmt.Println(str)
+}
+
+func get() {
+	<-clk
+	symbol := <-ch
+	defer wg.Done()
+	fmt.Println("Symbol = ", symbol)
+
+	err := get_(symbol)
+	if err != nil {
+		fmt.Printf("Get error (%s): %s\n", symbol, err)
+		return
+	}
+}
+
+func clock() {
+	for {
+		if (throttle) {
+			time.Sleep(20 * time.Second)
+			throttle = false
+		} else {
+			clk <- true
+		}
+		time.Sleep(freq)
+	}
+}
+
+func main() {
+	fmt.Println("goget\n")
+	ch = make(chan string)
+	clk = make(chan bool)
+	errs = make(map[string]int)
+
+	if len(os.Args) < 2 {
+		fmt.Println("Usage: goget symbol1 [symbol2...]")
+		os.Exit(-1)
+	}
+
+	go clock()
+	for i := 0; i < concurrency; i++ {
+		go func() {
+			for {
+				get()
+			}
+		}()
+	}
+
+	for _, x := range(os.Args[1:]) {
+		wg.Add(1)
+		ch <- x
+	}
+
+	wg.Wait()
+
+	var inc []string
+
+	for k,v := range errs {
+		if v >= 5 {
+			inc = append(inc, k)
+		}
+	}
+	if len(inc) > 0 {
+		fmt.Println("Failed downloads: ", inc)
+	}
+	os.Exit(0)
+}