From 882486c0406e3851c6a3a5898459f67ba617b7cf Mon Sep 17 00:00:00 2001 From: Greg Pomerantz Date: Mon, 31 Aug 2026 16:08:10 -0400 Subject: [PATCH] Format dates in the exchange timezone; sort and dedup events - Dates were formatted with time.Unix (machine local zone), shifting the trading day by one for any exchange in a different timezone. Use meta.exchangeTimezoneName (IANA), falling back to meta.gmtoffset, then UTC. - Dividend/capital-gain/split CSVs were emitted in random order (JSON object iteration). Sort rows by date. - Yahoo occasionally emits the same chart timestamp twice; keep the last occurrence (this drops the spurious CVSIX 2008-12-18 0.292 row whose date is duplicated by the real 0.641 distribution). - nil volume no longer panics (prints 0). - Missing dividend/cap-gain amount still reports an error. --- main.go | 230 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 149 insertions(+), 81 deletions(-) diff --git a/main.go b/main.go index c97254d..8881201 100644 --- a/main.go +++ b/main.go @@ -4,9 +4,71 @@ import ( "encoding/json" "fmt" "os" + "sort" "time" ) +// exchangeTimezone returns the exchange's IANA zone (from meta), falling back +// to the most recent gmtoffset, then UTC. The old code formatted dates in the +// machine's local timezone, which shifts dates by a day for any exchange in +// a different zone. +func exchangeTimezone(result map[string]interface{}) *time.Location { + if meta, ok := result["meta"].(map[string]interface{}); ok { + if name, ok := meta["exchangeTimezoneName"].(string); ok && name != "" { + if loc, err := time.LoadLocation(name); err == nil { + return loc + } + } + if off, ok := meta["gmtoffset"].(float64); ok { + return time.FixedZone("gmt", int(off)) + } + } + return time.UTC +} + +// eventRow is one dividend/capital-gain/split, flattened for sorting. +type eventRow struct { + epoch float64 + date string + amount float64 + ratio string // splits only +} + +// eventRows flattens an events map, sorts rows by date (JSON object iteration +// order is random, so sorting is required for stable output) and formats each +// date in the exchange zone. kind is used in error messages. +func eventRows(name, kind string, events map[string]interface{}, loc *time.Location) []eventRow { + rows := make([]eventRow, 0, len(events)) + for _, v := range events { + v2, ok := v.(map[string]interface{}) + if !ok { + fmt.Println(name + ": Bad format (" + kind + ")") + return nil + } + t, ok := v2["date"].(float64) + if !ok { + fmt.Println(name + ": Bad format (" + kind + " date)") + return nil + } + r := eventRow{epoch: t, date: time.Unix(int64(t), 0).In(loc).Format("2006-01-02")} + if v2["amount"] != nil { + r.amount = v2["amount"].(float64) + } else if kind != "split" { + fmt.Println(name + ": Bad format (" + kind + " amount)") + return nil + } + if v2["splitRatio"] != nil { + r.ratio = v2["splitRatio"].(string) + } else if kind == "split" { + fmt.Println(name + ": Bad format (split ratio)") + return nil + } + rows = append(rows, r) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].epoch < rows[j].epoch }) + return rows +} + func Conv(name string) { text, err := os.ReadFile(name + ".json") if err != nil { @@ -22,22 +84,40 @@ func Conv(name string) { } j, ok := i.(map[string]interface{}) - if !ok { fmt.Println(name + ": Bad format"); return } + if !ok { + fmt.Println(name + ": Bad format") + return + } j, ok = j["chart"].(map[string]interface{}) - if !ok { fmt.Println(name + ": Bad format (chart)"); return } + if !ok { + fmt.Println(name + ": Bad format (chart)") + return + } x, ok := j["result"].([]interface{}) - if !ok { fmt.Println(name + ": Bad format (result)"); return } + if !ok { + fmt.Println(name + ": Bad format (result)") + return + } result, ok := x[0].(map[string]interface{}) - if !ok { fmt.Println(name + ": Bad format (result[0])"); return } + if !ok { + fmt.Println(name + ": Bad format (result[0])") + return + } timestamp, ok := result["timestamp"].([]interface{}) - if !ok { fmt.Println(name + ": Bad format (timestamp)"); return } + if !ok { + fmt.Println(name + ": Bad format (timestamp)") + return + } indicators, ok := result["indicators"].(map[string]interface{}) - if !ok { fmt.Println(name + ": Bad format (indicators)"); return } + if !ok { + fmt.Println(name + ": Bad format (indicators)") + return + } events, ok := result["events"].(map[string]interface{}) @@ -48,34 +128,66 @@ func Conv(name string) { splits, ok := events["splits"].(map[string]interface{}) x, ok = indicators["quote"].([]interface{}) - if !ok { fmt.Println(name + ": Bad format (quote)"); return } + if !ok { + fmt.Println(name + ": Bad format (quote)") + return + } quote, ok := x[0].(map[string]interface{}) - if !ok { fmt.Println(name + ": Bad format (quote[0])"); return } + if !ok { + fmt.Println(name + ": Bad format (quote[0])") + return + } x, ok = indicators["adjclose"].([]interface{}) - if !ok { fmt.Println(name + ": Bad format (adjclose)"); return } + if !ok { + fmt.Println(name + ": Bad format (adjclose)") + return + } a, ok := x[0].(map[string]interface{}) - if !ok { fmt.Println(name + ": Bad format (adjclose[0])"); return } + if !ok { + fmt.Println(name + ": Bad format (adjclose[0])") + return + } adjclose, ok := a["adjclose"].([]interface{}) - if !ok { fmt.Println(name + ": Bad format (adjclose[adjclose])"); return } + if !ok { + fmt.Println(name + ": Bad format (adjclose[adjclose])") + return + } open, ok := quote["open"].([]interface{}) - if !ok { fmt.Println(name + ": Bad format (open)"); return } + if !ok { + fmt.Println(name + ": Bad format (open)") + return + } high, ok := quote["high"].([]interface{}) - if !ok { fmt.Println(name + ": Bad format (high)"); return } + if !ok { + fmt.Println(name + ": Bad format (high)") + return + } low, ok := quote["low"].([]interface{}) - if !ok { fmt.Println(name + ": Bad format (low)"); return } + if !ok { + fmt.Println(name + ": Bad format (low)") + return + } close, ok := quote["close"].([]interface{}) - if !ok { fmt.Println(name + ": Bad format (close)"); return } + if !ok { + fmt.Println(name + ": Bad format (close)") + return + } volume, ok := quote["volume"].([]interface{}) - if !ok { fmt.Println(name + ": Bad format (volume)"); return } + if !ok { + fmt.Println(name + ": Bad format (volume)") + return + } + + loc := exchangeTimezone(result) out, err := os.Create(name + "-history.csv") if err != nil { @@ -85,15 +197,26 @@ func Conv(name string) { defer out.Close() out.WriteString("Date,Open,High,Low,Close,Adj Close,Volume\n") + // Yahoo occasionally emits the same timestamp twice; keep the last one. + seen := make(map[int64]int, len(timestamp)) for k, v := range timestamp { - unixTime := time.Unix(int64(v.(float64)),0) + seen[int64(v.(float64))] = k + } + for k, v := range timestamp { + if seen[int64(v.(float64))] != k { + continue + } if open[k] == nil { continue } + vol := 0 + if volume[k] != nil { + vol = int(volume[k].(float64)) + } out.WriteString(fmt.Sprintf("%s,%f,%f,%f,%f,%f,%d\n", - unixTime.Format("2006-01-02"), - open[k],high[k],low[k],close[k], - adjclose[k],int(volume[k].(float64)))) + time.Unix(int64(v.(float64)), 0).In(loc).Format("2006-01-02"), + open[k], high[k], low[k], close[k], + adjclose[k], vol)) } if dividends != nil { out, err = os.Create(name + "-dividend.csv") @@ -105,26 +228,8 @@ func Conv(name string) { defer out.Close() out.WriteString("Date,Dividends\n") - for _,v := range dividends { - v2, ok := v.(map[string]interface{}) - if !ok { - fmt.Println(name + ": Bad format (dividends)") - return - } - amount, ok := v2["amount"] - if !ok { - fmt.Println(name + ": Bad format (dividend amount)") - return - } - t, ok := v2["date"].(float64) - if !ok { - fmt.Println(name + ": Bad format (dividend date)") - return - } - unixTime := time.Unix(int64(t),0) - out.WriteString(fmt.Sprintf("%s,%f\n", - unixTime.Format("2006-01-02"), - amount.(float64))) + for _, r := range eventRows(name, "dividend", dividends, loc) { + out.WriteString(fmt.Sprintf("%s,%f\n", r.date, r.amount)) } } @@ -138,26 +243,8 @@ func Conv(name string) { defer out.Close() out.WriteString("Date,Capital Gains\n") - for _,v := range capgains { - v2, ok := v.(map[string]interface{}) - if !ok { - fmt.Println(name + ": Bad format (capital gains)") - return - } - amount, ok := v2["amount"] - if !ok { - fmt.Println(name + ": Bad format (capital gains amount)") - return - } - t, ok := v2["date"].(float64) - if !ok { - fmt.Println(name + ": Bad format (capital gains date)") - return - } - unixTime := time.Unix(int64(t),0) - out.WriteString(fmt.Sprintf("%s,%f\n", - unixTime.Format("2006-01-02"), - amount.(float64))) + for _, r := range eventRows(name, "capital gain", capgains, loc) { + out.WriteString(fmt.Sprintf("%s,%f\n", r.date, r.amount)) } } @@ -171,27 +258,8 @@ func Conv(name string) { defer out.Close() out.WriteString("Date,Stock Splits\n") - for _,v := range splits { - v2, ok := v.(map[string]interface{}) - if !ok { - fmt.Println(name + ": Bad format (splits)") - return - } - splitRatio, ok := v2["splitRatio"] - if !ok { - fmt.Println(name + ": Bad format (split ratio)") - return - } - t, ok := v2["date"].(float64) - if !ok { - fmt.Println(name + ": Bad format (split date)") - return - } - unixTime := time.Unix(int64(t),0) - out.WriteString(fmt.Sprintf("%s,%s\n", - unixTime.Format("2006-01-02"), - splitRatio)) + for _, r := range eventRows(name, "split", splits, loc) { + out.WriteString(fmt.Sprintf("%s,%s\n", r.date, r.ratio)) } } } -