Merge pull request 'Format dates in the exchange timezone; sort and dedup events' (#1) from tz-and-event-ordering into main
Reviewed-on: #1
This commit is contained in:
commit
584ec8753c
228
main.go
228
main.go
|
|
@ -4,9 +4,71 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// exchangeTimezone returns the exchange's IANA zone (from meta), falling back
|
||||
// to the most recent gmtoffset, then UTC. The old code formatted dates in the
|
||||
// machine's local timezone, which shifts dates by a day for any exchange in
|
||||
// a different zone.
|
||||
func exchangeTimezone(result map[string]interface{}) *time.Location {
|
||||
if meta, ok := result["meta"].(map[string]interface{}); ok {
|
||||
if name, ok := meta["exchangeTimezoneName"].(string); ok && name != "" {
|
||||
if loc, err := time.LoadLocation(name); err == nil {
|
||||
return loc
|
||||
}
|
||||
}
|
||||
if off, ok := meta["gmtoffset"].(float64); ok {
|
||||
return time.FixedZone("gmt", int(off))
|
||||
}
|
||||
}
|
||||
return time.UTC
|
||||
}
|
||||
|
||||
// eventRow is one dividend/capital-gain/split, flattened for sorting.
|
||||
type eventRow struct {
|
||||
epoch float64
|
||||
date string
|
||||
amount float64
|
||||
ratio string // splits only
|
||||
}
|
||||
|
||||
// eventRows flattens an events map, sorts rows by date (JSON object iteration
|
||||
// order is random, so sorting is required for stable output) and formats each
|
||||
// date in the exchange zone. kind is used in error messages.
|
||||
func eventRows(name, kind string, events map[string]interface{}, loc *time.Location) []eventRow {
|
||||
rows := make([]eventRow, 0, len(events))
|
||||
for _, v := range events {
|
||||
v2, ok := v.(map[string]interface{})
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (" + kind + ")")
|
||||
return nil
|
||||
}
|
||||
t, ok := v2["date"].(float64)
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (" + kind + " date)")
|
||||
return nil
|
||||
}
|
||||
r := eventRow{epoch: t, date: time.Unix(int64(t), 0).In(loc).Format("2006-01-02")}
|
||||
if v2["amount"] != nil {
|
||||
r.amount = v2["amount"].(float64)
|
||||
} else if kind != "split" {
|
||||
fmt.Println(name + ": Bad format (" + kind + " amount)")
|
||||
return nil
|
||||
}
|
||||
if v2["splitRatio"] != nil {
|
||||
r.ratio = v2["splitRatio"].(string)
|
||||
} else if kind == "split" {
|
||||
fmt.Println(name + ": Bad format (split ratio)")
|
||||
return nil
|
||||
}
|
||||
rows = append(rows, r)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].epoch < rows[j].epoch })
|
||||
return rows
|
||||
}
|
||||
|
||||
func Conv(name string) {
|
||||
text, err := os.ReadFile(name + ".json")
|
||||
if err != nil {
|
||||
|
|
@ -22,22 +84,40 @@ func Conv(name string) {
|
|||
}
|
||||
|
||||
j, ok := i.(map[string]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format")
|
||||
return
|
||||
}
|
||||
|
||||
j, ok = j["chart"].(map[string]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (chart)"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (chart)")
|
||||
return
|
||||
}
|
||||
|
||||
x, ok := j["result"].([]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (result)"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (result)")
|
||||
return
|
||||
}
|
||||
|
||||
result, ok := x[0].(map[string]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (result[0])"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (result[0])")
|
||||
return
|
||||
}
|
||||
|
||||
timestamp, ok := result["timestamp"].([]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (timestamp)"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (timestamp)")
|
||||
return
|
||||
}
|
||||
|
||||
indicators, ok := result["indicators"].(map[string]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (indicators)"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (indicators)")
|
||||
return
|
||||
}
|
||||
|
||||
events, ok := result["events"].(map[string]interface{})
|
||||
|
||||
|
|
@ -48,34 +128,66 @@ func Conv(name string) {
|
|||
splits, ok := events["splits"].(map[string]interface{})
|
||||
|
||||
x, ok = indicators["quote"].([]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (quote)"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (quote)")
|
||||
return
|
||||
}
|
||||
|
||||
quote, ok := x[0].(map[string]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (quote[0])"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (quote[0])")
|
||||
return
|
||||
}
|
||||
|
||||
x, ok = indicators["adjclose"].([]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (adjclose)"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (adjclose)")
|
||||
return
|
||||
}
|
||||
|
||||
a, ok := x[0].(map[string]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (adjclose[0])"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (adjclose[0])")
|
||||
return
|
||||
}
|
||||
|
||||
adjclose, ok := a["adjclose"].([]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (adjclose[adjclose])"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (adjclose[adjclose])")
|
||||
return
|
||||
}
|
||||
|
||||
open, ok := quote["open"].([]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (open)"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (open)")
|
||||
return
|
||||
}
|
||||
|
||||
high, ok := quote["high"].([]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (high)"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (high)")
|
||||
return
|
||||
}
|
||||
|
||||
low, ok := quote["low"].([]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (low)"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (low)")
|
||||
return
|
||||
}
|
||||
|
||||
close, ok := quote["close"].([]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (close)"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (close)")
|
||||
return
|
||||
}
|
||||
|
||||
volume, ok := quote["volume"].([]interface{})
|
||||
if !ok { fmt.Println(name + ": Bad format (volume)"); return }
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (volume)")
|
||||
return
|
||||
}
|
||||
|
||||
loc := exchangeTimezone(result)
|
||||
|
||||
out, err := os.Create(name + "-history.csv")
|
||||
if err != nil {
|
||||
|
|
@ -85,15 +197,26 @@ func Conv(name string) {
|
|||
defer out.Close()
|
||||
|
||||
out.WriteString("Date,Open,High,Low,Close,Adj Close,Volume\n")
|
||||
// Yahoo occasionally emits the same timestamp twice; keep the last one.
|
||||
seen := make(map[int64]int, len(timestamp))
|
||||
for k, v := range timestamp {
|
||||
unixTime := time.Unix(int64(v.(float64)),0)
|
||||
seen[int64(v.(float64))] = k
|
||||
}
|
||||
for k, v := range timestamp {
|
||||
if seen[int64(v.(float64))] != k {
|
||||
continue
|
||||
}
|
||||
if open[k] == nil {
|
||||
continue
|
||||
}
|
||||
vol := 0
|
||||
if volume[k] != nil {
|
||||
vol = int(volume[k].(float64))
|
||||
}
|
||||
out.WriteString(fmt.Sprintf("%s,%f,%f,%f,%f,%f,%d\n",
|
||||
unixTime.Format("2006-01-02"),
|
||||
time.Unix(int64(v.(float64)), 0).In(loc).Format("2006-01-02"),
|
||||
open[k], high[k], low[k], close[k],
|
||||
adjclose[k],int(volume[k].(float64))))
|
||||
adjclose[k], vol))
|
||||
}
|
||||
if dividends != nil {
|
||||
out, err = os.Create(name + "-dividend.csv")
|
||||
|
|
@ -105,26 +228,8 @@ func Conv(name string) {
|
|||
defer out.Close()
|
||||
|
||||
out.WriteString("Date,Dividends\n")
|
||||
for _,v := range dividends {
|
||||
v2, ok := v.(map[string]interface{})
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (dividends)")
|
||||
return
|
||||
}
|
||||
amount, ok := v2["amount"]
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (dividend amount)")
|
||||
return
|
||||
}
|
||||
t, ok := v2["date"].(float64)
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (dividend date)")
|
||||
return
|
||||
}
|
||||
unixTime := time.Unix(int64(t),0)
|
||||
out.WriteString(fmt.Sprintf("%s,%f\n",
|
||||
unixTime.Format("2006-01-02"),
|
||||
amount.(float64)))
|
||||
for _, r := range eventRows(name, "dividend", dividends, loc) {
|
||||
out.WriteString(fmt.Sprintf("%s,%f\n", r.date, r.amount))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -138,26 +243,8 @@ func Conv(name string) {
|
|||
defer out.Close()
|
||||
|
||||
out.WriteString("Date,Capital Gains\n")
|
||||
for _,v := range capgains {
|
||||
v2, ok := v.(map[string]interface{})
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (capital gains)")
|
||||
return
|
||||
}
|
||||
amount, ok := v2["amount"]
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (capital gains amount)")
|
||||
return
|
||||
}
|
||||
t, ok := v2["date"].(float64)
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (capital gains date)")
|
||||
return
|
||||
}
|
||||
unixTime := time.Unix(int64(t),0)
|
||||
out.WriteString(fmt.Sprintf("%s,%f\n",
|
||||
unixTime.Format("2006-01-02"),
|
||||
amount.(float64)))
|
||||
for _, r := range eventRows(name, "capital gain", capgains, loc) {
|
||||
out.WriteString(fmt.Sprintf("%s,%f\n", r.date, r.amount))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -171,27 +258,8 @@ func Conv(name string) {
|
|||
defer out.Close()
|
||||
|
||||
out.WriteString("Date,Stock Splits\n")
|
||||
for _,v := range splits {
|
||||
v2, ok := v.(map[string]interface{})
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (splits)")
|
||||
return
|
||||
}
|
||||
splitRatio, ok := v2["splitRatio"]
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (split ratio)")
|
||||
return
|
||||
}
|
||||
t, ok := v2["date"].(float64)
|
||||
if !ok {
|
||||
fmt.Println(name + ": Bad format (split date)")
|
||||
return
|
||||
}
|
||||
unixTime := time.Unix(int64(t),0)
|
||||
out.WriteString(fmt.Sprintf("%s,%s\n",
|
||||
unixTime.Format("2006-01-02"),
|
||||
splitRatio))
|
||||
for _, r := range eventRows(name, "split", splits, loc) {
|
||||
out.WriteString(fmt.Sprintf("%s,%s\n", r.date, r.ratio))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user