Initial commit.

.gitignore

@@ -0,0 +1 @@
+cget

go.mod

@@ -0,0 +1,9 @@
+module git.wow.st/gmp/cget
+
+go 1.14
+
+require (
+	github.com/BurntSushi/toml v0.3.1
+	github.com/domodwyer/mailyak v3.1.1+incompatible
+	github.com/mitchellh/go-homedir v1.1.0
+)

go.sum

@@ -0,0 +1,7 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/domodwyer/mailyak v1.2.0 h1:xCFvdXmHg3hwVey0OXQJrmKuDwcHfRZx7toiIaSi8hU=
+github.com/domodwyer/mailyak v3.1.1+incompatible h1:oPtXn3+56LEFbdqH0bpuPRsqtijW9l2POpQe9sTUsSI=
+github.com/domodwyer/mailyak v3.1.1+incompatible/go.mod h1:5NNYkn9hxcdNEOmmMx0yultN5VLorZQ+AWQo9iya+UY=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=

main.go

@@ -0,0 +1,159 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/smtp"
+	"os"
+	"os/exec"
+	"path"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/domodwyer/mailyak"
+)
+
+type Config struct {
+	MailHost, Username, Password string
+	MailPort int
+	Len int64
+}
+
+var conf Config
+var confFile string
+
+func getConf() Config {
+	confDir := path.Dir(confFile)
+
+	if _, err := os.Stat(confDir); os.IsNotExist(err) {
+		log.Printf("Creating config directory")
+		err = os.MkdirAll(confDir, 0777)
+		if err != nil {
+			log.Fatal("Cannot create config directory: ", err)
+		}
+	}
+	if _, err := os.Stat(confFile); os.IsNotExist(err) {
+		log.Printf("Creating config file")
+		f, err := os.Create(confFile)
+		if err != nil {
+			log.Fatal("Cannot create config file: ", err)
+		}
+		err = f.Close()
+		if err != nil {
+			log.Fatal("Error closing config file: ", err)
+		}
+	}
+	if _, err := toml.DecodeFile(confFile, &conf); err != nil {
+		log.Fatal("Error reading config file: ", err)
+	}
+	of, err := os.Create(path.Join(confDir, "c.R"))
+	if err != nil {
+		log.Fatal("Error creating R source file: ", err)
+	}
+	_, err = of.WriteString(rCode())
+	if err != nil {
+		log.Fatal("Error writing R source code: ", err)
+	}
+	return conf
+}
+
+func save() {
+	of, err := os.Create(confFile)
+	if err != nil {
+		log.Fatal("Cannot open config file: ", err)
+	}
+	var buf bytes.Buffer
+	w := bufio.NewWriter(&buf)
+	enc := toml.NewEncoder(w)
+	enc.Encode(conf)
+	io.Copy(of, &buf)
+	err = of.Close()
+	if err != nil {
+		log.Fatal("Error closing config file: ", err)
+	}
+}
+
+func fetch() {
+	url := "https://health.data.ny.gov/api/views/xdss-u53e/rows.csv?accessType=DOWNLOAD"
+
+	resp, err := http.Get(url)
+	if err != nil {
+		log.Printf("HTTP error: ", err)
+		return
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Printf("Read error: ", err)
+		return
+	}
+	if int64(len(body)) == conf.Len {
+		log.Printf("Equal length, returning")
+		return
+	}
+	log.Printf("Length differs: body = %d", len(body))
+	conf.Len = int64(len(body))
+	save()
+
+	of, err := os.Create(path.Join(path.Dir(confFile), "rows.csv"))
+	if err != nil {
+		log.Fatal("Cannot create rows.csv: ", err)
+	}
+	_, err = of.Write(body)
+	if err != nil {
+		log.Fatal("Error writing rows.csv: ", err)
+	}
+
+	confDir := path.Dir(confFile)
+	cmd := exec.Command("R", "--no-save", "-f", path.Join(confDir, "c.R"))
+	log.Printf("Running R")
+	err = cmd.Run()
+	if err != nil {
+		log.Fatal("Error running R: ", err)
+	}
+	log.Printf("R completed")
+	if conf.MailHost == "" {
+		log.Print("No mail host set, returning")
+		return
+	}
+	f, err := os.Open("Rplots.pdf")
+	if err != nil {
+		log.Fatal("Error opening Rplots.pdf: ", err)
+	}
+	defer f.Close()
+	email := mailyak.New(fmt.Sprintf("%s:%d", conf.MailHost, conf.MailPort),
+		smtp.PlainAuth("", conf.Username, conf.Password, conf.MailHost))
+	email.To("gmp@wow.st")
+	email.From("covid@wow.st")
+	msg := "Covid dashboard update for " + time.Now().Format("Monday, January 2 2006")
+	email.Subject(msg)
+	email.Plain().Set(msg)
+	email.Attach("dashboard.pdf", f)
+	err = email.Send()
+	if err != nil {
+		log.Print("Error sending email: ", err)
+	}
+}
+
+func main() {
+	log.Printf("starting")
+	homeDir, err := homedir.Dir()
+	if err != nil {
+		log.Fatal("Cannot locate user's home directory")
+	}
+	confFile = path.Join(homeDir, ".config", "cget", "cget.conf")
+	os.Chdir(path.Dir(confFile))
+	getConf()
+
+	for {
+		fetch()
+		time.Sleep(time.Minute * 30)
+	}
+	log.Printf("done")
+}
+

rcode.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"path"
+)
+
+func rCode() string {
+	return `
+library(tidyverse)
+library(ggplot2)
+library(gridExtra)
+library(lubridate)
+
+gc()
+
+options(scipen=100000)
+
+d <- read_csv('`+ path.Dir(confFile) + `/rows.csv')
+d <- pivot_longer(d, c('New Positives', 'Total Number of Tests Performed'))
+d <- transmute(d, date=mdy(d$'Test Date'), county=County, name=name, value=value)
+queens <- subset(d, county=='Queens')
+qrate <- pivot_wider(queens, names_from='name') %>% transmute(date=date,county=county,name='Positive Rate', value=ifelse(`+"`"+`New Positives`+"`"+`==0,0,`+"`"+`New Positives`+"`"+`/`+"`"+`Total Number of Tests Performed`+"`"+`))
+
+totals <- group_by(d, date, county='New York', name) %>% summarize(value=sum(value)) %>% ungroup
+trate <- pivot_wider(totals, names_from='name') %>% transmute(date=date,county=county,name='Positive Rate', value=ifelse(`+"`"+`New Positives`+"`"+`==0,0,`+"`"+`New Positives`+"`"+`/`+"`"+`Total Number of Tests Performed`+"`"+`))
+
+d2 <- rbind(queens, totals)
+
+p1 <- ggplot(queens, aes(x=date, y=value, color=name))+geom_line()+scale_y_log10(n.breaks=6)+labs(y='',color='')+ggtitle('Queens')+theme(plot.title=element_text(hjust = 0.5))+theme(legend.position='bottom')
+#p2 <- ggplot(d, aes(x=date, y=value, color=name))+geom_line(data=queens)+stat_smooth(geom='line', linetype='dotted', data=queens, method='gam', se=FALSE)+geom_line(alpha=1/3, data=totals)+stat_smooth(geom='line', linetype='dotted', data=totals, method='gam', se=FALSE)+scale_y_log10(n.breaks=6)+labs(y='')+labs(color='')+ggtitle('New York')+theme(plot.title=element_text(hjust = 0.5))+theme(legend.position='bottom')
+p2 <- ggplot(totals, aes(x=date, y=value, color=name))+geom_line()+stat_smooth(geom='line', linetype='dotted', method='gam', se=FALSE)+scale_y_log10(n.breaks=6)+labs(y='')+labs(color='')+ggtitle('New York')+theme(plot.title=element_text(hjust = 0.5))+theme(legend.position='bottom')
+
+pr1 <- ggplot(qrate, aes(x=date, y=value)) + geom_line()+geom_smooth(method='gam',formula=y~s(x, bs="cs"),se=FALSE)+scale_y_continuous(labels = scales::percent,)+labs(y='Positive Rate')+coord_cartesian(ylim=c(0,0.04))+ggtitle('Queens')+theme(plot.title=element_text(hjust = 0.5))
+pr2 <- ggplot(trate, aes(x=date, y=value)) + geom_line()+geom_smooth(method='gam',formula=y~s(x, bs="cs"),se=FALSE)+scale_y_continuous(labels = scales::percent)+labs(y='Positive Rate')+coord_cartesian(ylim=c(0,0.04))+ggtitle('New York')+theme(plot.title=element_text(hjust = 0.5))
+
+p <- grid.arrange(p1,p2,pr1,pr2,ncol=2)
+
+zip <- read_csv('~/prog/covid/nyopendata/timeseries.csv')
+zip <- pivot_longer(zip, c('COVID_CASE_COUNT','COVID_DEATH_COUNT','TOTAL_COVID_TESTS'))
+zip$date <- mdy(zip$date)
+`
+}