gmp/cget
1
0
Fork 0
Code Issues Pull Requests Wiki Activity

Initial commit.

Browse Source
This commit is contained in:
Greg 2020-08-27 10:38:30 -04:00
commit 2d9aa5d1cd
5 changed files with 202 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
cget

9
go.mod Normal file
View File

@ -0,0 +1,9 @@
module git.wow.st/gmp/cget
go 1.14
require (
github.com/BurntSushi/toml v0.3.1
github.com/domodwyer/mailyak v3.1.1+incompatible
github.com/mitchellh/go-homedir v1.1.0
)

7
go.sum Normal file
View File

@ -0,0 +1,7 @@
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/domodwyer/mailyak v1.2.0 h1:xCFvdXmHg3hwVey0OXQJrmKuDwcHfRZx7toiIaSi8hU=
github.com/domodwyer/mailyak v3.1.1+incompatible h1:oPtXn3+56LEFbdqH0bpuPRsqtijW9l2POpQe9sTUsSI=
github.com/domodwyer/mailyak v3.1.1+incompatible/go.mod h1:5NNYkn9hxcdNEOmmMx0yultN5VLorZQ+AWQo9iya+UY=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=

149
main.go Normal file
View File

@ -0,0 +1,149 @@
package main
import (
"bufio"
"bytes"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/smtp"
"os"
"os/exec"
"path"
"time"
"github.com/BurntSushi/toml"
homedir "github.com/mitchellh/go-homedir"
"github.com/domodwyer/mailyak"
)
type Config struct {
MailHost, Username, Password string
MailPort int
Len int64
}
var conf Config
var confFile string
func getConf() Config {
confDir := path.Dir(confFile)
if _, err := os.Stat(confDir); os.IsNotExist(err) {
log.Printf("Creating config directory")
err = os.MkdirAll(confDir, 0777)
if err != nil {
log.Fatal("Cannot create config directory: ", err)
}
}
if _, err := os.Stat(confFile); os.IsNotExist(err) {
log.Printf("Creating config file")
f, err := os.Create(confFile)
if err != nil {
log.Fatal("Cannot create config file: ", err)
}
err = f.Close()
if err != nil {
log.Fatal("Error closing config file: ", err)
}
}
if _, err := toml.DecodeFile(confFile, &conf); err != nil {
log.Fatal("Error reading config file: ", err)
}
of, err := os.Create(path.Join(confDir, "c.R"))
if err != nil {
log.Fatal("Error creating R source file: ", err)
}
_, err = of.WriteString(rCode)
if err != nil {
log.Fatal("Error writing R source code: ", err)
}
return conf
}
func save() {
of, err := os.Create(confFile)
if err != nil {
log.Fatal("Cannot open config file: ", err)
}
var buf bytes.Buffer
w := bufio.NewWriter(&buf)
enc := toml.NewEncoder(w)
enc.Encode(conf)
io.Copy(of, &buf)
err = of.Close()
if err != nil {
log.Fatal("Error closing config file: ", err)
}
}
func fetch() {
url := "https://health.data.ny.gov/api/views/xdss-u53e/rows.csv?accessType=DOWNLOAD"
resp, err := http.Get(url)
if err != nil {
log.Printf("HTTP error: ", err)
return
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Read error: ", err)
return
}
if int64(len(body)) == conf.Len {
log.Printf("Equal length, returning")
return
}
log.Printf("Length differs: body = %d", len(body))
conf.Len = int64(len(body))
save()
confDir := path.Dir(confFile)
cmd := exec.Command("R", "--no-save", "-f", path.Join(confDir, "c.R"))
log.Printf("Running R")
err = cmd.Run()
if err != nil {
log.Fatal("Error running R: ", err)
}
log.Printf("R completed")
if conf.MailHost == "" {
log.Print("No mail host set, returning")
return
}
f, err := os.Open("Rplots.pdf")
if err != nil {
log.Fatal("Error opening Rplots.pdf: ", err)
}
defer f.Close()
email := mailyak.New(fmt.Sprintf("%s:%d", conf.MailHost, conf.MailPort),
smtp.PlainAuth("", conf.Username, conf.Password, conf.MailHost))
email.To("gmp@wow.st")
email.From("covid@wow.st")
msg := "Covid dashboard update for " + time.Now().Format("Monday, January 2 2006")
email.Subject(msg)
email.Plain().Set(msg)
email.Attach("dashboard.pdf", f)
err = email.Send()
if err != nil {
log.Print("Error sending email: ", err)
}
}
func main() {
log.Printf("starting")
homeDir, err := homedir.Dir()
if err != nil {
log.Fatal("Cannot locate user's home directory")
}
confFile = path.Join(homeDir, ".config", "cget", "cget.conf")
os.Chdir(path.Dir(confFile))
getConf()
for {
fetch()
time.Sleep(time.Minute * 30)
}
log.Printf("done")
}

36
rcode.go Normal file
View File

@ -0,0 +1,36 @@
package main
var rCode string = `
library(tidyverse)
library(ggplot2)
library(gridExtra)
library(lubridate)
gc()
options(scipen=100000)
d <- read_csv('~/prog/covid/nyopendata/rows.csv')
d <- pivot_longer(d, c('New Positives', 'Total Number of Tests Performed'))
d <- transmute(d, date=mdy(d$'Test Date'), county=County, name=name, value=value)
queens <- subset(d, county=='Queens')
qrate <- pivot_wider(queens, names_from='name') %>% transmute(date=date,county=county,name='Positive Rate', value=ifelse(`+"`"+`New Positives`+"`"+`==0,0,`+"`"+`New Positives`+"`"+`/`+"`"+`Total Number of Tests Performed`+"`"+`))
totals <- group_by(d, date, county='New York', name) %>% summarize(value=sum(value)) %>% ungroup
trate <- pivot_wider(totals, names_from='name') %>% transmute(date=date,county=county,name='Positive Rate', value=ifelse(`+"`"+`New Positives`+"`"+`==0,0,`+"`"+`New Positives`+"`"+`/`+"`"+`Total Number of Tests Performed`+"`"+`))
d2 <- rbind(queens, totals)
p1 <- ggplot(queens, aes(x=date, y=value, color=name))+geom_line()+scale_y_log10(n.breaks=6)+labs(y='',color='')+ggtitle('Queens')+theme(plot.title=element_text(hjust = 0.5))+theme(legend.position='bottom')
#p2 <- ggplot(d, aes(x=date, y=value, color=name))+geom_line(data=queens)+stat_smooth(geom='line', linetype='dotted', data=queens, method='gam', se=FALSE)+geom_line(alpha=1/3, data=totals)+stat_smooth(geom='line', linetype='dotted', data=totals, method='gam', se=FALSE)+scale_y_log10(n.breaks=6)+labs(y='')+labs(color='')+ggtitle('New York')+theme(plot.title=element_text(hjust = 0.5))+theme(legend.position='bottom')
p2 <- ggplot(totals, aes(x=date, y=value, color=name))+geom_line()+stat_smooth(geom='line', linetype='dotted', method='gam', se=FALSE)+scale_y_log10(n.breaks=6)+labs(y='')+labs(color='')+ggtitle('New York')+theme(plot.title=element_text(hjust = 0.5))+theme(legend.position='bottom')
pr1 <- ggplot(qrate, aes(x=date, y=value)) + geom_line()+geom_smooth(method='gam',formula=y~s(x, bs="cs"),se=FALSE)+scale_y_continuous(labels = scales::percent,)+labs(y='Positive Rate')+coord_cartesian(ylim=c(0,0.04))+ggtitle('Queens')+theme(plot.title=element_text(hjust = 0.5))
pr2 <- ggplot(trate, aes(x=date, y=value)) + geom_line()+geom_smooth(method='gam',formula=y~s(x, bs="cs"),se=FALSE)+scale_y_continuous(labels = scales::percent)+labs(y='Positive Rate')+coord_cartesian(ylim=c(0,0.04))+ggtitle('New York')+theme(plot.title=element_text(hjust = 0.5))
p <- grid.arrange(p1,p2,pr1,pr2,ncol=2)
zip <- read_csv('~/prog/covid/nyopendata/timeseries.csv')
zip <- pivot_longer(zip, c('COVID_CASE_COUNT','COVID_DEATH_COUNT','TOTAL_COVID_TESTS'))
zip$date <- mdy(zip$date)
`