Scraping 2020 US Election Data

# Zoe Phin
# 2020/11/11

require() {
   sudo apt-get install curl jq gnuplot
}

download() {
   curl -O 'https://static01.nyt.com/elections-assets/2020/data/api/2020-11-03/state-page/{alabama,alaska,arizona,arkansas,california,colorado,connecticut,delaware,florida,georgia,hawaii,idaho,illinois,indiana,iowa,kansas,kentucky,louisiana,maine,maryland,massachusetts,michigan,minnesota,mississippi,missouri,montana,nebraska,nevada,new-hampshire,new-jersey,new-mexico,new-york,north-carolina,north-dakota,ohio,oklahoma,oregon,pennsylvania,rhode-island,south-carolina,south-dakota,tennessee,texas,utah,vermont,virginia,washington,west-virginia,wisconsin,wyoming}.json'
}

timelines() {
   for state in `ls *.json | sed s/.json//`; do
      jq -Mc '.data.races[0].timeseries[]|[.timestamp,.votes,.votes*.vote_shares.trumpd,.votes*.vote_shares.bidenj]' $state.json | tr -d '["]' |\
      awk -F, '{"TZ=US/Eastern date +%d%H%M%S -d "$1 | getline t; printf "%s %8d %8d %8d\n",t,$2-vlast,$3-tlast,$4-blast; vlast=$2; tlast=$3; blast=$4}' > $state.csv
   done
}

merged() {
   sort -n *.csv | awk '{vtotal+=$2; ttotal+=$3; btotal+=$4; printf "%s %9d %9d %9d\n",$1,vtotal,ttotal,btotal}' > elec.dat
}

plot() {
   echo 'set term png size 740,740
   set key top left 
   set grid xtics ytics
   set ylabel "Million Votes"
   set timefmt "%d%H%M%S"
   set xdata time
   set xtics format "11/%d\n%H:%M"
   set ytics format "%.0f"
   set xrange ["03190000":"04040000"]
   plot "elec.dat" u 1:($3/1e6) t "Trump" w lines lc rgb "red" lw 1,\
                "" u 1:($4/1e6) t "Biden" w lines lc rgb "blue" lw 1
   ' | gnuplot > elec.png
}

counties() {
   echo " County              |    Votes |    Votes    Votes    Votes |      %          %         %   "
   echo " Name                |    Total |    Biden    Trump    Jorg. |    Biden      Trump     Jorg."
   for state in `ls *.json | sed s/.json//`; do
      echo -e "\n--- $state ---\n"
      jq -Mc '.data.races[0].counties[]|[.name,.votes,.results.bidenj,.results.trumpd,.results.jorgensenj]' $state.json | tr -d '["]' |\
      awk -F, -vs=$state '$2!=0{ v+=$2; b+=$3; t+=$4; j+=$5;
         printf "%-20s | %8s | %8s %8s %8s | %9.3f %9.3f %9.3f\n",$1,$2,$3,$4,$5,$3/$2*100,$4/$2*100,$5/$2*100} END {
         printf "\nTotal                | %8s | %8s %8s %8s | %9.3f %9.3f %9.3f\n",v,b,t,j,b/v*100,t/v*100,j/v*100
      }'
   done
}