Update download scripts to include installing aria2

censusreporter · Sep 15, 2016 · 58dd56b · 58dd56b
1 parent f4cf0aa
commit 58dd56b
Show file tree

Hide file tree

Showing 22 changed files with 25 additions and 25 deletions.
diff --git a/02_download_acs_2007_1yr.sh b/02_download_acs_2007_1yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2007_1yr
 cd acs2007_1yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2007_1yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2007_1yr/summaryfile/Alabama/all_al.zip" \
     "http://www2.census.gov/acs2007_1yr/summaryfile/Alabama/g20071al.txt" \
@@ -244,4 +244,4 @@ done
 for i in /mnt/tmp/acs2007_1yr/tab4/sumfile/prod/2007/data/m20071*0141000.txt; do
     python /home/ubuntu/census-postgres/meta-scripts/fix_csv.py --columns 44 $i $i.fixed.txt
     mv $i.fixed.txt $i
-done
+done
diff --git a/02_download_acs_2007_3yr.sh b/02_download_acs_2007_3yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2007_3yr
 cd acs2007_3yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2007_3yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2007_3yr/summaryfile/Alabama/all_al.zip" \
     "http://www2.census.gov/acs2007_3yr/summaryfile/Alabama/g20073al.txt" \
@@ -235,4 +235,4 @@ done
 for i in /mnt/tmp/acs2007_3yr/tab4/sumfile/prod/2005thru2007/data/m20073*0141000.txt; do
     python /home/ubuntu/census-postgres/meta-scripts/fix_csv.py --columns 43 $i $i.fixed.txt
     mv $i.fixed.txt $i
-done
+done
diff --git a/02_download_acs_2008_1yr.sh b/02_download_acs_2008_1yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2008_1yr
 cd acs2008_1yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2008_1yr --max-connection-per-server=5 \
     "http://www2.census.gov/acs2008_1yr/summaryfile/all_ACSSF.zip"
 unzip -q all_ACSSF.zip

diff --git a/02_download_acs_2008_3yr.sh b/02_download_acs_2008_3yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2008_3yr
 cd acs2008_3yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2008_3yr --max-connection-per-server=5 \
     "http://www2.census.gov/acs2008_3yr/summaryfile/all_ACSSF.zip"
 unzip -q all_ACSSF.zip

diff --git a/02_download_acs_2009_1yr.sh b/02_download_acs_2009_1yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2009_1yr
 cd acs2009_1yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2009_1yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2009_1yr/summaryfile/Entire_SF/20091YRSF.zip" \
     "http://www2.census.gov/acs2009_1yr/summaryfile/UserTools/merge_5_6.txt"

diff --git a/02_download_acs_2009_3yr.sh b/02_download_acs_2009_3yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2009_3yr
 cd acs2009_3yr
-sudo apt-get -y install unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2009_3yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2009_3yr/summaryfile/2007-2009_ACSSF_All_In_1_Giant_File(Experienced-Users-Only)/All_Geographies.zip" \
     "http://www2.census.gov/acs2009_3yr/summaryfile/UserTools/Sequence_Number_and_Table_Number_Lookup.txt"

diff --git a/02_download_acs_2009_5yr.sh b/02_download_acs_2009_5yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2009_5yr/group{1,2}
 cd acs2009_5yr/group1
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2009_5yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2009_5yr/summaryfile/2005-2009_ACSSF_All_In_2_Giant_Files(Experienced-Users-Only)/All_Geographies_Not_Tracts_Block_Groups.zip" \
     "http://www2.census.gov/acs2009_5yr/summaryfile/2005-2009_ACSSF_All_In_2_Giant_Files(Experienced-Users-Only)/Tracts_Block_Groups_Only.zip" \
@@ -22,4 +22,4 @@ cd ..
 
 # Let the Postgres user access this data
 chmod 777 /mnt/tmp/acs2009_5yr/group1 /mnt/tmp/acs2009_5yr/group2
-chmod 644 /mnt/tmp/acs2009_5yr/group1/* /mnt/tmp/acs2009_5yr/group2/*
+chmod 644 /mnt/tmp/acs2009_5yr/group1/* /mnt/tmp/acs2009_5yr/group2/*
diff --git a/02_download_acs_2010_1yr.sh b/02_download_acs_2010_1yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2010_1yr
 cd acs2010_1yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2010_1yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2010_1yr/summaryfile/2010_ACSSF_All_In_1_Giant_File(Experienced-Users-Only)/All_Geographies.zip" \
     "http://www2.census.gov/acs2010_1yr/summaryfile/Sequence_Number_and_Table_Number_Lookup.txt"

diff --git a/02_download_acs_2010_3yr.sh b/02_download_acs_2010_3yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2010_3yr
 cd acs2010_3yr
-sudo apt-get -y install unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2010_3yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2010_3yr/summaryfile/2008-2010_ACSSF_All_In_2_Giant_Files(Experienced-Users-Only)/All_Geographies.zip" \
     "http://www2.census.gov/acs2010_3yr/summaryfile/Sequence_Number_and_Table_Number_Lookup.txt"

diff --git a/02_download_acs_2010_5yr.sh b/02_download_acs_2010_5yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2010_5yr
 cd acs2010_5yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2010_5yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2010_5yr/summaryfile/2006-2010_ACSSF_All_In_2_Giant_Files(Experienced-Users-Only)/All_Geographies_Not_Tracts_Block_Groups.zip" \
     "http://www2.census.gov/acs2010_5yr/summaryfile/2006-2010_ACSSF_All_In_2_Giant_Files(Experienced-Users-Only)/Tracts_Block_Groups_Only.zip" \

diff --git a/02_download_acs_2011_1yr.sh b/02_download_acs_2011_1yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2011_1yr
 cd acs2011_1yr
-sudo apt-get -y install unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2011_1yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2011_1yr/summaryfile/2011_ACSSF_All_In_1_Giant_File(Experienced-Users-Only)/All_Geographies.zip" \
     "http://www2.census.gov/acs2011_1yr/summaryfile/Sequence_Number_and_Table_Number_Lookup.txt"

diff --git a/02_download_acs_2011_3yr.sh b/02_download_acs_2011_3yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2011_3yr
 cd acs2011_3yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2011_3yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2011_3yr/summaryfile/2009-2011_ACSSF_All_In_1_Giant_File(Experienced-Users-Only)/All_Geographies.zip" \
     "http://www2.census.gov/acs2011_3yr/summaryfile/Sequence_Number_and_Table_Number_Lookup.txt"

diff --git a/02_download_acs_2011_5yr.sh b/02_download_acs_2011_5yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2011_5yr
 cd acs2011_5yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2011_5yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2011_5yr/summaryfile/2007-2011_ACSSF_All_In_2_Giant_Files(Experienced-Users-Only)/All_Geographies_Not_Tracts_Block_Groups.tar.gz" \
     "http://www2.census.gov/acs2011_5yr/summaryfile/2007-2011_ACSSF_All_In_2_Giant_Files(Experienced-Users-Only)/Tracts_Block_Groups_Only.tar.gz" \

diff --git a/02_download_acs_2012_1yr.sh b/02_download_acs_2012_1yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2012_1yr
 cd acs2012_1yr
-sudo apt-get -y install unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2012_1yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2012_1yr/summaryfile/2012_ACSSF_All_In_1_Giant_File(Experienced-Users-Only)/All_Geographies.zip" \
     "http://www2.census.gov/acs2012_1yr/summaryfile/Sequence_Number_and_Table_Number_Lookup.txt"

diff --git a/02_download_acs_2012_3yr.sh b/02_download_acs_2012_3yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2012_3yr
 cd acs2012_3yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2012_3yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2012_3yr/summaryfile/2010-2012_ACSSF_All_In_1_Giant_Files(Experienced-Users-Only)/All_Geographies.zip" \
     "http://www2.census.gov/acs2012_3yr/summaryfile/Sequence_Number_and_Table_Number_Lookup.txt"

diff --git a/02_download_acs_2012_5yr.sh b/02_download_acs_2012_5yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2012_5yr
 cd acs2012_5yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2012_5yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2012_5yr/summaryfile/2008-2012_ACSSF_All_In_2_Giant_Files(Experienced-Users-Only)/All_Geographies_Not_Tracts_Block_Groups.tar.gz" \
     "http://www2.census.gov/acs2012_5yr/summaryfile/2008-2012_ACSSF_All_In_2_Giant_Files(Experienced-Users-Only)/Tracts_Block_Groups_Only.tar.gz" \

diff --git a/02_download_acs_2013_1yr.sh b/02_download_acs_2013_1yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2013_1yr
 cd acs2013_1yr
-sudo apt-get -y install unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2013_1yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2013_1yr/summaryfile/2013_ACSSF_All_In_1_Giant_File(Experienced-Users-Only)/All_Geographies.zip" \
     "http://www2.census.gov/acs2013_1yr/summaryfile/Sequence_Number_and_Table_Number_Lookup.xls"

diff --git a/02_download_acs_2013_3yr.sh b/02_download_acs_2013_3yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2013_3yr
 cd acs2013_3yr
-sudo apt-get -y install unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2013_3yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/acs2013_3yr/summaryfile/2011-2013_ACSSF_All_In_1_Giant_File(Experienced-Users-Only)/All_Geographies.zip" \
     "http://www2.census.gov/acs2013_3yr/summaryfile/Sequence_Number_and_Table_Number_Lookup.xls"

diff --git a/02_download_acs_2013_5yr.sh b/02_download_acs_2013_5yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2013_5yr
 cd acs2013_5yr
-sudo apt-get -y install aria2 unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2013_5yr --max-connection-per-server=5 --force-sequential=true --file-allocation=none \
     "http://www2.census.gov/acs2013_5yr/summaryfile/2009-2013_ACSSF_All_In_2_Giant_Files(Experienced-Users-Only)/All_Geographies_Not_Tracts_Block_Groups.tar.gz" \
     "http://www2.census.gov/acs2013_5yr/summaryfile/2009-2013_ACSSF_All_In_2_Giant_Files(Experienced-Users-Only)/Tracts_Block_Groups_Only.tar.gz" \

diff --git a/02_download_acs_2014_1yr.sh b/02_download_acs_2014_1yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2014_1yr
 cd acs2014_1yr
-sudo apt-get -y install unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2014_1yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/programs-surveys/acs/summary_file/2014/data/1_year_entire_sf/All_Geographies.zip" \
     "http://www2.census.gov/programs-surveys/acs/summary_file/2014/documentation/user_tools/ACS_1yr_Seq_Table_Number_Lookup.txt"

diff --git a/02_download_acs_2014_5yr.sh b/02_download_acs_2014_5yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2014_5yr
 cd acs2014_5yr
-sudo apt-get -y install unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2014_5yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/programs-surveys/acs/summary_file/2014/data/5_year_entire_sf/All_Geographies_Not_Tracts_Block_Groups.tar.gz" \
     "http://www2.census.gov/programs-surveys/acs/summary_file/2014/data/5_year_entire_sf/Tracts_Block_Groups_Only.tar.gz" \

diff --git a/02_download_acs_2015_1yr.sh b/02_download_acs_2015_1yr.sh
@@ -8,7 +8,7 @@ sudo chown ubuntu /mnt/tmp
 cd /mnt/tmp
 mkdir -p acs2015_1yr
 cd acs2015_1yr
-sudo apt-get -y install unzip
+sudo apt-get -y install unzip aria2
 aria2c --dir=/mnt/tmp/acs2015_1yr --max-connection-per-server=5 --force-sequential=true \
     "http://www2.census.gov/programs-surveys/acs/summary_file/2015/data/1_year_entire_sf/All_Geographies.zip" \
     "http://www2.census.gov/programs-surveys/acs/summary_file/2015/documentation/user_tools/ACS_1yr_Seq_Table_Number_Lookup.txt"