Skip to content

Commit 8315f1c

Browse files
pdilpdil
authored
Implement automated shapefile updates (#7)
2 parents c1bdcf2 + 337135d commit 8315f1c

7 files changed

Lines changed: 220 additions & 0 deletions

File tree

.github/workflows/R-CMD-check.yaml renamed to .github/workflows/R-CMD-check.yml

File renamed without changes.

.github/workflows/test-coverage.yaml renamed to .github/workflows/test-coverage.yml

File renamed without changes.

‎.github/workflows/update-maps.yml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
2+
name: Check for new shapefiles
3+
4+
on:
5+
schedule:
6+
# runs at midnight on the 1st of March and September
7+
- cron: '0 0 1 3,9 *'
8+
workflow_dispatch:
9+
10+
jobs:
11+
check:
12+
runs-on: ubuntu-latest
13+
env:
14+
PUSHOVER_API_KEY: ${{ secrets.PUSHOVER_API_KEY }}
15+
PUSHOVER_USER_KEY: ${{ secrets.PUSHOVER_USER_KEY }}
16+
17+
steps:
18+
- name: Checkout
19+
uses: actions/checkout@v4
20+
21+
- name: Setup Python
22+
uses: actions/setup-python@v4
23+
with:
24+
python-version: '3.9'
25+
cache: 'pip'
26+
27+
- name: Download shapefiles
28+
run: |
29+
python data-raw/scripts/shapefiles.py
30+
echo "python_exit_code=$?" >> $GITHUB_ENV
31+
continue-on-error: true
32+
33+
- name: Send failure notification
34+
if: ${{ failure() }}
35+
run: python pushover.py "⚠️ usmapdata updater failed to find new shapefiles." "LOW"
36+
37+
- name: Setup usmapdata
38+
if: ${{ success() }}
39+
uses: r-lib/actions/setup-r-dependencies@v2
40+
41+
- name: Modify shapefiles
42+
if: ${{ success() }}
43+
env:
44+
STATE_SHP_DIR: "data-raw/scripts/shapefiles/$state_shp_path"
45+
COUNTY_SHP_DIR: "data-raw/scripts/shapefiles/$county_shp_path"
46+
STATE_OUTPUT: "inst/extdata/us_states.gpkg"
47+
COUNTY_OUTPUT: "inst/extdata/us_counties.gpkg"
48+
run: |
49+
Rscript -e "usmapdata:::create_us_map('states', Sys.getenv('STATE_SHP_DIR'), Sys.getenv('STATE_OUTPUT'))"
50+
Rscript -e "usmapdata:::create_us_map('counties', Sys.getenv('COUNTY_SHP_DIR'), Sys.getenv('COUNTY_OUTPUT'))"
51+
52+
- name: Determine pull request parameters
53+
if: ${{ success() }}
54+
run: |
55+
echo "branch_name=data-update/$(date +'%B-%Y')" >> "$GITHUB_ENV"
56+
echo "pr_title=Update map data - $(date +'%B %Y')" >> "$GITHUB_ENV"
57+
58+
- name: Open pull request
59+
if: ${{ success() }}
60+
uses: peter-evans/create-pull-request@v5
61+
with:
62+
commit-message: Update map data based on latest shapefiles
63+
branch: ${{ env.branch_name }}
64+
title: ${{ env.pr_title }}
65+
body: |
66+
Updated map data based on latest shapefiles from
67+
the US Census Bureau's [cartographic boundary files][1].
68+
69+
### Review Checklist
70+
- [ ] Ensure all checks and tests pass
71+
- [ ] Load current branch with `devtools::install_github("usmapdata", "${{ env.branch_name }}")` and test `usmap`
72+
- [ ] Perform smoke test of all plotting features to ensure consistency
73+
- [ ] Update data file changelog in [`usmap` `README.md`][2]
74+
75+
[1]: https://www.census.gov/geographies/mapping-files/time-series/geo/cartographic-boundary.html
76+
[2]: https://github.com/pdil/usmap/blob/master/README.md
77+
assignees: pdil
78+
labels: data update
79+
reviewers: pdil
80+
delete-branch: true
81+
82+
- name: Send success notification
83+
if: ${{ success() }}
84+
run: python pushover.py "✅ usmapdata has updated its data files, a PR review is needed."

‎NEWS.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
* Once the upgrade is complete, this parameter will be removed and the new functionality will be the default.
88
* The new map files are smaller in size while maintaining the same resolution.
99
* The format of the data also allows for easier manipulation in the future using the `sf` package.
10+
* Add scripts to perform automated map data updates, see [Issue #5](https://github.com/pdil/usmapdata/issues/5).
11+
* This is not yet fully functional and will be refined over time independent of `usmapdata` package updates.
1012

1113
# usmapdata 0.1.2
1214
Released Monday, December 11, 2023.

‎data-raw/scripts/config.ini‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[shapefiles]
2+
url = https://www2.census.gov/geo/tiger/GENZ{year}/shp/cb_{year}_us_{entity}_{res}.zip
3+
current_year = 2021
4+
entities = state,county
5+
res = 20m
6+

‎data-raw/scripts/pushover.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
2+
import os
3+
import requests
4+
from strenum import StrEnum
5+
6+
class Pushover:
7+
8+
class Priority(StrEnum):
9+
LOWEST = "-2"
10+
LOW = "-1"
11+
NORMAL = "0"
12+
HIGH = "1"
13+
EMERGENCY = "2"
14+
15+
def __init__(self, token: str, user: str):
16+
self._token = token
17+
self._user = user
18+
19+
# Send a Pushover notification
20+
def send(self, message: str, attachment_url: str=None, priority=Priority.NORMAL):
21+
MESSAGES_URL = "https://api.pushover.net/1/messages.json"
22+
data = {
23+
"token": self._token,
24+
"user": self._user,
25+
"message": message,
26+
"priority": priority
27+
}
28+
29+
files = None
30+
if attachment_url and os.path.isfile(attachment_url):
31+
files = {
32+
"attachment": ("image.jpg", open(attachment_url, "rb"), "image/jpeg")
33+
}
34+
35+
requests.post(MESSAGES_URL, data=data, files=files)
36+
37+
if __name__ == "__main__":
38+
api_key = os.environ["PUSHOVER_API_KEY"]
39+
user_key = os.environ["PUSHOVER_USER_KEY"]
40+
41+
pushover = Pushover(token=api_key, user=user_key)
42+
args = sys.argv
43+
44+
try:
45+
message = args[1]
46+
except IndexError:
47+
raise SystemExit("Required message parameter not supplied")
48+
49+
priority = getattr(Pushover.Priority, sys.argv[2]) if len(args) >= 3 else Pushover.Priority.NORMAL)
50+
51+
Pushover.send(message, attachment_url, priority)

‎data-raw/scripts/shapefiles.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
2+
from configparser import ConfigParser
3+
import os
4+
import requests
5+
import shutil
6+
import sys
7+
from zipfile import ZipFile
8+
9+
class DownloadError(Exception):
10+
def __init__(self, message, code=None):
11+
super().__init__(message)
12+
self.code = code
13+
14+
def _download_and_extract(file_url: str, extract_dir: str) -> bool:
15+
response = requests.get(file_url)
16+
LOCAL_FILE = "download.zip"
17+
18+
if response.status_code == 200:
19+
with open(LOCAL_FILE, "wb") as f:
20+
f.write(response.content)
21+
print(f"{LOCAL_FILE} downloaded from {file_url}.")
22+
23+
with ZipFile(LOCAL_FILE, "r") as z:
24+
z.extractall(extract_dir)
25+
print(f"{LOCAL_FILE} extracted to {extract_dir}.")
26+
27+
os.remove(LOCAL_FILE)
28+
else:
29+
raise DownloadError(f"Failed to download {file_url}.", code=response.status_code)
30+
31+
def download_shapefiles():
32+
# create output directory
33+
script_dir = os.path.abspath(os.path.dirname(__file__))
34+
extract_dir = os.path.join(script_dir, "..", "shapefiles")
35+
36+
if os.path.exists(extract_dir):
37+
shutil.rmtree(extract_dir)
38+
shutil.os.makedirs(extract_dir)
39+
40+
# get current configuration
41+
CONFIG_FILE = "config.ini"
42+
config = ConfigParser()
43+
config.read(os.path.join(script_dir, CONFIG_FILE))
44+
SECTION = "shapefiles"
45+
46+
url_template = config.get(SECTION, "url")
47+
current_year = config.getint(SECTION, "current_year")
48+
entities = config.get(SECTION, "entities").split(",")
49+
res = config.get(SECTION, "res")
50+
51+
year = current_year + 1
52+
53+
try:
54+
# attempt shapefile downloads
55+
for entity in entities:
56+
url = url_template.format(year=year, entity=entity, res=res)
57+
_download_and_extract(url, extract_dir)
58+
59+
if (gh_env := os.getenv("GITHUB_ENV")):
60+
with open(gh_env, "a") as f:
61+
f.write(f"{entity}_shp_path=cb_{year}_us_{entity}_{res}.shp")
62+
63+
# update current year
64+
config.set(SECTION, "current_year", f"{year}")
65+
with open(CONFIG_FILE, "w") as f:
66+
config.write(f)
67+
except DownloadError as e:
68+
if e.code == 404: # i.e. shapefiles not found
69+
print(f"The shapefiles for {year} were not found. Better luck next time!")
70+
else: # other download errors
71+
print(e)
72+
73+
sys.exit(e.code)
74+
75+
76+
if __name__ == "__main__":
77+
download_shapefiles()

0 commit comments

Comments
 (0)