Skip to content

extract module

Module for constructing data URLs and fetching crime data CSV files from SLMPD.

ExtractionError

Bases: Exception

Raised if the target URL cannot be formed or network operations fail.

Source code in src/geocrime_stl/etl/extract.py
15
16
17
18
class ExtractionError(Exception):
    """Raised if the target URL cannot be formed or network operations fail."""

    pass

construct_url(month_input=None, year_input=None)

Parses flexible date inputs and builds the target SLMPD data asset URL.

Parameters:

Name Type Description Default
month_input str | int | None

The month to target, specified as an integer, string number, or full/abbreviated month name. Defaults to the previous month.

None
year_input str | int | None

The year to target, specified as an integer or string. Defaults to the current year (or previous year if the current month is January).

None

Returns:

Type Description
tuple[str, int, int]

A tuple containing: - url (str): The constructed download URL for the target CSV asset. - month_int (int): The validated numerical month (1-12). - year_int (int): The validated four-digit year.

Raises:

Type Description
ExtractionError

If the month or year input cannot be parsed or mapped properly.

Source code in src/geocrime_stl/etl/extract.py
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
def construct_url(
    month_input: str | int | None = None, 
    year_input: str | int | None = None
) -> tuple[str, int, int]:
    """Parses flexible date inputs and builds the target SLMPD data asset URL.

    Args:
        month_input: The month to target, specified as an integer, string number, 
            or full/abbreviated month name. Defaults to the previous month.
        year_input: The year to target, specified as an integer or string. 
            Defaults to the current year (or previous year if the current month is January).

    Returns:
        A tuple containing:
            - url (str): The constructed download URL for the target CSV asset.
            - month_int (int): The validated numerical month (1-12).
            - year_int (int): The validated four-digit year.

    Raises:
        ExtractionError: If the month or year input cannot be parsed or mapped properly.
    """
    try:
        now = datetime.now()
        if now.month == 1:
            default_month = 12
            default_year = now.year - 1
        else:
            default_month = now.month - 1
            default_year = now.year

        # Parse year input into consistent types
        if year_input is None:
            year_int = default_year
            year_str = str(year_int)
        else:
            year_str = str(year_input).strip()
            year_int = int(year_str)

        # Parse month input into consistent types
        if month_input is None:
            month_int = default_month
            month_str = datetime.strptime(str(month_int), "%m").strftime("%B")
        else:
            clean_month = str(month_input).strip()
            try:
                month_int = int(clean_month)
                month_str = datetime.strptime(str(month_int), "%m").strftime("%B")
            except ValueError:
                try:
                    dt = datetime.strptime(clean_month.capitalize(), "%B")
                except ValueError:
                    dt = datetime.strptime(clean_month.capitalize(), "%b")
                month_int = dt.month
                month_str = dt.strftime("%B")

        # A few early months were published under one-off names; look those up first
        special = SPECIAL_URLS.get((year_int, month_int))
        if special:
            return special, month_int, year_int

        # Handle the city's specific upload folder schedule offset
        folder_month_int = (month_int % 12) + 1
        folder_month_str = f"{folder_month_int:02d}"
        folder_year_str = str(year_int + 1) if month_int == 12 else year_str

        url = f"{BASE_URL}/{folder_year_str}/{folder_month_str}/{month_str}{year_str}.csv"
        return url, month_int, year_int

    except Exception as e:
        raise ExtractionError(
            f"Invalid month ('{month_input}') or year ('{year_input}') mapping: {e}"
        ) from e

fetch_crime_data_csv(url, keep_raw_csv=False)

Downloads raw crime data CSV bytes from the specified URL.

Parameters:

Name Type Description Default
url str

The target HTTP URL to fetch data from.

required
keep_raw_csv bool

If True, saves a copy of the downloaded raw bytes locally using the filename derived from the URL.

False

Returns:

Name Type Description
bytes bytes

The raw content bytes of the downloaded CSV file.

Raises:

Type Description
ExtractionError

If an HTTP, network, or unexpected error occurs during download.

Source code in src/geocrime_stl/etl/extract.py
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
def fetch_crime_data_csv(url: str, keep_raw_csv: bool = False) -> bytes:
    """Downloads raw crime data CSV bytes from the specified URL.

    Args:
        url: The target HTTP URL to fetch data from.
        keep_raw_csv: If True, saves a copy of the downloaded raw bytes locally 
            using the filename derived from the URL.

    Returns:
        bytes: The raw content bytes of the downloaded CSV file.

    Raises:
        ExtractionError: If an HTTP, network, or unexpected error occurs during download.
    """
    headers = {"User-Agent": "Mozilla/5.0"}
    filename = url.split("/")[-1] or "downloaded_data.csv"

    logger.info("Attempting network download from: %s", url)
    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        raw_bytes = response.content
    except requests.exceptions.HTTPError as http_err:
        raise ExtractionError(f"HTTP error downloading data asset: {http_err}") from http_err
    except requests.exceptions.RequestException as net_err:
        raise ExtractionError(f"Network transport error reaching server: {net_err}") from net_err
    except Exception as e:
        raise ExtractionError(f"Unexpected extraction execution failure: {e}") from e

    if keep_raw_csv:
        try:
            with open(filename, "wb") as f:
                f.write(raw_bytes)
            logger.info("💾 Raw CSV backup cached locally: %s", filename)
        except IOError as io_err:
            logger.warning("⚠️ Could not write raw data local cache: %s", io_err)

    return raw_bytes