Skip to content

transform module

Module for processing, cleaning, and spatially clipping raw crime data.

CrimeDataPackage dataclass

Wrapper that bundles structured monthly crime data with its target temporal metadata.

Attributes:

Name Type Description
df DataFrame

The processed pandas DataFrame containing crime records.

month int

The numerical month (1-12) associated with the dataset.

year int

The four-digit year associated with the dataset.

Source code in src/geocrime_stl/etl/transform.py
24
25
26
27
28
29
30
31
32
33
34
35
36
@dataclass(frozen=True)
class CrimeDataPackage:
    """Wrapper that bundles structured monthly crime data with its target temporal metadata.

    Attributes:
        df: The processed pandas DataFrame containing crime records.
        month: The numerical month (1-12) associated with the dataset.
        year: The four-digit year associated with the dataset.
    """

    df: pd.DataFrame
    month: int
    year: int

clean_and_spatial_clip(df, month, year)

Normalizes features, handles type enforcement, and spatially clips observations.

Normalizes records to St. Louis City limits and returns a populated CrimeDataPackage wrapper.

Parameters:

Name Type Description Default
df DataFrame

The raw pandas DataFrame to be cleaned.

required
month int

The target month integer.

required
year int

The target year integer.

required

Returns:

Name Type Description
CrimeDataPackage CrimeDataPackage

A package containing the cleaned, clipped DataFrame and metadata.

Source code in src/geocrime_stl/etl/transform.py
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
def clean_and_spatial_clip(
    df: pd.DataFrame, month: int, year: int
) -> CrimeDataPackage:
    """Normalizes features, handles type enforcement, and spatially clips observations.

    Normalizes records to St. Louis City limits and returns a populated 
    CrimeDataPackage wrapper.

    Args:
        df: The raw pandas DataFrame to be cleaned.
        month: The target month integer.
        year: The target year integer.

    Returns:
        CrimeDataPackage: A package containing the cleaned, clipped DataFrame and metadata.
    """
    # Handle the original empty CSV dataset scenario gracefully
    if df.empty:
        logger.warning(
            "⚠️ Ingested DataFrame was empty. Skipping cleaning pipelines for %d/%d.",
            month,
            year,
        )
        return CrimeDataPackage(df=df, month=month, year=year)

    # Date Alignments & Conversions
    df["IncidentDate"] = parse_incident_dates(df["IncidentDate"])

    # Filter by date alignment (keeps only requested month/year observations)
    df = df[(df["IncidentDate"].dt.month == month) & (df["IncidentDate"].dt.year == year)].copy()

    # String Timestamp Synthesis
    df["OccurredFromTime"] = df["OccurredFromTime"].fillna("00:00")
    combined_time_str = (
        df["IncidentDate"].dt.strftime("%Y-%m-%d") + " " + df["OccurredFromTime"].astype(str)
    )
    df["date_time"] = pd.to_datetime(combined_time_str, errors="coerce")

    # Structural Re-mapping & Column Dropping
    df = df.rename(columns=COLUMN_RENAMES)
    df = df.drop(columns=COLUMNS_TO_DROP, errors="ignore")

    # Data Type Enforcement
    df["district"] = pd.to_numeric(df["district"], errors="coerce")
    df["district"] = df["district"].astype("Int64").astype("string")
    df["inc_#"] = df["inc_#"].astype("string")

    # Regex Character Stripping (e.g., "G-88" -> 88)
    df["nbhd_num"] = df["nbhd_num"].astype(str).str.extract(r"(\d+)", expand=False)
    df["nbhd_num"] = pd.to_numeric(df["nbhd_num"], errors="coerce").astype("Int64")
    df["off_type"] = df["off_type"].replace("Unspecified", "Other")

    # Coordinate Sanitization
    df["lon"] = pd.to_numeric(df["lon"], errors="coerce")
    df["lat"] = pd.to_numeric(df["lat"], errors="coerce")
    df = df.dropna(subset=["lon", "lat"]).copy()

    # Apply Final Schema Alignment from global configuration
    existing_cols = [col for col in FINAL_SCHEMA_COLUMNS if col in df.columns]
    df = df[existing_cols]

    # Spatial Boundary Clipping Execution
    try:
        clipped_df = clip_df(df)
    except Exception as e:
        logger.warning("⚠️ Spatial fencing bypassed due to pipeline engine block: %s", e)
        logger.info("▶️ Flowing original full metropolitan boundary coordinates forward.")
        return CrimeDataPackage(df=df, month=month, year=year)

    # Ward comes from the coordinates (neighborhoods are split across wards). If this step fails the
    # month is still kept; the missing column is filled in later from the master dataset.
    try:
        clipped_df = assign_wards(clipped_df)
    except Exception as e:
        logger.warning("⚠️ Ward assignment skipped: %s", e)
    return CrimeDataPackage(df=clipped_df, month=month, year=year)

clip_df(df)

Restricts point coordinates to the formal polygon limits of St. Louis City.

Uses an inner spatial join against the city boundary file.

Parameters:

Name Type Description Default
df DataFrame

The pandas DataFrame containing latitude and longitude columns.

required

Returns:

Type Description
DataFrame

pd.DataFrame: A filtered DataFrame containing only points within city limits.

Raises:

Type Description
FileNotFoundError

If the city boundary configuration file cannot be found.

Source code in src/geocrime_stl/etl/transform.py
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
def clip_df(df: pd.DataFrame) -> pd.DataFrame:
    """Restricts point coordinates to the formal polygon limits of St. Louis City.

    Uses an inner spatial join against the city boundary file.

    Args:
        df: The pandas DataFrame containing latitude and longitude columns.

    Returns:
        pd.DataFrame: A filtered DataFrame containing only points within city limits.

    Raises:
        FileNotFoundError: If the city boundary configuration file cannot be found.
    """
    if not CITY_BNDY_WGS84.exists():
        raise FileNotFoundError(
            f"Could not locate boundary file asset at designated config path: {CITY_BNDY_WGS84}"
        )

    # Temporarily switch structure to an explicit GeoDataFrame ('gdf')
    gdf = gpd.GeoDataFrame(
        df, geometry=gpd.points_from_xy(df.lon, df.lat), crs="EPSG:4326"
    )

    # Ingest the boundary file directly using your pathlib setup
    stl_boundary = gpd.read_file(CITY_BNDY_WGS84).to_crs("EPSG:4326")

    # Inner join drops any points falling outside the boundary polygon limits
    gdf = gpd.sjoin(gdf, stl_boundary[["geometry"]], how="inner", predicate="within").copy()

    if "index_right" in gdf.columns:
        gdf = gdf.drop(columns=["index_right"])

    # Revert back to standard DataFrame structure and throw away spatial metadata
    return pd.DataFrame(gdf.drop(columns=["geometry"])).copy()

parse_incident_dates(values)

Parse the IncidentDate column, which SLMPD has published in two layouts.

Most monthly files use m/d/YYYY (5/1/2024). The January-April 2024 files use ISO dates (2024-01-05). The m/d/YYYY layout is tried first, so months that already worked parse exactly as before; anything it cannot read is then tried as ISO. Values that match neither become NaT.

Source code in src/geocrime_stl/etl/transform.py
58
59
60
61
62
63
64
65
66
67
68
69
70
def parse_incident_dates(values: pd.Series) -> pd.Series:
    """Parse the ``IncidentDate`` column, which SLMPD has published in two layouts.

    Most monthly files use ``m/d/YYYY`` (``5/1/2024``). The January-April 2024 files use ISO dates
    (``2024-01-05``). The ``m/d/YYYY`` layout is tried first, so months that already worked parse
    exactly as before; anything it cannot read is then tried as ISO. Values that match neither become NaT.
    """
    parsed = pd.to_datetime(values, format="%m/%d/%Y", exact=False, errors="coerce")
    unread = parsed.isna() & values.notna()
    if unread.any():
        parsed = parsed.copy()
        parsed[unread] = pd.to_datetime(values[unread], format="ISO8601", errors="coerce")
    return parsed

raw_csv_to_dataframe(csv_bytes, **kwargs)

Converts raw byte stream cleanly into an in-memory Pandas DataFrame.

Parameters:

Name Type Description Default
csv_bytes bytes

The raw byte content of the downloaded CSV file.

required
**kwargs

Additional keyword arguments passed directly to pd.read_csv.

{}

Returns:

Type Description
DataFrame

pd.DataFrame: The loaded DataFrame parsed from the byte stream.

Raises:

Type Description
ValueError

If the byte stream fails to decode into a valid DataFrame.

Source code in src/geocrime_stl/etl/transform.py
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
def raw_csv_to_dataframe(csv_bytes: bytes, **kwargs) -> pd.DataFrame:
    """Converts raw byte stream cleanly into an in-memory Pandas DataFrame.

    Args:
        csv_bytes: The raw byte content of the downloaded CSV file.
        **kwargs: Additional keyword arguments passed directly to `pd.read_csv`.

    Returns:
        pd.DataFrame: The loaded DataFrame parsed from the byte stream.

    Raises:
        ValueError: If the byte stream fails to decode into a valid DataFrame.
    """
    try:
        return pd.read_csv(io.BytesIO(csv_bytes), **kwargs)
    except Exception as e:
        raise ValueError(f"Failed to decode raw byte stream to DataFrame: {e}") from e

run_pipeline(month, year)

Runs the entire ingestion, cleaning, and spatial-clipping pipeline.

Parameters:

Name Type Description Default
month int

The target month integer to process.

required
year int

The target year integer to process.

required

Returns:

Name Type Description
CrimeDataPackage CrimeDataPackage

The resulting packaged dataset containing cleaned data and metadata.

Source code in src/geocrime_stl/etl/transform.py
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
def run_pipeline(month: int, year: int) -> CrimeDataPackage:
    """Runs the entire ingestion, cleaning, and spatial-clipping pipeline.

    Args:
        month: The target month integer to process.
        year: The target year integer to process.

    Returns:
        CrimeDataPackage: The resulting packaged dataset containing cleaned data and metadata.
    """
    from geocrime_stl.etl import extract

    # Build the target URL
    url, m_int, y_int = extract.construct_url(month_input=month, year_input=year)

    # Fetch the data stream from SLMPD with built-in safety net
    try:
        raw_bytes = extract.fetch_crime_data_csv(url, keep_raw_csv=False)
    except extract.ExtractionError as e:
        logger.warning(
            "🛑 Unable to retrieve data for %d/%d. The asset might not be published yet. Details: %s",
            month,
            year,
            e,
        )
        return CrimeDataPackage(df=pd.DataFrame(), month=m_int, year=y_int)

    # Decode bytes to DataFrame (Only runs if extraction succeeded!)
    raw_df = raw_csv_to_dataframe(raw_bytes)

    # Clean, type-enforce, and clip to city boundaries
    data_package = clean_and_spatial_clip(raw_df, m_int, y_int)

    return data_package