Pandas 3.1 release - Address deprecation warning

Makefile

@@ -1,104 +1,43 @@
-.PHONY: clean-pyc clean-build docs clean
-define BROWSER_PYSCRIPT
-import os, webbrowser, sys
-try:
-	from urllib import pathname2url
-except:
-	from urllib.request import pathname2url
+.PHONY: clean-pyc clean-build clean-cython clean help tests tests-fast-fail tests-no-build install install-dev build_extensions
 
-webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
-endef
-export BROWSER_PYSCRIPT
-BROWSER := python3 -c "$$BROWSER_PYSCRIPT"
+help: ## Display this help screen
+	@sed -n 's/^\([A-Za-z0-9_.-]*\):.*## \(.*\)$$/\1:\2/p' $(MAKEFILE_LIST) | sort | column -t -s ':'
 
-help:
-	@echo "build_extensions - Build Cython extensions"
-	@echo "clean - remove all build, test, coverage and Python artifacts"
-	@echo "clean-build - remove build artifacts"
-	@echo "clean-pyc - remove Python file artifacts"
-	@echo "clean-test - remove test and coverage artifacts"
-	@echo "lint - check style with flake8"
-	@echo "test - run tests quickly with the default Python"
-	@echo "test-all - run tests on every Python version with tox"
-	@echo "coverage - check code coverage quickly with the default Python"
-	@echo "docs - generate Sphinx HTML documentation, including API docs"
-	@echo "release - package and upload a release"
-	@echo "dist - package"
-	@echo "install - install the package to the active Python's site-packages"
+build_extensions: clean ## Build Cython extensions
+	python setup.py build_ext --inplace
+
+clean: clean-build clean-pyc ## Remove all build, test, coverage and Python artifacts
 
-clean_cython:
+clean-cython: ## Clean up Cython generated C files
 	find . -name detrended_kriging.c -type f -delete
 	find . -name envphys_c.c -type f -delete
 	find . -name wind_c.c -type f -delete
 
-build_extensions: clean
-	python setup.py build_ext --inplace
-
-clean: clean-build clean-pyc clean-test
-
-clean-build:
+clean-build: ## Remove build artifacts
 	rm -fr build/
 	rm -fr dist/
 	rm -fr .eggs/
 	find . -name '*.egg-info' -exec rm -fr {} +
 	find . -name '*.egg' -exec rm -f {} +
 	find . -name '*.so' -type f -delete
 
-clean-pyc:
+clean-pyc: ## Remove Python file artifacts
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
 	find . -name '*~' -exec rm -f {} +
 	find . -name '__pycache__' -exec rm -fr {} +
 
-clean-test:
-	rm -f .coverage
-	rm -fr htmlcov/
-
-isort: ## using isort to sort imports
-	isort -rc -v .
-
-lint:
-	flake8 smrf
-
-tests: build_extensions
+tests: build_extensions ## Run tests and build extensions if necessary
 	python3 -m unittest discover -s smrf/tests/
 
-tests_fast_fail: build_extensions
+tests_fast_fail: build_extensions ## Run tests and stop at first failure
 	python3 -m unittest discover -s smrf/tests/ -f
 
-coverage: ## run coverage and submit
-	coverage run --source smrf setup.py test
-	coverage report --fail-under=75
-
-coveralls: coverage ## run coveralls
-	coveralls
-
-coverage-html: coverage ## check code coverage quickly with the default Python
-	coverage html
-	$(BROWSER) htmlcov/index.html
-
-docs:
-	rm -f docs/smrf.rst
-	rm -f docs/modules.rst
-	sphinx-apidoc -o docs/api smrf smrf/tests/*
-	$(MAKE) -C docs clean
-	$(MAKE) -C docs html
-	$(BROWSER) docs/_build/html/index.html
-
-servedocs: docs
-	watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
-
-release: clean
-	python setup.py sdist upload
-	python setup.py bdist_wheel upload
-
-dist: clean
-	python setup.py sdist
-	python setup.py bdist_wheel
-	ls -l dist
+tests-no-build: ## Run tests without building extensions
+	python3 -m unittest discover -s smrf/tests/
 
-install: clean build_extensions
+install: clean build_extensions ## Install the package with dependencies
 	python3 -m pip install .[dev]
 
-install-dev: clean
+install-dev: clean ## Install the package without dependencies for development
 	python3 -m pip install --no-deps -e .[dev]

pyproject.toml

@@ -1,32 +1,31 @@
 [build-system]
-requires = ["setuptools", "setuptools_scm>=8", "wheel", "numpy<1.23", "cython"]
+requires = ["cython<4", "numpy<3", "setuptools", "setuptools_scm", "wheel"]
 
 [project]
 name = "smrf"
 description = "Forcing input framework for iSnobal"
 dynamic = ["version"]
-requires-python = ">=3.9"
+requires-python = ">=3.14"
 readme = "README.md"
 keywords = ['iSnobal', 'Snow', 'Modeling']
 classifiers = [
     "Intended Audience :: Science/Research",
     "Natural Language :: English",
-    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.14",
 ]
 dependencies = [
     "coloredlogs",
     "cfgrib",
-    "inicheck<0.10.0",
+    "m3w-inicheck",
     "netCDF4",
-    "numpy<1.23",
-    "pandas<1.4",
-    "pykrige>=1.5.0",
+    "numpy<3",
+    "pandas<4",
+    "pykrige",
     "pytz",
-    "regex<2021",
-    "scipy<1.12",
+    "scipy<2",
     "topocalc @ git+https://github.com/iSnobal/topocalc.git@20260526",
-    "utm==0.5",
-    "xarray<2023",
+    "utm",
+    "xarray<2027",
 ]
 
 [project.scripts]

smrf/data/hrrr/file_loader.py

@@ -1,11 +1,11 @@
 import os
 from datetime import datetime, timedelta
-from typing import Dict
 
 import numpy as np
 import pandas as pd
 import utm
 import xarray as xr
+
 from smrf.data.load_topo import Topo
 
 from .file_handler import FileHandler
@@ -69,7 +69,7 @@ def data_for_time_and_topo(
         start_date: datetime,
         bbox: list[float],
         topo: Topo,
-    ) -> Dict[str, pd.DataFrame]:
+    ) -> dict[str, pd.DataFrame]:
         """
         Load variables from HRRR either using Xarray (default) or GDAL.
 
@@ -80,7 +80,7 @@ def data_for_time_and_topo(
         :return:
           Dict - Dataframe dictionary with variable names as keys
         """
-        self.log.debug("Reading GRIB file for date: {}".format(datetime))
+        self.log.debug(f"Reading GRIB file for date: {datetime}")
 
         data = self.xarray(start_date, bbox)
         if len(self._load_gdal) > 0:
@@ -124,7 +124,7 @@ def xarray(
         self,
         date: datetime,
         bbox: list[float],
-    ) -> Dict[str, pd.DataFrame]:
+    ) -> dict[str, pd.DataFrame]:
         """
         Get the data for given time range and a specified bounding box using Xarray.
         Read data is stored on instance attribute.
@@ -161,20 +161,16 @@ def xarray(
                     )
                 )
         except Exception as e:
-            self.log.error(
-                "  Could not load forecast for date {} successfully".format(date)
-            )
+            self.log.error(f"  Could not load forecast for date {date} successfully")
             raise e
 
         try:
             return self.convert_to_dataframes(data)
         except Exception as e:
-            self.log.debug(
-                "  Could not combine forecast data for given date: {} ".format(date)
-            )
+            self.log.debug(f"  Could not combine forecast data for given date: {date} ")
             raise e
 
-    def convert_to_dataframes(self, data: xr.Dataset) -> Dict[str, pd.DataFrame]:
+    def convert_to_dataframes(self, data: xr.Dataset) -> dict[str, pd.DataFrame]:
         """
         Convert the loaded data from a dataframe to a dictionary of Pandas dataframes
 
@@ -187,16 +183,14 @@ def convert_to_dataframes(self, data: xr.Dataset) -> Dict[str, pd.DataFrame]:
         dataframe = {}
 
         for variable in list(data.data_vars):
-            # convert from a row multi-index to a column multi-index
-            df = data[variable].to_dataframe().unstack(level=[1, 2])
-
-            df = df.loc[:, variable]
-
-            df.columns = self.format_column_names(df)
-            df.index.rename("date_time", inplace=True)
+            values, rows, cols = self._grid_values(data[variable])
+            has_data = ~np.isnan(values).all(axis=0)
 
-            df.dropna(axis=1, how="all", inplace=True)
-            df.sort_index(axis=0, inplace=True)
+            df = pd.DataFrame(
+                values[:, has_data],
+                index=pd.DatetimeIndex(data[variable].time.values, name="date_time"),
+                columns=self._grid_names(rows[has_data], cols[has_data]),
+            )
             dataframe[variable] = df
 
             # TODO - Move to the corresponding variable distribution class
@@ -227,7 +221,7 @@ def get_metadata(
 
         file = self._get_file_path(date, self._forecast_hour)
 
-        self.log.debug("Loading metadata from file: {}".format(file))
+        self.log.debug(f"Loading metadata from file: {file}")
 
         if os.path.exists(file):
             data = file_loader.load_variable_level(
@@ -248,59 +242,63 @@ def get_metadata(
             )
         data = file_loader.crop_to_bbox(data)
 
-        df = data[self.HRRR_METADATA_NAME].to_dataframe().unstack(level=[1, 2])
-
-        metadata = []
-        for mm in self.METADATA_VARIABLES:
-            df_tmp = df[mm]
-            df_tmp.columns = self.format_column_names(df_tmp)
-            df_tmp = df_tmp.iloc[0]
-            df_tmp.name = mm
-            metadata.append(df_tmp)
+        # The metadata is static, so only the first time step is needed
+        metadata_values = {}
+        for variable in self.METADATA_VARIABLES:
+            values, rows, cols = self._grid_values(data[variable])
+            metadata_values[variable] = values[0]
+
+        # float64 keeps coordinates and elevation at the precision the grid expects
+        metadata = (
+            pd.DataFrame(metadata_values, index=self._grid_names(rows, cols))
+            .dropna(subset=[self.HRRR_METADATA_NAME])
+            .astype(np.float64)
+        )
 
-        metadata = pd.concat(metadata, axis=1).dropna(subset=[self.HRRR_METADATA_NAME])
-        metadata = metadata.apply(
-            FileLoader.apply_utm, args=(utm_zone_number,), axis=1
+        # HRRR longitudes run 0 to 360 degrees east, so shift them to -180 to 180
+        metadata["longitude"] -= 360
+        utm_x, utm_y, *_ = utm.from_latlon(
+            metadata["latitude"].to_numpy(),
+            metadata["longitude"].to_numpy(),
+            force_zone_number=utm_zone_number,
         )
+        metadata["utm_x"] = utm_x
+        metadata["utm_y"] = utm_y
 
         return metadata
 
     @staticmethod
-    def format_column_names(dataframe):
+    def _grid_values(array: xr.DataArray) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
         """
-        Make new names for the columns as grid_y_x
+        Flatten the y and x dimensions of a DataArray into one grid dimension.
+        Any leading dimension (time) is kept as the row dimension.
 
-        :param dataframe:
-        :return: Array - New column names including the y and x GRIB pixel
-                         index. Example: grid_0_1 for y at 0 and x at 1
+        Args:
+            array: DataArray with y and x as its last two dimensions
+
+        Returns:
+            (values, rows, cols) where values is a 2D array of shape
+            (time, y * x), and rows and cols give the y and x index of
+            each grid cell
         """
-        return [
-            "grid_{c[0]}_{c[1]}".format(c=col)
-            for col in dataframe.columns.to_flat_index()
-        ]
+        array = array.transpose(..., "y", "x")
+        rows, cols = np.indices(array.shape[-2:])
+        values = array.values.reshape(-1, rows.size)
+        return values, rows.ravel(), cols.ravel()
 
     @staticmethod
-    def apply_utm(dataframe, utm_zone_number):
+    def _grid_names(rows: np.ndarray, cols: np.ndarray) -> list[str]:
         """
-        Ufunc to calculate the utm from lat/lon for a series
+        Build a column name for each grid cell from its y and x index
 
         Args:
-            dataframe: pandas series with fields latitude and longitude
-            utm_zone_number: Zone number to force to
+            rows: y index of each grid cell
+            cols: x index of each grid cell
 
         Returns:
-            Pandas series entry with fields 'utm_x' and 'utm_y' filled
+            List of names as grid_{y}_{x}, e.g. grid_0_1 for y=0 and x=1
         """
-        # HRRR has longitude reporting in degrees from the east
-        dataframe["longitude"] -= 360
-
-        (dataframe["utm_x"], dataframe["utm_y"], *unused) = utm.from_latlon(
-            dataframe["latitude"],
-            dataframe["longitude"],
-            force_zone_number=utm_zone_number,
-        )
-
-        return dataframe
+        return [f"grid_{row}_{col}" for row, col in zip(rows, cols)]
 
     def gdal(self, date: datetime, topo: Topo) -> dict[str, np.ndarray]:
         """