Enable processing of CM4 abrupt 4xCO2 ensemble

scripts/data_process/Makefile

@@ -356,6 +356,14 @@ cm4_piControl_ocean_stats:
 cm4_1pctCO2_ocean_stats:
 	./compute_stats.sh --config configs/CM4-1pctCO2-ocean-$(RESOLUTION)-140yr.yaml
 
+.PHONY: cm4_abrupt_4xCO2_ensemble_ocean_stats
+cm4_abrupt_4xCO2_ensemble_ocean_stats:
+	./compute_stats.sh --config configs/CM4-abrupt-4xCO2-ensemble-ocean-$(RESOLUTION).yaml
+
+.PHONY: cm4_abrupt_4xCO2_ensemble_ocean_1daily_stats
+cm4_abrupt_4xCO2_ensemble_ocean_1daily_stats:
+	./compute_stats.sh --config configs/CM4-abrupt-4xCO2-ensemble-ocean-$(RESOLUTION)-1daily.yaml
+
 .PHONY: cm4_piControl_ocean_1daily_stats
 cm4_piControl_ocean_1daily_stats:
 	./compute_stats.sh --config configs/CM4-piControl-ocean-$(RESOLUTION)-1daily-200yr.yaml
@@ -377,6 +385,8 @@ CONFIG_CM4_1PCTCO2_COUPLED_1DAILY = configs/CM4-1pctCO2-coupled-$(RESOLUTION)-1d
 CONFIG_E3SMV3_PICONTROL_COUPLED = configs/E3SMv3-piControl-100yr-coupled.yaml
 CONFIG_CM4_LIKE_AM4_RANDCO2_COUPLED = configs/CM4-like-AM4-random-CO2-ensemble-coupled.yaml
 CONFIG_CM4_LIKE_AM4_RANDCO2_COUPLED_4DEG = configs/CM4-like-AM4-random-CO2-ensemble-coupled-4deg.yaml
+CONFIG_CM4_ABRUPT_4XCO2_COUPLED = configs/CM4-abrupt-4xCO2-ensemble-coupled-$(RESOLUTION).yaml
+CONFIG_CM4_ABRUPT_4XCO2_COUPLED_1DAILY = configs/CM4-abrupt-4xCO2-ensemble-coupled-$(RESOLUTION)-1daily.yaml
 
 .PHONY: cm4_piControl_coupled
 cm4_piControl_coupled:
@@ -428,6 +438,10 @@ e3smv3_piControl_100yr_coupled:
 cm4_like_am4_random_co2_ensemble_atmosphere:
 	./compute_dataset.sh --dataset --stats $(if $(filter 4deg,$(RESOLUTION)),--time-coarsen) --config configs/CM4-like-AM4-random-CO2-ensemble-atmosphere-$(RESOLUTION)-8layer.yaml
 
+.PHONY: cm4_abrupt_4xCO2_ensemble_atmosphere
+cm4_abrupt_4xCO2_ensemble_atmosphere:
+	./compute_dataset.sh --dataset --stats $(if $(filter 4deg,$(RESOLUTION)),--time-coarsen) --config configs/CM4-abrupt-4xCO2-ensemble-atmosphere-$(RESOLUTION)-8layer.yaml
+
 .PHONY: cm4_like_am4_random_co2_ensemble_coupled
 cm4_like_am4_random_co2_ensemble_coupled:
 	conda run --no-capture-output -n create_coupled_datasets python ./create_coupled_datasets.py --yaml $(CONFIG_CM4_LIKE_AM4_RANDCO2_COUPLED)
@@ -440,6 +454,22 @@ cm4_like_am4_random_co2_ensemble_coupled_argo:
 cm4_like_am4_random_co2_ensemble_coupled_4deg_argo:
 	./create_coupled_datasets.sh --config $(CONFIG_CM4_LIKE_AM4_RANDCO2_COUPLED_4DEG) --image $(COUPLED_PROCESSING_IMAGE):$(COUPLED_PROCESSING_IMAGE_VERSION)
 
+.PHONY: cm4_abrupt_4xCO2_ensemble_coupled
+cm4_abrupt_4xCO2_ensemble_coupled:
+	conda run --no-capture-output -n create_coupled_datasets python ./create_coupled_datasets.py --yaml $(CONFIG_CM4_ABRUPT_4XCO2_COUPLED)
+
+.PHONY: cm4_abrupt_4xCO2_ensemble_coupled_argo
+cm4_abrupt_4xCO2_ensemble_coupled_argo:
+	./create_coupled_datasets.sh --config $(CONFIG_CM4_ABRUPT_4XCO2_COUPLED) --image $(COUPLED_PROCESSING_IMAGE):$(COUPLED_PROCESSING_IMAGE_VERSION)
+
+.PHONY: cm4_abrupt_4xCO2_ensemble_coupled_1daily
+cm4_abrupt_4xCO2_ensemble_coupled_1daily:
+	conda run --no-capture-output -n create_coupled_datasets python ./create_coupled_datasets.py --yaml $(CONFIG_CM4_ABRUPT_4XCO2_COUPLED_1DAILY)
+
+.PHONY: cm4_abrupt_4xCO2_ensemble_coupled_1daily_argo
+cm4_abrupt_4xCO2_ensemble_coupled_1daily_argo:
+	./create_coupled_datasets.sh --config $(CONFIG_CM4_ABRUPT_4XCO2_COUPLED_1DAILY) --image $(COUPLED_PROCESSING_IMAGE):$(COUPLED_PROCESSING_IMAGE_VERSION)
+
 .PHONY: shield_amip_constant_co2_c96_dataset
 shield_amip_constant_co2_c96_dataset:
 	./compute_dataset.sh --dataset --stats --time-coarsen --config configs/shield-amip-constant-co2-c96-$(RESOLUTION)-$(LAYERS).yaml

scripts/data_process/combine_stats.py

@@ -9,6 +9,7 @@
 import fsspec
 import xarray as xr
 import yaml
+from fs_utils import load_netcdf
 
 
 def copy(source: str, destination: str):
@@ -59,12 +60,7 @@ def _make_history_string(config_filename: str, stats_output_dir: str):
 
 
 def open_datasets(roots: List[str], filename: str) -> List[xr.Dataset]:
-    datasets = []
-    for root in roots:
-        with fsspec.open(root + filename) as file:
-            ds = xr.open_dataset(file, decode_timedelta=False).load()
-        datasets.append(ds)
-    return datasets
+    return [load_netcdf(root + filename) for root in roots]
 
 
 def _combined_stats_exist(output_directory: str, subdirectory: str) -> bool:

scripts/data_process/configs/CM4-abrupt-4xCO2-ensemble-atmosphere-1deg-8layer.yaml

@@ -0,0 +1,146 @@
+runs:
+  ic_0001: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0001
+  ic_0002: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0002
+  ic_0003: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0003
+  ic_0004: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0004
+  ic_0005: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0005
+  ic_0006: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0006
+  ic_0007: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0007
+  ic_0008: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0008
+  ic_0009: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0009
+  ic_0010: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0010
+  ic_0011: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0011
+  ic_0012: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0012
+  ic_0013: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0013
+  ic_0014: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0014
+  ic_0015: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0015
+  ic_0016: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0016
+  ic_0017: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0017
+  ic_0018: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0018
+  ic_0019: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0019
+  ic_0020: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0020
+  ic_0021: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0021
+  ic_0022: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0022
+  ic_0023: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0023
+  ic_0024: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0024
+  ic_0025: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0025
+  ic_0026: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0026
+  ic_0027: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0027
+  ic_0028: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0028
+  ic_0029: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0029
+  ic_0030: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0030
+  ic_0031: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0031
+  ic_0032: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0032
+  ic_0033: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0033
+  ic_0034: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0034
+  ic_0035: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0035
+  ic_0036: gs://vcm-ml-raw-flexible-retention/2026-10-16-CM4-abrupt-4xCO2-ensemble/regridded-zarrs/gaussian_grid_180_by_360/abrupt-4xCO2-ic_0036
+data_output_directory: gs://vcm-ml-intermediate/2026-10-16-CM4-abrupt-4xCO2-ensemble
+stats:
+  output_directory: gs://vcm-ml-intermediate/2026-10-16-CM4-abrupt-4xCO2-ensemble-stats
+  data_type: CM4
+  beaker_dataset: 2026-10-16-CM4-abrupt-4xCO2-ensemble-stats
+dataset_computation:
+  n_split: 10
+  reference_vertical_coordinate_file: gs://vcm-ml-raw-flexible-retention/2024-08-10-pre-industrial-CM4-simulation/vertical-coordinate-file/fv_core.res.nc
+  reference_vertical_coordinate_file_land: gs://vcm-ml-raw-flexible-retention/2024-11-01-CM4-piControl/land-vertical-coordinate-file/land-vertical-coordinate.nc
+  vertical_coarsening_indices:
+    # computed here: https://github.com/ai2cm/explore/blob/master/jamesd/2024-08-13-pre-industiral-CM4-eda/2024-08-28-AM4-vertical-indices.ipynb
+    - [0, 7]
+    - [7, 10]
+    - [10, 13]
+    - [13, 16]
+    - [16, 18]
+    - [18, 22]
+    - [22, 25]
+    - [25, 33]
+  vertical_coarsening_indices_land:
+    # computed here: https://github.com/ai2cm/explore2/blob/main/troya/2025-02-28-LS-evaluation/soil-levels-lm4.ipynb
+    - [0, 2]
+    - [2, 6]
+    - [6, 10]
+    - [10, 15]
+  validate_vertical_coarsening_indices_land: false  # land vertical coarsening indices intentionally do not span full depth
+  mask_soil_moisture: true
+  renaming:
+    specific_humidity_at_two_meters: Q2m
+    air_temperature_at_two_meters: TMP2m
+    eastward_wind_at_ten_meters: UGRD10m
+    northward_wind_at_ten_meters: VGRD10m
+    interval_average_carbon_dioxide: carbon_dioxide
+  variable_sources:
+    fluxes_2d.zarr:
+      - PRATEsfc
+      - SHTFLsfc
+      - DLWRFsfc
+      - DSWRFsfc
+      - DSWRFtoa
+      - ULWRFsfc
+      - ULWRFtoa
+      - USWRFsfc
+      - USWRFtoa
+      - eastward_surface_wind_stress
+      - northward_surface_wind_stress
+      - surface_evaporation_rate
+      - total_energy
+      - total_frozen_precipitation_rate
+    full_state.zarr:
+      # 2D vars
+      - HGTsfc # static
+      - PRESsfc
+      - PRMSL
+      - surface_temperature
+      - air_temperature_at_two_meters
+      - specific_humidity_at_two_meters
+      - eastward_wind_at_ten_meters
+      - northward_wind_at_ten_meters
+      - h500
+      - TMP850
+      # 3D vars:
+      - air_temperature
+      - specific_humidity # water species
+      - cloud_water_mixing_ratio # water species
+      - cloud_ice_mixing_ratio # water species
+      - eastward_wind
+      - northward_wind
+    full_state_land.zarr:
+      - column_soil_moisture
+      - GHTFLsfc
+      - surface_snow_amount
+      - surface_snow_area_fraction
+      - total_land_column_energy_without_rivers
+      - total_precipitation_heat_flux
+      - total_vapor_sensible_heat_flux
+      - downward_longwave_radiative_flux
+      - net_surface_longwave_radiative_flux
+    full_state_soil.zarr:
+      # below are vertically resolved soil layer variables -- need to be coarsened
+      - total_moisture_content_of_soil_layer
+      - temperature_of_soil_layer
+    land_static.zarr:
+      - land_fraction
+    ice_6hourly_snap.zarr:
+      - sea_ice_fraction
+    # Constructed an interval-average CO2 concentration from 6-hour snapshots
+    # prior to uploading since AM4 has a subtle off-by-one bug when computing
+    # interval averages of the scalar CO2 concentration online.
+    scalar_6hourly_snap.zarr:
+      - interval_average_carbon_dioxide
+    land_surface_type_fractions.zarr:
+      - glac_fraction
+      - soil_fraction
+      - lake_fraction
+  standard_names:
+    longitude_dim: lon
+    latitude_dim: lat
+    graupel_mixing_ratio: None
+    rain_mixing_ratio: None
+    snow_mixing_ratio: None
+    precipitable_water_path: None
+    surface_snow_rate: None
+    surface_ice_rate: None
+    surface_graupel_rate: None
+    land_names_to_vertically_coarsen_by_height_weighting:
+      - temperature_of_soil_layer
+    land_names_to_vertically_coarsen_by_sum:
+      - total_moisture_content_of_soil_layer