From bc3a056efb60161770841cd017a5dc21f894b082 Mon Sep 17 00:00:00 2001 From: acollow Date: Tue, 9 Dec 2025 13:11:04 -0500 Subject: [PATCH 01/10] initial check in of AERONET evaluation codes --- src/pyobs/evaluation/AERONET/README | 5 + .../evaluation/AERONET/plotagainstm21c.py | 945 +++++++++++++++++ .../AERONET/plotsinglestationonly.py | 320 ++++++ .../evaluation/AERONET/processaeronet.py | 492 +++++++++ .../evaluation/AERONET/station_analysis.py | 670 ++++++++++++ .../AERONET/station_analysis_withm2.py | 987 ++++++++++++++++++ 6 files changed, 3419 insertions(+) create mode 100644 src/pyobs/evaluation/AERONET/README create mode 100644 src/pyobs/evaluation/AERONET/plotagainstm21c.py create mode 100644 src/pyobs/evaluation/AERONET/plotsinglestationonly.py create mode 100644 src/pyobs/evaluation/AERONET/processaeronet.py create mode 100644 src/pyobs/evaluation/AERONET/station_analysis.py create mode 100644 src/pyobs/evaluation/AERONET/station_analysis_withm2.py diff --git a/src/pyobs/evaluation/AERONET/README b/src/pyobs/evaluation/AERONET/README new file mode 100644 index 0000000..4fb35a8 --- /dev/null +++ b/src/pyobs/evaluation/AERONET/README @@ -0,0 +1,5 @@ +This set of codes is currently set up to evaluate MERRA-21C using Lunar AERONET observations but can adapted for standard AERONET observations or other GEOS simulations. + +The first step is to pre-process the model data using processaeronet.py. This code will generate csv files that include sampled model data according to the availability of observations. The code will loop through all AERONET files that match the specified pattern so that all stations are processed. There are command line arguments to change the filepath of the input model data and the output directory where you want to csv files stored. The AERONET observation files used were directly downloaded from the AERONET website (https://aeronet.gsfc.nasa.gov/) rather than ODS files generated for aerosol data assimilation. + +There are two codes that can be used to generate figures. plotagainstm21c.py will generate a global summary figure with a map showing the bias and correlation for AOD and Angtrom exponent for all available stations. plotsinglestationonly.py will produce two figures (one for AOD, one for Angstrom exponent) with a time series of the full period, a mean annual cycle, a 2d kernel density estimate with AERONET on the x axis and the model on the y axis. The flag "--include--merra2" will add a second experiment should you want to compare. diff --git a/src/pyobs/evaluation/AERONET/plotagainstm21c.py b/src/pyobs/evaluation/AERONET/plotagainstm21c.py new file mode 100644 index 0000000..bdda9e2 --- /dev/null +++ b/src/pyobs/evaluation/AERONET/plotagainstm21c.py @@ -0,0 +1,945 @@ +import os +import glob +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import cartopy.crs as ccrs +import cartopy.feature as cfeature +from matplotlib.colors import LinearSegmentedColormap, ListedColormap +from scipy.stats import pearsonr +import argparse +import warnings +from scipy.stats import gaussian_kde +import matplotlib.patches as patches +import matplotlib.ticker as ticker + +warnings.filterwarnings('ignore') + +def create_white_viridis_colormap(): + """Create a custom colormap that starts with white for low densities and transitions to viridis""" + # Get viridis colormap + viridis = plt.cm.get_cmap('viridis', 256) + + # Create new colormap that starts with white + # Take viridis colors but replace the lowest values with white + colors = viridis(np.linspace(0, 1, 256)) + + # Replace first 20% of colors with white to white-to-viridis transition + n_white = int(0.15 * 256) # 15% white transition + for i in range(n_white): + # Interpolate from white to first viridis color + alpha = i / n_white + colors[i] = (1-alpha) * np.array([1, 1, 1, 1]) + alpha * colors[n_white] + + return ListedColormap(colors, name='white_viridis') + +def generate_comparison_maps(data_dir="./aeronet_merra21c_comparison/", + output_dir="./figures/", + min_points=30, + years=None, + file_pattern=None, + debug=False): + """ + Generate global maps showing bias and correlation between AERONET and MERRA-21C data. + + Parameters: + ----------- + data_dir : str + Directory containing processed CSV files + output_dir : str + Directory to save output figures + min_points : int + Minimum number of data points required for a station to be included + years : list or None + List of years to include in analysis. If None, uses all available data. + file_pattern : str or None + Custom file pattern to match CSV files. If None, uses default pattern. + debug : bool + If True, print additional debugging information + """ + # Create custom colormap + white_viridis = create_white_viridis_colormap() + + # Create output directory + os.makedirs(output_dir, exist_ok=True) + + # Check if the data directory exists + if not os.path.exists(data_dir): + print(f"Error: Data directory '{data_dir}' does not exist.") + return + + # List all files in the directory + all_files = os.listdir(data_dir) + csv_files_in_dir = [f for f in all_files if f.endswith('.csv')] + + if debug: + print(f"Found {len(csv_files_in_dir)} total CSV files in directory.") + if csv_files_in_dir: + print(f"Sample filenames: {csv_files_in_dir[:5]}") + + # Determine file pattern based on years + if file_pattern is None: + if years is not None: + if len(years) == 1: + # Try different patterns for single year + patterns = [ + f"*_{years[0]}_{years[0]}.csv", # station_2018_2018.csv + f"*_{years[0]}.csv", # station_2018.csv + "*.csv" # Any CSV file + ] + else: + # Multiple years case - try different patterns + patterns = [ + f"*_{min(years)}_{max(years)}.csv", # station_2018_2020.csv + "*.csv" # Any CSV file + ] + else: + patterns = ["*.csv"] # Default pattern - match all CSV files + else: + patterns = [file_pattern] + + # Try each pattern until we find files + csv_files = [] + used_pattern = None + + for pattern in patterns: + csv_files = glob.glob(os.path.join(data_dir, pattern)) + if csv_files: + used_pattern = pattern + break + + if not csv_files: + print(f"No CSV files found in {data_dir} matching any of these patterns: {patterns}") + print(f"Available CSV files: {csv_files_in_dir if csv_files_in_dir else 'No CSV files in directory'}") + return + + print(f"Found {len(csv_files)} CSV files matching pattern '{used_pattern}'") + + if debug and csv_files: + print("Sample filenames:") + for file in csv_files[:5]: + print(f" {os.path.basename(file)}") + + # Initialize lists to store data for each station + stations = [] + lats = [] + lons = [] + mean_aod_biases = [] + mean_angstrom_biases = [] + aod_correlations = [] + angstrom_correlations = [] + mean_aeronet_aods = [] + mean_merra_aods = [] + data_counts = [] + valid_data_counts = [] # Count of data points after quality filtering + aod_sources = [] + angstrom_sources = [] + + # Process each station file + processed_count = 0 + skipped_files = [] + error_files = [] + + for csv_file in csv_files: + try: + # Read data + df = pd.read_csv(csv_file) + + # Debug: Show column names and data quality for first file + if debug and processed_count == 0: + print(f"\nColumns in CSV file: {list(df.columns)}") + print(f"Data shape: {df.shape}") + nan_counts = df.isnull().sum() + if nan_counts.sum() > 0: + print(f"NaN counts per column:\n{nan_counts[nan_counts > 0]}") + print(f"First few rows of data:\n{df.head(2)}") + + # Check for required metadata columns + if 'station' not in df.columns or 'lat' not in df.columns or 'lon' not in df.columns: + msg = "missing required metadata columns (station, lat, lon)" + skipped_files.append((os.path.basename(csv_file), msg)) + continue + + # Check for required data columns + required_cols = ['aeronet_aod_550', 'merra_aod_550', 'aeronet_angstrom', 'merra_angstrom'] + missing_cols = [col for col in required_cols if col not in df.columns] + + if missing_cols: + msg = f"missing columns: {', '.join(missing_cols)}" + skipped_files.append((os.path.basename(csv_file), msg)) + continue + + # Extract station metadata first + station = df['station'].iloc[0] if not df['station'].isna().iloc[0] else "Unknown" + lat = df['lat'].iloc[0] + lon = df['lon'].iloc[0] + + if np.isnan(lat) or np.isnan(lon): + msg = "invalid lat/lon coordinates" + skipped_files.append((os.path.basename(csv_file), msg)) + continue + + # Count original data points + original_count = len(df) + + # Apply comprehensive quality filters + quality_mask = ( + # Remove NaN values + (~df['aeronet_aod_550'].isna()) & + (~df['merra_aod_550'].isna()) & + (~df['aeronet_angstrom'].isna()) & + (~df['merra_angstrom'].isna()) & + # Remove negative AOD values and unreasonably high values + (df['aeronet_aod_550'] >= 0) & (df['aeronet_aod_550'] < 10) & + (df['merra_aod_550'] >= 0) & (df['merra_aod_550'] < 10) & + # Remove unreasonable Angstrom exponent values + (df['aeronet_angstrom'] >= -1) & (df['aeronet_angstrom'] <= 3) & + (df['merra_angstrom'] >= -1) & (df['merra_angstrom'] <= 3) & + # Remove infinite values + (np.isfinite(df['aeronet_aod_550'])) & + (np.isfinite(df['merra_aod_550'])) & + (np.isfinite(df['aeronet_angstrom'])) & + (np.isfinite(df['merra_angstrom'])) + ) + + df_quality = df[quality_mask].copy() + + if debug and processed_count == 0: + print(f"Quality filtering removed {original_count - len(df_quality)} out of {original_count} data points") + + # Skip if too few data points after quality filtering + if len(df_quality) < min_points: + msg = f"only {len(df_quality)} quality-filtered data points (minimum: {min_points})" + skipped_files.append((os.path.basename(csv_file), msg)) + continue + + # Filter by years if specified + if years is not None: + if 'datetime' not in df_quality.columns: + msg = "missing 'datetime' column" + skipped_files.append((os.path.basename(csv_file), msg)) + continue + + try: + df_quality['datetime'] = pd.to_datetime(df_quality['datetime']) + except: + msg = "unable to parse datetime column" + skipped_files.append((os.path.basename(csv_file), msg)) + continue + + df_year = df_quality[df_quality['datetime'].dt.year.isin(years)] + + if len(df_year) < min_points: + msg = f"only {len(df_year)} data points for years {years} after quality filtering" + skipped_files.append((os.path.basename(csv_file), msg)) + continue + + # Use the year-filtered dataframe + df_quality = df_year + + # Get source information if available + aod_source = df_quality['aod_source'].iloc[0] if 'aod_source' in df_quality.columns else 'Unknown' + angstrom_source = df_quality['angstrom_source'].iloc[0] if 'angstrom_source' in df_quality.columns else 'Unknown' + + # Calculate bias columns if they don't exist + if 'aod_bias' not in df_quality.columns: + df_quality['aod_bias'] = df_quality['merra_aod_550'] - df_quality['aeronet_aod_550'] + + if 'angstrom_bias' not in df_quality.columns: + df_quality['angstrom_bias'] = df_quality['merra_angstrom'] - df_quality['aeronet_angstrom'] + + # Calculate metrics using quality-filtered data + mean_aod_bias = df_quality['aod_bias'].mean() + mean_angstrom_bias = df_quality['angstrom_bias'].mean() + + # Calculate correlations with additional error handling + try: + if len(df_quality) < 3: # Need at least 3 points for meaningful correlation + raise ValueError("Insufficient data points for correlation") + + # Check for zero variance (constant values) + if (df_quality['aeronet_aod_550'].std() == 0 or + df_quality['merra_aod_550'].std() == 0): + aod_corr = np.nan + else: + aod_corr, _ = pearsonr(df_quality['aeronet_aod_550'], df_quality['merra_aod_550']) + + if (df_quality['aeronet_angstrom'].std() == 0 or + df_quality['merra_angstrom'].std() == 0): + angstrom_corr = np.nan + else: + angstrom_corr, _ = pearsonr(df_quality['aeronet_angstrom'], df_quality['merra_angstrom']) + + except Exception as e: + msg = f"correlation calculation failed: {str(e)}" + skipped_files.append((os.path.basename(csv_file), msg)) + continue + + # Check if correlations are valid (not NaN) + if np.isnan(aod_corr) and np.isnan(angstrom_corr): + msg = "both correlations are NaN" + skipped_files.append((os.path.basename(csv_file), msg)) + continue + + # Store data + stations.append(station) + lats.append(lat) + lons.append(lon) + mean_aod_biases.append(mean_aod_bias) + mean_angstrom_biases.append(mean_angstrom_bias) + aod_correlations.append(aod_corr if not np.isnan(aod_corr) else 0) # Replace NaN with 0 for plotting + angstrom_correlations.append(angstrom_corr if not np.isnan(angstrom_corr) else 0) + mean_aeronet_aods.append(df_quality['aeronet_aod_550'].mean()) + mean_merra_aods.append(df_quality['merra_aod_550'].mean()) + data_counts.append(original_count) + valid_data_counts.append(len(df_quality)) + aod_sources.append(aod_source) + angstrom_sources.append(angstrom_source) + + processed_count += 1 + + except Exception as e: + error_files.append((os.path.basename(csv_file), str(e))) + if debug: + print(f"Error processing {csv_file}: {e}") + + # Report on processing results + if skipped_files and debug: + print(f"\nSkipped {len(skipped_files)} files:") + for filename, reason in skipped_files[:10]: # Show only first 10 + print(f" {filename}: {reason}") + if len(skipped_files) > 10: + print(f" ... and {len(skipped_files) - 10} more") + + if error_files and debug: + print(f"\nErrors in {len(error_files)} files:") + for filename, error in error_files[:10]: # Show only first 10 + print(f" {filename}: {error}") + if len(error_files) > 10: + print(f" ... and {len(error_files) - 10} more") + + if processed_count == 0: + print("No stations were successfully processed. Check your data files and parameters.") + return + + print(f"Successfully processed {processed_count} stations") + + # Create dataframe with all station metrics + station_metrics = pd.DataFrame({ + 'station': stations, + 'latitude': lats, + 'longitude': lons, + 'mean_aod_bias': mean_aod_biases, + 'mean_angstrom_bias': mean_angstrom_biases, + 'aod_correlation': aod_correlations, + 'angstrom_correlation': angstrom_correlations, + 'mean_aeronet_aod': mean_aeronet_aods, + 'mean_merra_aod': mean_merra_aods, + 'total_data_points': data_counts, + 'valid_data_points': valid_data_counts, + 'aod_source': aod_sources, + 'angstrom_source': angstrom_sources + }) + + # Add year info to filename + year_str = f"_{min(years)}_{max(years)}" if years and len(years) > 1 else f"_{years[0]}" if years else "" + + # Save metrics to CSV + metrics_file = os.path.join(output_dir, f"station_metrics_summary{year_str}.csv") + station_metrics.to_csv(metrics_file, index=False) + print(f"Saved metrics summary to {metrics_file}") + + # Print some summary statistics + print(f"\nSummary Statistics:") + print(f"AOD Bias: mean = {np.mean(mean_aod_biases):.4f}, std = {np.std(mean_aod_biases):.4f}") + + # Handle potential NaN values in correlations for statistics + valid_aod_corrs = [c for c in aod_correlations if not np.isnan(c)] + valid_ang_corrs = [c for c in angstrom_correlations if not np.isnan(c)] + + if valid_aod_corrs: + print(f"AOD Correlation: mean = {np.mean(valid_aod_corrs):.3f}, std = {np.std(valid_aod_corrs):.3f}") + else: + print("AOD Correlation: no valid correlations") + + print(f"Angstrom Bias: mean = {np.mean(mean_angstrom_biases):.4f}, std = {np.std(mean_angstrom_biases):.4f}") + + if valid_ang_corrs: + print(f"Angstrom Correlation: mean = {np.mean(valid_ang_corrs):.3f}, std = {np.std(valid_ang_corrs):.3f}") + else: + print("Angstrom Correlation: no valid correlations") + + print(f"Data Points: mean = {np.mean(valid_data_counts):.1f}, std = {np.std(valid_data_counts):.1f}") + print(f"Data Points: min = {np.min(valid_data_counts)}, max = {np.max(valid_data_counts)}") + + # Create custom diverging colormap for bias (blue-white-red) + bias_cmap = LinearSegmentedColormap.from_list( + 'bias_cmap', ['blue', 'white', 'red'] + ) + + # Create custom sequential colormap for correlation (white-green) + corr_cmap = LinearSegmentedColormap.from_list( + 'corr_cmap', ['white', 'green'] + ) + + # Create custom colormap for data counts (white to purple) + count_cmap = LinearSegmentedColormap.from_list( + 'count_cmap', ['lightblue', 'blue', 'darkblue', 'purple'] + ) + + # Generate 4-panel comparison figure + fig = plt.figure(figsize=(24, 16)) # Increased height for better spacing + + # Set up the 2x2 subplot layout with cartopy projections + ax1 = plt.subplot(2, 2, 1, projection=ccrs.PlateCarree()) + ax2 = plt.subplot(2, 2, 2, projection=ccrs.PlateCarree()) + ax3 = plt.subplot(2, 2, 3, projection=ccrs.PlateCarree()) + ax4 = plt.subplot(2, 2, 4, projection=ccrs.PlateCarree()) + + axes = [ax1, ax2, ax3, ax4] + panel_labels = ['a', 'b', 'c', 'd'] + + # Add map features to all subplots + for i, ax in enumerate(axes): + ax.add_feature(cfeature.COASTLINE) + ax.add_feature(cfeature.BORDERS, linestyle=':') + ax.add_feature(cfeature.LAND, alpha=0.3) + ax.add_feature(cfeature.OCEAN, alpha=0.3) + ax.set_global() + + # Add gridlines but make them less prominent + gl = ax.gridlines(draw_labels=False, alpha=0.2) + + # Add panel labels to the TOP LEFT corner + ax.text(0.03, 1.05, f"({panel_labels[i]})", transform=ax.transAxes, + fontsize=16, fontweight='bold', ha='left', va='top', + bbox=dict(facecolor='white', alpha=0.7, pad=0.1, edgecolor='none')) + + # Debug information about bias distribution + if debug: + bias_data = station_metrics['mean_aod_bias'] + print(f"\nAOD Bias Statistics:") + print(f"Mean: {np.mean(bias_data):.4f}") + print(f"Median: {np.median(bias_data):.4f}") + print(f"Std: {np.std(bias_data):.4f}") + print(f"Min: {np.min(bias_data):.4f}") + print(f"Max: {np.max(bias_data):.4f}") + print(f"5th percentile: {np.percentile(bias_data, 5):.4f}") + print(f"95th percentile: {np.percentile(bias_data, 95):.4f}") + + # Panel 1: AOD Bias with percentile-based scaling + bias_data = station_metrics['mean_aod_bias'] + # Use percentiles to handle outliers + p5, p95 = np.percentile(bias_data, [2, 98]) + # Optional: make it symmetric around zero + max_abs_bias_clipped = max(abs(p5), abs(p95)) + + sc1 = ax1.scatter( + station_metrics['longitude'], + station_metrics['latitude'], + c=station_metrics['mean_aod_bias'], + cmap=bias_cmap, + vmin=-max_abs_bias_clipped, + vmax=max_abs_bias_clipped, + s=60, + edgecolor='black', + linewidth=0.5, + transform=ccrs.PlateCarree() + ) + ax1.set_title('Nighttime AOD Bias (MERRA-21C - AERONET)', fontsize=18, pad=10) + + # Panel 2: AOD Correlation + valid_aod_mask = ~np.isnan(station_metrics['aod_correlation']) + if valid_aod_mask.sum() > 0: + sc2 = ax2.scatter( + station_metrics.loc[valid_aod_mask, 'longitude'], + station_metrics.loc[valid_aod_mask, 'latitude'], + c=station_metrics.loc[valid_aod_mask, 'aod_correlation'], + cmap=corr_cmap, + vmin=0, + vmax=1, + s=60, + edgecolor='black', + linewidth=0.5, + transform=ccrs.PlateCarree() + ) + ax2.set_title('Nighttime AOD Temporal Correlation', fontsize=18, pad=10) + + # Panel 3: Angstrom Bias + max_abs_angstrom_bias = max(abs(np.array(mean_angstrom_biases))) + sc3 = ax3.scatter( + station_metrics['longitude'], + station_metrics['latitude'], + c=station_metrics['mean_angstrom_bias'], + cmap=bias_cmap, + vmin=-max_abs_angstrom_bias, + vmax=max_abs_angstrom_bias, + s=60, + edgecolor='black', + linewidth=0.5, + transform=ccrs.PlateCarree() + ) + ax3.set_title('Nighttime Angstrom Exponent Bias', fontsize=18, pad=10) + + # Panel 4: Angstrom Correlation + valid_ang_mask = ~np.isnan(station_metrics['angstrom_correlation']) + if valid_ang_mask.sum() > 0: + sc4 = ax4.scatter( + station_metrics.loc[valid_ang_mask, 'longitude'], + station_metrics.loc[valid_ang_mask, 'latitude'], + c=station_metrics.loc[valid_ang_mask, 'angstrom_correlation'], + cmap=corr_cmap, + vmin=0, + vmax=1, + s=60, + edgecolor='black', + linewidth=0.5, + transform=ccrs.PlateCarree() + ) + ax4.set_title('Nighttime Angstrom Exponent Temporal Correlation', fontsize=18, pad=10) + + # Adjust layout to reduce spacing between columns + plt.subplots_adjust(wspace=0.05, hspace=0.3) + + # Get positions of the axes to create properly sized colorbar axes + pos1 = ax1.get_position() + pos2 = ax2.get_position() + pos3 = ax3.get_position() + pos4 = ax4.get_position() + + # Create small colorbar axes below each panel + # [left, bottom, width, height] + cbar_height = 0.025 # Slightly increased for larger fonts + cbar_gap = 0.03 # Increased gap + + cbar_ax1 = fig.add_axes([pos1.x0, pos1.y0 - cbar_gap - cbar_height, pos1.width, cbar_height]) + cbar_ax2 = fig.add_axes([pos2.x0, pos2.y0 - cbar_gap - cbar_height, pos2.width, cbar_height]) + cbar_ax3 = fig.add_axes([pos3.x0, pos3.y0 - cbar_gap - cbar_height, pos3.width, cbar_height]) + cbar_ax4 = fig.add_axes([pos4.x0, pos4.y0 - cbar_gap - cbar_height, pos4.width, cbar_height]) + + # Add colorbars to the custom axes with font size 18 + cbar1 = plt.colorbar(sc1, cax=cbar_ax1, orientation='horizontal') + cbar1.ax.tick_params(labelsize=18) + cbar1.set_label('AOD Bias', fontsize=18) + + if valid_aod_mask.sum() > 0: + cbar2 = plt.colorbar(sc2, cax=cbar_ax2, orientation='horizontal') + cbar2.ax.tick_params(labelsize=18) + cbar2.set_label('Correlation', fontsize=18) + + cbar3 = plt.colorbar(sc3, cax=cbar_ax3, orientation='horizontal') + cbar3.ax.tick_params(labelsize=18) + cbar3.set_label('Angstrom Bias', fontsize=18) + + if valid_ang_mask.sum() > 0: + cbar4 = plt.colorbar(sc4, cax=cbar_ax4, orientation='horizontal') + cbar4.ax.tick_params(labelsize=18) + cbar4.set_label('Correlation', fontsize=18) + + # Add overall title with larger font + title_year = f" ({years[0]})" if years and len(years) == 1 else f" ({min(years)}-{max(years)})" if years else "" + unique_stations = station_metrics['station'].nunique() + fig.suptitle(f'Lunar AERONET vs MERRA-21C Comparison{title_year}\n({unique_stations} stations)', + fontsize=20, fontweight='bold', y=0.92) + + # Save the 4-panel figure with high resolution + plt.savefig(os.path.join(output_dir, f'comparison_4panel{year_str}.png'), + dpi=300, bbox_inches='tight') + plt.close() + + # Generate 4-panel kernel density estimate figure + fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(16, 16)) + + # Define regional boundaries + regions = { + 'US': {'name': 'United States', 'lat_range': (25, 50), 'lon_range': (-130, -65)}, + 'Europe': {'name': 'Europe', 'lat_range': (35, 70), 'lon_range': (-10, 40)}, + 'Africa_South': {'name': 'Africa (South of Equator)', 'lat_range': (-35, 0), 'lon_range': (-20, 55)}, + 'Asia': {'name': 'Asia (15-40ยฐN, 65-120ยฐE)', 'lat_range': (15, 40), 'lon_range': (65, 120)} + } + + axes_map = [ax1, ax2, ax3, ax4] + region_keys = ['US', 'Europe', 'Africa_South', 'Asia'] + panel_labels = ['a', 'b', 'c', 'd'] + + # Collect all regional data first to determine global axis ranges + all_regional_data = {} + all_aeronet_log = [] + all_merra_log = [] + + for region_key in region_keys: + region = regions[region_key] + + # Filter stations by region + lat_mask = ((station_metrics['latitude'] >= region['lat_range'][0]) & + (station_metrics['latitude'] <= region['lat_range'][1])) + lon_mask = ((station_metrics['longitude'] >= region['lon_range'][0]) & + (station_metrics['longitude'] <= region['lon_range'][1])) + regional_mask = lat_mask & lon_mask + + regional_aeronet_data = [] + regional_merra_data = [] + + if regional_mask.sum() > 0: + regional_stations = station_metrics[regional_mask] + + for _, station_row in regional_stations.iterrows(): + station_name = station_row['station'] + station_files = [f for f in csv_files if station_name in os.path.basename(f)] + + if station_files: + try: + station_df = pd.read_csv(station_files[0]) + + # Apply same quality filters as before + quality_mask = ( + (~station_df['aeronet_aod_550'].isna()) & + (~station_df['merra_aod_550'].isna()) & + (station_df['aeronet_aod_550'] >= 0) & (station_df['aeronet_aod_550'] < 10) & + (station_df['merra_aod_550'] >= 0) & (station_df['merra_aod_550'] < 10) & + (np.isfinite(station_df['aeronet_aod_550'])) & + (np.isfinite(station_df['merra_aod_550'])) + ) + + clean_data = station_df[quality_mask] + + # Filter by years if specified + if years is not None: + clean_data['datetime'] = pd.to_datetime(clean_data['datetime']) + clean_data = clean_data[clean_data['datetime'].dt.year.isin(years)] + + if len(clean_data) > 0: + regional_aeronet_data.extend(clean_data['aeronet_aod_550'].values) + regional_merra_data.extend(clean_data['merra_aod_550'].values) + + except Exception as e: + if debug: + print(f"Error reading data for {station_name}: {e}") + continue + + # Store regional data and add to global collection + all_regional_data[region_key] = { + 'aeronet': regional_aeronet_data, + 'merra': regional_merra_data, + 'mask': regional_mask + } + + if len(regional_aeronet_data) > 0: + all_aeronet_log.extend(np.log10(np.array(regional_aeronet_data) + 0.01)) + all_merra_log.extend(np.log10(np.array(regional_merra_data) + 0.01)) + + # Determine global axis ranges in log space + if len(all_aeronet_log) > 0: + global_x_min = min(all_aeronet_log) + global_x_max = max(all_aeronet_log) + global_y_min = min(all_merra_log) + global_y_max = max(all_merra_log) + + # Make ranges symmetric and add some padding + global_min = min(global_x_min, global_y_min) + global_max = max(global_x_max, global_y_max) + + # Add 10% padding + range_size = global_max - global_min + global_min -= 0.1 * range_size + global_max += 0.1 * range_size + else: + # Fallback ranges if no data + global_min = -2.5 + global_max = 0.5 + + # Custom formatter to convert log values back to AOD values + def log_to_aod_formatter(x, pos): + aod_val = 10**x - 0.01 + if aod_val < 0.001: + return f'{aod_val:.4f}' + elif aod_val < 0.01: + return f'{aod_val:.3f}' + elif aod_val < 0.1: + return f'{aod_val:.2f}' + else: + return f'{aod_val:.1f}' + + # Store all contourf objects and their density ranges for shared colorbar + all_contourfs = [] + all_densities = [] + + # Create plots for each region + for i, (region_key, ax) in enumerate(zip(region_keys, axes_map)): + region = regions[region_key] + regional_data = all_regional_data[region_key] + + # Initialize statistics variables + correlation = np.nan + bias = np.nan + n_points = len(regional_data['aeronet']) + n_stations = regional_data['mask'].sum() + + if len(regional_data['aeronet']) < 50: # Need minimum data for KDE + ax.text(0.5, 0.5, f'Insufficient data in\n{region["name"]}\n({n_points} points)', + transform=ax.transAxes, ha='center', va='center', fontsize=18) + all_contourfs.append(None) + else: + # Convert to log space + aeronet_log = np.log10(np.array(regional_data['aeronet']) + 0.01) + merra_log = np.log10(np.array(regional_data['merra']) + 0.01) + + # Calculate statistics in log space + try: + correlation, _ = pearsonr(aeronet_log, merra_log) + bias = np.mean(merra_log - aeronet_log) # Mean bias in log space + except Exception as e: + if debug: + print(f"Error calculating statistics for {region['name']}: {e}") + correlation = np.nan + bias = np.nan + + try: + # Create kernel density estimate + data_points = np.vstack([aeronet_log, merra_log]) + kde = gaussian_kde(data_points) + + # Create meshgrid using global ranges + xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] + positions = np.vstack([xx.ravel(), yy.ravel()]) + + # Evaluate KDE + density = kde(positions).reshape(xx.shape) + all_densities.append(density) + + # Plot KDE as contours (no individual colorbars) + contour = ax.contour(xx, yy, density, colors='black', alpha=0.6, linewidths=0.8) + + # Store contourf for shared colorbar (but don't create individual colorbars yet) + all_contourfs.append((xx, yy, density)) + + except Exception as e: + # Fallback to scatter plot if KDE fails + if debug: + print(f"KDE failed for {region['name']}, using scatter plot: {e}") + ax.scatter(aeronet_log, merra_log, alpha=0.5, s=1) + all_contourfs.append(None) + + # Set consistent axis ranges for all panels + ax.set_xlim(global_min, global_max) + ax.set_ylim(global_min, global_max) + + # Add 1:1 line + ax.plot([global_min, global_max], [global_min, global_max], 'r--', linewidth=2, alpha=0.8) + + # Set up custom tick formatting to show AOD values + ax.xaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) + ax.yaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) + + # Set appropriate tick locations + log_ticks = np.arange(np.ceil(global_min), np.floor(global_max) + 0.5, 0.5) + ax.set_xticks(log_ticks) + ax.set_yticks(log_ticks) + + # Set labels + ax.set_xlabel('AERONET AOD', fontsize=18) + ax.set_ylabel('MERRA-21C AOD', fontsize=18) + ax.tick_params(labelsize=16) + ax.grid(True, alpha=0.3) + + # Add panel label + ax.text(0.03, 0.95, f"({panel_labels[i]})", transform=ax.transAxes, + fontsize=18, fontweight='bold', ha='left', va='top', + bbox=dict(facecolor='white', alpha=0.8, pad=0.1, edgecolor='none')) + + # Add region name, data count, correlation, and bias + # Format statistics text + if not np.isnan(correlation): + corr_text = f"r = {correlation:.3f}" + else: + corr_text = "r = N/A" + + if not np.isnan(bias): + bias_text = f"bias = {bias:.3f}" + else: + bias_text = "bias = N/A" + + stats_text = f"{region['name']}\n{n_stations} stations\n{n_points:,} points\n{corr_text}\n{bias_text}" + + ax.text(0.97, 0.03, stats_text, + transform=ax.transAxes, ha='right', va='bottom', fontsize=14, + bbox=dict(facecolor='white', alpha=0.8, pad=0.1, edgecolor='none')) + + # Adjust layout to make room for shared colorbar + plt.tight_layout(rect=[0, 0.08, 1, 0.92]) + + # Create shared colorbar with white-viridis colormap + if any(cf is not None for cf in all_contourfs): + # Determine global density range for consistent colorbar + valid_densities = [density for density in all_densities if density is not None] + if valid_densities: + global_density_min = min(np.min(d) for d in valid_densities) + global_density_max = max(np.max(d) for d in valid_densities) + + # Create contourf plots with consistent density range using white-viridis colormap + for i, (cf, ax) in enumerate(zip(all_contourfs, axes_map)): + if cf is not None: + xx, yy, density = cf + # Create contourf with global density range and white-viridis colormap + contourf = ax.contourf(xx, yy, density, alpha=0.7, cmap=white_viridis, + levels=np.linspace(global_density_min, global_density_max, 20), + vmin=global_density_min, vmax=global_density_max) + + # Create single horizontal colorbar below the bottom row + # Position: [left, bottom, width, height] + cbar_ax = fig.add_axes([0.15, 0.02, 0.7, 0.03]) + cbar = plt.colorbar(contourf, cax=cbar_ax, orientation='horizontal') + cbar.set_label('Density', fontsize=16) + cbar.ax.tick_params(labelsize=14) + + # Set overall title with unique station count + title_year = f" ({years[0]})" if years and len(years) == 1 else f" ({min(years)}-{max(years)})" if years else "" + unique_stations = station_metrics['station'].nunique() + fig.suptitle(f'Regional AOD Density Distributions{title_year}\n({unique_stations} stations)', + fontsize=20, fontweight='bold', y=0.96) + + # Save the figure + plt.savefig(os.path.join(output_dir, f'regional_kde_plots{year_str}.png'), + dpi=300, bbox_inches='tight') + plt.close() + + print(f"Generated regional KDE plots: regional_kde_plots{year_str}.png") + + # Generate data coverage map (separate figure) + plt.figure(figsize=(15, 10)) + ax = plt.axes(projection=ccrs.PlateCarree()) + ax.add_feature(cfeature.COASTLINE) + ax.add_feature(cfeature.BORDERS, linestyle=':') + ax.add_feature(cfeature.LAND, alpha=0.5) + ax.add_feature(cfeature.OCEAN, alpha=0.5) + ax.set_global() + + # Add gridlines + gl = ax.gridlines(draw_labels=True, alpha=0.3) + gl.top_labels = False + gl.right_labels = False + + # Create scatter plot with point sizes and colors based on data count + data_counts_array = np.array(valid_data_counts) + min_count = np.min(data_counts_array) + max_count = np.max(data_counts_array) + + # Use different sizing strategies based on the range of data counts + if max_count > 10 * min_count and min_count > 0: + # Wide range - use log scale for sizing + sizes = 20 + 100 * (np.log10(data_counts_array) - np.log10(min_count)) / (np.log10(max_count) - np.log10(min_count)) + size_label = "Log-scaled by data count" + else: + # Narrow range - use linear scale + if max_count > min_count: + sizes = 20 + 100 * (data_counts_array - min_count) / (max_count - min_count) + else: + sizes = np.full_like(data_counts_array, 60) # Uniform size if all same + size_label = "Scaled by data count" + + sc = ax.scatter( + station_metrics['longitude'], + station_metrics['latitude'], + c=station_metrics['valid_data_points'], + s=sizes, + cmap=count_cmap, + alpha=0.8, + edgecolor='black', + linewidth=0.5, + transform=ccrs.PlateCarree() + ) + + # Add colorbar + cbar = plt.colorbar(sc, label='Number of Valid Data Points', shrink=0.8) + + title_year = f" ({years[0]})" if years and len(years) == 1 else f" ({min(years)}-{max(years)})" if years else "" + unique_stations = station_metrics['station'].nunique() + plt.title(f'Data Point Coverage at AERONET Stations{title_year}\n({unique_stations} stations, {size_label})', fontsize=14) + + # Add text annotation for size scale + plt.text(0.02, 0.02, f'Point size: {min_count}-{max_count} data points', + transform=ax.transAxes, fontsize=10, + bbox=dict(boxstyle="round,pad=0.3", facecolor="white", alpha=0.8)) + + plt.savefig(os.path.join(output_dir, f'data_coverage_map{year_str}.png'), dpi=300, bbox_inches='tight') + plt.close() + + print(f"Generated all maps in {output_dir}") + print(f"Main comparison figure: comparison_4panel{year_str}.png") + print(f"Data coverage figure: data_coverage_map{year_str}.png") + + # Generate scatter plots for overall comparison with unique station handling + fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(12, 10)) + + # Group by station and aggregate metrics for unique stations + unique_station_metrics = station_metrics.groupby('station').agg({ + 'mean_aeronet_aod': 'mean', + 'mean_merra_aod': 'mean', + 'mean_aod_bias': 'mean', + 'aod_correlation': 'mean', + 'valid_data_points': 'sum' # Sum data points across files for same station + }).reset_index() + + # AOD scatter plot + ax1.scatter(unique_station_metrics['mean_aeronet_aod'], unique_station_metrics['mean_merra_aod'], alpha=0.6) + max_aod = max(unique_station_metrics['mean_aeronet_aod'].max(), unique_station_metrics['mean_merra_aod'].max()) + ax1.plot([0, max_aod], [0, max_aod], 'k--', alpha=0.8) + ax1.set_xlabel('AERONET AOD 550nm') + ax1.set_ylabel('MERRA-21C AOD 550nm') + ax1.set_title('AOD Comparison') + ax1.grid(True, alpha=0.3) + + # AOD bias histogram + ax2.hist(unique_station_metrics['mean_aod_bias'], bins=20, alpha=0.7, edgecolor='black') + ax2.axvline(0, color='red', linestyle='--', alpha=0.8) + ax2.set_xlabel('AOD Bias (MERRA-21C - AERONET)') + ax2.set_ylabel('Number of Stations') + ax2.set_title('AOD Bias Distribution') + ax2.grid(True, alpha=0.3) + + # AOD correlation histogram (filter out NaN values) + valid_aod_correlations = unique_station_metrics['aod_correlation'][~np.isnan(unique_station_metrics['aod_correlation'])] + if len(valid_aod_correlations) > 0: + ax3.hist(valid_aod_correlations, bins=20, alpha=0.7, edgecolor='black') + ax3.set_xlabel('AOD Correlation') + ax3.set_ylabel('Number of Stations') + ax3.set_title('AOD Correlation Distribution') + ax3.grid(True, alpha=0.3) + + # Data points histogram + ax4.hist(unique_station_metrics['valid_data_points'], bins=20, alpha=0.7, edgecolor='black') + ax4.set_xlabel('Number of Valid Data Points') + ax4.set_ylabel('Number of Stations') + ax4.set_title('Data Points Distribution') + ax4.grid(True, alpha=0.3) + + plt.tight_layout() + plt.savefig(os.path.join(output_dir, f'comparison_summary{year_str}.png'), dpi=300, bbox_inches='tight') + plt.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Generate comparison maps between AERONET and MERRA-21C data.') + parser.add_argument('--data-dir', type=str, default="./aeronet_merra21c_comparison/", + help='Directory containing processed CSV files') + parser.add_argument('--output-dir', type=str, default="./comparison_figures/", + help='Directory to save output figures') + parser.add_argument('--min-points', type=int, default=30, + help='Minimum number of data points required for a station') + parser.add_argument('--years', nargs='+', type=int, default=None, + help='Years to include in analysis (e.g., --years 2018 2019)') + parser.add_argument('--file-pattern', type=str, default=None, + help='Custom file pattern to match CSV files') + parser.add_argument('--debug', action='store_true', + help='Print additional debugging information') + + args = parser.parse_args() + + generate_comparison_maps( + data_dir=args.data_dir, + output_dir=args.output_dir, + min_points=args.min_points, + years=args.years, + file_pattern=args.file_pattern, + debug=args.debug + ) diff --git a/src/pyobs/evaluation/AERONET/plotsinglestationonly.py b/src/pyobs/evaluation/AERONET/plotsinglestationonly.py new file mode 100644 index 0000000..0871260 --- /dev/null +++ b/src/pyobs/evaluation/AERONET/plotsinglestationonly.py @@ -0,0 +1,320 @@ +import os +import glob +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import argparse +import warnings +import importlib + +warnings.filterwarnings('ignore') + +def generate_station_analysis_only(data_dir="./aeronet_merra21c_comparison/", + output_dir="./figures/", + min_points=30, + years=None, + file_pattern=None, + debug=False, + target_station="Mauna_Loa", + include_merra2=False, + merra2_dir="./aeronet_merra2_comparison/"): + """ + Generate only a single station analysis figure. + + Parameters: + ----------- + data_dir : str + Directory containing processed MERRA-21C CSV files + merra2_dir : str + Directory containing processed MERRA-2 CSV files + output_dir : str + Directory to save output figures + min_points : int + Minimum number of data points required for a station to be included + years : list or None + List of years to include in analysis. If None, uses all available data. + file_pattern : str or None + Custom file pattern to match CSV files. If None, uses default pattern. + debug : bool + If True, print additional debugging information + target_station : str + Name of the station to analyze + include_merra2 : bool + If True, use station_analysis_withm2.py to include MERRA-2 in plots + """ + # Create output directory + os.makedirs(output_dir, exist_ok=True) + + # Dynamically import the appropriate station analysis module + if include_merra2: + try: + station_analysis = importlib.import_module('station_analysis_withm2') + print("๐Ÿ”„ Using station_analysis_withm2.py (includes MERRA-2 data)") + except ImportError: + print("โŒ Error: station_analysis_withm2.py not found!") + print(" Falling back to station_analysis.py (MERRA-21C only)") + station_analysis = importlib.import_module('station_analysis') + include_merra2 = False + else: + station_analysis = importlib.import_module('station_analysis') + print("๐Ÿ”„ Using station_analysis.py (MERRA-21C only)") + + # Collect CSV files from both directories if MERRA-2 is included + all_csv_files = [] + + # Get MERRA-21C files + if os.path.exists(data_dir): + m21c_files = glob.glob(os.path.join(data_dir, "*.csv")) + all_csv_files.extend(m21c_files) + print(f"๐Ÿ“‚ Found {len(m21c_files)} MERRA-21C files in {data_dir}") + else: + print(f"โš ๏ธ MERRA-21C directory not found: {data_dir}") + + # Get MERRA-2 files if requested + if include_merra2: + if os.path.exists(merra2_dir): + m2_files = glob.glob(os.path.join(merra2_dir, "*.csv")) + all_csv_files.extend(m2_files) + print(f"๐Ÿ“‚ Found {len(m2_files)} MERRA-2 files in {merra2_dir}") + else: + print(f"โš ๏ธ MERRA-2 directory not found: {merra2_dir}") + + if not all_csv_files: + print("โŒ No CSV files found in any of the specified directories") + return + + # Filter files by station name and years if specified + station_files = [] + for csv_file in all_csv_files: + filename = os.path.basename(csv_file) + if target_station in filename: + # Check year filtering + if years is not None: + # Extract year from filename (assuming format like Station_YYYY.csv) + try: + year_in_filename = int(filename.split('_')[-1].replace('.csv', '')) + if year_in_filename in years: + station_files.append(csv_file) + except: + # If year extraction fails, include the file + station_files.append(csv_file) + else: + station_files.append(csv_file) + + if not station_files: + print(f"โŒ No CSV files found for station '{target_station}'") + available_stations = set() + for csv_file in all_csv_files[:20]: + filename = os.path.basename(csv_file) + station_name = filename.split('_')[0] if '_' in filename else filename.replace('.csv', '') + available_stations.add(station_name) + + if available_stations: + print("๐Ÿ“ Available stations:") + for station in sorted(available_stations): + print(f" ๐Ÿ“Š {station}") + return + + print(f"โœ… Found {len(station_files)} files for station '{target_station}'") + if debug: + for f in station_files: + print(f" ๐Ÿ“„ {f}") + + # If we're including MERRA-2, we need to merge the data + if include_merra2: + merged_files = merge_merra_datasets(station_files, target_station, debug) + if not merged_files: + print("โŒ Failed to merge MERRA-21C and MERRA-2 datasets") + return + csv_files_to_use = merged_files + else: + csv_files_to_use = station_files + + # Create a minimal station_metrics dataframe for the target station + station_metrics = [] + + for csv_file in csv_files_to_use: + try: + df = pd.read_csv(csv_file) + if len(df) > 0 and 'station' in df.columns and 'lat' in df.columns and 'lon' in df.columns: + station_info = { + 'station': df['station'].iloc[0], + 'latitude': df['lat'].iloc[0], + 'longitude': df['lon'].iloc[0] + } + station_metrics.append(station_info) + break # Found our station + except Exception as e: + if debug: + print(f"โš ๏ธ Error reading {csv_file}: {e}") + continue + + if not station_metrics: + print(f"โŒ Could not extract station metadata from CSV files") + return + + # Convert to DataFrame + station_metrics_df = pd.DataFrame(station_metrics) + + print(f"๐ŸŽฏ Analyzing station: {target_station}") + if include_merra2: + print("๐Ÿ“ˆ Analysis will include both MERRA-21C and MERRA-2 data") + else: + print("๐Ÿ“ˆ Analysis will include MERRA-21C data only") + + # Create station analysis using the dynamically imported module + analyzer = station_analysis.StationAnalyzer(station_metrics_df, csv_files_to_use, output_dir, years, debug) + + # Generate AOD figure + success = analyzer.create_station_figure(target_station) + + if success: + print(f"โœ… Successfully created AOD station analysis for {target_station}") + + # Generate Angstrom Exponent figure + print(f"๐Ÿ”„ Creating Angstrom Exponent analysis for {target_station}") + angstrom_success = analyzer.create_angstrom_figure(target_station) + + if angstrom_success: + print(f"โœ… Successfully created Angstrom Exponent analysis for {target_station}") + else: + print(f"โš ๏ธ Failed to create Angstrom Exponent figure for {target_station}") + else: + print(f"โŒ Failed to create AOD station figure for {target_station}") + +def merge_merra_datasets(station_files, target_station, debug=False): + """Merge MERRA-21C and MERRA-2 CSV files for the same station and years""" + import tempfile + + # Separate files by source + m21c_files = [f for f in station_files if 'aeronet_merra21c_comparison' in f] + m2_files = [f for f in station_files if 'aeronet_merra2_comparison' in f] + + if debug: + print(f"๐Ÿ”„ MERRA-21C files: {len(m21c_files)}") + print(f"๐Ÿ”„ MERRA-2 files: {len(m2_files)}") + + merged_files = [] + + # Process each year + years_processed = set() + + # Get all years from both datasets + all_years = set() + for f in m21c_files + m2_files: + try: + year = int(os.path.basename(f).split('_')[-1].replace('.csv', '')) + all_years.add(year) + except: + continue + + for year in sorted(all_years): + # Find corresponding files for this year + m21c_year_file = None + m2_year_file = None + + for f in m21c_files: + if f.endswith(f'{year}.csv'): + m21c_year_file = f + break + + for f in m2_files: + if f.endswith(f'{year}.csv'): + m2_year_file = f + break + + if debug: + print(f"๐Ÿ”„ Year {year}: M21C={m21c_year_file is not None}, M2={m2_year_file is not None}") + + # Merge data for this year + try: + merged_df = None + + if m21c_year_file and m2_year_file: + # Both datasets available - merge them + df_m21c = pd.read_csv(m21c_year_file) + df_m2 = pd.read_csv(m2_year_file) + + # Rename MERRA-2 columns to avoid conflicts + df_m2_renamed = df_m2.rename(columns={ + 'merra_aod_550': 'merra2_aod_550', + 'merra_angstrom': 'merra2_angstrom' + }) + + # Merge on datetime + merged_df = pd.merge(df_m21c, df_m2_renamed[['datetime', 'merra2_aod_550', 'merra2_angstrom']], + on='datetime', how='outer') + + print(f"โœ… Merged {year}: {len(merged_df)} combined records") + + elif m21c_year_file: + # Only MERRA-21C available + merged_df = pd.read_csv(m21c_year_file) + # Add empty MERRA-2 columns + merged_df['merra2_aod_550'] = np.nan + merged_df['merra2_angstrom'] = np.nan + + print(f"โœ… MERRA-21C only {year}: {len(merged_df)} records") + + elif m2_year_file: + # Only MERRA-2 available + df_m2 = pd.read_csv(m2_year_file) + # Rename and add missing columns + merged_df = df_m2.rename(columns={ + 'merra_aod_550': 'merra2_aod_550', + 'merra_angstrom': 'merra2_angstrom' + }) + # Add empty MERRA-21C columns + merged_df['merra_aod_550'] = np.nan + merged_df['merra_angstrom'] = np.nan + + print(f"โœ… MERRA-2 only {year}: {len(merged_df)} records") + + if merged_df is not None and len(merged_df) > 0: + # Save merged file to temporary location + temp_file = tempfile.NamedTemporaryFile(mode='w', suffix=f'_{target_station}_{year}_merged.csv', + delete=False) + merged_df.to_csv(temp_file.name, index=False) + merged_files.append(temp_file.name) + temp_file.close() + + except Exception as e: + print(f"โš ๏ธ Error merging data for year {year}: {e}") + continue + + return merged_files + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Generate single station analysis between AERONET and MERRA data.') + parser.add_argument('--data-dir', type=str, default="./aeronet_merra21c_comparison/", + help='Directory containing processed MERRA-21C CSV files') + parser.add_argument('--merra2-dir', type=str, default="./aeronet_merra2_comparison/", + help='Directory containing processed MERRA-2 CSV files') + parser.add_argument('--output-dir', type=str, default="./station_figures/", + help='Directory to save output figures') + parser.add_argument('--min-points', type=int, default=30, + help='Minimum number of data points required for a station') + parser.add_argument('--years', nargs='+', type=int, default=None, + help='Years to include in analysis (e.g., --years 2018 2019)') + parser.add_argument('--file-pattern', type=str, default=None, + help='Custom file pattern to match CSV files') + parser.add_argument('--station', type=str, default="Mauna_Loa", + help='Station name for analysis (default: Mauna_Loa)') + parser.add_argument('--debug', action='store_true', + help='Print additional debugging information') + parser.add_argument('--include-merra2', action='store_true', + help='Include MERRA-2 data in plots (uses station_analysis_withm2.py)') + + args = parser.parse_args() + + generate_station_analysis_only( + data_dir=args.data_dir, + merra2_dir=args.merra2_dir, + output_dir=args.output_dir, + min_points=args.min_points, + years=args.years, + file_pattern=args.file_pattern, + debug=args.debug, + target_station=args.station, + include_merra2=args.include_merra2 + ) diff --git a/src/pyobs/evaluation/AERONET/processaeronet.py b/src/pyobs/evaluation/AERONET/processaeronet.py new file mode 100644 index 0000000..3968df3 --- /dev/null +++ b/src/pyobs/evaluation/AERONET/processaeronet.py @@ -0,0 +1,492 @@ +import os +import glob +import numpy as np +import pandas as pd +import xarray as xr +from datetime import datetime, timedelta +import warnings +import multiprocessing as mp +from functools import partial +import time +import sys +warnings.filterwarnings('ignore') + +def process_station(station, years_to_process, aeronet_dir, merra_dir, output_dir): + """ + Process a single AERONET station and match with MERRA-21C data + + Parameters: + ----------- + station : str + Station name to process + years_to_process : list + List of years to process + aeronet_dir : str + Directory containing AERONET lunar AOD data files + merra_dir : str + Directory containing MERRA-21C data files + output_dir : str + Directory to save output CSV files + + Returns: + -------- + tuple + (station_name, status, message) where status is True if successful + """ + + try: + # Station name might contain underscores + station_file = os.path.join(aeronet_dir, f"20130101_20250920_{station}.lev20") + + if not os.path.exists(station_file): + return station, False, "File not found" + + # Read AERONET data with different encodings if needed + try: + # First try UTF-8 + with open(station_file, 'r', encoding='utf-8') as f: + lines = f.readlines() + header_line = lines[6].strip() + aeronet_data = pd.read_csv(station_file, skiprows=7, header=None, sep=',', encoding='utf-8') + except UnicodeDecodeError: + # If UTF-8 fails, try Latin-1 which is more permissive + with open(station_file, 'r', encoding='latin-1') as f: + lines = f.readlines() + header_line = lines[6].strip() + aeronet_data = pd.read_csv(station_file, skiprows=7, header=None, sep=',', encoding='latin-1') + + # Assign column names based on header line + column_names = header_line.split(',') + aeronet_data.columns = column_names + + # Convert date and time to datetime + aeronet_data['DateTime'] = pd.to_datetime( + aeronet_data['Date(dd:mm:yyyy)'] + ' ' + aeronet_data['Time(hh:mm:ss)'], + format='%d:%m:%Y %H:%M:%S' + ) + + # Filter for years we're interested in + year_mask = aeronet_data['DateTime'].dt.year.isin(years_to_process) + aeronet_data = aeronet_data[year_mask] + + if len(aeronet_data) == 0: + return station, False, f"No data for years {years_to_process}" + + # Extract lat/lon for the station + lat = float(aeronet_data['Site_Latitude(Degrees)'].iloc[0]) + lon = float(aeronet_data['Site_Longitude(Degrees)'].iloc[0]) + + # More robust approach to get AOD at 550nm + # Check all possible columns that could be used for 550nm AOD + aod_columns = ['AOD_551nm', 'AOD_550nm', 'AOD_532nm', 'AOD_531nm', 'AOD_555nm', 'AOD_560nm'] + aod_550 = None + used_column = None + + for col in aod_columns: + if col in aeronet_data.columns and not aeronet_data[col].isna().all(): + # Filter out negative values and very large values (likely fill values) + valid_aod = aeronet_data[col][(aeronet_data[col] >= 0) & (aeronet_data[col] < 10)] + if len(valid_aod) > 0: + aod_550 = aeronet_data[col].copy() + # Set negative and unreasonable values to NaN + aod_550[(aeronet_data[col] < 0) | (aeronet_data[col] >= 10)] = np.nan + used_column = col + break + + # If no direct measurement near 550nm, interpolate + if aod_550 is None: + # Try common wavelength pairs for interpolation + interpolation_pairs = [ + ('AOD_500nm', 'AOD_675nm'), + ('AOD_500nm', 'AOD_667nm'), + ('AOD_500nm', 'AOD_620nm'), + ('AOD_490nm', 'AOD_675nm'), + ('AOD_440nm', 'AOD_675nm') + ] + + for short_col, long_col in interpolation_pairs: + if (short_col in aeronet_data.columns and long_col in aeronet_data.columns and + not aeronet_data[short_col].isna().all() and not aeronet_data[long_col].isna().all()): + + # Extract wavelengths from column names + lambda1 = float(short_col.replace('AOD_', '').replace('nm', '')) / 1000.0 # Convert to ยตm + lambda2 = float(long_col.replace('AOD_', '').replace('nm', '')) / 1000.0 # Convert to ยตm + target_lambda = 0.550 # 550nm in ยตm + + # Apply quality filters: AOD >= 0, AOD < 10, both wavelengths valid + valid_mask = ((aeronet_data[short_col] >= 0) & (aeronet_data[short_col] < 10) & + (aeronet_data[long_col] >= 0) & (aeronet_data[long_col] < 10) & + (~aeronet_data[short_col].isna()) & (~aeronet_data[long_col].isna())) + + if valid_mask.sum() == 0: + continue + + aod_short = aeronet_data[short_col][valid_mask] + aod_long = aeronet_data[long_col][valid_mask] + + # Calculate Angstrom exponent + alpha = -np.log(aod_long/aod_short) / np.log(lambda2/lambda1) + + # Filter out unreasonable Angstrom exponents + reasonable_alpha_mask = (alpha >= -1) & (alpha <= 3) # Reasonable range for alpha + + if reasonable_alpha_mask.sum() == 0: + continue + + # Interpolate to get AOD at 550nm + aod_550_valid = aod_short[reasonable_alpha_mask] * (target_lambda/lambda1)**(-alpha[reasonable_alpha_mask]) + + # Create full array with NaNs for invalid points + aod_550 = pd.Series(np.nan, index=aeronet_data.index) + valid_indices = valid_mask[valid_mask].index[reasonable_alpha_mask] + aod_550[valid_indices] = aod_550_valid + + used_column = f"Interpolated from {short_col} and {long_col}" + break + + # If we still don't have AOD at 550nm, give up + if aod_550 is None: + return station, False, "Cannot obtain AOD at 550nm from available wavelengths" + + # Get Angstrom exponent with quality filtering + angstrom_columns = ['440-870_Angstrom_Exponent', '500-870_Angstrom_Exponent', '440-675_Angstrom_Exponent'] + ang_exponent = None + ang_source = None + + for col in angstrom_columns: + if col in aeronet_data.columns and not aeronet_data[col].isna().all(): + # Filter out unreasonable Angstrom values + valid_ang = aeronet_data[col][(aeronet_data[col] >= -1) & (aeronet_data[col] <= 3)] + if len(valid_ang) > 0: + ang_exponent = aeronet_data[col].copy() + # Set unreasonable values to NaN + ang_exponent[(aeronet_data[col] < -1) | (aeronet_data[col] > 3)] = np.nan + ang_source = col + break + + # If no direct Angstrom measurement, calculate it + if ang_exponent is None: + # Try common wavelength pairs for Angstrom calculation + angstrom_pairs = [ + ('AOD_440nm', 'AOD_870nm'), # Close to desired 470-870 + ('AOD_443nm', 'AOD_870nm'), + ('AOD_500nm', 'AOD_870nm'), + ('AOD_440nm', 'AOD_675nm') + ] + + for short_col, long_col in angstrom_pairs: + if (short_col in aeronet_data.columns and long_col in aeronet_data.columns and + not aeronet_data[short_col].isna().all() and not aeronet_data[long_col].isna().all()): + + # Extract wavelengths from column names + lambda1 = float(short_col.replace('AOD_', '').replace('nm', '')) / 1000.0 # Convert to ยตm + lambda2 = float(long_col.replace('AOD_', '').replace('nm', '')) / 1000.0 # Convert to ยตm + + # Apply quality filters + valid_mask = ((aeronet_data[short_col] >= 0) & (aeronet_data[short_col] < 10) & + (aeronet_data[long_col] >= 0) & (aeronet_data[long_col] < 10) & + (~aeronet_data[short_col].isna()) & (~aeronet_data[long_col].isna())) + + if valid_mask.sum() == 0: + continue + + aod_short = aeronet_data[short_col][valid_mask] + aod_long = aeronet_data[long_col][valid_mask] + + alpha_valid = -np.log(aod_long/aod_short) / np.log(lambda2/lambda1) + + # Filter reasonable Angstrom values + reasonable_mask = (alpha_valid >= -1) & (alpha_valid <= 3) + + if reasonable_mask.sum() == 0: + continue + + # Create full array with NaNs for invalid points + ang_exponent = pd.Series(np.nan, index=aeronet_data.index) + valid_indices = valid_mask[valid_mask].index[reasonable_mask] + ang_exponent[valid_indices] = alpha_valid[reasonable_mask] + + ang_source = f"Calculated from {short_col} and {long_col}" + break + + # If we still don't have Angstrom exponent, give up + if ang_exponent is None: + return station, False, "Cannot obtain Angstrom exponent from available wavelengths" + + # Create hourly means + aeronet_data['hour'] = aeronet_data['DateTime'].dt.floor('H') + + # Group by hour and calculate means (using nanmean to handle NaN values) + hourly_groups = aeronet_data.groupby('hour') + + results = pd.DataFrame({ + 'datetime': hourly_groups.groups.keys(), + 'aeronet_aod_550': hourly_groups.apply(lambda x: np.nanmean(aod_550.loc[x.index])), + 'aeronet_angstrom': hourly_groups.apply(lambda x: np.nanmean(ang_exponent.loc[x.index])), + 'station': station, + 'lat': lat, + 'lon': lon, + 'aod_source': used_column, + 'angstrom_source': ang_source + }) + + results = results.reset_index(drop=True) + + # Filter out hours where we couldn't calculate meaningful averages + results = results[~np.isnan(results['aeronet_aod_550']) & ~np.isnan(results['aeronet_angstrom'])] + + if len(results) == 0: + return station, False, "No valid hourly averages after quality filtering" + + # Add MERRA-21C data for each hourly point + merra_aod = [] + merra_angstrom = [] + merra_cache = {} + merra_file_found = 0 + merra_file_missing = 0 + + # Process MERRA data for each datetime in the AERONET dataset + for dt in results['datetime']: + year = dt.year + month = dt.month + day = dt.day + hour = dt.hour + + # Construct MERRA file path + merra_file = os.path.join( + merra_dir, + f"Y{year}", + f"M{month:02d}", + f"e5303_m21c_jan18.aer_inst_1hr_glo_L1152x721_slv.{year}-{month:02d}-{day:02d}T{hour:02d}00Z.nc4" + ) + + # Check if file exists + if not os.path.exists(merra_file): + merra_aod.append(np.nan) + merra_angstrom.append(np.nan) + merra_file_missing += 1 + continue + + merra_file_found += 1 + + try: + # Use cached dataset if available, otherwise open the file + if merra_file in merra_cache: + ds = merra_cache[merra_file] + else: + # Limit cache size to avoid memory issues + if len(merra_cache) > 10: + # Close oldest file and remove from cache + oldest_file = list(merra_cache.keys())[0] + merra_cache[oldest_file].close() + del merra_cache[oldest_file] + + ds = xr.open_dataset(merra_file) + merra_cache[merra_file] = ds + + # Find closest grid point to station location + # Convert longitude to 0-360 if MERRA uses that convention + merra_lon = lon + if ds.lon.min() >= 0 and lon < 0: + merra_lon = lon + 360 + + # Get MERRA-21C data at station location + aod_at_station = ds['TOTEXTTAU'].sel(lat=lat, lon=merra_lon, method='nearest').values + angstrom_at_station = ds['TOTANGSTR'].sel(lat=lat, lon=merra_lon, method='nearest').values + + # Apply quality filters to MERRA data too + if aod_at_station < 0 or aod_at_station >= 10: + aod_at_station = np.nan + if angstrom_at_station < -1 or angstrom_at_station > 3: + angstrom_at_station = np.nan + + merra_aod.append(float(aod_at_station)) + merra_angstrom.append(float(angstrom_at_station)) + + except Exception as e: + merra_aod.append(np.nan) + merra_angstrom.append(np.nan) + + # Close all open datasets + for ds in merra_cache.values(): + ds.close() + + # Add MERRA data to results + results['merra_aod_550'] = merra_aod + results['merra_angstrom'] = merra_angstrom + + # Calculate bias and other metrics + results['aod_bias'] = results['merra_aod_550'] - results['aeronet_aod_550'] + results['aod_rel_bias'] = (results['merra_aod_550'] / results['aeronet_aod_550']) * 100 - 100 + results['angstrom_bias'] = results['merra_angstrom'] - results['aeronet_angstrom'] + + # Final quality check - remove any remaining invalid data + valid_mask = (~np.isnan(results['aeronet_aod_550']) & + ~np.isnan(results['merra_aod_550']) & + ~np.isnan(results['aeronet_angstrom']) & + ~np.isnan(results['merra_angstrom']) & + (results['aeronet_aod_550'] >= 0) & + (results['merra_aod_550'] >= 0) & + (results['aeronet_angstrom'] >= -1) & (results['aeronet_angstrom'] <= 3) & + (results['merra_angstrom'] >= -1) & (results['merra_angstrom'] <= 3)) + + results_clean = results[valid_mask].copy() + + # Only proceed if we have enough valid data points + if len(results_clean) < 10: + return station, False, f"Insufficient matched data points ({len(results_clean)})" + + # Create a safe filename (replace characters that might be problematic in filenames) + safe_station_name = station.replace('/', '_').replace('\\', '_') + + # Save to CSV + year_str = f"{years_to_process[0]}_{years_to_process[-1]}" if len(years_to_process) > 1 else f"{years_to_process[0]}" + output_file = os.path.join(output_dir, f"{safe_station_name}_{year_str}.csv") + results.to_csv(output_file, index=False) + + return station, True, f"Processed successfully with {len(results_clean)} valid comparison points (total: {len(results)})" + + except Exception as e: + return station, False, f"Error: {str(e)}" + +def process_aeronet_merra_data_parallel(years_to_process=[2018, 2019, 2020], + station_names=None, + aeronet_dir="/discover/nobackup/acollow/aeroeval/aeronet_lunar/AOD_LUNAR/AOD20/ALL_POINTS/", + merra_dir="/discover/nobackup/projects/gmao/merra21c/archive/e5303_m21c_jan18/chem/", + output_dir="./processed_data/", + n_processes=None): + """ + Process AERONET lunar AOD data and MERRA-21C data for specified years and stations in parallel. + + Parameters: + ----------- + years_to_process : list + Years to process + station_names : list or None + List of station names to process. If None, process all stations. + aeronet_dir : str + Directory containing AERONET lunar AOD data files + merra_dir : str + Directory containing MERRA-21C data files + output_dir : str + Directory to save output CSV files + n_processes : int or None + Number of parallel processes to use. If None, will use CPU count - 1 + """ + start_time = time.time() + + # Create output directory if it doesn't exist + os.makedirs(output_dir, exist_ok=True) + + # Get list of all AERONET files + aeronet_files = glob.glob(os.path.join(aeronet_dir, "20130101_20250920_*.lev20")) + + if len(aeronet_files) == 0: + print(f"No AERONET files found in {aeronet_dir}") + print(f"Checking if directory exists: {os.path.exists(aeronet_dir)}") + if os.path.exists(aeronet_dir): + print(f"Directory contents: {os.listdir(aeronet_dir)[:10]} ...") + return + + # Extract station names from filenames if not specified + if station_names is None: + station_names = [] + for file_path in aeronet_files: + filename = os.path.basename(file_path) + # More robust station name extraction + # Format is: 20130101_20250920_STATIONNAME.lev20 + parts = filename.split("_", 2) # Split on first two underscores only + if len(parts) >= 3: + station_with_extension = parts[2] + station = station_with_extension.split(".")[0] # Remove .lev20 + station_names.append(station) + + station_names = list(set(station_names)) + + print(f"Processing {len(station_names)} stations for years {years_to_process}") + + # Determine number of processes to use + if n_processes is None: + n_processes = max(1, mp.cpu_count() - 1) # Leave one CPU free for system processes + + print(f"Using {n_processes} parallel processes") + + # Create a partial function with fixed parameters + process_station_partial = partial( + process_station, + years_to_process=years_to_process, + aeronet_dir=aeronet_dir, + merra_dir=merra_dir, + output_dir=output_dir + ) + + # Create a multiprocessing pool + with mp.Pool(processes=n_processes) as pool: + # Process stations in parallel with progress updates + results = [] + for i, result in enumerate(pool.imap_unordered(process_station_partial, station_names)): + results.append(result) + if (i+1) % 10 == 0 or (i+1) == len(station_names): + print(f"Progress: {i+1}/{len(station_names)} stations processed") + + # Summarize results + successful = 0 + failed = 0 + failure_reasons = {} + + for station, status, message in results: + if status: + successful += 1 + print(f"โœ“ {station}: {message}") + else: + failed += 1 + print(f"โœ— {station}: {message}") + + # Count failure reasons + reason = message.split(':')[0] if ':' in message else message + failure_reasons[reason] = failure_reasons.get(reason, 0) + 1 + + end_time = time.time() + elapsed_time = end_time - start_time + + print(f"\nProcessing complete in {elapsed_time:.1f} seconds") + print(f"Successfully processed {successful} stations") + print(f"Failed to process {failed} stations") + + # Print summary of failure reasons + if failure_reasons: + print("\nFailure reasons summary:") + for reason, count in sorted(failure_reasons.items(), key=lambda x: x[1], reverse=True): + print(f" {reason}: {count} stations") + +if __name__ == "__main__": + # Set up command line arguments + import argparse + + parser = argparse.ArgumentParser(description='Process AERONET lunar AOD data and match with MERRA-21C.') + parser.add_argument('--years', nargs='+', type=int, default=[2018, 2019, 2020], + help='Years to process') + parser.add_argument('--stations', nargs='+', type=str, default=None, + help='Specific stations to process (default: all stations)') + parser.add_argument('--aeronet-dir', type=str, + default="/discover/nobackup/acollow/aeroeval/aeronet_lunar/AOD_LUNAR/AOD20/ALL_POINTS/", + help='Directory containing AERONET lunar AOD data files') + parser.add_argument('--merra-dir', type=str, + default="/discover/nobackup/projects/gmao/merra21c/archive/e5303_m21c_jan18/chem/", + help='Directory containing MERRA-21C data files') + parser.add_argument('--output-dir', type=str, default="./aeronet_merra21c_comparison/", + help='Directory to save output CSV files') + parser.add_argument('--processes', type=int, default=None, + help='Number of parallel processes to use (default: CPU count - 1)') + + args = parser.parse_args() + + # Execute with command line arguments + process_aeronet_merra_data_parallel( + years_to_process=args.years, + station_names=args.stations, + aeronet_dir=args.aeronet_dir, + merra_dir=args.merra_dir, + output_dir=args.output_dir, + n_processes=args.processes + ) diff --git a/src/pyobs/evaluation/AERONET/station_analysis.py b/src/pyobs/evaluation/AERONET/station_analysis.py new file mode 100644 index 0000000..3e25a36 --- /dev/null +++ b/src/pyobs/evaluation/AERONET/station_analysis.py @@ -0,0 +1,670 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib.dates as mdates +import matplotlib.ticker as ticker +from matplotlib.colors import LinearSegmentedColormap +from scipy.stats import gaussian_kde, pearsonr +import os + +def create_white_viridis_cmap(): + """Create a custom colormap that starts with white and transitions to viridis""" + # Get the viridis colormap + viridis = plt.cm.get_cmap('viridis') + + # Create colors: more white values at the beginning for low densities + n_white = 50 # Number of white/near-white colors for low densities + n_viridis = 206 # Remaining colors for viridis + + # Create white to light colors transition + white_colors = [] + for i in range(n_white): + # Transition from pure white to very light viridis + alpha = i / n_white + viridis_light = viridis(0.1) # Very light viridis color + white_colors.append([ + 1 - alpha * (1 - viridis_light[0]), # R + 1 - alpha * (1 - viridis_light[1]), # G + 1 - alpha * (1 - viridis_light[2]), # B + 1.0 # Alpha + ]) + + # Add viridis colors for higher densities + viridis_colors = [viridis(i) for i in np.linspace(0.1, 1, n_viridis)] + + # Combine all colors + all_colors = white_colors + viridis_colors + + # Create the custom colormap + white_viridis = LinearSegmentedColormap.from_list('white_viridis', all_colors, N=256) + + return white_viridis + +class StationAnalyzer: + def __init__(self, station_metrics, csv_files, output_dir, years=None, debug=False): + self.station_metrics = station_metrics + self.csv_files = csv_files + self.output_dir = output_dir + self.years = years + self.debug = debug + self.white_viridis = create_white_viridis_cmap() + + def load_station_data(self, station_name): + """Load and combine all CSV files for a station""" + # Find all files for this station + station_files = [f for f in self.csv_files if station_name in os.path.basename(f)] + if not station_files: + return None, f"No CSV files found for {station_name}" + + # Read and combine all files + combined_data = [] + for file_path in station_files: + try: + df = pd.read_csv(file_path) + combined_data.append(df) + except Exception as e: + if self.debug: + print(f"Error reading {file_path}: {e}") + continue + + if not combined_data: + return None, "No valid data files" + + # Combine and clean data + data = pd.concat(combined_data, ignore_index=True) + data['datetime'] = pd.to_datetime(data['datetime']) + data = data.drop_duplicates(subset=['datetime']) + + # Filter by years if specified + if self.years is not None: + data = data[data['datetime'].dt.year.isin(self.years)] + + return data, "Success" + + def apply_quality_filters(self, data): + """Apply quality filters to the data""" + quality_mask = ( + (data['aeronet_aod_550'] >= 0) & (data['aeronet_aod_550'] < 10) & + (data['merra_aod_550'] >= 0) & (data['merra_aod_550'] < 10) & + (np.isfinite(data['aeronet_aod_550'])) & + (np.isfinite(data['merra_aod_550'])) & + (~data['aeronet_aod_550'].isna()) & + (~data['merra_aod_550'].isna()) + ) + return quality_mask + + def apply_angstrom_quality_filters(self, data): + """Apply quality filters to the Angstrom Exponent data""" + quality_mask = ( + (data['aeronet_angstrom'] >= -0.5) & (data['aeronet_angstrom'] <= 3.0) & + (data['merra_angstrom'] >= -0.5) & (data['merra_angstrom'] <= 3.0) & + (np.isfinite(data['aeronet_angstrom'])) & + (np.isfinite(data['merra_angstrom'])) & + (~data['aeronet_angstrom'].isna()) & + (~data['merra_angstrom'].isna()) + ) + return quality_mask + + def create_daily_timeseries(self, data): + """Create daily mean time series with proper gap handling""" + # Create daily means + data['date'] = data['datetime'].dt.date + daily_means = data.groupby('date').agg({ + 'aeronet_aod_550': 'mean', + 'merra_aod_550': 'mean' + }).reset_index() + + # Create complete date range + if len(daily_means) > 0: + start_date = daily_means['date'].min() + end_date = daily_means['date'].max() + complete_dates = pd.date_range(start=start_date, end=end_date, freq='D') + complete_df = pd.DataFrame({'date': complete_dates.date}) + daily_complete = complete_df.merge(daily_means, on='date', how='left') + daily_complete['date_dt'] = pd.to_datetime(daily_complete['date']) + else: + daily_complete = pd.DataFrame() + + return daily_complete + + def create_angstrom_daily_timeseries(self, data): + """Create daily mean time series for Angstrom Exponent with proper gap handling""" + # Create daily means + data['date'] = data['datetime'].dt.date + daily_means = data.groupby('date').agg({ + 'aeronet_angstrom': 'mean', + 'merra_angstrom': 'mean' + }).reset_index() + + # Create complete date range + if len(daily_means) > 0: + start_date = daily_means['date'].min() + end_date = daily_means['date'].max() + complete_dates = pd.date_range(start=start_date, end=end_date, freq='D') + complete_df = pd.DataFrame({'date': complete_dates.date}) + daily_complete = complete_df.merge(daily_means, on='date', how='left') + daily_complete['date_dt'] = pd.to_datetime(daily_complete['date']) + else: + daily_complete = pd.DataFrame() + + return daily_complete + + def calculate_seasonal_cycle(self, data): + """Calculate monthly seasonal cycle with percentiles""" + if data is None or len(data) == 0: + return pd.DataFrame() + + # Apply quality filters + quality_mask = self.apply_quality_filters(data) + valid_data = data[quality_mask].copy() + + if len(valid_data) == 0: + return pd.DataFrame() + + # Add month column + valid_data['month'] = valid_data['datetime'].dt.month + + # Calculate monthly statistics + monthly_stats = valid_data.groupby('month').agg({ + 'aeronet_aod_550': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'], + 'merra_aod_550': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] + }).reset_index() + + # Flatten column names + monthly_stats.columns = [ + 'month', + 'aeronet_median', 'aeronet_p25', 'aeronet_p75', 'aeronet_count', + 'merra_median', 'merra_p25', 'merra_p75', 'merra_count' + ] + + # Ensure all months are present (fill with NaN if missing) + all_months = pd.DataFrame({'month': range(1, 13)}) + monthly_stats = all_months.merge(monthly_stats, on='month', how='left') + + return monthly_stats + + def calculate_angstrom_seasonal_cycle(self, data): + """Calculate monthly seasonal cycle for Angstrom Exponent with percentiles""" + if data is None or len(data) == 0: + return pd.DataFrame() + + # Apply quality filters + quality_mask = self.apply_angstrom_quality_filters(data) + valid_data = data[quality_mask].copy() + + if len(valid_data) == 0: + return pd.DataFrame() + + # Add month column + valid_data['month'] = valid_data['datetime'].dt.month + + # Calculate monthly statistics + monthly_stats = valid_data.groupby('month').agg({ + 'aeronet_angstrom': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'], + 'merra_angstrom': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] + }).reset_index() + + # Flatten column names + monthly_stats.columns = [ + 'month', + 'aeronet_median', 'aeronet_p25', 'aeronet_p75', 'aeronet_count', + 'merra_median', 'merra_p25', 'merra_p75', 'merra_count' + ] + + # Ensure all months are present (fill with NaN if missing) + all_months = pd.DataFrame({'month': range(1, 13)}) + monthly_stats = all_months.merge(monthly_stats, on='month', how='left') + + return monthly_stats + + def calculate_statistics(self, aeronet_values, merra_values): + """Calculate comparison statistics""" + try: + correlation, _ = pearsonr(aeronet_values, merra_values) + bias = np.mean(merra_values - aeronet_values) + rmse = np.sqrt(np.mean((merra_values - aeronet_values)**2)) + return correlation, bias, rmse + except: + return np.nan, np.nan, np.nan + + def plot_timeseries(self, ax, daily_data, station_name): + """Plot the time series panel""" + ax.plot(daily_data['date_dt'], daily_data['merra_aod_550'], + 'k-', linewidth=1.5, label='MERRA-21C', alpha=0.8, marker='o', markersize=2) + ax.plot(daily_data['date_dt'], daily_data['aeronet_aod_550'], + 'r-', linewidth=1.5, label='AERONET', alpha=0.8, marker='o', markersize=2) + + # Format axes + if not daily_data.empty: + y_max = max( + daily_data['merra_aod_550'].max() if not daily_data['merra_aod_550'].isna().all() else 0, + daily_data['aeronet_aod_550'].max() if not daily_data['aeronet_aod_550'].isna().all() else 0 + ) + ax.set_ylim(0, y_max * 1.1) + + ax.set_xlabel('Date', fontsize=16) + ax.set_ylabel('AOD 550nm', fontsize=16) + ax.set_title('(a) Daily Mean AOD Time Series', fontsize=16, pad=10) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + ax.legend(fontsize=14) + + # Format dates + if len(daily_data) > 100: + ax.xaxis.set_major_locator(mdates.MonthLocator(interval=2)) + else: + ax.xaxis.set_major_locator(mdates.MonthLocator()) + ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) + plt.setp(ax.xaxis.get_majorticklabels(), rotation=45) + + def plot_angstrom_timeseries(self, ax, daily_data, station_name): + """Plot the Angstrom Exponent time series panel""" + ax.plot(daily_data['date_dt'], daily_data['merra_angstrom'], + 'k-', linewidth=1.5, label='MERRA-21C', alpha=0.8, marker='o', markersize=2) + ax.plot(daily_data['date_dt'], daily_data['aeronet_angstrom'], + 'r-', linewidth=1.5, label='AERONET', alpha=0.8, marker='o', markersize=2) + + # Format axes + if not daily_data.empty: + y_min = min( + daily_data['merra_angstrom'].min() if not daily_data['merra_angstrom'].isna().all() else 0, + daily_data['aeronet_angstrom'].min() if not daily_data['aeronet_angstrom'].isna().all() else 0 + ) + y_max = max( + daily_data['merra_angstrom'].max() if not daily_data['merra_angstrom'].isna().all() else 2, + daily_data['aeronet_angstrom'].max() if not daily_data['aeronet_angstrom'].isna().all() else 2 + ) + ax.set_ylim(y_min - 0.1, y_max + 0.1) + + ax.set_xlabel('Date', fontsize=16) + ax.set_ylabel('Angstrom Exponent', fontsize=16) + ax.set_title('(a) Daily Mean Angstrom Exponent Time Series', fontsize=16, pad=10) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + ax.legend(fontsize=14) + + # Format dates + if len(daily_data) > 100: + ax.xaxis.set_major_locator(mdates.MonthLocator(interval=2)) + else: + ax.xaxis.set_major_locator(mdates.MonthLocator()) + ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) + plt.setp(ax.xaxis.get_majorticklabels(), rotation=45) + + def plot_kde(self, ax, valid_data, station_info): + """Plot the KDE panel with white-to-viridis colormap using the proven approach""" + if len(valid_data) < 20: + ax.text(0.5, 0.5, f"Insufficient data\n({len(valid_data)} points)", + ha='center', va='center', fontsize=14, transform=ax.transAxes) + stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data)} points" + else: + # Log transform and calculate stats + aeronet_log = np.log10(valid_data['aeronet_aod_550'] + 0.01) + merra_log = np.log10(valid_data['merra_aod_550'] + 0.01) + correlation, bias, rmse = self.calculate_statistics(aeronet_log, merra_log) + + # Create KDE plot using the proven approach + try: + data_points = np.vstack([aeronet_log, merra_log]) + kde = gaussian_kde(data_points) + + x_min, x_max = aeronet_log.min(), aeronet_log.max() + y_min, y_max = merra_log.min(), merra_log.max() + global_min = min(x_min, y_min) - 0.1 * (max(x_max, y_max) - min(x_min, y_min)) + global_max = max(x_max, y_max) + 0.1 * (max(x_max, y_max) - min(x_min, y_min)) + + xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] + positions = np.vstack([xx.ravel(), yy.ravel()]) + density = kde(positions).reshape(xx.shape) + + # Apply the proven white-to-viridis approach + f_min = np.min(density) + f_max = np.max(density) + f_range = f_max - f_min + + # Set the minimum level higher to push low densities to white + # This makes the bottom 20% of the density range appear white/very light + threshold_factor = 0.2 # Adjust this to control how much appears white + adjusted_min = f_min + threshold_factor * f_range + + # Create levels starting from the adjusted minimum + levels = np.linspace(adjusted_min, f_max, 15) + + # Create filled contour plot with custom colormap + contour = ax.contour(xx, yy, density, colors='black', alpha=0.6, linewidths=0.8) + contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, + alpha=0.95, extend='min') + + cbar = plt.colorbar(contourf, ax=ax, shrink=0.8, extend='min') + cbar.set_label('Density', fontsize=14) + cbar.ax.tick_params(labelsize=12) + + ax.set_xlim(global_min, global_max) + ax.set_ylim(global_min, global_max) + except: + ax.scatter(aeronet_log, merra_log, alpha=0.6, s=20) + + # Add 1:1 line + xlim, ylim = ax.get_xlim(), ax.get_ylim() + min_lim, max_lim = min(xlim[0], ylim[0]), max(xlim[1], ylim[1]) + ax.plot([min_lim, max_lim], [min_lim, max_lim], 'r--', linewidth=2, alpha=0.8) + + # Format stats + corr_text = f"r = {correlation:.3f}" if not np.isnan(correlation) else "r = N/A" + bias_text = f"bias = {bias:.3f}" if not np.isnan(bias) else "bias = N/A" + rmse_text = f"RMSE = {rmse:.3f}" if not np.isnan(rmse) else "RMSE = N/A" + + stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data):,} points\n{corr_text}\n{bias_text}\n{rmse_text}" + + # Format axes + def log_to_aod_formatter(x, pos): + aod_val = 10**x - 0.01 + if aod_val < 0.001: return f'{aod_val:.4f}' + elif aod_val < 0.01: return f'{aod_val:.3f}' + elif aod_val < 0.1: return f'{aod_val:.2f}' + else: return f'{aod_val:.1f}' + + ax.xaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) + ax.yaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) + ax.set_xlabel('AERONET AOD', fontsize=16) + ax.set_ylabel('MERRA-21C AOD', fontsize=16) + ax.set_title('(b) AOD Density Distribution', fontsize=16, pad=10) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + + # Add stats box + ax.text(0.97, 0.03, stats_text, transform=ax.transAxes, ha='right', va='bottom', + fontsize=12, bbox=dict(facecolor='white', alpha=0.8, pad=0.5, edgecolor='black')) + + def plot_angstrom_kde(self, ax, valid_data, station_info): + """Plot the Angstrom Exponent KDE panel with white-to-viridis colormap using the proven approach""" + if len(valid_data) < 20: + ax.text(0.5, 0.5, f"Insufficient data\n({len(valid_data)} points)", + ha='center', va='center', fontsize=14, transform=ax.transAxes) + stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data)} points" + else: + # Calculate stats (no log transform needed for Angstrom) + aeronet_angstrom = valid_data['aeronet_angstrom'] + merra_angstrom = valid_data['merra_angstrom'] + correlation, bias, rmse = self.calculate_statistics(aeronet_angstrom, merra_angstrom) + + # Create KDE plot using the proven approach + try: + data_points = np.vstack([aeronet_angstrom, merra_angstrom]) + kde = gaussian_kde(data_points) + + x_min, x_max = aeronet_angstrom.min(), aeronet_angstrom.max() + y_min, y_max = merra_angstrom.min(), merra_angstrom.max() + global_min = min(x_min, y_min) - 0.1 * (max(x_max, y_max) - min(x_min, y_min)) + global_max = max(x_max, y_max) + 0.1 * (max(x_max, y_max) - min(x_min, y_min)) + + xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] + positions = np.vstack([xx.ravel(), yy.ravel()]) + density = kde(positions).reshape(xx.shape) + + # Apply the proven white-to-viridis approach + f_min = np.min(density) + f_max = np.max(density) + f_range = f_max - f_min + + # Set the minimum level higher to push low densities to white + # This makes the bottom 20% of the density range appear white/very light + threshold_factor = 0.2 # Adjust this to control how much appears white + adjusted_min = f_min + threshold_factor * f_range + + # Create levels starting from the adjusted minimum + levels = np.linspace(adjusted_min, f_max, 15) + + # Create filled contour plot with custom colormap + contour = ax.contour(xx, yy, density, colors='black', alpha=0.6, linewidths=0.8) + contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, + alpha=0.95, extend='min') + + cbar = plt.colorbar(contourf, ax=ax, shrink=0.8, extend='min') + cbar.set_label('Density', fontsize=14) + cbar.ax.tick_params(labelsize=12) + + ax.set_xlim(global_min, global_max) + ax.set_ylim(global_min, global_max) + except: + ax.scatter(aeronet_angstrom, merra_angstrom, alpha=0.6, s=20) + + # Add 1:1 line + xlim, ylim = ax.get_xlim(), ax.get_ylim() + min_lim, max_lim = min(xlim[0], ylim[0]), max(xlim[1], ylim[1]) + ax.plot([min_lim, max_lim], [min_lim, max_lim], 'r--', linewidth=2, alpha=0.8) + + # Format stats + corr_text = f"r = {correlation:.3f}" if not np.isnan(correlation) else "r = N/A" + bias_text = f"bias = {bias:.3f}" if not np.isnan(bias) else "bias = N/A" + rmse_text = f"RMSE = {rmse:.3f}" if not np.isnan(rmse) else "RMSE = N/A" + + stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data):,} points\n{corr_text}\n{bias_text}\n{rmse_text}" + + # Format axes + ax.set_xlabel('AERONET Angstrom Exponent', fontsize=16) + ax.set_ylabel('MERRA-21C Angstrom Exponent', fontsize=16) + ax.set_title('(b) Angstrom Exponent Density Distribution', fontsize=16, pad=10) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + + # Add stats box + ax.text(0.97, 0.03, stats_text, transform=ax.transAxes, ha='right', va='bottom', + fontsize=12, bbox=dict(facecolor='white', alpha=0.8, pad=0.5, edgecolor='black')) + + def plot_seasonal_cycle(self, ax, monthly_stats, station_info): + """Plot the seasonal cycle panel""" + if monthly_stats.empty or monthly_stats['aeronet_median'].isna().all(): + ax.text(0.5, 0.5, 'No seasonal data available', + ha='center', va='center', fontsize=14, transform=ax.transAxes) + ax.set_title('(c) Seasonal Cycle', fontsize=16, pad=10) + return + + months = monthly_stats['month'] + month_names = ['J', 'F', 'M', 'A', 'M', 'J', 'J', 'A', 'S', 'O', 'N', 'D'] + + # Plot AERONET seasonal cycle + aeronet_median = monthly_stats['aeronet_median'] + aeronet_p25 = monthly_stats['aeronet_p25'] + aeronet_p75 = monthly_stats['aeronet_p75'] + + # Only plot where we have data + valid_aeronet = ~aeronet_median.isna() + if valid_aeronet.any(): + ax.plot(months[valid_aeronet], aeronet_median[valid_aeronet], + 'ro-', linewidth=2, markersize=6, label='AERONET', alpha=0.8) + ax.fill_between(months[valid_aeronet], + aeronet_p25[valid_aeronet], + aeronet_p75[valid_aeronet], + alpha=0.3, color='red', label='AERONET 25-75%') + + # Plot MERRA seasonal cycle + merra_median = monthly_stats['merra_median'] + merra_p25 = monthly_stats['merra_p25'] + merra_p75 = monthly_stats['merra_p75'] + + # Only plot where we have data + valid_merra = ~merra_median.isna() + if valid_merra.any(): + ax.plot(months[valid_merra], merra_median[valid_merra], + 'ko-', linewidth=2, markersize=6, label='MERRA-21C', alpha=0.8) + ax.fill_between(months[valid_merra], + merra_p25[valid_merra], + merra_p75[valid_merra], + alpha=0.3, color='black', label='MERRA-21C 25-75%') + + # Format axes + ax.set_xlim(0.5, 12.5) + ax.set_xticks(range(1, 13)) + ax.set_xticklabels(month_names) + ax.set_xlabel('Month', fontsize=16) + ax.set_ylabel('AOD 550nm', fontsize=16) + ax.set_title('(c) Seasonal Cycle', fontsize=16, pad=10) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + ax.legend(fontsize=12, loc='upper right') + + # Set y-axis to start from 0 + current_ylim = ax.get_ylim() + ax.set_ylim(0, current_ylim[1]) + + + def plot_angstrom_seasonal_cycle(self, ax, monthly_stats, station_info): + """Plot the Angstrom Exponent seasonal cycle panel""" + if monthly_stats.empty or monthly_stats['aeronet_median'].isna().all(): + ax.text(0.5, 0.5, 'No seasonal data available', + ha='center', va='center', fontsize=14, transform=ax.transAxes) + ax.set_title('(c) Seasonal Cycle', fontsize=16, pad=10) + return + + months = monthly_stats['month'] + month_names = ['J', 'F', 'M', 'A', 'M', 'J', 'J', 'A', 'S', 'O', 'N', 'D'] + + # Plot AERONET seasonal cycle + aeronet_median = monthly_stats['aeronet_median'] + aeronet_p25 = monthly_stats['aeronet_p25'] + aeronet_p75 = monthly_stats['aeronet_p75'] + + # Only plot where we have data + valid_aeronet = ~aeronet_median.isna() + if valid_aeronet.any(): + ax.plot(months[valid_aeronet], aeronet_median[valid_aeronet], + 'ro-', linewidth=2, markersize=6, label='AERONET', alpha=0.8) + ax.fill_between(months[valid_aeronet], + aeronet_p25[valid_aeronet], + aeronet_p75[valid_aeronet], + alpha=0.3, color='red', label='AERONET 25-75%') + + # Plot MERRA seasonal cycle + merra_median = monthly_stats['merra_median'] + merra_p25 = monthly_stats['merra_p25'] + merra_p75 = monthly_stats['merra_p75'] + + # Only plot where we have data + valid_merra = ~merra_median.isna() + if valid_merra.any(): + ax.plot(months[valid_merra], merra_median[valid_merra], + 'ko-', linewidth=2, markersize=6, label='MERRA-21C', alpha=0.8) + ax.fill_between(months[valid_merra], + merra_p25[valid_merra], + merra_p75[valid_merra], + alpha=0.3, color='black', label='MERRA-21C 25-75%') + + # Format axes + ax.set_xlim(0.5, 12.5) + ax.set_xticks(range(1, 13)) + ax.set_xticklabels(month_names) + ax.set_xlabel('Month', fontsize=16) + ax.set_ylabel('Angstrom Exponent', fontsize=16) + ax.set_title('(c) Seasonal Cycle', fontsize=16, pad=10) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + ax.legend(fontsize=12, loc='upper right') + + + def create_station_figure(self, station_name): + """Main function to create station analysis figure with three panels""" + # Check if station exists + station_mask = self.station_metrics['station'] == station_name + if not station_mask.any(): + print(f"Station '{station_name}' not found") + return False + + # Load data + data, message = self.load_station_data(station_name) + if data is None: + print(f"Failed to load data for {station_name}: {message}") + return False + + # Get station info + station_info = { + 'name': station_name.replace('_', ' '), + 'lat': self.station_metrics[station_mask].iloc[0]['latitude'], + 'lon': self.station_metrics[station_mask].iloc[0]['longitude'] + } + + # Prepare data + quality_mask = self.apply_quality_filters(data) + daily_data = self.create_daily_timeseries(data) + valid_data = data[quality_mask] + monthly_stats = self.calculate_seasonal_cycle(data) + + # Create figure with three panels + fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(24, 8)) + + # Plot panels + self.plot_timeseries(ax1, daily_data, station_name) + self.plot_kde(ax2, valid_data, station_info) + self.plot_seasonal_cycle(ax3, monthly_stats, station_info) + + # Overall formatting + year_str = f" ({self.years[0]})" if self.years and len(self.years) == 1 else f" ({min(self.years)}-{max(self.years)})" if self.years else "" + fig.suptitle(f'Station Analysis: {station_info["name"]}{year_str}', fontsize=22, fontweight='bold', y=0.95) + plt.tight_layout(rect=[0, 0, 1, 0.92]) + + # Save + station_filename = station_name.replace('_', '-').lower() + year_suffix = f"_{self.years[0]}_{self.years[-1]}" if self.years and len(self.years) > 1 else f"_{self.years[0]}" if self.years else "" + plt.savefig(os.path.join(self.output_dir, f'station_analysis_{station_filename}{year_suffix}.png'), + dpi=300, bbox_inches='tight') + plt.close() + + print(f"Generated station analysis: station_analysis_{station_filename}{year_suffix}.png") + return True + + def create_angstrom_figure(self, station_name): + """Main function to create Angstrom Exponent analysis figure with three panels""" + # Check if station exists + station_mask = self.station_metrics['station'] == station_name + if not station_mask.any(): + print(f"Station '{station_name}' not found") + return False + + # Load data + data, message = self.load_station_data(station_name) + if data is None: + print(f"Failed to load data for {station_name}: {message}") + return False + + # Check if Angstrom columns exist + required_cols = ['aeronet_angstrom', 'merra_angstrom'] + missing_cols = [col for col in required_cols if col not in data.columns] + if missing_cols: + print(f"Missing Angstrom Exponent columns: {missing_cols}") + return False + + # Get station info + station_info = { + 'name': station_name.replace('_', ' '), + 'lat': self.station_metrics[station_mask].iloc[0]['latitude'], + 'lon': self.station_metrics[station_mask].iloc[0]['longitude'] + } + + # Prepare data + quality_mask = self.apply_angstrom_quality_filters(data) + daily_data = self.create_angstrom_daily_timeseries(data) + valid_data = data[quality_mask] + monthly_stats = self.calculate_angstrom_seasonal_cycle(data) + + # Create figure with three panels + fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(24, 8)) + + # Plot panels + self.plot_angstrom_timeseries(ax1, daily_data, station_name) + self.plot_angstrom_kde(ax2, valid_data, station_info) + self.plot_angstrom_seasonal_cycle(ax3, monthly_stats, station_info) + + # Overall formatting + year_str = f" ({self.years[0]})" if self.years and len(self.years) == 1 else f" ({min(self.years)}-{max(self.years)})" if self.years else "" + fig.suptitle(f'Angstrom Exponent Analysis: {station_info["name"]}{year_str}', fontsize=22, fontweight='bold', y=0.95) + plt.tight_layout(rect=[0, 0, 1, 0.92]) + + # Save + station_filename = station_name.replace('_', '-').lower() + year_suffix = f"_{self.years[0]}_{self.years[-1]}" if self.years and len(self.years) > 1 else f"_{self.years[0]}" if self.years else "" + plt.savefig(os.path.join(self.output_dir, f'angstrom_analysis_{station_filename}{year_suffix}.png'), + dpi=300, bbox_inches='tight') + plt.close() + + print(f"Generated Angstrom analysis: angstrom_analysis_{station_filename}{year_suffix}.png") + return True diff --git a/src/pyobs/evaluation/AERONET/station_analysis_withm2.py b/src/pyobs/evaluation/AERONET/station_analysis_withm2.py new file mode 100644 index 0000000..0bf2e5e --- /dev/null +++ b/src/pyobs/evaluation/AERONET/station_analysis_withm2.py @@ -0,0 +1,987 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib.dates as mdates +import matplotlib.ticker as ticker +from matplotlib.colors import LinearSegmentedColormap +from scipy.stats import gaussian_kde, pearsonr +import os + +def create_white_viridis_cmap(): + """Create a custom colormap that starts with white and transitions to viridis""" + # Get the viridis colormap + viridis = plt.cm.get_cmap('viridis') + + # Create colors: more white values at the beginning for low densities + n_white = 50 # Number of white/near-white colors for low densities + n_viridis = 206 # Remaining colors for viridis + + # Create white to light colors transition + white_colors = [] + for i in range(n_white): + # Transition from pure white to very light viridis + alpha = i / n_white + viridis_light = viridis(0.1) # Very light viridis color + white_colors.append([ + 1 - alpha * (1 - viridis_light[0]), # R + 1 - alpha * (1 - viridis_light[1]), # G + 1 - alpha * (1 - viridis_light[2]), # B + 1.0 # Alpha + ]) + + # Add viridis colors for higher densities + viridis_colors = [viridis(i) for i in np.linspace(0.1, 1, n_viridis)] + + # Combine all colors + all_colors = white_colors + viridis_colors + + # Create the custom colormap + white_viridis = LinearSegmentedColormap.from_list('white_viridis', all_colors, N=256) + + return white_viridis + +class StationAnalyzer: + def __init__(self, station_metrics, csv_files, output_dir, years=None, debug=False): + self.station_metrics = station_metrics + self.csv_files = csv_files + self.output_dir = output_dir + self.years = years + self.debug = debug + self.white_viridis = create_white_viridis_cmap() + + def load_station_data(self, station_name): + """Load and combine all CSV files for a station""" + # Find all files for this station + station_files = [f for f in self.csv_files if station_name in os.path.basename(f)] + if not station_files: + return None, f"No CSV files found for {station_name}" + + # Read and combine all files + combined_data = [] + for file_path in station_files: + try: + df = pd.read_csv(file_path) + combined_data.append(df) + except Exception as e: + if self.debug: + print(f"Error reading {file_path}: {e}") + continue + + if not combined_data: + return None, "No valid data files" + + # Combine and clean data + data = pd.concat(combined_data, ignore_index=True) + data['datetime'] = pd.to_datetime(data['datetime']) + data = data.drop_duplicates(subset=['datetime']) + + # Filter by years if specified + if self.years is not None: + data = data[data['datetime'].dt.year.isin(self.years)] + + return data, "Success" + + def apply_quality_filters(self, data): + """Apply quality filters to the data - supports both MERRA-21C and MERRA-2""" + # Check which columns exist + has_m21c = 'merra_aod_550' in data.columns + has_m2 = 'merra2_aod_550' in data.columns + + base_mask = ( + (data['aeronet_aod_550'] >= 0) & (data['aeronet_aod_550'] < 10) & + (np.isfinite(data['aeronet_aod_550'])) & + (~data['aeronet_aod_550'].isna()) + ) + + if has_m21c and has_m2: + # Both datasets present + quality_mask = base_mask & ( + (data['merra_aod_550'] >= 0) & (data['merra_aod_550'] < 10) & + (data['merra2_aod_550'] >= 0) & (data['merra2_aod_550'] < 10) & + (np.isfinite(data['merra_aod_550'])) & + (np.isfinite(data['merra2_aod_550'])) & + (~data['merra_aod_550'].isna()) & + (~data['merra2_aod_550'].isna()) + ) + elif has_m21c: + # Only MERRA-21C + quality_mask = base_mask & ( + (data['merra_aod_550'] >= 0) & (data['merra_aod_550'] < 10) & + (np.isfinite(data['merra_aod_550'])) & + (~data['merra_aod_550'].isna()) + ) + elif has_m2: + # Only MERRA-2 + quality_mask = base_mask & ( + (data['merra2_aod_550'] >= 0) & (data['merra2_aod_550'] < 10) & + (np.isfinite(data['merra2_aod_550'])) & + (~data['merra2_aod_550'].isna()) + ) + else: + # No MERRA data + quality_mask = base_mask + + return quality_mask + + def apply_angstrom_quality_filters(self, data): + """Apply quality filters to the Angstrom Exponent data""" + # Check which columns exist + has_m21c = 'merra_angstrom' in data.columns + has_m2 = 'merra2_angstrom' in data.columns + + base_mask = ( + (data['aeronet_angstrom'] >= -0.5) & (data['aeronet_angstrom'] <= 3.0) & + (np.isfinite(data['aeronet_angstrom'])) & + (~data['aeronet_angstrom'].isna()) + ) + + if has_m21c and has_m2: + # Both datasets present + quality_mask = base_mask & ( + (data['merra_angstrom'] >= -0.5) & (data['merra_angstrom'] <= 3.0) & + (data['merra2_angstrom'] >= -0.5) & (data['merra2_angstrom'] <= 3.0) & + (np.isfinite(data['merra_angstrom'])) & + (np.isfinite(data['merra2_angstrom'])) & + (~data['merra_angstrom'].isna()) & + (~data['merra2_angstrom'].isna()) + ) + elif has_m21c: + # Only MERRA-21C + quality_mask = base_mask & ( + (data['merra_angstrom'] >= -0.5) & (data['merra_angstrom'] <= 3.0) & + (np.isfinite(data['merra_angstrom'])) & + (~data['merra_angstrom'].isna()) + ) + elif has_m2: + # Only MERRA-2 + quality_mask = base_mask & ( + (data['merra2_angstrom'] >= -0.5) & (data['merra2_angstrom'] <= 3.0) & + (np.isfinite(data['merra2_angstrom'])) & + (~data['merra2_angstrom'].isna()) + ) + else: + # No MERRA data + quality_mask = base_mask + + return quality_mask + + def create_daily_timeseries(self, data): + """Create daily mean time series with proper gap handling for all datasets""" + # Create daily means + data['date'] = data['datetime'].dt.date + + # Determine which columns to aggregate + agg_dict = {'aeronet_aod_550': 'mean'} + if 'merra_aod_550' in data.columns: + agg_dict['merra_aod_550'] = 'mean' + if 'merra2_aod_550' in data.columns: + agg_dict['merra2_aod_550'] = 'mean' + + daily_means = data.groupby('date').agg(agg_dict).reset_index() + + # Create complete date range + if len(daily_means) > 0: + start_date = daily_means['date'].min() + end_date = daily_means['date'].max() + complete_dates = pd.date_range(start=start_date, end=end_date, freq='D') + complete_df = pd.DataFrame({'date': complete_dates.date}) + daily_complete = complete_df.merge(daily_means, on='date', how='left') + daily_complete['date_dt'] = pd.to_datetime(daily_complete['date']) + else: + daily_complete = pd.DataFrame() + + return daily_complete + + def create_angstrom_daily_timeseries(self, data): + """Create daily mean time series for Angstrom Exponent with proper gap handling""" + # Create daily means + data['date'] = data['datetime'].dt.date + + # Determine which columns to aggregate + agg_dict = {'aeronet_angstrom': 'mean'} + if 'merra_angstrom' in data.columns: + agg_dict['merra_angstrom'] = 'mean' + if 'merra2_angstrom' in data.columns: + agg_dict['merra2_angstrom'] = 'mean' + + daily_means = data.groupby('date').agg(agg_dict).reset_index() + + # Create complete date range + if len(daily_means) > 0: + start_date = daily_means['date'].min() + end_date = daily_means['date'].max() + complete_dates = pd.date_range(start=start_date, end=end_date, freq='D') + complete_df = pd.DataFrame({'date': complete_dates.date}) + daily_complete = complete_df.merge(daily_means, on='date', how='left') + daily_complete['date_dt'] = pd.to_datetime(daily_complete['date']) + else: + daily_complete = pd.DataFrame() + + return daily_complete + + def calculate_seasonal_cycle(self, data): + """Calculate monthly seasonal cycle with percentiles for all datasets""" + if data is None or len(data) == 0: + return pd.DataFrame() + + # Apply quality filters + quality_mask = self.apply_quality_filters(data) + valid_data = data[quality_mask].copy() + + if len(valid_data) == 0: + return pd.DataFrame() + + # Add month column + valid_data['month'] = valid_data['datetime'].dt.month + + # Determine which columns to aggregate + agg_dict = { + 'aeronet_aod_550': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] + } + + if 'merra_aod_550' in valid_data.columns: + agg_dict['merra_aod_550'] = ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] + + if 'merra2_aod_550' in valid_data.columns: + agg_dict['merra2_aod_550'] = ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] + + # Calculate monthly statistics + monthly_stats = valid_data.groupby('month').agg(agg_dict).reset_index() + + # Flatten column names + flattened_columns = ['month'] + if 'aeronet_aod_550' in agg_dict: + flattened_columns.extend(['aeronet_median', 'aeronet_p25', 'aeronet_p75', 'aeronet_count']) + if 'merra_aod_550' in agg_dict: + flattened_columns.extend(['merra_median', 'merra_p25', 'merra_p75', 'merra_count']) + if 'merra2_aod_550' in agg_dict: + flattened_columns.extend(['merra2_median', 'merra2_p25', 'merra2_p75', 'merra2_count']) + + monthly_stats.columns = flattened_columns + + # Ensure all months are present (fill with NaN if missing) + all_months = pd.DataFrame({'month': range(1, 13)}) + monthly_stats = all_months.merge(monthly_stats, on='month', how='left') + + return monthly_stats + + def calculate_angstrom_seasonal_cycle(self, data): + """Calculate monthly seasonal cycle for Angstrom Exponent with percentiles for all datasets""" + if data is None or len(data) == 0: + return pd.DataFrame() + + # Apply quality filters + quality_mask = self.apply_angstrom_quality_filters(data) + valid_data = data[quality_mask].copy() + + if len(valid_data) == 0: + return pd.DataFrame() + + # Add month column + valid_data['month'] = valid_data['datetime'].dt.month + + # Determine which columns to aggregate + agg_dict = { + 'aeronet_angstrom': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] + } + + if 'merra_angstrom' in valid_data.columns: + agg_dict['merra_angstrom'] = ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] + + if 'merra2_angstrom' in valid_data.columns: + agg_dict['merra2_angstrom'] = ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] + + # Calculate monthly statistics + monthly_stats = valid_data.groupby('month').agg(agg_dict).reset_index() + + # Flatten column names + flattened_columns = ['month'] + if 'aeronet_angstrom' in agg_dict: + flattened_columns.extend(['aeronet_median', 'aeronet_p25', 'aeronet_p75', 'aeronet_count']) + if 'merra_angstrom' in agg_dict: + flattened_columns.extend(['merra_median', 'merra_p25', 'merra_p75', 'merra_count']) + if 'merra2_angstrom' in agg_dict: + flattened_columns.extend(['merra2_median', 'merra2_p25', 'merra2_p75', 'merra2_count']) + + monthly_stats.columns = flattened_columns + + # Ensure all months are present (fill with NaN if missing) + all_months = pd.DataFrame({'month': range(1, 13)}) + monthly_stats = all_months.merge(monthly_stats, on='month', how='left') + + return monthly_stats + + def calculate_statistics(self, aeronet_values, model_values): + """Calculate comparison statistics""" + try: + correlation, _ = pearsonr(aeronet_values, model_values) + bias = np.mean(model_values - aeronet_values) + rmse = np.sqrt(np.mean((model_values - aeronet_values)**2)) + return correlation, bias, rmse + except: + return np.nan, np.nan, np.nan + + def get_axis_limits(self, valid_data): + """Get consistent axis limits for KDE plots""" + # Log transform all data to determine global limits + aeronet_log = np.log10(valid_data['aeronet_aod_550'] + 0.01) + + all_model_values = [] + if 'merra_aod_550' in valid_data.columns: + merra21c_log = np.log10(valid_data['merra_aod_550'] + 0.01) + all_model_values.extend(merra21c_log) + if 'merra2_aod_550' in valid_data.columns: + merra2_log = np.log10(valid_data['merra2_aod_550'] + 0.01) + all_model_values.extend(merra2_log) + + if all_model_values: + all_values = np.concatenate([aeronet_log, all_model_values]) + else: + all_values = aeronet_log + + # Calculate global limits with some padding + global_min = np.min(all_values) + global_max = np.max(all_values) + data_range = global_max - global_min + + # Add padding + padded_min = global_min - 0.1 * data_range + padded_max = global_max + 0.1 * data_range + + return padded_min, padded_max + + def plot_timeseries(self, ax, daily_data, station_name): + """Plot the time series panel with all three datasets""" + # Plot AERONET first (red) + ax.plot(daily_data['date_dt'], daily_data['aeronet_aod_550'], + 'r-', linewidth=1.5, label='AERONET', alpha=0.8, marker='o', markersize=2) + + # Plot MERRA-21C (black) + if 'merra_aod_550' in daily_data.columns: + ax.plot(daily_data['date_dt'], daily_data['merra_aod_550'], + 'k-', linewidth=1.5, label='MERRA-21C', alpha=0.8, marker='s', markersize=2) + + # Plot MERRA-2 (blue) + if 'merra2_aod_550' in daily_data.columns: + ax.plot(daily_data['date_dt'], daily_data['merra2_aod_550'], + 'b-', linewidth=1.5, label='MERRA-2', alpha=0.8, marker='^', markersize=2) + + # Format axes + if not daily_data.empty: + y_values = [] + for col in ['aeronet_aod_550', 'merra_aod_550', 'merra2_aod_550']: + if col in daily_data.columns and not daily_data[col].isna().all(): + y_values.extend(daily_data[col].dropna()) + + if y_values: + y_max = max(y_values) + ax.set_ylim(0, y_max * 1.1) + + ax.set_xlabel('Date', fontsize=16) + ax.set_ylabel('AOD 550nm', fontsize=16) + ax.set_title('(a) Daily Mean AOD Time Series', fontsize=16, pad=10) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + ax.legend(fontsize=14) + + # Format dates + if len(daily_data) > 100: + ax.xaxis.set_major_locator(mdates.MonthLocator(interval=2)) + else: + ax.xaxis.set_major_locator(mdates.MonthLocator()) + ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) + plt.setp(ax.xaxis.get_majorticklabels(), rotation=45) + + def plot_angstrom_timeseries(self, ax, daily_data, station_name): + """Plot the Angstrom Exponent time series panel with all three datasets""" + # Plot AERONET first (red) + ax.plot(daily_data['date_dt'], daily_data['aeronet_angstrom'], + 'r-', linewidth=1.5, label='AERONET', alpha=0.8, marker='o', markersize=2) + + # Plot MERRA-21C (black) + if 'merra_angstrom' in daily_data.columns: + ax.plot(daily_data['date_dt'], daily_data['merra_angstrom'], + 'k-', linewidth=1.5, label='MERRA-21C', alpha=0.8, marker='s', markersize=2) + + # Plot MERRA-2 (blue) + if 'merra2_angstrom' in daily_data.columns: + ax.plot(daily_data['date_dt'], daily_data['merra2_angstrom'], + 'b-', linewidth=1.5, label='MERRA-2', alpha=0.8, marker='^', markersize=2) + + # Format axes + if not daily_data.empty: + y_values = [] + for col in ['aeronet_angstrom', 'merra_angstrom', 'merra2_angstrom']: + if col in daily_data.columns and not daily_data[col].isna().all(): + y_values.extend(daily_data[col].dropna()) + + if y_values: + y_min = min(y_values) + y_max = max(y_values) + ax.set_ylim(y_min - 0.1, y_max + 0.1) + + ax.set_xlabel('Date', fontsize=16) + ax.set_ylabel('Angstrom Exponent', fontsize=16) + ax.set_title('(a) Daily Mean Angstrom Exponent Time Series', fontsize=16, pad=10) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + ax.legend(fontsize=14) + + # Format dates + if len(daily_data) > 100: + ax.xaxis.set_major_locator(mdates.MonthLocator(interval=2)) + else: + ax.xaxis.set_major_locator(mdates.MonthLocator()) + ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) + plt.setp(ax.xaxis.get_majorticklabels(), rotation=45) + + def plot_seasonal_cycle(self, ax, monthly_stats, station_info): + """Plot the seasonal cycle panel with all three datasets""" + if monthly_stats.empty or monthly_stats['aeronet_median'].isna().all(): + ax.text(0.5, 0.5, 'No seasonal data available', + ha='center', va='center', fontsize=14, transform=ax.transAxes) + ax.set_title('(b) Seasonal Cycle', fontsize=16, pad=10) + return + + months = monthly_stats['month'] + month_names = ['J', 'F', 'M', 'A', 'M', 'J', 'J', 'A', 'S', 'O', 'N', 'D'] + + # Plot AERONET seasonal cycle (red) + aeronet_median = monthly_stats['aeronet_median'] + aeronet_p25 = monthly_stats['aeronet_p25'] + aeronet_p75 = monthly_stats['aeronet_p75'] + + valid_aeronet = ~aeronet_median.isna() + if valid_aeronet.any(): + ax.plot(months[valid_aeronet], aeronet_median[valid_aeronet], + 'ro-', linewidth=2, markersize=6, label='AERONET', alpha=0.8) + ax.fill_between(months[valid_aeronet], + aeronet_p25[valid_aeronet], + aeronet_p75[valid_aeronet], + alpha=0.3, color='red') + + # Plot MERRA-21C seasonal cycle (black) + if 'merra_median' in monthly_stats.columns: + merra_median = monthly_stats['merra_median'] + merra_p25 = monthly_stats['merra_p25'] + merra_p75 = monthly_stats['merra_p75'] + + valid_merra = ~merra_median.isna() + if valid_merra.any(): + ax.plot(months[valid_merra], merra_median[valid_merra], + 'ko-', linewidth=2, markersize=6, label='MERRA-21C', alpha=0.8) + ax.fill_between(months[valid_merra], + merra_p25[valid_merra], + merra_p75[valid_merra], + alpha=0.3, color='black') + + # Plot MERRA-2 seasonal cycle (blue) + if 'merra2_median' in monthly_stats.columns: + merra2_median = monthly_stats['merra2_median'] + merra2_p25 = monthly_stats['merra2_p25'] + merra2_p75 = monthly_stats['merra2_p75'] + + valid_merra2 = ~merra2_median.isna() + if valid_merra2.any(): + ax.plot(months[valid_merra2], merra2_median[valid_merra2], + 'bo-', linewidth=2, markersize=6, label='MERRA-2', alpha=0.8) + ax.fill_between(months[valid_merra2], + merra2_p25[valid_merra2], + merra2_p75[valid_merra2], + alpha=0.3, color='blue') + + # Format axes + ax.set_xlim(0.5, 12.5) + ax.set_xticks(range(1, 13)) + ax.set_xticklabels(month_names) + ax.set_xlabel('Month', fontsize=16) + ax.set_ylabel('AOD 550nm', fontsize=16) + ax.set_title('(b) Seasonal Cycle', fontsize=16, pad=10) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + ax.legend(fontsize=12, loc='upper right') + + # Set y-axis to start from 0 + current_ylim = ax.get_ylim() + ax.set_ylim(0, current_ylim[1]) + + def plot_angstrom_seasonal_cycle(self, ax, monthly_stats, station_info): + """Plot the Angstrom Exponent seasonal cycle panel with all three datasets""" + if monthly_stats.empty or monthly_stats['aeronet_median'].isna().all(): + ax.text(0.5, 0.5, 'No seasonal data available', + ha='center', va='center', fontsize=14, transform=ax.transAxes) + ax.set_title('(b) Seasonal Cycle', fontsize=16, pad=10) + return + + months = monthly_stats['month'] + month_names = ['J', 'F', 'M', 'A', 'M', 'J', 'J', 'A', 'S', 'O', 'N', 'D'] + + # Plot AERONET seasonal cycle (red) + aeronet_median = monthly_stats['aeronet_median'] + aeronet_p25 = monthly_stats['aeronet_p25'] + aeronet_p75 = monthly_stats['aeronet_p75'] + + valid_aeronet = ~aeronet_median.isna() + if valid_aeronet.any(): + ax.plot(months[valid_aeronet], aeronet_median[valid_aeronet], + 'ro-', linewidth=2, markersize=6, label='AERONET', alpha=0.8) + ax.fill_between(months[valid_aeronet], + aeronet_p25[valid_aeronet], + aeronet_p75[valid_aeronet], + alpha=0.3, color='red') + + # Plot MERRA-21C seasonal cycle (black) + if 'merra_median' in monthly_stats.columns: + merra_median = monthly_stats['merra_median'] + merra_p25 = monthly_stats['merra_p25'] + merra_p75 = monthly_stats['merra_p75'] + + valid_merra = ~merra_median.isna() + if valid_merra.any(): + ax.plot(months[valid_merra], merra_median[valid_merra], + 'ko-', linewidth=2, markersize=6, label='MERRA-21C', alpha=0.8) + ax.fill_between(months[valid_merra], + merra_p25[valid_merra], + merra_p75[valid_merra], + alpha=0.3, color='black') + + # Plot MERRA-2 seasonal cycle (blue) + if 'merra2_median' in monthly_stats.columns: + merra2_median = monthly_stats['merra2_median'] + merra2_p25 = monthly_stats['merra2_p25'] + merra2_p75 = monthly_stats['merra2_p75'] + + valid_merra2 = ~merra2_median.isna() + if valid_merra2.any(): + ax.plot(months[valid_merra2], merra2_median[valid_merra2], + 'bo-', linewidth=2, markersize=6, label='MERRA-2', alpha=0.8) + ax.fill_between(months[valid_merra2], + merra2_p25[valid_merra2], + merra2_p75[valid_merra2], + alpha=0.3, color='blue') + + # Format axes + ax.set_xlim(0.5, 12.5) + ax.set_xticks(range(1, 13)) + ax.set_xticklabels(month_names) + ax.set_xlabel('Month', fontsize=16) + ax.set_ylabel('Angstrom Exponent', fontsize=16) + ax.set_title('(b) Seasonal Cycle', fontsize=16, pad=10) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + ax.legend(fontsize=12, loc='upper right') + + def plot_kde_panel(self, ax, valid_data, station_info, model_col, model_name, global_min, global_max, vmin=None, vmax=None): + """Plot a single KDE panel for model vs AERONET comparison""" + if len(valid_data) < 20: + ax.text(0.5, 0.5, f"Insufficient data\n({len(valid_data)} points)", + ha='center', va='center', fontsize=14, transform=ax.transAxes) + stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data)} points" + else: + # Log transform and calculate stats + aeronet_log = np.log10(valid_data['aeronet_aod_550'] + 0.01) + model_log = np.log10(valid_data[model_col] + 0.01) + correlation, bias, rmse = self.calculate_statistics(aeronet_log, model_log) + + # Create KDE plot + try: + data_points = np.vstack([aeronet_log, model_log]) + kde = gaussian_kde(data_points) + + xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] + positions = np.vstack([xx.ravel(), yy.ravel()]) + density = kde(positions).reshape(xx.shape) + + # Apply consistent density scaling if provided + if vmin is not None and vmax is not None: + # Use global levels + f_range = vmax - vmin + threshold_factor = 0.2 + adjusted_min = vmin + threshold_factor * f_range + levels = np.linspace(adjusted_min, vmax, 15) + contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, + alpha=0.95, extend='min', vmin=adjusted_min, vmax=vmax) + else: + f_min = np.min(density) + f_max = np.max(density) + f_range = f_max - f_min + threshold_factor = 0.2 + adjusted_min = f_min + threshold_factor * f_range + levels = np.linspace(adjusted_min, f_max, 15) + contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, + alpha=0.95, extend='min') + + contour = ax.contour(xx, yy, density, colors='black', alpha=0.6, linewidths=0.8) + + cbar = plt.colorbar(contourf, ax=ax, shrink=0.8, extend='min') + cbar.set_label('Density', fontsize=14) + cbar.ax.tick_params(labelsize=12) + + ax.set_xlim(global_min, global_max) + ax.set_ylim(global_min, global_max) + except: + ax.scatter(aeronet_log, model_log, alpha=0.6, s=20) + + # Add 1:1 line + ax.plot([global_min, global_max], [global_min, global_max], 'r--', linewidth=2, alpha=0.8) + + # Format stats + corr_text = f"r = {correlation:.3f}" if not np.isnan(correlation) else "r = N/A" + bias_text = f"bias = {bias:.3f}" if not np.isnan(bias) else "bias = N/A" + rmse_text = f"RMSE = {rmse:.3f}" if not np.isnan(rmse) else "RMSE = N/A" + + stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data):,} points\n{corr_text}\n{bias_text}\n{rmse_text}" + + # Format axes + def log_to_aod_formatter(x, pos): + aod_val = 10**x - 0.01 + if aod_val < 0.001: return f'{aod_val:.4f}' + elif aod_val < 0.01: return f'{aod_val:.3f}' + elif aod_val < 0.1: return f'{aod_val:.2f}' + else: return f'{aod_val:.1f}' + + ax.xaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) + ax.yaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) + ax.set_xlabel('AERONET AOD', fontsize=16) + ax.set_ylabel(f'{model_name} AOD', fontsize=16) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + + # Add stats box + ax.text(0.97, 0.03, stats_text, transform=ax.transAxes, ha='right', va='bottom', + fontsize=12, bbox=dict(facecolor='white', alpha=0.8, pad=0.5, edgecolor='black')) + + return density if 'density' in locals() else None + + def plot_angstrom_kde_panel(self, ax, valid_data, station_info, model_col, model_name, global_min, global_max, vmin=None, vmax=None): + """Plot a single Angstrom KDE panel for model vs AERONET comparison""" + if len(valid_data) < 20: + ax.text(0.5, 0.5, f"Insufficient data\n({len(valid_data)} points)", + ha='center', va='center', fontsize=14, transform=ax.transAxes) + stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data)} points" + else: + # No log transform for Angstrom data + aeronet_angstrom = valid_data['aeronet_angstrom'] + model_angstrom = valid_data[model_col] + correlation, bias, rmse = self.calculate_statistics(aeronet_angstrom, model_angstrom) + + # Create KDE plot + try: + data_points = np.vstack([aeronet_angstrom, model_angstrom]) + kde = gaussian_kde(data_points) + + xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] + positions = np.vstack([xx.ravel(), yy.ravel()]) + density = kde(positions).reshape(xx.shape) + + # Apply consistent density scaling if provided + if vmin is not None and vmax is not None: + f_range = vmax - vmin + threshold_factor = 0.2 + adjusted_min = vmin + threshold_factor * f_range + levels = np.linspace(adjusted_min, vmax, 15) + contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, + alpha=0.95, extend='min', vmin=adjusted_min, vmax=vmax) + else: + f_min = np.min(density) + f_max = np.max(density) + f_range = f_max - f_min + threshold_factor = 0.2 + adjusted_min = f_min + threshold_factor * f_range + levels = np.linspace(adjusted_min, f_max, 15) + contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, + alpha=0.95, extend='min') + + contour = ax.contour(xx, yy, density, colors='black', alpha=0.6, linewidths=0.8) + + cbar = plt.colorbar(contourf, ax=ax, shrink=0.8, extend='min') + cbar.set_label('Density', fontsize=14) + cbar.ax.tick_params(labelsize=12) + + ax.set_xlim(global_min, global_max) + ax.set_ylim(global_min, global_max) + except: + ax.scatter(aeronet_angstrom, model_angstrom, alpha=0.6, s=20) + + # Add 1:1 line + ax.plot([global_min, global_max], [global_min, global_max], 'r--', linewidth=2, alpha=0.8) + + # Format stats + corr_text = f"r = {correlation:.3f}" if not np.isnan(correlation) else "r = N/A" + bias_text = f"bias = {bias:.3f}" if not np.isnan(bias) else "bias = N/A" + rmse_text = f"RMSE = {rmse:.3f}" if not np.isnan(rmse) else "RMSE = N/A" + + stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data):,} points\n{corr_text}\n{bias_text}\n{rmse_text}" + + # Format axes + ax.set_xlabel('AERONET Angstrom Exponent', fontsize=16) + ax.set_ylabel(f'{model_name} Angstrom Exponent', fontsize=16) + ax.tick_params(labelsize=14) + ax.grid(True, alpha=0.3) + + # Add stats box + ax.text(0.97, 0.03, stats_text, transform=ax.transAxes, ha='right', va='bottom', + fontsize=12, bbox=dict(facecolor='white', alpha=0.8, pad=0.5, edgecolor='black')) + + return density if 'density' in locals() else None + + def create_station_figure(self, station_name): + """Main function to create 4-panel station analysis figure""" + # Check if station exists + station_mask = self.station_metrics['station'] == station_name + if not station_mask.any(): + print(f"Station '{station_name}' not found") + return False + + # Load data + data, message = self.load_station_data(station_name) + if data is None: + print(f"Failed to load data for {station_name}: {message}") + return False + + # Check which model datasets are available + has_m21c = 'merra_aod_550' in data.columns + has_m2 = 'merra2_aod_550' in data.columns + + if not (has_m21c or has_m2): + print(f"No MERRA data found for {station_name}") + return False + + # Get station info + station_info = { + 'name': station_name.replace('_', ' '), + 'lat': self.station_metrics[station_mask].iloc[0]['latitude'], + 'lon': self.station_metrics[station_mask].iloc[0]['longitude'] + } + + # Prepare data + quality_mask = self.apply_quality_filters(data) + daily_data = self.create_daily_timeseries(data) + valid_data = data[quality_mask] + monthly_stats = self.calculate_seasonal_cycle(data) + + if len(valid_data) == 0: + print(f"No valid data after quality filtering for {station_name}") + return False + + # Get consistent axis limits for KDE plots + global_min, global_max = self.get_axis_limits(valid_data) + + # Calculate global density range for consistent colorbars + all_densities = [] + if has_m21c and len(valid_data) >= 20: + try: + aeronet_log = np.log10(valid_data['aeronet_aod_550'] + 0.01) + m21c_log = np.log10(valid_data['merra_aod_550'] + 0.01) + data_points = np.vstack([aeronet_log, m21c_log]) + kde = gaussian_kde(data_points) + xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] + positions = np.vstack([xx.ravel(), yy.ravel()]) + density = kde(positions).reshape(xx.shape) + all_densities.extend(density.flatten()) + except: + pass + + if has_m2 and len(valid_data) >= 20: + try: + aeronet_log = np.log10(valid_data['aeronet_aod_550'] + 0.01) + m2_log = np.log10(valid_data['merra2_aod_550'] + 0.01) + data_points = np.vstack([aeronet_log, m2_log]) + kde = gaussian_kde(data_points) + xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] + positions = np.vstack([xx.ravel(), yy.ravel()]) + density = kde(positions).reshape(xx.shape) + all_densities.extend(density.flatten()) + except: + pass + + # Calculate global density limits + if all_densities: + global_vmin = np.min(all_densities) + global_vmax = np.max(all_densities) + else: + global_vmin = global_vmax = None + + # Create figure with 2x2 layout + fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(20, 16)) + + # Top row: Time series (left) and Seasonal cycle (right) + self.plot_timeseries(ax1, daily_data, station_name) + self.plot_seasonal_cycle(ax2, monthly_stats, station_info) + + # Bottom row: KDE plots + if has_m21c and has_m2: + # Both datasets available + self.plot_kde_panel(ax3, valid_data, station_info, 'merra_aod_550', 'MERRA-21C', + global_min, global_max, global_vmin, global_vmax) + self.plot_kde_panel(ax4, valid_data, station_info, 'merra2_aod_550', 'MERRA-2', + global_min, global_max, global_vmin, global_vmax) + ax3.set_title('(c) MERRA-21C vs AERONET AOD', fontsize=16, pad=10) + ax4.set_title('(d) MERRA-2 vs AERONET AOD', fontsize=16, pad=10) + elif has_m21c: + # Only MERRA-21C available + self.plot_kde_panel(ax3, valid_data, station_info, 'merra_aod_550', 'MERRA-21C', + global_min, global_max) + ax4.axis('off') + ax4.text(0.5, 0.5, 'MERRA-2 data\nnot available', ha='center', va='center', + fontsize=16, transform=ax4.transAxes) + ax3.set_title('(c) MERRA-21C vs AERONET AOD', fontsize=16, pad=10) + elif has_m2: + # Only MERRA-2 available + ax3.axis('off') + ax3.text(0.5, 0.5, 'MERRA-21C data\nnot available', ha='center', va='center', + fontsize=16, transform=ax3.transAxes) + self.plot_kde_panel(ax4, valid_data, station_info, 'merra2_aod_550', 'MERRA-2', + global_min, global_max) + ax4.set_title('(d) MERRA-2 vs AERONET AOD', fontsize=16, pad=10) + + # Overall formatting + year_str = f" ({self.years[0]})" if self.years and len(self.years) == 1 else f" ({min(self.years)}-{max(self.years)})" if self.years else "" + fig.suptitle(f'Station Analysis: {station_info["name"]}{year_str}', fontsize=22, fontweight='bold', y=0.95) + plt.tight_layout(rect=[0, 0, 1, 0.92]) + + # Save + station_filename = station_name.replace('_', '-').lower() + year_suffix = f"_{self.years[0]}_{self.years[-1]}" if self.years and len(self.years) > 1 else f"_{self.years[0]}" if self.years else "" + plt.savefig(os.path.join(self.output_dir, f'station_analysis_{station_filename}{year_suffix}_withm2.png'), + dpi=300, bbox_inches='tight') + plt.close() + + print(f"Generated 4-panel station analysis: station_analysis_{station_filename}{year_suffix}_withm2.png") + return True + + def create_angstrom_figure(self, station_name): + """Main function to create 4-panel Angstrom Exponent analysis figure""" + # Check if station exists + station_mask = self.station_metrics['station'] == station_name + if not station_mask.any(): + print(f"Station '{station_name}' not found") + return False + + # Load data + data, message = self.load_station_data(station_name) + if data is None: + print(f"Failed to load data for {station_name}: {message}") + return False + + # Check if Angstrom columns exist + has_aeronet = 'aeronet_angstrom' in data.columns + has_m21c = 'merra_angstrom' in data.columns + has_m2 = 'merra2_angstrom' in data.columns + + if not has_aeronet: + print(f"Missing AERONET Angstrom Exponent data for {station_name}") + return False + + if not (has_m21c or has_m2): + print(f"No MERRA Angstrom Exponent data found for {station_name}") + return False + + # Get station info + station_info = { + 'name': station_name.replace('_', ' '), + 'lat': self.station_metrics[station_mask].iloc[0]['latitude'], + 'lon': self.station_metrics[station_mask].iloc[0]['longitude'] + } + + # Prepare data + quality_mask = self.apply_angstrom_quality_filters(data) + daily_data = self.create_angstrom_daily_timeseries(data) + valid_data = data[quality_mask] + monthly_stats = self.calculate_angstrom_seasonal_cycle(data) + + if len(valid_data) == 0: + print(f"No valid Angstrom data after quality filtering for {station_name}") + return False + + # Get consistent axis limits for KDE plots (no log transform for Angstrom) + aeronet_vals = valid_data['aeronet_angstrom'] + all_model_values = [] + + if has_m21c: + all_model_values.extend(valid_data['merra_angstrom']) + if has_m2: + all_model_values.extend(valid_data['merra2_angstrom']) + + all_values = np.concatenate([aeronet_vals, all_model_values]) + global_min = np.min(all_values) - 0.1 * (np.max(all_values) - np.min(all_values)) + global_max = np.max(all_values) + 0.1 * (np.max(all_values) - np.min(all_values)) + + # Calculate global density range for consistent colorbars + all_densities = [] + if has_m21c and len(valid_data) >= 20: + try: + data_points = np.vstack([valid_data['aeronet_angstrom'], valid_data['merra_angstrom']]) + kde = gaussian_kde(data_points) + xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] + positions = np.vstack([xx.ravel(), yy.ravel()]) + density = kde(positions).reshape(xx.shape) + all_densities.extend(density.flatten()) + except: + pass + + if has_m2 and len(valid_data) >= 20: + try: + data_points = np.vstack([valid_data['aeronet_angstrom'], valid_data['merra2_angstrom']]) + kde = gaussian_kde(data_points) + xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] + positions = np.vstack([xx.ravel(), yy.ravel()]) + density = kde(positions).reshape(xx.shape) + all_densities.extend(density.flatten()) + except: + pass + + # Calculate global density limits + if all_densities: + global_vmin = np.min(all_densities) + global_vmax = np.max(all_densities) + else: + global_vmin = global_vmax = None + + # Create figure with 2x2 layout + fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(20, 16)) + + # Top row: Time series (left) and Seasonal cycle (right) + self.plot_angstrom_timeseries(ax1, daily_data, station_name) + self.plot_angstrom_seasonal_cycle(ax2, monthly_stats, station_info) + + # Bottom row: KDE plots for Angstrom + if has_m21c and has_m2: + # Both datasets available + self.plot_angstrom_kde_panel(ax3, valid_data, station_info, 'merra_angstrom', 'MERRA-21C', + global_min, global_max, global_vmin, global_vmax) + self.plot_angstrom_kde_panel(ax4, valid_data, station_info, 'merra2_angstrom', 'MERRA-2', + global_min, global_max, global_vmin, global_vmax) + ax3.set_title('(c) MERRA-21C vs AERONET Angstrom', fontsize=16, pad=10) + ax4.set_title('(d) MERRA-2 vs AERONET Angstrom', fontsize=16, pad=10) + elif has_m21c: + # Only MERRA-21C available + self.plot_angstrom_kde_panel(ax3, valid_data, station_info, 'merra_angstrom', 'MERRA-21C', + global_min, global_max) + ax4.axis('off') + ax4.text(0.5, 0.5, 'MERRA-2 Angstrom data\nnot available', ha='center', va='center', + fontsize=16, transform=ax4.transAxes) + ax3.set_title('(c) MERRA-21C vs AERONET Angstrom', fontsize=16, pad=10) + elif has_m2: + # Only MERRA-2 available + ax3.axis('off') + ax3.text(0.5, 0.5, 'MERRA-21C Angstrom data\nnot available', ha='center', va='center', + fontsize=16, transform=ax3.transAxes) + self.plot_angstrom_kde_panel(ax4, valid_data, station_info, 'merra2_angstrom', 'MERRA-2', + global_min, global_max) + ax4.set_title('(d) MERRA-2 vs AERONET Angstrom', fontsize=16, pad=10) + + # Overall formatting + year_str = f" ({self.years[0]})" if self.years and len(self.years) == 1 else f" ({min(self.years)}-{max(self.years)})" if self.years else "" + fig.suptitle(f'Angstrom Exponent Analysis: {station_info["name"]}{year_str}', fontsize=22, fontweight='bold', y=0.95) + plt.tight_layout(rect=[0, 0, 1, 0.92]) + + # Save + station_filename = station_name.replace('_', '-').lower() + year_suffix = f"_{self.years[0]}_{self.years[-1]}" if self.years and len(self.years) > 1 else f"_{self.years[0]}" if self.years else "" + plt.savefig(os.path.join(self.output_dir, f'angstrom_analysis_{station_filename}{year_suffix}_withm2.png'), + dpi=300, bbox_inches='tight') + plt.close() + + print(f"Generated 4-panel Angstrom analysis: angstrom_analysis_{station_filename}{year_suffix}_withm2.png") + return True From 73025c990e0b5c43a213a7ae87a4d778adb70a60 Mon Sep 17 00:00:00 2001 From: acollow Date: Tue, 9 Dec 2025 13:34:37 -0500 Subject: [PATCH 02/10] initial check in of MODIS evaluation codes --- .../MODIS_NNR/monthlygeossample_speciated.py | 474 +++++++ .../MODIS_NNR/plotregionalcomparison.py | 1238 +++++++++++++++++ 2 files changed, 1712 insertions(+) create mode 100755 src/pyobs/evaluation/MODIS_NNR/monthlygeossample_speciated.py create mode 100755 src/pyobs/evaluation/MODIS_NNR/plotregionalcomparison.py diff --git a/src/pyobs/evaluation/MODIS_NNR/monthlygeossample_speciated.py b/src/pyobs/evaluation/MODIS_NNR/monthlygeossample_speciated.py new file mode 100755 index 0000000..fc70d48 --- /dev/null +++ b/src/pyobs/evaluation/MODIS_NNR/monthlygeossample_speciated.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python +""" +Monthly AOD Model-Observation Comparison Tool +A script to compare GEOS model AOD to monthly mean MODIS NNR Retrievals +Original by Sampa Das (May 2020), Refactored +""" + +import numpy as np +import os +import logging +from datetime import datetime +from scipy.interpolate import RegularGridInterpolator +from netCDF4 import Dataset + +# Constants +MODEL_LAT_SIZE = 361 +MODEL_LON_SIZE = 720 +DAILY_FILES = 8 # number of MODIS files per day +TIME_STEPS = range(0, 24, 3) +FILL_VALUE = 999.0 +AOD_THRESHOLD = 100.0 + +# Set up logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + + +class AODProcessor: + """Class to handle AOD processing operations""" + + def __init__(self): + self.variable_names = ['tot', 'bc', 'oc', 'br', 'ss', 'su', 'du', 'ni'] + self.netcdf_mapping = { + 'tot': 'TOTEXTTAU', + 'bc': 'BCEXTTAU', + 'oc': 'OCEXTTAU', + 'br': 'BREXTTAU', + 'ss': 'SSEXTTAU', + 'su': 'SUEXTTAU', + 'du': 'DUEXTTAU', + 'ni': 'NIEXTTAU' + } + + def initialize_arrays(self, shape): + """Initialize arrays with NaN values""" + arrays = {} + for name in self.variable_names: + arrays[name] = np.full(shape, np.nan, dtype=np.float32) + return arrays + + def validate_inputs(self, yy, mm, EXPID, sat): + """Validate input parameters""" + if not (1 <= mm <= 12): + raise ValueError(f"Month must be between 1-12, got {mm}") + if yy < 1900 or yy > 2100: + raise ValueError(f"Year seems unrealistic: {yy}") + if not EXPID.strip(): + raise ValueError("EXPID cannot be empty") + if not sat.strip(): + raise ValueError("Satellite identifier cannot be empty") + + def check_directories(self, dir_obs, dirm): + """Check if required directories exist""" + if not os.path.exists(dir_obs): + raise FileNotFoundError(f"Observation directory not found: {dir_obs}") + if not os.path.exists(dirm): + raise FileNotFoundError(f"Model directory not found: {dirm}") + + def parse_file_dates(self, MOD_files): + """Parse start and end dates from file list""" + if len(MOD_files) < 2: + raise ValueError("Insufficient files found") + + try: + dds = int(MOD_files[0][30:32]) + dde = int(MOD_files[-1][30:32]) # Use last file instead of [-2] + return dds, dde + except (IndexError, ValueError) as e: + raise ValueError(f"Error parsing file names: {e}") + + def interpolate_observations(self, lons, lats, tau_nnr_L, lonm, latm): + """Interpolate observation data to model grid""" + # Replace NaN with fill value for interpolation + tau_for_interp = tau_nnr_L.copy() + tau_for_interp[np.isnan(tau_for_interp)] = FILL_VALUE + + # Create interpolator + interpolator = RegularGridInterpolator( + (lats, lons), tau_for_interp, + method='linear', + bounds_error=False, + fill_value=np.nan + ) + + # Create coordinate arrays for interpolation + lonm_grid, latm_grid = np.meshgrid(lonm, latm) + points = np.column_stack([latm_grid.ravel(), lonm_grid.ravel()]) + + # Perform interpolation + tau_obs_modelres = interpolator(points).reshape(latm_grid.shape) + + # Clean up unreasonable values + tau_obs_modelres[tau_obs_modelres >= AOD_THRESHOLD] = np.nan + + return tau_obs_modelres + + def process_single_timestep(self, nc_fileL, nc_fileM, lonm, latm): + """Process a single timestep of data""" + result = { + 'success': False, + 'tau_obs': None, + 'model_data': {} + } + + if not os.path.isfile(nc_fileL): + return result + + if not os.path.isfile(nc_fileM): + logger.warning(f"Model file missing: {nc_fileM}") + return result + + try: + # Read observation data + with Dataset(nc_fileL, 'r') as ncid: + lons = ncid.variables['lon'][:] + lats = ncid.variables['lat'][:] + tau_nnr_L = np.squeeze(ncid.variables['tau'][:]) + + # Read model data + with Dataset(nc_fileM, 'r') as ncid: + lonm = ncid.variables['lon'][:] + latm = ncid.variables['lat'][:] + + model_data = {} + for var_name in self.variable_names: + nc_var_name = self.netcdf_mapping[var_name] + model_data[var_name] = np.squeeze(ncid.variables[nc_var_name][:]) + + # Interpolate observations to model grid + tau_obs_modelres = self.interpolate_observations(lons, lats, tau_nnr_L, lonm, latm) + + # Apply observation mask to model data + for var_name in self.variable_names: + model_data[var_name][np.isnan(tau_obs_modelres)] = np.nan + + result.update({ + 'success': True, + 'tau_obs': tau_obs_modelres, + 'model_data': model_data + }) + + except Exception as e: + logger.error(f"Error processing files {nc_fileL}, {nc_fileM}: {e}") + + return result + + def process_daily_data(self, dd, dds, dir_obs, dirm, yy, mm, EXPID, sat, lonm, latm): + """Process data for a single day""" + hourly_shape = (MODEL_LAT_SIZE, MODEL_LON_SIZE, DAILY_FILES) + + # Initialize arrays + mod_arrays = self.initialize_arrays(hourly_shape) + tau_nnrLOD = np.full(hourly_shape, np.nan, dtype=np.float32) + + valid_timesteps = 0 + + for i, t in enumerate(TIME_STEPS): + nc_fileL = (f"{dir_obs}nnr_003.{sat}_L3a.blend." + f"{yy:04d}{mm:02d}{dd:02d}_{t:02d}00z.nc4") + nc_fileM = (f"{dirm}{EXPID}.inst2d_hwl_x." + f"{yy:04d}{mm:02d}{dd:02d}_{t:02d}00z.nc4") + + result = self.process_single_timestep(nc_fileL, nc_fileM, lonm, latm) + + if result['success']: + tau_nnrLOD[:, :, i] = result['tau_obs'] + for var_name in self.variable_names: + mod_arrays[var_name][:, :, i] = result['model_data'][var_name] + valid_timesteps += 1 + + if valid_timesteps == 0: + logger.warning(f"No valid data found for day {dd}") + + return mod_arrays, tau_nnrLOD + + def write_netcdf_output(self, filename, data_dict, lonm, latm, yy, mm, EXPID, sat): + """Write data to NetCDF file with proper metadata""" + try: + with Dataset(filename, mode='w', format='NETCDF4_CLASSIC') as ncfile: + # Create dimensions + ncfile.createDimension('lat', MODEL_LAT_SIZE) + ncfile.createDimension('lon', MODEL_LON_SIZE) + ncfile.createDimension('time', 1) + + # Variable definitions with metadata + var_info = { + 'MODtau': { + 'data': data_dict['obs'], + 'long_name': 'MODIS AOD', + 'units': '1', + 'description': 'Monthly mean MODIS Neural Network Retrieval AOD' + }, + 'GEOStau': { + 'data': data_dict['tot'], + 'long_name': 'GEOS Total AOD', + 'units': '1', + 'description': 'GEOS model total aerosol optical depth' + }, + 'bcexttau': { + 'data': data_dict['bc'], + 'long_name': 'Black Carbon AOD', + 'units': '1' + }, + 'ocexttau': { + 'data': data_dict['oc'], + 'long_name': 'Organic Carbon AOD', + 'units': '1' + }, + 'brexttau': { + 'data': data_dict['br'], + 'long_name': 'Brown Carbon AOD', + 'units': '1' + }, + 'ssexttau': { + 'data': data_dict['ss'], + 'long_name': 'Sea Salt AOD', + 'units': '1' + }, + 'duexttau': { + 'data': data_dict['du'], + 'long_name': 'Dust AOD', + 'units': '1' + }, + 'suexttau': { + 'data': data_dict['su'], + 'long_name': 'Sulfate AOD', + 'units': '1' + }, + 'niexttau': { + 'data': data_dict['ni'], + 'long_name': 'Nitrate AOD', + 'units': '1' + } + } + + # Create data variables + for var_name, info in var_info.items(): + var = ncfile.createVariable( + var_name, 'f', ('lat', 'lon'), + compression='zlib', complevel=4, + fill_value=np.nan + ) + var[:] = info['data'] + var.long_name = info['long_name'] + var.units = info['units'] + if 'description' in info: + var.description = info['description'] + + # Coordinate variables + lat_var = ncfile.createVariable('lat', np.float32, ('lat',)) + lat_var.units = 'degrees_north' + lat_var.long_name = 'latitude' + lat_var.standard_name = 'latitude' + lat_var[:] = latm + + lon_var = ncfile.createVariable('lon', np.float32, ('lon',)) + lon_var.units = 'degrees_east' + lon_var.long_name = 'longitude' + lon_var.standard_name = 'longitude' + lon_var[:] = lonm + + time_var = ncfile.createVariable('time', np.float64, ('time',)) + time_var.units = f'hours since {yy}-{mm:02d}-01' + time_var.long_name = 'time' + time_var.standard_name = 'time' + time_var[:] = [0] + + # Global attributes + ncfile.title = f'Monthly AOD comparison for {yy}-{mm:02d}' + ncfile.institution = 'NASA GSFC' + ncfile.source = f'GEOS model experiment {EXPID}' + ncfile.satellite_data = f'{sat} MODIS Neural Network Retrievals' + ncfile.history = f'Created on {datetime.now().isoformat()}' + ncfile.conventions = 'CF-1.6' + ncfile.contact = 'geosaerosols@lists.nasa.gov' + + logger.info(f"Successfully wrote: {filename}") + + except Exception as e: + logger.error(f"Error writing NetCDF file {filename}: {e}") + raise + + +def monthlyAOD_mod_obs(yy, mm, EXPID, sat, output_dir="./"): + """ + Main function to process monthly AOD model-observation comparison + + Parameters: + ----------- + yy : int + Year + mm : int + Month (1-12) + EXPID : str + Model experiment ID + sat : str + Satellite identifier (e.g., 'MYD04', 'MOD04') + output_dir : str + Output directory for results + + Returns: + -------- + tuple : (tau_nnrLOD_mm, Mod_TotAOD_mm, lonm, latm) + Monthly mean observations, model total AOD, and coordinates + """ + + processor = AODProcessor() + + # Validate inputs + processor.validate_inputs(yy, mm, EXPID, sat) + + # Set up directories + dir_obs = f"/css/gmao/dp/gds/AeroObs/nnr_003_blend/{sat}/Y{yy:04d}/M{mm:02d}/" + dirm = f"/discover/nobackup/acollow/geos_aerosols/acollow/{EXPID}/holding/inst2d_hwl_x/{yy:04d}{mm:02d}/" + + # Check directories exist + processor.check_directories(dir_obs, dirm) + + # Get file list and parse dates + try: + MOD_files = sorted([f for f in os.listdir(dir_obs) if f.endswith('.nc4')]) + dds, dde = processor.parse_file_dates(MOD_files) + logger.info(f"Processing {yy}-{mm:02d}: Days {dds} to {dde}") + except Exception as e: + logger.error(f"Error processing file list: {e}") + raise + + # Initialize arrays + num_days = dde - dds + 1 + daily_shape = (MODEL_LAT_SIZE, MODEL_LON_SIZE, num_days) + + mod_arrays_dd = processor.initialize_arrays(daily_shape) + tau_nnrLOD_dd = np.full(daily_shape, np.nan, dtype=np.float32) + + # Get coordinate arrays from first available file + lonm, latm = None, None + for dd in range(dds, dde + 1): + for t in TIME_STEPS: + nc_fileM = (f"{dirm}{EXPID}.inst2d_hwl_x." + f"{yy:04d}{mm:02d}{dd:02d}_{t:02d}00z.nc4") + if os.path.isfile(nc_fileM): + try: + with Dataset(nc_fileM, 'r') as ncid: + lonm = ncid.variables['lon'][:] + latm = ncid.variables['lat'][:] + break + except Exception as e: + logger.warning(f"Error reading coordinates from {nc_fileM}: {e}") + continue + if lonm is not None: + break + + if lonm is None: + raise FileNotFoundError("Could not find valid model file to read coordinates") + + # Process each day + valid_days = 0 + for dd in range(dds, dde + 1): + try: + day_index = dd - dds + mod_arrays, tau_nnrLOD = processor.process_daily_data( + dd, dds, dir_obs, dirm, yy, mm, EXPID, sat, lonm, latm + ) + + # Store daily means + for var_name in processor.variable_names: + mod_arrays_dd[var_name][:, :, day_index] = np.nanmean(mod_arrays[var_name], axis=2) + tau_nnrLOD_dd[:, :, day_index] = np.nanmean(tau_nnrLOD, axis=2) + + valid_days += 1 + logger.info(f"Processed day {dd}") + + except Exception as e: + logger.error(f"Error processing day {dd}: {e}") + continue + + if valid_days == 0: + raise ValueError(f"No valid days processed for {yy}-{mm:02d}") + + logger.info(f"Successfully processed {valid_days} out of {num_days} days") + + # Calculate monthly means + tau_nnrLOD_mm = np.nanmean(tau_nnrLOD_dd, axis=2) + mod_monthly = {} + for var_name in processor.variable_names: + mod_monthly[var_name] = np.nanmean(mod_arrays_dd[var_name], axis=2) + + # Prepare data for NetCDF output + output_data = { + 'obs': tau_nnrLOD_mm, + 'tot': mod_monthly['tot'], + 'bc': mod_monthly['bc'], + 'oc': mod_monthly['oc'], + 'br': mod_monthly['br'], + 'ss': mod_monthly['ss'], + 'du': mod_monthly['du'], + 'su': mod_monthly['su'], + 'ni': mod_monthly['ni'] + } + + # Write NetCDF output + output_filename = f"{output_dir}/{EXPID}.tavgM_aod_{sat}filtered.{yy}{mm:02d}.nc4" + processor.write_netcdf_output(output_filename, output_data, lonm, latm, yy, mm, EXPID, sat) + + return tau_nnrLOD_mm, mod_monthly['tot'], lonm, latm + + +def main(): + """Main execution function""" + # Configuration + config = { + 'year': 2024, + 'experiment_id': "c180R_qfed3igbp_xf", + 'satellite': "MYD04", + 'months': range(1, 13), + 'output_dir': './sampledGEOS/c180R_qfed3igbp_xf/' + } + + # Create output directory + os.makedirs(config['output_dir'], exist_ok=True) + + logger.info(f"Starting AOD processing for {config['year']}") + logger.info(f"Experiment: {config['experiment_id']}") + logger.info(f"Satellite: {config['satellite']}") + logger.info(f"Months: {list(config['months'])}") + + successful_months = [] + failed_months = [] + + for mm in config['months']: + try: + logger.info(f"Processing {config['year']}-{mm:02d}") + start_time = datetime.now() + + result = monthlyAOD_mod_obs( + config['year'], mm, + config['experiment_id'], + config['satellite'], + config['output_dir'] + ) + + end_time = datetime.now() + duration = (end_time - start_time).total_seconds() + + logger.info(f"Successfully completed {config['year']}-{mm:02d} in {duration:.1f} seconds") + successful_months.append(mm) + + except Exception as e: + logger.error(f"Failed to process {config['year']}-{mm:02d}: {e}") + failed_months.append(mm) + continue + + # Summary + logger.info(f"Processing complete!") + logger.info(f"Successful months: {successful_months}") + if failed_months: + logger.warning(f"Failed months: {failed_months}") + + +if __name__ == "__main__": + main() diff --git a/src/pyobs/evaluation/MODIS_NNR/plotregionalcomparison.py b/src/pyobs/evaluation/MODIS_NNR/plotregionalcomparison.py new file mode 100755 index 0000000..79d7e44 --- /dev/null +++ b/src/pyobs/evaluation/MODIS_NNR/plotregionalcomparison.py @@ -0,0 +1,1238 @@ +#!/usr/bin/env python +""" +This code uses the output generated by monthlygeossample_speciated.py to +create figures comparing GEOS with MODIS NNR data. Plots include global maps +of regional summaries (bias, correlation, scaling), and individual figures +showing the annual cycle of total AOD and how that is broken down by species in GEOS. +""" + +import numpy as np +import xarray as xr +import matplotlib.pyplot as plt +import matplotlib.colors as mcolors +import matplotlib.patches as patches +from matplotlib.patches import Rectangle +from matplotlib.colors import TwoSlopeNorm +import cartopy.crs as ccrs +import cartopy.feature as cfeature +from pathlib import Path +import pandas as pd +from datetime import datetime +import seaborn as sns +import argparse +import sys +from scipy.stats import pearsonr + +# Define regions (including global) +REGIONS = { + 0: {'name': 'Global', 'lon': [-180, 180], 'lat': [-90, 90]}, + 1: {'name': 'Alaska', 'lon': [-170, -140], 'lat': [50, 70]}, + 2: {'name': 'Canada', 'lon': [-140, -80], 'lat': [50, 70]}, + 3: {'name': 'Quebec', 'lon': [-80, -55], 'lat': [45, 65]}, + 4: {'name': 'USWest', 'lon': [-130, -105], 'lat': [30, 50]}, + 5: {'name': 'USCentral', 'lon': [-105, -90], 'lat': [30, 50]}, + 6: {'name': 'USEast', 'lon': [-90, -70], 'lat': [25, 45]}, + 7: {'name': 'Mexico', 'lon': [-120, -85], 'lat': [10, 30]}, + 8: {'name': 'BrazilFor', 'lon': [-75, -50], 'lat': [-15, 5]}, + 9: {'name': 'BrazilCer', 'lon': [-50, -30], 'lat': [-20, 0]}, + 10: {'name': 'Argentina', 'lon': [-75, -50], 'lat': [-60, -15]}, + 11: {'name': 'AfricaWest', 'lon': [-20, 15], 'lat': [0, 15]}, + 12: {'name': 'AfricaCent', 'lon': [15, 30], 'lat': [5, 15]}, + 13: {'name': 'AfricaEast', 'lon': [30, 50], 'lat': [-10, 15]}, + 14: {'name': 'Congo', 'lon': [10, 30], 'lat': [-10, 5]}, + 15: {'name': 'Zambia', 'lon': [22, 35], 'lat': [-18, -8]}, + 16: {'name': 'AfricaSouth', 'lon': [10, 35], 'lat': [-35, -20]}, + 17: {'name': 'Madagascar', 'lon': [42, 50], 'lat': [-25, -12]}, + 18: {'name': 'Scandinavia', 'lon': [0, 35], 'lat': [55, 75]}, + 19: {'name': 'Moscow', 'lon': [30, 60], 'lat': [45, 60]}, + 20: {'name': 'SiberiaWest', 'lon': [35, 90], 'lat': [60, 75]}, + 21: {'name': 'SiberiaEast', 'lon': [90, 140], 'lat': [60, 75]}, + 22: {'name': 'EuropeWest', 'lon': [-10, 30], 'lat': [35, 55]}, + 23: {'name': 'MiddleEast', 'lon': [30, 60], 'lat': [30, 45]}, + 24: {'name': 'AsiaCent', 'lon': [60, 110], 'lat': [35, 50]}, + 25: {'name': 'ChinaEast', 'lon': [110, 150], 'lat': [35, 60]}, + 26: {'name': 'Nepal', 'lon': [65, 95], 'lat': [25, 35]}, + 27: {'name': 'India', 'lon': [70, 90], 'lat': [5, 25]}, + 28: {'name': 'ChinaSouth', 'lon': [100, 125], 'lat': [20, 40]}, + 29: {'name': 'Indochina', 'lon': [90, 110], 'lat': [10, 25]}, + 30: {'name': 'Philippines', 'lon': [115, 130], 'lat': [5, 20]}, + 31: {'name': 'Sumatra', 'lon': [95, 110], 'lat': [-10, 10]}, + 32: {'name': 'Borneo', 'lon': [110, 120], 'lat': [-5, 8]}, + 33: {'name': 'Indonesia', 'lon': [120, 160], 'lat': [-10, 5]}, + 34: {'name': 'AustraliaN', 'lon': [120, 150], 'lat': [-20, -10]}, + 35: {'name': 'AustraliaW', 'lon': [110, 130], 'lat': [-35, -20]}, + 36: {'name': 'AustraliaE', 'lon': [135, 155], 'lat': [-45, -20]}, + 37: {'name': 'SiberiaFE', 'lon': [140, 170], 'lat': [60, 75]}, + 38: {'name': 'Sahara', 'lon': [-15, 30], 'lat': [13, 35]}, + 39: {'name': 'Sahel', 'lon': [-15, 35], 'lat': [12, 13]}, + 40: {'name': 'CapeVerde', 'lon': [-26, -20], 'lat': [10, 20]}, + 41: {'name': 'RedSea', 'lon': [30, 45], 'lat': [10, 30]}, + 42: {'name': 'PersianGulf', 'lon': [45, 60], 'lat': [20, 30]}, + 43: {'name': 'ArabSea', 'lon': [60, 70], 'lat': [10, 20]}, + 44: {'name': 'Caribbean', 'lon': [-80, -60], 'lat': [13, 23]}, + 45: {'name': 'SALDust', 'lon': [-60, -26], 'lat': [13, 30]}, + 46: {'name': 'SAmerBB', 'lon': [-45, -20], 'lat': [-45, -25]} +} + +def load_monthly_data(base_path='sampledGEOS/c180R_qfed3igbp_allviirs', sensor='both'): + """Load all monthly data files and combine into datasets.""" + base_path = Path(base_path) + sensor_lower = sensor.lower() + + # Get all MOD and MYD files + mod_files = sorted(base_path.glob('*MOD04filtered*.nc4')) + myd_files = sorted(base_path.glob('*MYD04filtered*.nc4')) + + # Filter based on sensor selection + if sensor_lower in ['terra', 'mod']: + myd_files = [] + print(f"Using Terra (MOD) data only") + elif sensor_lower in ['aqua', 'myd']: + mod_files = [] + print(f"Using Aqua (MYD) data only") + elif sensor_lower == 'both': + print(f"Using both Terra (MOD) and Aqua (MYD) data separately") + else: + raise ValueError(f"Invalid sensor option: {sensor}. Use 'terra', 'aqua', or 'both'") + + print(f"Found {len(mod_files)} MOD files and {len(myd_files)} MYD files") + + # Load and combine data + mod_data = [] + myd_data = [] + + for f in mod_files: + try: + ds = xr.open_dataset(f) + # Extract month from filename + month_str = f.name.split('.')[-2] # e.g., '202401' + month = int(month_str[-2:]) + ds = ds.assign_coords(month=month) + mod_data.append(ds) + print(f" Loaded Terra month {month}") + except Exception as e: + print(f"Error loading {f}: {e}") + + for f in myd_files: + try: + ds = xr.open_dataset(f) + month_str = f.name.split('.')[-2] + month = int(month_str[-2:]) + ds = ds.assign_coords(month=month) + myd_data.append(ds) + print(f" Loaded Aqua month {month}") + except Exception as e: + print(f"Error loading {f}: {e}") + + # Combine datasets + if mod_data: + mod_combined = xr.concat(mod_data, dim='month') + mod_combined.attrs['sensor'] = 'Terra' + else: + mod_combined = None + + if myd_data: + myd_combined = xr.concat(myd_data, dim='month') + myd_combined.attrs['sensor'] = 'Aqua' + else: + myd_combined = None + + return mod_combined, myd_combined + +def calculate_regional_means(data, region_id): + """Calculate regional means for a specific region.""" + region = REGIONS[region_id] + lon_min, lon_max = region['lon'] + lat_min, lat_max = region['lat'] + + sensor_name = data.attrs.get('sensor', 'MODIS') + print(f"Calculating means for {region['name']} using {sensor_name}") + + if region_id == 0: # Global region + region_data = data + print(" Using global data (no spatial subsetting)") + else: + # Handle longitude conversion for regional data + data_lon = data.lon.values + if np.any(data_lon > 180): + # Data is in 0-360 format, convert region bounds + if lon_min < 0: + lon_min += 360 + if lon_max < 0: + lon_max += 360 + + # Select region + if lon_min > lon_max: # Crossing dateline + region_data = data.where( + ((data.lon >= lon_min) | (data.lon <= lon_max)) & + (data.lat >= lat_min) & (data.lat <= lat_max) + ) + else: + region_data = data.where( + (data.lon >= lon_min) & (data.lon <= lon_max) & + (data.lat >= lat_min) & (data.lat <= lat_max) + ) + + # Calculate means + means = {} + for var in ['MODtau', 'GEOStau', 'bcexttau', 'ocexttau', 'brexttau', + 'ssexttau', 'duexttau', 'suexttau', 'niexttau']: + if var in region_data.data_vars: + var_mean = region_data[var].mean(dim=['lat', 'lon'], skipna=True) + means[var] = var_mean + + return means + +def calculate_regional_statistics(mod_data, myd_data, sensor='both'): + """Calculate correlation, bias, and scaling factor statistics for all regions.""" + print("Calculating regional statistics...") + + stats_data = [] + + for region_id, region_info in REGIONS.items(): + region_name = region_info['name'] + print(f" Processing {region_name}...") + + # Process each sensor + sensors_to_process = [] + if sensor.lower() in ['terra', 'both'] and mod_data is not None: + sensors_to_process.append(('Terra', mod_data)) + if sensor.lower() in ['aqua', 'both'] and myd_data is not None: + sensors_to_process.append(('Aqua', myd_data)) + + for sensor_name, data in sensors_to_process: + try: + means = calculate_regional_means(data, region_id) + + if not means or all(var.isnull().all() for var in means.values()): + continue + + # Extract valid data + mod_vals = means['MODtau'].values + geos_vals = means['GEOStau'].values + valid_mask = ~(np.isnan(mod_vals) | np.isnan(geos_vals)) + + if np.sum(valid_mask) < 2: # Need at least 2 points for correlation + continue + + valid_mod = mod_vals[valid_mask] + valid_geos = geos_vals[valid_mask] + + # Calculate statistics + corr, p_value = pearsonr(valid_mod, valid_geos) + bias = np.mean(valid_geos - valid_mod) + rmse = np.sqrt(np.mean((valid_geos - valid_mod)**2)) + mean_obs = np.mean(valid_mod) + mean_model = np.mean(valid_geos) + n_points = len(valid_mod) + + # Calculate scaling factor chi = exp(log(MODtau+0.01) - log(GEOStau+0.01)) + # This is equivalent to (MODtau+0.01)/(GEOStau+0.01) + mod_adjusted = valid_mod + 0.01 + geos_adjusted = valid_geos + 0.01 + + # Calculate chi for each time point + chi_values = np.exp(np.log(mod_adjusted) - np.log(geos_adjusted)) + + # Calculate statistics of chi + chi_mean = np.mean(chi_values) + chi_median = np.median(chi_values) + chi_std = np.std(chi_values) + chi_min = np.min(chi_values) + chi_max = np.max(chi_values) + + stats_data.append({ + 'region_id': region_id, + 'region_name': region_name, + 'sensor': sensor_name, + 'correlation': corr, + 'p_value': p_value, + 'bias': bias, + 'rmse': rmse, + 'mean_obs': mean_obs, + 'mean_model': mean_model, + 'n_points': n_points, + 'chi_mean': chi_mean, + 'chi_median': chi_median, + 'chi_std': chi_std, + 'chi_min': chi_min, + 'chi_max': chi_max, + 'lon_min': region_info['lon'][0], + 'lon_max': region_info['lon'][1], + 'lat_min': region_info['lat'][0], + 'lat_max': region_info['lat'][1] + }) + + except Exception as e: + print(f" Error processing {sensor_name} for {region_name}: {e}") + continue + + return pd.DataFrame(stats_data) + +def create_map_plot(stats_df, metric='correlation', sensor='both', vmin=None, vmax=None, + title_suffix='', output_dir='regional_analysis', experiment_name='', + show_values=False): + """Create map visualization of regional statistics with optional value annotations.""" + + # Filter data by sensor if needed + if sensor.lower() in ['terra', 'aqua']: + plot_data = stats_df[stats_df['sensor'].str.lower() == sensor.lower()].copy() + sensor_title = sensor.capitalize() + else: + # For 'both', average the metrics across sensors for each region + if len(stats_df) == 0: + print("No data available for mapping") + return None + + # Group by region and average metrics + numeric_cols = ['correlation', 'bias', 'rmse', 'mean_obs', 'mean_model', + 'chi_mean', 'chi_median', 'chi_std', 'chi_min', 'chi_max'] + plot_data = stats_df.groupby('region_id').agg({ + 'region_name': 'first', + 'lon_min': 'first', 'lon_max': 'first', + 'lat_min': 'first', 'lat_max': 'first', + 'n_points': 'sum', + **{col: 'mean' for col in numeric_cols if col in stats_df.columns} + }).reset_index() + sensor_title = 'Combined (Terra+Aqua)' + + if len(plot_data) == 0: + print(f"No data available for sensor: {sensor}") + return None + + # Set up the plot + fig = plt.figure(figsize=(18, 12)) # Slightly larger for text annotations + ax = plt.axes(projection=ccrs.PlateCarree()) + + # Add map features + ax.add_feature(cfeature.COASTLINE, alpha=0.5) + ax.add_feature(cfeature.BORDERS, alpha=0.3) + ax.add_feature(cfeature.OCEAN, color='lightblue', alpha=0.3) + ax.add_feature(cfeature.LAND, color='lightgray', alpha=0.3) + + # Set global extent + ax.set_global() + + # Define color mapping based on metric + if metric == 'correlation': + if vmin is None or vmax is None: + vmin, vmax = -1, 1 + cmap = plt.cm.RdBu_r # Red for negative, blue for positive + cmap_label = 'Correlation Coefficient' + title_metric = 'Correlation' + elif metric == 'bias': + if vmin is None or vmax is None: + abs_max = max(abs(plot_data[metric].min()), abs(plot_data[metric].max())) + vmin, vmax = -abs_max, abs_max + cmap = plt.cm.RdBu # Blue for negative bias, red for positive + cmap_label = 'Bias (GEOS - MODIS)' + title_metric = 'Bias' + elif metric == 'rmse': + if vmin is None or vmax is None: + vmin, vmax = 0, plot_data[metric].max() + cmap = plt.cm.Reds # White to red for RMSE + cmap_label = 'Root Mean Square Error' + title_metric = 'RMSE' + elif metric == 'chi_mean': + if vmin is None or vmax is None: + vmin, vmax = 0.5, 2.0 # Reasonable default range for scaling factors + cmap = plt.cm.RdYlBu_r # Red for high scaling, blue for low scaling + cmap_label = 'Scaling Factor (ฯ‡)' + title_metric = 'Scaling Factor (ฯ‡)' + show_values = True # Always show values for chi + else: + # Default settings + if vmin is None or vmax is None: + vmin, vmax = plot_data[metric].min(), plot_data[metric].max() + cmap = plt.cm.viridis + cmap_label = metric.replace('_', ' ').title() + title_metric = metric.replace('_', ' ').title() + + # Create color normalization + norm = plt.Normalize(vmin=vmin, vmax=vmax) + + # Plot regions with different styles for better visibility + for idx, row in plot_data.iterrows(): + # Skip global region (region_id = 0) for mapping + if row['region_id'] == 0: + continue + + lon_min, lon_max = row['lon_min'], row['lon_max'] + lat_min, lat_max = row['lat_min'], row['lat_max'] + + # Handle longitude wrapping + if lon_min > lon_max: # Crosses dateline + # Split into two rectangles + rect1 = Rectangle((lon_min, lat_min), 180 - lon_min, lat_max - lat_min, + transform=ccrs.PlateCarree(), alpha=0.7, + edgecolor='black', linewidth=1.5) + rect2 = Rectangle((-180, lat_min), lon_max + 180, lat_max - lat_min, + transform=ccrs.PlateCarree(), alpha=0.7, + edgecolor='black', linewidth=1.5) + + color = cmap(norm(row[metric])) + rect1.set_facecolor(color) + rect2.set_facecolor(color) + ax.add_patch(rect1) + ax.add_patch(rect2) + else: + # Regular rectangle + rect = Rectangle((lon_min, lat_min), lon_max - lon_min, lat_max - lat_min, + transform=ccrs.PlateCarree(), alpha=0.7, + edgecolor='black', linewidth=1.5) + + color = cmap(norm(row[metric])) + rect.set_facecolor(color) + ax.add_patch(rect) + + # Calculate center for text placement + center_lon = (lon_min + lon_max) / 2 + center_lat = (lat_min + lat_max) / 2 + + # Adjust for dateline crossing + if lon_min > lon_max: + if center_lon < 0: + center_lon += 180 + else: + center_lon -= 180 + + if show_values: + # Add both region name and metric value + value_text = f"{row['region_name']}\nฯ‡ = {row[metric]:.3f}" + fontsize = 9 + bbox_props = dict(boxstyle='round,pad=0.4', facecolor='white', + alpha=0.9, edgecolor='black', linewidth=0.5) + else: + # Add just region name + value_text = row['region_name'] + fontsize = 8 + bbox_props = dict(boxstyle='round,pad=0.3', facecolor='white', + alpha=0.8, edgecolor='none') + + ax.text(center_lon, center_lat, value_text, + transform=ccrs.PlateCarree(), + ha='center', va='center', fontsize=fontsize, fontweight='bold', + bbox=bbox_props) + + # Add colorbar + sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm) + sm.set_array([]) + cbar = plt.colorbar(sm, ax=ax, shrink=0.6, aspect=20, pad=0.02) + cbar.set_label(cmap_label, fontsize=12) + + # Add gridlines + gl = ax.gridlines(draw_labels=True, alpha=0.5) + gl.top_labels = False + gl.right_labels = False + + # Set title + title = f'{title_metric} - {sensor_title}' + if title_suffix: + title += f' - {title_suffix}' + if experiment_name: + title += f' ({experiment_name})' + + plt.title(title, fontsize=14, fontweight='bold', pad=20) + + # Save the plot + output_dir = Path(output_dir) + output_dir.mkdir(exist_ok=True) + + if show_values and metric == 'chi_mean': + filename = f"{experiment_name}_{metric}_{sensor.lower()}_map_with_values.png" + else: + filename = f"{experiment_name}_{metric}_{sensor.lower()}_map.png" + + save_path = output_dir / filename + plt.savefig(save_path, dpi=300, bbox_inches='tight', facecolor='white') + print(f"Map saved to: {save_path}") + + return fig, save_path + +def create_chi_value_map(stats_df, sensor='both', output_dir='regional_analysis', + experiment_name='', title_suffix=''): + """Create a dedicated map showing chi values with annotations.""" + + print("Creating chi scaling factor map with values...") + + # Filter data by sensor if needed + if sensor.lower() in ['terra', 'aqua']: + plot_data = stats_df[stats_df['sensor'].str.lower() == sensor.lower()].copy() + sensor_title = sensor.capitalize() + else: + # For 'both', average the metrics across sensors for each region + if len(stats_df) == 0: + print("No data available for mapping") + return None + + # Group by region and average chi values + numeric_cols = ['chi_mean', 'chi_median', 'chi_std', 'chi_min', 'chi_max'] + plot_data = stats_df.groupby('region_id').agg({ + 'region_name': 'first', + 'lon_min': 'first', 'lon_max': 'first', + 'lat_min': 'first', 'lat_max': 'first', + 'n_points': 'sum', + **{col: 'mean' for col in numeric_cols if col in stats_df.columns} + }).reset_index() + sensor_title = 'Combined (Terra+Aqua)' + + if len(plot_data) == 0 or 'chi_mean' not in plot_data.columns: + print(f"No chi data available for sensor: {sensor}") + return None + + # Set up the plot with larger size for better text visibility + fig = plt.figure(figsize=(20, 14)) + ax = plt.axes(projection=ccrs.PlateCarree()) + + # Add map features + ax.add_feature(cfeature.COASTLINE, alpha=0.5) + ax.add_feature(cfeature.BORDERS, alpha=0.3) + ax.add_feature(cfeature.OCEAN, color='lightblue', alpha=0.3) + ax.add_feature(cfeature.LAND, color='lightgray', alpha=0.3) + + # Set global extent + ax.set_global() + + # Define chi-specific color mapping + chi_min = plot_data['chi_mean'].min() + chi_max = plot_data['chi_mean'].max() + + # Set reasonable limits for chi visualization + vmin = max(0.2, chi_min * 0.9) # Don't go below 0.2 + vmax = min(3.0, chi_max * 1.1) # Don't go above 3.0 + + # Use a diverging colormap centered at 1.0 (perfect scaling) + norm = TwoSlopeNorm(vmin=vmin, vcenter=1.0, vmax=vmax) + cmap = plt.cm.RdYlBu_r # Red for overestimate (chi > 1), blue for underestimate (chi < 1) + + # Plot regions + for idx, row in plot_data.iterrows(): + # Skip global region for mapping + if row['region_id'] == 0: + continue + + lon_min, lon_max = row['lon_min'], row['lon_max'] + lat_min, lat_max = row['lat_min'], row['lat_max'] + + # Check if chi_mean is valid, skip region if not + if 'chi_mean' not in row or pd.isna(row['chi_mean']): + print(f"Warning: No valid chi value for region {row['region_name']}") + continue + + chi_value = row['chi_mean'] + + # Handle longitude wrapping + if lon_min > lon_max: # Crosses dateline + # Split into two rectangles + rect1 = Rectangle((lon_min, lat_min), 180 - lon_min, lat_max - lat_min, + transform=ccrs.PlateCarree(), alpha=0.8, + edgecolor='black', linewidth=2) + rect2 = Rectangle((-180, lat_min), lon_max + 180, lat_max - lat_min, + transform=ccrs.PlateCarree(), alpha=0.8, + edgecolor='black', linewidth=2) + + color = cmap(norm(chi_value)) + rect1.set_facecolor(color) + rect2.set_facecolor(color) + ax.add_patch(rect1) + ax.add_patch(rect2) + else: + # Regular rectangle + rect = Rectangle((lon_min, lat_min), lon_max - lon_min, lat_max - lat_min, + transform=ccrs.PlateCarree(), alpha=0.8, + edgecolor='black', linewidth=2) + + color = cmap(norm(chi_value)) + rect.set_facecolor(color) + ax.add_patch(rect) + + # Calculate center for text placement + center_lon = (lon_min + lon_max) / 2 + center_lat = (lat_min + lat_max) / 2 + + # Adjust for dateline crossing + if lon_min > lon_max: + if center_lon < 0: + center_lon += 180 + else: + center_lon -= 180 + + # Create text with region name and chi value + # Ensure chi value is properly formatted to handle extreme values + chi_text = f"{row['region_name']}\nฯ‡ = {chi_value:.3f}" + + # Choose text color based on background + if chi_value > 1.5: # Red background + text_color = 'white' + elif chi_value < 0.7: # Blue background + text_color = 'white' + else: # Light background + text_color = 'black' + + ax.text(center_lon, center_lat, chi_text, + transform=ccrs.PlateCarree(), + ha='center', va='center', fontsize=10, fontweight='bold', + color=text_color, + bbox=dict(boxstyle='round,pad=0.4', facecolor='white', + alpha=0.9, edgecolor='black', linewidth=0.8)) + + # Add colorbar with custom ticks + sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm) + sm.set_array([]) + cbar = plt.colorbar(sm, ax=ax, shrink=0.7, aspect=25, pad=0.02) + + # Use the exact requested formula in the colorbar label + cbar.set_label('Scaling Factor (ฯ‡ = exp(log(MODtau+0.01)-log(GEOStau+0.01)))', + fontsize=12, fontweight='bold') + + # Add interpretation text to colorbar + cbar.ax.text(1.15, 1.02, 'GEOS underestimates', transform=cbar.ax.transAxes, + rotation=0, ha='left', va='bottom', fontsize=10, color='red', fontweight='bold') + cbar.ax.text(1.15, -0.02, 'GEOS overestimates', transform=cbar.ax.transAxes, + rotation=0, ha='left', va='top', fontsize=10, color='blue', fontweight='bold') + + # Add reference line at chi=1 + cbar.ax.axhline(y=norm(1.0), color='black', linestyle='--', linewidth=2, alpha=0.7) + cbar.ax.text(1.05, norm(1.0), 'Perfect match (ฯ‡=1)', transform=cbar.ax.get_yaxis_transform(), + ha='left', va='center', fontsize=10, fontweight='bold') + + # Add gridlines + gl = ax.gridlines(draw_labels=True, alpha=0.5) + gl.top_labels = False + gl.right_labels = False + + # Set title + title = f'Scaling Factor (ฯ‡) - {sensor_title}' + if title_suffix: + title += f' - {title_suffix}' + if experiment_name: + title += f' ({experiment_name})' + + plt.title(title, fontsize=16, fontweight='bold', pad=25) + + # Add subtitle with interpretation + plt.figtext(0.5, 0.02, 'ฯ‡ > 1: GEOS underestimates AOD relative to MODIS | ฯ‡ < 1: GEOS overestimates AOD relative to MODIS', + ha='center', va='bottom', fontsize=12, style='italic') + + # Save the plot + output_dir = Path(output_dir) + output_dir.mkdir(exist_ok=True) + + filename = f"{experiment_name}_chi_scaling_factor_{sensor.lower()}_map_with_values.png" + save_path = output_dir / filename + plt.savefig(save_path, dpi=300, bbox_inches='tight', facecolor='white') + print(f"Chi scaling factor map with values saved to: {save_path}") + + return fig, save_path + +def create_statistical_summary_table(stats_df, output_dir='regional_analysis', experiment_name=''): + """Create a comprehensive statistical summary table including scaling factors.""" + + if len(stats_df) == 0: + print("No data available for summary table") + return None + + # Create output directory + output_dir = Path(output_dir) + output_dir.mkdir(exist_ok=True) + + # Prepare data for table + summary_data = [] + + for sensor in stats_df['sensor'].unique(): + sensor_data = stats_df[stats_df['sensor'] == sensor].copy() + + for idx, row in sensor_data.iterrows(): + summary_data.append({ + 'Region ID': row['region_id'], + 'Region Name': row['region_name'], + 'Sensor': row['sensor'], + 'Correlation': f"{row['correlation']:.3f}", + 'P-value': f"{row['p_value']:.4f}" if not np.isnan(row['p_value']) else 'N/A', + 'Bias': f"{row['bias']:.4f}", + 'RMSE': f"{row['rmse']:.4f}", + 'Mean MODIS': f"{row['mean_obs']:.4f}", + 'Mean GEOS': f"{row['mean_model']:.4f}", + 'Chi Mean': f"{row['chi_mean']:.4f}", + 'Chi Median': f"{row['chi_median']:.4f}", + 'Chi Std': f"{row['chi_std']:.4f}", + 'Chi Min': f"{row['chi_min']:.4f}", + 'Chi Max': f"{row['chi_max']:.4f}", + 'N Points': int(row['n_points']) + }) + + # Convert to DataFrame and save + summary_df = pd.DataFrame(summary_data) + + # Save as CSV + csv_path = output_dir / f"{experiment_name}_regional_statistics_with_chi.csv" + summary_df.to_csv(csv_path, index=False) + print(f"Statistical summary with scaling factors saved to: {csv_path}") + + # Create a separate chi-focused table + chi_data = [] + for sensor in stats_df['sensor'].unique(): + sensor_data = stats_df[stats_df['sensor'] == sensor].copy() + + for idx, row in sensor_data.iterrows(): + chi_data.append({ + 'Region ID': row['region_id'], + 'Region Name': row['region_name'], + 'Sensor': row['sensor'], + 'Chi Mean': f"{row['chi_mean']:.4f}", + 'Chi Median': f"{row['chi_median']:.4f}", + 'Chi Std': f"{row['chi_std']:.4f}", + 'Chi Range': f"[{row['chi_min']:.4f}, {row['chi_max']:.4f}]", + 'N Points': int(row['n_points']) + }) + + chi_df = pd.DataFrame(chi_data) + chi_csv_path = output_dir / f"{experiment_name}_scaling_factors_chi.csv" + chi_df.to_csv(chi_csv_path, index=False) + print(f"Scaling factor (chi) table saved to: {chi_csv_path}") + + # Create a formatted version for display + print("\n" + "="*120) + print("REGIONAL STATISTICS SUMMARY WITH SCALING FACTORS") + print("="*120) + print(summary_df.to_string(index=False)) + + print("\n" + "="*80) + print("SCALING FACTOR (CHI) SUMMARY") + print("="*80) + print(chi_df.to_string(index=False)) + + return summary_df, csv_path, chi_df, chi_csv_path + +def create_all_maps(mod_data, myd_data, sensor='both', output_dir='regional_analysis', + experiment_name='', title_suffix=''): + """Create maps for all key metrics including chi values.""" + + # Calculate statistics + stats_df = calculate_regional_statistics(mod_data, myd_data, sensor=sensor) + + if len(stats_df) == 0: + print("No statistics calculated - no valid data found") + return None, None + + # Create statistical summary + summary_result = create_statistical_summary_table(stats_df, output_dir, experiment_name) + + # Handle both old and new return formats + if len(summary_result) == 4: + summary_df, csv_path, chi_df, chi_csv_path = summary_result + else: + summary_df, csv_path = summary_result + + # Create maps for different metrics + metrics = ['correlation', 'bias', 'rmse', 'chi_mean'] + figures = {} + + for metric in metrics: + if metric in stats_df.columns: + print(f"\nCreating {metric} map...") + try: + fig, save_path = create_map_plot(stats_df, metric=metric, sensor=sensor, + title_suffix=title_suffix, + output_dir=output_dir, + experiment_name=experiment_name) + if fig: + figures[metric] = (fig, save_path) + plt.close(fig) # Close to save memory + except Exception as e: + print(f"Error creating {metric} map: {e}") + + # Create special chi value map with annotations + if 'chi_mean' in stats_df.columns: + print(f"\nCreating detailed chi scaling factor map with values...") + try: + chi_fig, chi_save_path = create_chi_value_map(stats_df, sensor=sensor, + output_dir=output_dir, + experiment_name=experiment_name, + title_suffix=title_suffix) + if chi_fig: + figures['chi_detailed'] = (chi_fig, chi_save_path) + plt.close(chi_fig) + except Exception as e: + print(f"Error creating detailed chi map: {e}") + + return figures, stats_df + +def plot_single_sensor_panels(data, region_id, axes, sensor_name, shared_limits=None, shared_bc_max=None, shared_y_limits=None, show_legend=True, legend_position='upper left'): + """Plot time series and scatter plot for a single sensor with shared scaling.""" + means = calculate_regional_means(data, region_id) + + if not means or all(var.isnull().all() for var in means.values()): + axes[0].text(0.5, 0.5, f'No valid data for {sensor_name}', + transform=axes[0].transAxes, ha='center', va='center', fontsize=12) + axes[1].text(0.5, 0.5, f'No valid data for {sensor_name}', + transform=axes[1].transAxes, ha='center', va='center', fontsize=12) + return False, None, None, None + + # Extract data + months = data.month.values + mod_vals = means['MODtau'].values + geos_vals = means['GEOStau'].values + + # Aerosol components with BETTER color separation + components = { + 'Black Carbon': means.get('bcexttau', xr.DataArray(np.zeros_like(months))).values, + 'Organic Carbon': means.get('ocexttau', xr.DataArray(np.zeros_like(months))).values, + 'Brown Carbon': means.get('brexttau', xr.DataArray(np.zeros_like(months))).values, + 'Sea Salt': means.get('ssexttau', xr.DataArray(np.zeros_like(months))).values, + 'Dust': means.get('duexttau', xr.DataArray(np.zeros_like(months))).values, + 'Sulfate': means.get('suexttau', xr.DataArray(np.zeros_like(months))).values, + 'Nitrate': means.get('niexttau', xr.DataArray(np.zeros_like(months))).values + } + + valid_mask = ~(np.isnan(mod_vals) | np.isnan(geos_vals)) + + if np.sum(valid_mask) > 0: + valid_months = months[valid_mask] + valid_mod = mod_vals[valid_mask] + valid_geos = geos_vals[valid_mask] + + # Time series with BETTER color separation + bottom = np.zeros(len(valid_months)) + component_colors = { + 'Black Carbon': '#2C2C2C', # Dark gray/black + 'Organic Carbon': '#228B22', # Forest green (changed from brown) + 'Brown Carbon': '#8B4513', # Saddle brown + 'Sea Salt': '#4682B4', # Steel blue + 'Dust': '#DAA520', # Goldenrod + 'Sulfate': '#FF6347', # Tomato red + 'Nitrate': '#9370DB' # Medium purple + } + + # Store the middle y-position of each component for labeling + component_middles = {} + + for comp_name, comp_vals in components.items(): + valid_comp = comp_vals[valid_mask] + valid_comp = np.nan_to_num(valid_comp, nan=0.0) + if np.any(valid_comp > 0): + axes[0].fill_between(valid_months, bottom, bottom + valid_comp, + alpha=0.8, color=component_colors[comp_name], + edgecolor='white', linewidth=0.5) + # Calculate middle position for this component + component_middles[comp_name] = np.mean(bottom + valid_comp/2) + bottom += valid_comp + + axes[0].plot(valid_months, valid_mod, 'ko-', linewidth=2, markersize=6, + label=f'MODIS AOD ({sensor_name})', zorder=10) + axes[0].plot(valid_months, valid_geos, 'r--', linewidth=2, + label='GEOS Total AOD', zorder=9) + + # Set shared y-axis limits for time series if provided + if shared_y_limits is not None: + axes[0].set_ylim(shared_y_limits) + + axes[0].set_xlabel('Month') + axes[0].set_ylabel('AOD') + axes[0].set_title(f'{sensor_name} - AOD Time Series') + axes[0].grid(True, alpha=0.3) + axes[0].set_xticks(valid_months) + + # Add legend conditionally and with specified position + if show_legend: + axes[0].legend(loc=legend_position, frameon=True, fancybox=True, shadow=True) + + # Add component labels directly on the plot + x_center = np.mean(valid_months) # Center x-position + for comp_name, y_middle in component_middles.items(): + # Only label if the component has significant contribution + if y_middle > 0.01: # Only label components with meaningful contribution + axes[0].text(x_center, y_middle, comp_name, + ha='center', va='center', fontsize=10, + fontweight='bold', color='white', + bbox=dict(boxstyle='round,pad=0.3', + facecolor='black', alpha=0.7, edgecolor='none')) + + # Scatter plot with SHARED sizing and limits + bc_fraction = means.get('brexttau', xr.DataArray(np.zeros_like(months))).values / np.maximum(means['GEOStau'].values, 1e-10) + bc_fraction = np.nan_to_num(bc_fraction, nan=0.0) + + # Use shared brown carbon maximum for consistent sizing + min_size = 30 + max_size = 200 + bc_frac_valid = bc_fraction[valid_mask] + + if shared_bc_max is not None and shared_bc_max > 0: + sizes = min_size + (bc_frac_valid / shared_bc_max) * (max_size - min_size) + elif np.max(bc_frac_valid) > 0: + sizes = min_size + (bc_frac_valid / np.max(bc_frac_valid)) * (max_size - min_size) + else: + sizes = np.full(len(bc_frac_valid), min_size) + + scatter = axes[1].scatter(valid_mod, valid_geos, c=valid_months, s=sizes, + alpha=0.7, cmap='viridis', edgecolors='black', linewidth=0.5) + + # Use shared axis limits if provided with EXPANDED RANGE (scatter plot only) + if shared_limits is not None: + # EXPAND the axis limits by 10% for scatter plot only + range_val = shared_limits[1] - shared_limits[0] + expanded_limits = [shared_limits[0] - range_val * 0.1, + shared_limits[1] + range_val * 0.1] + axes[1].set_xlim(expanded_limits) + axes[1].set_ylim(expanded_limits) + + # 1:1 line using expanded limits + axes[1].plot(expanded_limits, expanded_limits, 'k--', alpha=0.5, label='1:1 line') + else: + # Calculate expanded limits for this panel + max_val = max(np.max(valid_mod), np.max(valid_geos)) + min_val = min(np.min(valid_mod), np.min(valid_geos)) + range_val = max_val - min_val + expanded_limits = [min_val - range_val * 0.1, max_val + range_val * 0.1] + axes[1].set_xlim(expanded_limits) + axes[1].set_ylim(expanded_limits) + axes[1].plot(expanded_limits, expanded_limits, 'k--', alpha=0.5, label='1:1 line') + + axes[1].set_xlabel(f'MODIS AOD ({sensor_name})') + axes[1].set_ylabel('GEOS AOD') + axes[1].set_title(f'{sensor_name} - MODIS vs GEOS AOD') + axes[1].grid(True, alpha=0.3) + axes[1].legend() + + # Size legend for brown carbon fraction (using shared scale) + bc_max_for_legend = shared_bc_max if shared_bc_max is not None else np.max(bc_frac_valid) + if bc_max_for_legend > 0: + size_legend_values = [0, bc_max_for_legend * 0.5, bc_max_for_legend] + size_legend_sizes = [min_size, (min_size + max_size) / 2, max_size] + size_legend_labels = [f'{val:.3f}' for val in size_legend_values] + + legend_elements = [] + for size, label in zip(size_legend_sizes, size_legend_labels): + legend_elements.append(plt.scatter([], [], s=size, c='gray', alpha=0.7, + edgecolors='black', linewidth=0.5)) + + size_legend = axes[1].legend(legend_elements, size_legend_labels, + title='Brown Carbon\nFraction', + loc='upper left', bbox_to_anchor=(0.02, 0.98), + frameon=True, fancybox=True, shadow=True) + axes[1].add_artist(size_legend) + + # Calculate and display chi scaling factor on the scatter plot + valid_mod_adj = valid_mod + 0.01 + valid_geos_adj = valid_geos + 0.01 + chi_values = np.exp(np.log(valid_mod_adj) - np.log(valid_geos_adj)) + chi_mean = np.mean(chi_values) + + # Statistics + try: + from scipy.stats import pearsonr + if len(valid_mod) > 1: + corr, _ = pearsonr(valid_mod, valid_geos) + bias = np.mean(valid_geos - valid_mod) + rmse = np.sqrt(np.mean((valid_geos - valid_mod)**2)) + + stats_text = f'R = {corr:.3f}\nBias = {bias:.3f}\nRMSE = {rmse:.3f}\nฯ‡ = {chi_mean:.3f}\nN = {len(valid_mod)}' + axes[1].text(0.98, 0.02, stats_text, transform=axes[1].transAxes, + verticalalignment='bottom', horizontalalignment='right', + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8)) + except ImportError: + pass + + # Calculate y-axis limits for time series (including stacked components) + # Find maximum values including stacked components + total_vals = np.zeros_like(valid_geos) + for comp_vals in components.values(): + valid_comp = comp_vals[valid_mask] + valid_comp = np.nan_to_num(valid_comp, nan=0.0) + total_vals += valid_comp + + # Include MODIS and GEOS values in y-range calculation + all_y_values = np.concatenate([valid_mod, valid_geos, total_vals]) + current_y_limits = [0, np.max(all_y_values) * 1.05] # Start from 0, add 5% padding at top + + # Return original data limits (not expanded) for proper shared scaling calculation + current_limits = [min(np.min(valid_mod), np.min(valid_geos)), + max(np.max(valid_mod), np.max(valid_geos))] + current_bc_max = np.max(bc_frac_valid) if len(bc_frac_valid) > 0 else 0 + + return True, current_limits, current_bc_max, current_y_limits + else: + axes[0].text(0.5, 0.5, f'No valid data for {sensor_name}', + transform=axes[0].transAxes, ha='center', va='center', fontsize=12) + axes[1].text(0.5, 0.5, f'No valid data for {sensor_name}', + transform=axes[1].transAxes, ha='center', va='center', fontsize=12) + return False, None, None, None + +def create_and_save_plot(mod_data, myd_data, region_id, sensor='both', output_dir='regional_analysis', experiment_name='c180R_qfed3igbp_allviirs'): + """Create and automatically save the analysis plot.""" + region = REGIONS[region_id] + region_name = region['name'] + + # Create output directory + output_dir = Path(output_dir) + output_dir.mkdir(exist_ok=True) + + print(f"Creating plot for {region_name}...") + + has_mod = mod_data is not None + has_myd = myd_data is not None + + if sensor.lower() == 'both' and has_mod and has_myd: + # 4-panel figure with IMPROVED layout and panel labels + fig = plt.figure(figsize=(20, 14)) # Increased width + + # Better layout: 2 rows, 2 real columns with uniform spacing + # Time series and scatter plot are treated as main columns + # Colorbar will be added later to use proper positioning + gs = fig.add_gridspec(2, 2, width_ratios=[1.2, 1], height_ratios=[1, 1], + left=0.06, right=0.9, top=0.90, bottom=0.08, + hspace=0.3, wspace=0.25) # Proper spacing between main columns + + # Create main axes + ax_terra_ts = fig.add_subplot(gs[0, 0]) + ax_terra_scatter = fig.add_subplot(gs[0, 1]) + ax_aqua_ts = fig.add_subplot(gs[1, 0]) + ax_aqua_scatter = fig.add_subplot(gs[1, 1]) + + axes_terra = [ax_terra_ts, ax_terra_scatter] + axes_aqua = [ax_aqua_ts, ax_aqua_scatter] + + # First pass: get data ranges for shared scaling (without labels) + terra_success, terra_limits, terra_bc_max, terra_y_limits = plot_single_sensor_panels( + mod_data, region_id, axes_terra, 'Terra') + aqua_success, aqua_limits, aqua_bc_max, aqua_y_limits = plot_single_sensor_panels( + myd_data, region_id, axes_aqua, 'Aqua') + + # Calculate shared limits and brown carbon maximum + shared_limits = None + shared_bc_max = None + shared_y_limits = None + + if terra_success and aqua_success: + if terra_limits and aqua_limits: + shared_limits = [min(terra_limits[0], aqua_limits[0]), + max(terra_limits[1], aqua_limits[1])] + if terra_bc_max and aqua_bc_max: + shared_bc_max = max(terra_bc_max, aqua_bc_max) + if terra_y_limits and aqua_y_limits: + shared_y_limits = [min(terra_y_limits[0], aqua_y_limits[0]), + max(terra_y_limits[1], aqua_y_limits[1])] + elif terra_success and terra_limits: + shared_limits = terra_limits + shared_bc_max = terra_bc_max + shared_y_limits = terra_y_limits + elif aqua_success and aqua_limits: + shared_limits = aqua_limits + shared_bc_max = aqua_bc_max + shared_y_limits = aqua_y_limits + + # Second pass: plot with shared scaling + if terra_success: + ax_terra_ts.clear() + ax_terra_scatter.clear() + + plot_single_sensor_panels(mod_data, region_id, axes_terra, 'Terra', + shared_limits, shared_bc_max, shared_y_limits, + show_legend=True, legend_position='upper left') + + if aqua_success: + ax_aqua_ts.clear() + ax_aqua_scatter.clear() + + plot_single_sensor_panels(myd_data, region_id, axes_aqua, 'Aqua', + shared_limits, shared_bc_max, shared_y_limits, + show_legend=False) + + # ADD CLEAN PANEL LABELS OUTSIDE the panels (after final plotting) + ax_terra_ts.text(-0.1, 1.02, 'a)', transform=ax_terra_ts.transAxes, + fontsize=12, fontweight='bold', va='bottom', ha='right') + ax_terra_scatter.text(-0.1, 1.02, 'b)', transform=ax_terra_scatter.transAxes, + fontsize=12, fontweight='bold', va='bottom', ha='right') + ax_aqua_ts.text(-0.1, 1.02, 'c)', transform=ax_aqua_ts.transAxes, + fontsize=12, fontweight='bold', va='bottom', ha='right') + ax_aqua_scatter.text(-0.1, 1.02, 'd)', transform=ax_aqua_scatter.transAxes, + fontsize=12, fontweight='bold', va='bottom', ha='right') + + # Add colorbar with proper positioning (minimal gap) + if aqua_success and len(ax_aqua_scatter.collections) > 0: + scatter = ax_aqua_scatter.collections[0] + cbar_ax = fig.add_axes([0.92, 0.15, 0.02, 0.7]) # [left, bottom, width, height] + cbar = fig.colorbar(scatter, cax=cbar_ax) + cbar.set_label('Month', rotation=270, labelpad=15) + elif terra_success and len(ax_terra_scatter.collections) > 0: + scatter = ax_terra_scatter.collections[0] + cbar_ax = fig.add_axes([0.92, 0.15, 0.02, 0.7]) # [left, bottom, width, height] + cbar = fig.colorbar(scatter, cax=cbar_ax) + cbar.set_label('Month', rotation=270, labelpad=15) + + fig.suptitle(f'{region_name} - Terra vs Aqua Comparison ({experiment_name})', fontsize=18) + sensor_str = 'terra_aqua' + + else: + # 2-panel figure with panel labels + fig, axes = plt.subplots(2, 1, figsize=(14, 12)) + + if sensor.lower() in ['terra', 'mod'] or (sensor.lower() == 'both' and has_mod and not has_myd): + if not has_mod: + print("No Terra data available") + return None, None + success, _, _, _ = plot_single_sensor_panels(mod_data, region_id, axes, 'Terra') + sensor_name = 'Terra' + sensor_str = 'terra' + elif sensor.lower() in ['aqua', 'myd'] or (sensor.lower() == 'both' and has_myd and not has_mod): + if not has_myd: + print("No Aqua data available") + return None, None + success, _, _, _ = plot_single_sensor_panels(myd_data, region_id, axes, 'Aqua') + sensor_name = 'Aqua' + sensor_str = 'aqua' + else: + print("No data available") + return None, None + + # ADD CLEAN PANEL LABELS for single sensor (after plotting, outside panels) + axes[0].text(-0.1, 1.02, 'a)', transform=axes[0].transAxes, + fontsize=12, fontweight='bold', va='bottom', ha='right') + axes[1].text(-0.1, 1.02, 'b)', transform=axes[1].transAxes, + fontsize=12, fontweight='bold', va='bottom', ha='right') + + if success: + # For single sensor plots, we don't need to remove the legend positioning + # since it's handled within the plotting function + if len(axes[1].collections) > 0: + cbar = plt.colorbar(axes[1].collections[0], ax=axes[1], shrink=0.8, aspect=20) + cbar.set_label('Month', rotation=270, labelpad=15) + + fig.suptitle(f'{region_name} - {sensor_name} Analysis ({experiment_name})', fontsize=18) + plt.subplots_adjust(left=0.08, right=0.85, top=0.92, bottom=0.08, hspace=0.3) + + # Save with experiment name in filename (NO region number) + filename = f"{experiment_name}_{region_name}_{sensor_str}.png" + save_path = output_dir / filename + plt.savefig(save_path, dpi=300, bbox_inches='tight', facecolor='white') + print(f"Plot saved to: {save_path}") + + return fig, save_path + +def main(): + """Enhanced main function with mapping capability.""" + parser = argparse.ArgumentParser(description='AOD Regional Analysis Tool') + + parser.add_argument('--region', '-r', type=int, + help='Single region ID to analyze. Use --list-regions to see options.') + parser.add_argument('--list-regions', action='store_true', + help='List all available regions and exit') + parser.add_argument('--sensor', '-s', choices=['terra', 'aqua', 'both'], default='both', + help='Which MODIS sensor(s) to use (default: both)') + parser.add_argument('--data-path', '-d', type=str, + default='sampledGEOS/c180R_qfed3igbp_allviirs', + help='Path to data directory') + parser.add_argument('--output', '-o', type=str, default='regional_analysis', + help='Output directory for plots') + parser.add_argument('--experiment', '-e', type=str, default='c180R_qfed3igbp_allviirs', + help='Experiment name for output filenames') + parser.add_argument('--no-display', action='store_true', + help='Do not display plots interactively') + + # Add mapping arguments + parser.add_argument('--create-maps', action='store_true', + help='Create regional statistics maps') + parser.add_argument('--maps-only', action='store_true', + help='Only create maps, skip individual region plots') + parser.add_argument('--chi-map', action='store_true', + help='Create dedicated chi scaling factor map with values') + + args = parser.parse_args() + + if args.list_regions: + print("Available regions:") + print("ID | Name") + print("----|" + "-"*30) + for region_id, region_info in REGIONS.items(): + print(f"{region_id:2d} | {region_info['name']}") + return + + if args.no_display: + import matplotlib + matplotlib.use('Agg') + + print(f"AOD Regional Analysis Tool") + print(f"=" * 50) + print(f"Experiment: {args.experiment}") + print(f"Data path: {args.data_path}") + print(f"Sensor: {args.sensor}") + print(f"Output directory: {args.output}") + + try: + # Load data + print(f"\nLoading data...") + mod_data, myd_data = load_monthly_data(base_path=args.data_path, sensor=args.sensor) + + if mod_data is None and myd_data is None: + print("ERROR: No data loaded!") + return 1 + + # Create maps if requested + if args.create_maps or args.maps_only or args.chi_map: + print("\nCreating regional statistics maps...") + figures, stats_df = create_all_maps(mod_data, myd_data, + sensor=args.sensor, + output_dir=args.output, + experiment_name=args.experiment) + + # If only chi map is requested, create it specifically + if args.chi_map and not args.create_maps: + if 'chi_mean' in stats_df.columns: + print("\nCreating dedicated chi scaling factor map...") + chi_fig, chi_save_path = create_chi_value_map( + stats_df, sensor=args.sensor, + output_dir=args.output, + experiment_name=args.experiment) + + if not args.no_display and chi_fig: + plt.figure(chi_fig.number) + plt.show() + elif not args.no_display and figures: + # Display one of the maps + correlation_fig = figures.get('correlation') + if correlation_fig: + plt.figure(correlation_fig[0].number) + plt.show() + + # Skip individual plots if maps-only is specified + if args.maps_only: + print(f"\nMaps created! Check {args.output}/ for results.") + return 0 + + if args.region is not None: + # Single region + if args.region not in REGIONS: + print(f"ERROR: Invalid region ID {args.region}") + return 1 + + fig, save_path = create_and_save_plot(mod_data, myd_data, args.region, + args.sensor, args.output, args.experiment) + + if fig and not args.no_display: + plt.show() + + else: + # All regions (only if not maps-only) + if not args.create_maps and not args.chi_map: # Avoid double processing + print(f"\nProcessing all regions...") + for region_id in REGIONS.keys(): + try: + fig, save_path = create_and_save_plot(mod_data, myd_data, region_id, + args.sensor, args.output, args.experiment) + if fig: + plt.close(fig) + except Exception as e: + print(f"Error processing region {region_id}: {e}") + + print(f"\nCompleted! Check {args.output}/ for results.") + + return 0 + + except Exception as e: + print(f"ERROR: {e}") + import traceback + traceback.print_exc() + return 1 + +if __name__ == "__main__": + sys.exit(main()) From d27f6188b8d746bb24e7fe100e83caec9d8762f9 Mon Sep 17 00:00:00 2001 From: acollow Date: Tue, 16 Dec 2025 11:51:48 -0500 Subject: [PATCH 03/10] check in of code to blend dark target and deep blue --- .../evaluation/MODIS_NNR/blendmodisstreams.py | 320 ++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 src/pyobs/evaluation/MODIS_NNR/blendmodisstreams.py diff --git a/src/pyobs/evaluation/MODIS_NNR/blendmodisstreams.py b/src/pyobs/evaluation/MODIS_NNR/blendmodisstreams.py new file mode 100644 index 0000000..8c22955 --- /dev/null +++ b/src/pyobs/evaluation/MODIS_NNR/blendmodisstreams.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +""" +Processes MODIS satellite AOD data by blending deep blue, land, and ocean retrievals based on observation count. +Example usage: python blendmodisstreams.py -y 2024 -m 1 -s MOD04 +""" + +import argparse +import calendar +import os +import sys +from pathlib import Path +import numpy as np +import netCDF4 as nc +from datetime import datetime +from concurrent.futures import ProcessPoolExecutor, as_completed +import multiprocessing as mp +from functools import partial + +def get_coordinates(satellite='MOD04'): + """ + Get longitude and latitude coordinates from reference file. + + Args: + satellite (str): Satellite identifier (MOD04 or MYD04) + + Returns: + tuple: (longitude, latitude) arrays + """ + ref_file = f'/css/gmao/dp/gds/AeroObs/nnr_003_{satellite}_061/Level3/Y2020/M02/nnr_003.{satellite}_L3a.ocean.20200224_2100z.nc4' + + try: + with nc.Dataset(ref_file, 'r') as ncfile: + lon = ncfile.variables['lon'][:] + lat = ncfile.variables['lat'][:] + return lon, lat + except (FileNotFoundError, KeyError) as e: + print(f"Error reading coordinates from {ref_file}: {e}") + sys.exit(1) + +def process_single_timestep(args_tuple): + """ + Process single timestep: + 1. Create weighted average of land+deep where they overlap + 2. Add in ocean data + """ + year, month, day, hour, satellite, output_dir, lon, lat = args_tuple + + # Format strings + year_str = f"{year:04d}" + month_str = f"{month:02d}" + day_str = f"{day:02d}" + hour_str = f"{hour:02d}" + + # Create output directory structure + output_path = Path(output_dir) / f"nnr_003_blend/{satellite}/Y{year_str}/M{month_str}" + output_path.mkdir(parents=True, exist_ok=True) + + # Construct file paths + base_path = f"/css/gmao/dp/gds/AeroObs/nnr_003_{satellite}_061/Level3/Y{year_str}/M{month_str}" + + deep_file = f"{base_path}/nnr_003.{satellite}_L3a.deep.{year_str}{month_str}{day_str}_{hour_str}00z.nc4" + land_file = f"{base_path}/nnr_003.{satellite}_L3a.land.{year_str}{month_str}{day_str}_{hour_str}00z.nc4" + ocean_file = f"{base_path}/nnr_003.{satellite}_L3a.ocean.{year_str}{month_str}{day_str}_{hour_str}00z.nc4" + + try: + # Read data + with nc.Dataset(deep_file, 'r') as ncfile: + deep = ncfile.variables['tau_'][:] + deep_nobs = ncfile.variables['count_tau_'][:] + + with nc.Dataset(land_file, 'r') as ncfile: + land = ncfile.variables['tau_'][:] + land_nobs = ncfile.variables['count_tau_'][:] + + with nc.Dataset(ocean_file, 'r') as ncfile: + ocean = ncfile.variables['tau_'][:] + + # Step 1: Create weighted average of land and deep (following MATLAB logic) + deep_processed = deep.copy() + land_processed = land.copy() + + # Set to 0 where no observations (MATLAB: deep(deep_nobs==0)=0) + deep_processed[deep_nobs == 0] = 0 + land_processed[land_nobs == 0] = 0 + + # Calculate weighted blend of land and deep + total_land_deep_obs = deep_nobs + land_nobs + land_deep_blend = np.full_like(deep, np.nan) + + # Only calculate where we have observations + mask = total_land_deep_obs > 0 + if np.sum(mask) > 0: + land_deep_blend[mask] = ((deep_processed[mask] * deep_nobs[mask]) + + (land_processed[mask] * land_nobs[mask])) / total_land_deep_obs[mask] + + # Set calculated zeros to NaN (MATLAB: blend(blend==0)=nan) + land_deep_blend[land_deep_blend == 0] = np.nan + + # Step 2: Start with land+deep blend as the foundation + final_blend = land_deep_blend.copy() + + # Step 3: Add ocean data ONLY where land+deep is NaN (no land/deep data available) + land_deep_missing = np.isnan(land_deep_blend) + ocean_available = ~np.isnan(ocean) + use_ocean = land_deep_missing & ocean_available + + final_blend[use_ocean] = ocean[use_ocean] + + # Debug output + print(f"=== BLEND {year_str}{month_str}{day_str}_{hour_str} ===") + land_deep_valid = ~np.isnan(land_deep_blend) + print(f"Land+Deep blend: {np.sum(land_deep_valid)} pixels") + print(f"Ocean fills gaps: {np.sum(use_ocean)} pixels") + print(f"Total combined: {np.sum(~np.isnan(final_blend))}") + + if np.sum(~np.isnan(final_blend)) > 0: + print(f"Final range: {np.nanmin(final_blend):.6f} to {np.nanmax(final_blend):.6f}") + + # Create output filename + output_file = output_path / f"nnr_003.{satellite}_L3a.blend.{year_str}{month_str}{day_str}_{hour_str}00z.nc4" + + if output_file.exists(): + output_file.unlink() + + # Write NetCDF file + with nc.Dataset(output_file, 'w') as ncfile: + ncfile.createDimension('lon', len(lon)) + ncfile.createDimension('lat', len(lat)) + + tau_var = ncfile.createVariable('tau', 'f4', ('lat', 'lon'), fill_value=np.nan) + lon_var = ncfile.createVariable('lon', 'f4', ('lon',)) + lat_var = ncfile.createVariable('lat', 'f4', ('lat',)) + + tau_var[:] = final_blend + lon_var[:] = lon + lat_var[:] = lat + + tau_var.long_name = "Aerosol Optical Depth at 550nm (blended)" + tau_var.units = "1" + lon_var.long_name = "Longitude" + lon_var.units = "degrees_east" + lat_var.long_name = "Latitude" + lat_var.units = "degrees_north" + + ncfile.title = f"Blended AOD from {satellite}" + ncfile.source = "Weighted land+deep blend with ocean gap-filling" + ncfile.created = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + return True, f"Processed: {output_file.name}" + + except FileNotFoundError as e: + return False, f"Missing input file: {str(e)}" + except Exception as e: + return False, f"Error processing: {str(e)}" + +def process_aod_blend_parallel(year, month, satellite, output_dir, max_workers=None): + """ + Process AOD blending for a given year, month, and satellite using parallel processing. + + Args: + year (int): Year to process + month (int): Month to process (1-12) + satellite (str): Satellite identifier (MOD04 or MYD04) + output_dir (str): Output directory path + max_workers (int): Maximum number of parallel workers (None for auto) + """ + # Get month length + month_length = calendar.monthrange(year, month)[1] + + # Get coordinates (only once) + lon, lat = get_coordinates(satellite) + + # Create list of all timesteps to process + timesteps = [] + for day in range(1, month_length + 1): + for hour in range(0, 24, 3): # 0, 3, 6, 9, 12, 15, 18, 21 + timesteps.append((year, month, day, hour, satellite, output_dir, lon, lat)) + + # Determine number of workers + if max_workers is None: + max_workers = min(mp.cpu_count(), len(timesteps)) + + print(f"Processing {len(timesteps)} timesteps using {max_workers} parallel workers...") + + # Process in parallel + successful = 0 + failed = 0 + + with ProcessPoolExecutor(max_workers=max_workers) as executor: + # Submit all jobs + future_to_timestep = { + executor.submit(process_single_timestep, timestep): timestep + for timestep in timesteps + } + + # Process completed jobs + for future in as_completed(future_to_timestep): + timestep = future_to_timestep[future] + try: + success, message = future.result() + if success: + successful += 1 + print(f"โœ“ {message}") + else: + failed += 1 + print(f"โœ— {message}") + except Exception as e: + failed += 1 + year, month, day, hour = timestep[:4] + print(f"โœ— Unexpected error for {year:04d}{month:02d}{day:02d}_{hour:02d}: {e}") + + print(f"\nProcessing summary:") + print(f" Successful: {successful}") + print(f" Failed: {failed}") + print(f" Total: {len(timesteps)}") + +def main(): + """Main function with command-line argument parsing.""" + parser = argparse.ArgumentParser( + description="Blend satellite AOD retrievals from deep blue, land, and ocean products (parallel version)", + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + '--year', '-y', + type=int, + required=True, + help='Year to process (e.g., 2024)' + ) + + parser.add_argument( + '--month', '-m', + type=int, + required=True, + choices=range(1, 13), + help='Month to process (1-12)' + ) + + parser.add_argument( + '--satellite', '-s', + type=str, + required=True, + choices=['MOD04', 'MYD04'], + help='Satellite identifier (MOD04 for Terra, MYD04 for Aqua)' + ) + + parser.add_argument( + '--output', '-o', + type=str, + default='reprocessedblend', + help='Base output directory path (default: ./reprocessedblend/)' + ) + + parser.add_argument( + '--workers', '-w', + type=int, + default=None, + help='Number of parallel workers (default: auto-detect based on CPU count)' + ) + + args = parser.parse_args() + + # Create base output directory if it doesn't exist + output_path = Path(args.output) + if not output_path.exists(): + try: + output_path.mkdir(parents=True) + except Exception as e: + print(f"Error creating output directory {output_path}: {e}") + sys.exit(1) + + # Satellite info + sat_info = { + 'MOD04': 'Terra', + 'MYD04': 'Aqua' + } + + # Determine worker count + if args.workers is None: + workers = mp.cpu_count() + worker_text = f"{workers} (auto-detected)" + else: + workers = args.workers + worker_text = f"{workers} (specified)" + + # Show full output path + year_str = f"{args.year:04d}" + month_str = f"{args.month:02d}" + full_output_path = Path(args.output) / f"{args.satellite}/Y{year_str}/M{month_str}" + + print(f"Processing AOD blending for:") + print(f" Year: {args.year}") + print(f" Month: {args.month}") + print(f" Satellite: {args.satellite} ({sat_info[args.satellite]})") + print(f" Base output: {args.output}") + print(f" Full output path: {full_output_path}") + print(f" Workers: {worker_text}") + print() + + # Process the data + try: + start_time = datetime.now() + + # Pass the base directory; the function will create the full structure + base_output = str(output_path.parent) if args.output == 'reprocessedblend' else args.output + process_aod_blend_parallel(args.year, args.month, args.satellite, base_output, args.workers) + + end_time = datetime.now() + duration = end_time - start_time + + print(f"\nCompleted processing for {args.year}-{args.month:02d} {args.satellite} ({sat_info[args.satellite]})") + print(f"Total processing time: {duration}") + print(f"Output files saved to: {full_output_path}") + + except Exception as e: + print(f"Error during processing: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() From da60798cff76c97e88d88b8ee7673138f1752ede Mon Sep 17 00:00:00 2001 From: acollow Date: Thu, 14 May 2026 13:03:46 -0400 Subject: [PATCH 04/10] fix nan issue in dial.py --- CHANGELOG.md | 1 + src/pyobs/dial.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b42bc80..a6ea8d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Added list parsing for variables in trajectory sampler +- dial.py to accomodate python change for nan ### Added - MPL reader and plot curtain - calculation of total backscatter coefficient in aop.py diff --git a/src/pyobs/dial.py b/src/pyobs/dial.py index 45e9201..699bc8f 100644 --- a/src/pyobs/dial.py +++ b/src/pyobs/dial.py @@ -6,7 +6,7 @@ import os import h5py -from numpy import ones, zeros, interp, NaN, isnan, array, ma +from numpy import ones, zeros, interp, nan as NaN, isnan, array, ma from datetime import datetime, timedelta from matplotlib.pyplot import imshow, xlabel, ylabel, title, colorbar, \ From e2a9f378537b56ea76052c6b993a3fb05f3355ee Mon Sep 17 00:00:00 2001 From: acollow Date: Fri, 15 May 2026 09:36:03 -0400 Subject: [PATCH 05/10] update hrsl.py to handle HALO --- src/pyobs/hsrl.py | 49 +++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/src/pyobs/hsrl.py b/src/pyobs/hsrl.py index 2086ce1..d412cbc 100644 --- a/src/pyobs/hsrl.py +++ b/src/pyobs/hsrl.py @@ -4,7 +4,7 @@ """ import h5py -from numpy import ones, zeros, interp, NaN, isnan, array +import numpy as np from datetime import datetime, timedelta from .config import strTemplate @@ -79,14 +79,21 @@ 'Relative_Humidity', ) } +SDS_HALO = { + '/': ('lat', 'lon', 'time', 'z', + '1064_bsc_cloud_screened', '1064_ext', '1064_aer_dep', + '532_bsc_cloud_screened', '532_ext', '532_aer_dep'), + 'Nav_Data': ('gps_date', 'date', 'gps_alt', 'gps_lat', 'gps_lon', 'gps_time'), + 'State': ('Pressure', 'Temperature', 'Relative_Humidity') +} -NAV = ( 'Altitude','date', 'gps_date','gps_lat','gps_lon','gps_time') +NAV = ( 'Altitude','date', 'gps_date','gps_lat','gps_lon','gps_time','lat','lon','time','z') # Short names # ----------- Short_Name = dict( gps_alt = 'lev', - gps_date = 'datex', + gps_date = 'date', gps_lat = 'lat', gps_lon = 'lon', gps_time = 'time', @@ -169,13 +176,23 @@ def __init__ (self,hsrl_filename,Nav_only=False,verbose=True,freq=3.0, # Handle incosistency of date across HSRL datasets # ----------------------------------------------- - if self.nt != self.date.shape[0]: + if getattr(self, 'date', None) is None: + import re + match = re.search(r'_(\d{8})_', hsrl_filename) + if match: + dt_str = match.group(1) + # Format as MM/DD/YYYY to match HSRL convention + dates = '%s/%s/%s' % (dt_str[4:6], dt_str[6:8], dt_str[0:4]) + self.date = np.array([dates for i in range(self.nt)]) + else: + raise ValueError(f"Missing date variable and could not parse YYYYMMDD from filename: {hsrl_filename}") + elif self.nt != self.date.shape[0]: date_ = self.date[0,0] - yy = int(date_)/10000 - mm = (int(date_) - yy * 10000)/100 + yy = int(date_)//10000 + mm = (int(date_) - yy * 10000)//100 dd = int(date_) - yy*10000 - mm*100 dates = '%02d/%02d/%4d'%(mm,dd,yy) - self.date = array([dates for i in range(self.nt)]) + self.date = np.array([dates for i in range(self.nt)]) # Create datetime # --------------- @@ -184,7 +201,7 @@ def __init__ (self,hsrl_filename,Nav_only=False,verbose=True,freq=3.0, dt = timedelta(seconds = int(self.time[i]* 60. * 60.+0.5)) mm, dd, yy = self.date[i].split('/') self.Time += [datetime(int(yy), int(mm), int(dd)) + dt,] - self.Time = array(self.Time) + self.Time = np.array(self.Time) self.tyme = self.Time.reshape((self.nt,1)) # Find bracketing synoptic times @@ -204,7 +221,7 @@ def __init__ (self,hsrl_filename,Nav_only=False,verbose=True,freq=3.0, # ------------------------------------------ self.IA = [] # index and weight for time interpolation for t in self.syn[:-1]: - a = ones(self.nt) + a = np.ones(self.nt) I = (a==1.) for n in range(self.nt): I[n] = (self.Time[n]>=t)&(self.Time[n]vmax] = NaN + v_[v_>vmax] = np.nan gca().set_axis_bgcolor('black') @@ -432,7 +449,7 @@ def _attachVarXYT (self,g5_filename,Vars): # Interpolate in each synoptic interval # ------------------------------------- s = 0 - v = NaN * zeros(V[0].shape) + v = np.nan * np.zeros(V[0].shape) for I, a in self.IA: a_ = a[I] if len(v.shape) == 1: From 2c2dc7b5046a755505891de38442429c3d5ba962 Mon Sep 17 00:00:00 2001 From: Allie Collow <31133739+acollow@users.noreply.github.com> Date: Fri, 15 May 2026 09:39:37 -0400 Subject: [PATCH 06/10] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6ea8d4..f206734 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - MPL reader and plot curtain - calculation of total backscatter coefficient in aop.py - xrctl supports providing a list of control files +- support for HALO in hsrl.py ### Changed - add auto chunking to TRAJECTORY and STATION. This enables dask - preload some key variables in aop.py so you don't hit dask repeatedly in for loop From 6566bf26cc86c3abae3fbcc50aab05b2f0787f90 Mon Sep 17 00:00:00 2001 From: acollow Date: Fri, 15 May 2026 09:45:44 -0400 Subject: [PATCH 07/10] remove codes that went to AeroApps instead --- src/pyobs/evaluation/AERONET/README | 5 - .../evaluation/AERONET/plotagainstm21c.py | 945 ------------- .../AERONET/plotsinglestationonly.py | 320 ----- .../evaluation/AERONET/processaeronet.py | 492 ------- .../evaluation/AERONET/station_analysis.py | 670 --------- .../AERONET/station_analysis_withm2.py | 987 ------------- .../evaluation/MODIS_NNR/blendmodisstreams.py | 320 ----- .../MODIS_NNR/monthlygeossample_speciated.py | 474 ------- .../MODIS_NNR/plotregionalcomparison.py | 1238 ----------------- 9 files changed, 5451 deletions(-) delete mode 100644 src/pyobs/evaluation/AERONET/README delete mode 100644 src/pyobs/evaluation/AERONET/plotagainstm21c.py delete mode 100644 src/pyobs/evaluation/AERONET/plotsinglestationonly.py delete mode 100644 src/pyobs/evaluation/AERONET/processaeronet.py delete mode 100644 src/pyobs/evaluation/AERONET/station_analysis.py delete mode 100644 src/pyobs/evaluation/AERONET/station_analysis_withm2.py delete mode 100644 src/pyobs/evaluation/MODIS_NNR/blendmodisstreams.py delete mode 100755 src/pyobs/evaluation/MODIS_NNR/monthlygeossample_speciated.py delete mode 100755 src/pyobs/evaluation/MODIS_NNR/plotregionalcomparison.py diff --git a/src/pyobs/evaluation/AERONET/README b/src/pyobs/evaluation/AERONET/README deleted file mode 100644 index 4fb35a8..0000000 --- a/src/pyobs/evaluation/AERONET/README +++ /dev/null @@ -1,5 +0,0 @@ -This set of codes is currently set up to evaluate MERRA-21C using Lunar AERONET observations but can adapted for standard AERONET observations or other GEOS simulations. - -The first step is to pre-process the model data using processaeronet.py. This code will generate csv files that include sampled model data according to the availability of observations. The code will loop through all AERONET files that match the specified pattern so that all stations are processed. There are command line arguments to change the filepath of the input model data and the output directory where you want to csv files stored. The AERONET observation files used were directly downloaded from the AERONET website (https://aeronet.gsfc.nasa.gov/) rather than ODS files generated for aerosol data assimilation. - -There are two codes that can be used to generate figures. plotagainstm21c.py will generate a global summary figure with a map showing the bias and correlation for AOD and Angtrom exponent for all available stations. plotsinglestationonly.py will produce two figures (one for AOD, one for Angstrom exponent) with a time series of the full period, a mean annual cycle, a 2d kernel density estimate with AERONET on the x axis and the model on the y axis. The flag "--include--merra2" will add a second experiment should you want to compare. diff --git a/src/pyobs/evaluation/AERONET/plotagainstm21c.py b/src/pyobs/evaluation/AERONET/plotagainstm21c.py deleted file mode 100644 index bdda9e2..0000000 --- a/src/pyobs/evaluation/AERONET/plotagainstm21c.py +++ /dev/null @@ -1,945 +0,0 @@ -import os -import glob -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -import cartopy.crs as ccrs -import cartopy.feature as cfeature -from matplotlib.colors import LinearSegmentedColormap, ListedColormap -from scipy.stats import pearsonr -import argparse -import warnings -from scipy.stats import gaussian_kde -import matplotlib.patches as patches -import matplotlib.ticker as ticker - -warnings.filterwarnings('ignore') - -def create_white_viridis_colormap(): - """Create a custom colormap that starts with white for low densities and transitions to viridis""" - # Get viridis colormap - viridis = plt.cm.get_cmap('viridis', 256) - - # Create new colormap that starts with white - # Take viridis colors but replace the lowest values with white - colors = viridis(np.linspace(0, 1, 256)) - - # Replace first 20% of colors with white to white-to-viridis transition - n_white = int(0.15 * 256) # 15% white transition - for i in range(n_white): - # Interpolate from white to first viridis color - alpha = i / n_white - colors[i] = (1-alpha) * np.array([1, 1, 1, 1]) + alpha * colors[n_white] - - return ListedColormap(colors, name='white_viridis') - -def generate_comparison_maps(data_dir="./aeronet_merra21c_comparison/", - output_dir="./figures/", - min_points=30, - years=None, - file_pattern=None, - debug=False): - """ - Generate global maps showing bias and correlation between AERONET and MERRA-21C data. - - Parameters: - ----------- - data_dir : str - Directory containing processed CSV files - output_dir : str - Directory to save output figures - min_points : int - Minimum number of data points required for a station to be included - years : list or None - List of years to include in analysis. If None, uses all available data. - file_pattern : str or None - Custom file pattern to match CSV files. If None, uses default pattern. - debug : bool - If True, print additional debugging information - """ - # Create custom colormap - white_viridis = create_white_viridis_colormap() - - # Create output directory - os.makedirs(output_dir, exist_ok=True) - - # Check if the data directory exists - if not os.path.exists(data_dir): - print(f"Error: Data directory '{data_dir}' does not exist.") - return - - # List all files in the directory - all_files = os.listdir(data_dir) - csv_files_in_dir = [f for f in all_files if f.endswith('.csv')] - - if debug: - print(f"Found {len(csv_files_in_dir)} total CSV files in directory.") - if csv_files_in_dir: - print(f"Sample filenames: {csv_files_in_dir[:5]}") - - # Determine file pattern based on years - if file_pattern is None: - if years is not None: - if len(years) == 1: - # Try different patterns for single year - patterns = [ - f"*_{years[0]}_{years[0]}.csv", # station_2018_2018.csv - f"*_{years[0]}.csv", # station_2018.csv - "*.csv" # Any CSV file - ] - else: - # Multiple years case - try different patterns - patterns = [ - f"*_{min(years)}_{max(years)}.csv", # station_2018_2020.csv - "*.csv" # Any CSV file - ] - else: - patterns = ["*.csv"] # Default pattern - match all CSV files - else: - patterns = [file_pattern] - - # Try each pattern until we find files - csv_files = [] - used_pattern = None - - for pattern in patterns: - csv_files = glob.glob(os.path.join(data_dir, pattern)) - if csv_files: - used_pattern = pattern - break - - if not csv_files: - print(f"No CSV files found in {data_dir} matching any of these patterns: {patterns}") - print(f"Available CSV files: {csv_files_in_dir if csv_files_in_dir else 'No CSV files in directory'}") - return - - print(f"Found {len(csv_files)} CSV files matching pattern '{used_pattern}'") - - if debug and csv_files: - print("Sample filenames:") - for file in csv_files[:5]: - print(f" {os.path.basename(file)}") - - # Initialize lists to store data for each station - stations = [] - lats = [] - lons = [] - mean_aod_biases = [] - mean_angstrom_biases = [] - aod_correlations = [] - angstrom_correlations = [] - mean_aeronet_aods = [] - mean_merra_aods = [] - data_counts = [] - valid_data_counts = [] # Count of data points after quality filtering - aod_sources = [] - angstrom_sources = [] - - # Process each station file - processed_count = 0 - skipped_files = [] - error_files = [] - - for csv_file in csv_files: - try: - # Read data - df = pd.read_csv(csv_file) - - # Debug: Show column names and data quality for first file - if debug and processed_count == 0: - print(f"\nColumns in CSV file: {list(df.columns)}") - print(f"Data shape: {df.shape}") - nan_counts = df.isnull().sum() - if nan_counts.sum() > 0: - print(f"NaN counts per column:\n{nan_counts[nan_counts > 0]}") - print(f"First few rows of data:\n{df.head(2)}") - - # Check for required metadata columns - if 'station' not in df.columns or 'lat' not in df.columns or 'lon' not in df.columns: - msg = "missing required metadata columns (station, lat, lon)" - skipped_files.append((os.path.basename(csv_file), msg)) - continue - - # Check for required data columns - required_cols = ['aeronet_aod_550', 'merra_aod_550', 'aeronet_angstrom', 'merra_angstrom'] - missing_cols = [col for col in required_cols if col not in df.columns] - - if missing_cols: - msg = f"missing columns: {', '.join(missing_cols)}" - skipped_files.append((os.path.basename(csv_file), msg)) - continue - - # Extract station metadata first - station = df['station'].iloc[0] if not df['station'].isna().iloc[0] else "Unknown" - lat = df['lat'].iloc[0] - lon = df['lon'].iloc[0] - - if np.isnan(lat) or np.isnan(lon): - msg = "invalid lat/lon coordinates" - skipped_files.append((os.path.basename(csv_file), msg)) - continue - - # Count original data points - original_count = len(df) - - # Apply comprehensive quality filters - quality_mask = ( - # Remove NaN values - (~df['aeronet_aod_550'].isna()) & - (~df['merra_aod_550'].isna()) & - (~df['aeronet_angstrom'].isna()) & - (~df['merra_angstrom'].isna()) & - # Remove negative AOD values and unreasonably high values - (df['aeronet_aod_550'] >= 0) & (df['aeronet_aod_550'] < 10) & - (df['merra_aod_550'] >= 0) & (df['merra_aod_550'] < 10) & - # Remove unreasonable Angstrom exponent values - (df['aeronet_angstrom'] >= -1) & (df['aeronet_angstrom'] <= 3) & - (df['merra_angstrom'] >= -1) & (df['merra_angstrom'] <= 3) & - # Remove infinite values - (np.isfinite(df['aeronet_aod_550'])) & - (np.isfinite(df['merra_aod_550'])) & - (np.isfinite(df['aeronet_angstrom'])) & - (np.isfinite(df['merra_angstrom'])) - ) - - df_quality = df[quality_mask].copy() - - if debug and processed_count == 0: - print(f"Quality filtering removed {original_count - len(df_quality)} out of {original_count} data points") - - # Skip if too few data points after quality filtering - if len(df_quality) < min_points: - msg = f"only {len(df_quality)} quality-filtered data points (minimum: {min_points})" - skipped_files.append((os.path.basename(csv_file), msg)) - continue - - # Filter by years if specified - if years is not None: - if 'datetime' not in df_quality.columns: - msg = "missing 'datetime' column" - skipped_files.append((os.path.basename(csv_file), msg)) - continue - - try: - df_quality['datetime'] = pd.to_datetime(df_quality['datetime']) - except: - msg = "unable to parse datetime column" - skipped_files.append((os.path.basename(csv_file), msg)) - continue - - df_year = df_quality[df_quality['datetime'].dt.year.isin(years)] - - if len(df_year) < min_points: - msg = f"only {len(df_year)} data points for years {years} after quality filtering" - skipped_files.append((os.path.basename(csv_file), msg)) - continue - - # Use the year-filtered dataframe - df_quality = df_year - - # Get source information if available - aod_source = df_quality['aod_source'].iloc[0] if 'aod_source' in df_quality.columns else 'Unknown' - angstrom_source = df_quality['angstrom_source'].iloc[0] if 'angstrom_source' in df_quality.columns else 'Unknown' - - # Calculate bias columns if they don't exist - if 'aod_bias' not in df_quality.columns: - df_quality['aod_bias'] = df_quality['merra_aod_550'] - df_quality['aeronet_aod_550'] - - if 'angstrom_bias' not in df_quality.columns: - df_quality['angstrom_bias'] = df_quality['merra_angstrom'] - df_quality['aeronet_angstrom'] - - # Calculate metrics using quality-filtered data - mean_aod_bias = df_quality['aod_bias'].mean() - mean_angstrom_bias = df_quality['angstrom_bias'].mean() - - # Calculate correlations with additional error handling - try: - if len(df_quality) < 3: # Need at least 3 points for meaningful correlation - raise ValueError("Insufficient data points for correlation") - - # Check for zero variance (constant values) - if (df_quality['aeronet_aod_550'].std() == 0 or - df_quality['merra_aod_550'].std() == 0): - aod_corr = np.nan - else: - aod_corr, _ = pearsonr(df_quality['aeronet_aod_550'], df_quality['merra_aod_550']) - - if (df_quality['aeronet_angstrom'].std() == 0 or - df_quality['merra_angstrom'].std() == 0): - angstrom_corr = np.nan - else: - angstrom_corr, _ = pearsonr(df_quality['aeronet_angstrom'], df_quality['merra_angstrom']) - - except Exception as e: - msg = f"correlation calculation failed: {str(e)}" - skipped_files.append((os.path.basename(csv_file), msg)) - continue - - # Check if correlations are valid (not NaN) - if np.isnan(aod_corr) and np.isnan(angstrom_corr): - msg = "both correlations are NaN" - skipped_files.append((os.path.basename(csv_file), msg)) - continue - - # Store data - stations.append(station) - lats.append(lat) - lons.append(lon) - mean_aod_biases.append(mean_aod_bias) - mean_angstrom_biases.append(mean_angstrom_bias) - aod_correlations.append(aod_corr if not np.isnan(aod_corr) else 0) # Replace NaN with 0 for plotting - angstrom_correlations.append(angstrom_corr if not np.isnan(angstrom_corr) else 0) - mean_aeronet_aods.append(df_quality['aeronet_aod_550'].mean()) - mean_merra_aods.append(df_quality['merra_aod_550'].mean()) - data_counts.append(original_count) - valid_data_counts.append(len(df_quality)) - aod_sources.append(aod_source) - angstrom_sources.append(angstrom_source) - - processed_count += 1 - - except Exception as e: - error_files.append((os.path.basename(csv_file), str(e))) - if debug: - print(f"Error processing {csv_file}: {e}") - - # Report on processing results - if skipped_files and debug: - print(f"\nSkipped {len(skipped_files)} files:") - for filename, reason in skipped_files[:10]: # Show only first 10 - print(f" {filename}: {reason}") - if len(skipped_files) > 10: - print(f" ... and {len(skipped_files) - 10} more") - - if error_files and debug: - print(f"\nErrors in {len(error_files)} files:") - for filename, error in error_files[:10]: # Show only first 10 - print(f" {filename}: {error}") - if len(error_files) > 10: - print(f" ... and {len(error_files) - 10} more") - - if processed_count == 0: - print("No stations were successfully processed. Check your data files and parameters.") - return - - print(f"Successfully processed {processed_count} stations") - - # Create dataframe with all station metrics - station_metrics = pd.DataFrame({ - 'station': stations, - 'latitude': lats, - 'longitude': lons, - 'mean_aod_bias': mean_aod_biases, - 'mean_angstrom_bias': mean_angstrom_biases, - 'aod_correlation': aod_correlations, - 'angstrom_correlation': angstrom_correlations, - 'mean_aeronet_aod': mean_aeronet_aods, - 'mean_merra_aod': mean_merra_aods, - 'total_data_points': data_counts, - 'valid_data_points': valid_data_counts, - 'aod_source': aod_sources, - 'angstrom_source': angstrom_sources - }) - - # Add year info to filename - year_str = f"_{min(years)}_{max(years)}" if years and len(years) > 1 else f"_{years[0]}" if years else "" - - # Save metrics to CSV - metrics_file = os.path.join(output_dir, f"station_metrics_summary{year_str}.csv") - station_metrics.to_csv(metrics_file, index=False) - print(f"Saved metrics summary to {metrics_file}") - - # Print some summary statistics - print(f"\nSummary Statistics:") - print(f"AOD Bias: mean = {np.mean(mean_aod_biases):.4f}, std = {np.std(mean_aod_biases):.4f}") - - # Handle potential NaN values in correlations for statistics - valid_aod_corrs = [c for c in aod_correlations if not np.isnan(c)] - valid_ang_corrs = [c for c in angstrom_correlations if not np.isnan(c)] - - if valid_aod_corrs: - print(f"AOD Correlation: mean = {np.mean(valid_aod_corrs):.3f}, std = {np.std(valid_aod_corrs):.3f}") - else: - print("AOD Correlation: no valid correlations") - - print(f"Angstrom Bias: mean = {np.mean(mean_angstrom_biases):.4f}, std = {np.std(mean_angstrom_biases):.4f}") - - if valid_ang_corrs: - print(f"Angstrom Correlation: mean = {np.mean(valid_ang_corrs):.3f}, std = {np.std(valid_ang_corrs):.3f}") - else: - print("Angstrom Correlation: no valid correlations") - - print(f"Data Points: mean = {np.mean(valid_data_counts):.1f}, std = {np.std(valid_data_counts):.1f}") - print(f"Data Points: min = {np.min(valid_data_counts)}, max = {np.max(valid_data_counts)}") - - # Create custom diverging colormap for bias (blue-white-red) - bias_cmap = LinearSegmentedColormap.from_list( - 'bias_cmap', ['blue', 'white', 'red'] - ) - - # Create custom sequential colormap for correlation (white-green) - corr_cmap = LinearSegmentedColormap.from_list( - 'corr_cmap', ['white', 'green'] - ) - - # Create custom colormap for data counts (white to purple) - count_cmap = LinearSegmentedColormap.from_list( - 'count_cmap', ['lightblue', 'blue', 'darkblue', 'purple'] - ) - - # Generate 4-panel comparison figure - fig = plt.figure(figsize=(24, 16)) # Increased height for better spacing - - # Set up the 2x2 subplot layout with cartopy projections - ax1 = plt.subplot(2, 2, 1, projection=ccrs.PlateCarree()) - ax2 = plt.subplot(2, 2, 2, projection=ccrs.PlateCarree()) - ax3 = plt.subplot(2, 2, 3, projection=ccrs.PlateCarree()) - ax4 = plt.subplot(2, 2, 4, projection=ccrs.PlateCarree()) - - axes = [ax1, ax2, ax3, ax4] - panel_labels = ['a', 'b', 'c', 'd'] - - # Add map features to all subplots - for i, ax in enumerate(axes): - ax.add_feature(cfeature.COASTLINE) - ax.add_feature(cfeature.BORDERS, linestyle=':') - ax.add_feature(cfeature.LAND, alpha=0.3) - ax.add_feature(cfeature.OCEAN, alpha=0.3) - ax.set_global() - - # Add gridlines but make them less prominent - gl = ax.gridlines(draw_labels=False, alpha=0.2) - - # Add panel labels to the TOP LEFT corner - ax.text(0.03, 1.05, f"({panel_labels[i]})", transform=ax.transAxes, - fontsize=16, fontweight='bold', ha='left', va='top', - bbox=dict(facecolor='white', alpha=0.7, pad=0.1, edgecolor='none')) - - # Debug information about bias distribution - if debug: - bias_data = station_metrics['mean_aod_bias'] - print(f"\nAOD Bias Statistics:") - print(f"Mean: {np.mean(bias_data):.4f}") - print(f"Median: {np.median(bias_data):.4f}") - print(f"Std: {np.std(bias_data):.4f}") - print(f"Min: {np.min(bias_data):.4f}") - print(f"Max: {np.max(bias_data):.4f}") - print(f"5th percentile: {np.percentile(bias_data, 5):.4f}") - print(f"95th percentile: {np.percentile(bias_data, 95):.4f}") - - # Panel 1: AOD Bias with percentile-based scaling - bias_data = station_metrics['mean_aod_bias'] - # Use percentiles to handle outliers - p5, p95 = np.percentile(bias_data, [2, 98]) - # Optional: make it symmetric around zero - max_abs_bias_clipped = max(abs(p5), abs(p95)) - - sc1 = ax1.scatter( - station_metrics['longitude'], - station_metrics['latitude'], - c=station_metrics['mean_aod_bias'], - cmap=bias_cmap, - vmin=-max_abs_bias_clipped, - vmax=max_abs_bias_clipped, - s=60, - edgecolor='black', - linewidth=0.5, - transform=ccrs.PlateCarree() - ) - ax1.set_title('Nighttime AOD Bias (MERRA-21C - AERONET)', fontsize=18, pad=10) - - # Panel 2: AOD Correlation - valid_aod_mask = ~np.isnan(station_metrics['aod_correlation']) - if valid_aod_mask.sum() > 0: - sc2 = ax2.scatter( - station_metrics.loc[valid_aod_mask, 'longitude'], - station_metrics.loc[valid_aod_mask, 'latitude'], - c=station_metrics.loc[valid_aod_mask, 'aod_correlation'], - cmap=corr_cmap, - vmin=0, - vmax=1, - s=60, - edgecolor='black', - linewidth=0.5, - transform=ccrs.PlateCarree() - ) - ax2.set_title('Nighttime AOD Temporal Correlation', fontsize=18, pad=10) - - # Panel 3: Angstrom Bias - max_abs_angstrom_bias = max(abs(np.array(mean_angstrom_biases))) - sc3 = ax3.scatter( - station_metrics['longitude'], - station_metrics['latitude'], - c=station_metrics['mean_angstrom_bias'], - cmap=bias_cmap, - vmin=-max_abs_angstrom_bias, - vmax=max_abs_angstrom_bias, - s=60, - edgecolor='black', - linewidth=0.5, - transform=ccrs.PlateCarree() - ) - ax3.set_title('Nighttime Angstrom Exponent Bias', fontsize=18, pad=10) - - # Panel 4: Angstrom Correlation - valid_ang_mask = ~np.isnan(station_metrics['angstrom_correlation']) - if valid_ang_mask.sum() > 0: - sc4 = ax4.scatter( - station_metrics.loc[valid_ang_mask, 'longitude'], - station_metrics.loc[valid_ang_mask, 'latitude'], - c=station_metrics.loc[valid_ang_mask, 'angstrom_correlation'], - cmap=corr_cmap, - vmin=0, - vmax=1, - s=60, - edgecolor='black', - linewidth=0.5, - transform=ccrs.PlateCarree() - ) - ax4.set_title('Nighttime Angstrom Exponent Temporal Correlation', fontsize=18, pad=10) - - # Adjust layout to reduce spacing between columns - plt.subplots_adjust(wspace=0.05, hspace=0.3) - - # Get positions of the axes to create properly sized colorbar axes - pos1 = ax1.get_position() - pos2 = ax2.get_position() - pos3 = ax3.get_position() - pos4 = ax4.get_position() - - # Create small colorbar axes below each panel - # [left, bottom, width, height] - cbar_height = 0.025 # Slightly increased for larger fonts - cbar_gap = 0.03 # Increased gap - - cbar_ax1 = fig.add_axes([pos1.x0, pos1.y0 - cbar_gap - cbar_height, pos1.width, cbar_height]) - cbar_ax2 = fig.add_axes([pos2.x0, pos2.y0 - cbar_gap - cbar_height, pos2.width, cbar_height]) - cbar_ax3 = fig.add_axes([pos3.x0, pos3.y0 - cbar_gap - cbar_height, pos3.width, cbar_height]) - cbar_ax4 = fig.add_axes([pos4.x0, pos4.y0 - cbar_gap - cbar_height, pos4.width, cbar_height]) - - # Add colorbars to the custom axes with font size 18 - cbar1 = plt.colorbar(sc1, cax=cbar_ax1, orientation='horizontal') - cbar1.ax.tick_params(labelsize=18) - cbar1.set_label('AOD Bias', fontsize=18) - - if valid_aod_mask.sum() > 0: - cbar2 = plt.colorbar(sc2, cax=cbar_ax2, orientation='horizontal') - cbar2.ax.tick_params(labelsize=18) - cbar2.set_label('Correlation', fontsize=18) - - cbar3 = plt.colorbar(sc3, cax=cbar_ax3, orientation='horizontal') - cbar3.ax.tick_params(labelsize=18) - cbar3.set_label('Angstrom Bias', fontsize=18) - - if valid_ang_mask.sum() > 0: - cbar4 = plt.colorbar(sc4, cax=cbar_ax4, orientation='horizontal') - cbar4.ax.tick_params(labelsize=18) - cbar4.set_label('Correlation', fontsize=18) - - # Add overall title with larger font - title_year = f" ({years[0]})" if years and len(years) == 1 else f" ({min(years)}-{max(years)})" if years else "" - unique_stations = station_metrics['station'].nunique() - fig.suptitle(f'Lunar AERONET vs MERRA-21C Comparison{title_year}\n({unique_stations} stations)', - fontsize=20, fontweight='bold', y=0.92) - - # Save the 4-panel figure with high resolution - plt.savefig(os.path.join(output_dir, f'comparison_4panel{year_str}.png'), - dpi=300, bbox_inches='tight') - plt.close() - - # Generate 4-panel kernel density estimate figure - fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(16, 16)) - - # Define regional boundaries - regions = { - 'US': {'name': 'United States', 'lat_range': (25, 50), 'lon_range': (-130, -65)}, - 'Europe': {'name': 'Europe', 'lat_range': (35, 70), 'lon_range': (-10, 40)}, - 'Africa_South': {'name': 'Africa (South of Equator)', 'lat_range': (-35, 0), 'lon_range': (-20, 55)}, - 'Asia': {'name': 'Asia (15-40ยฐN, 65-120ยฐE)', 'lat_range': (15, 40), 'lon_range': (65, 120)} - } - - axes_map = [ax1, ax2, ax3, ax4] - region_keys = ['US', 'Europe', 'Africa_South', 'Asia'] - panel_labels = ['a', 'b', 'c', 'd'] - - # Collect all regional data first to determine global axis ranges - all_regional_data = {} - all_aeronet_log = [] - all_merra_log = [] - - for region_key in region_keys: - region = regions[region_key] - - # Filter stations by region - lat_mask = ((station_metrics['latitude'] >= region['lat_range'][0]) & - (station_metrics['latitude'] <= region['lat_range'][1])) - lon_mask = ((station_metrics['longitude'] >= region['lon_range'][0]) & - (station_metrics['longitude'] <= region['lon_range'][1])) - regional_mask = lat_mask & lon_mask - - regional_aeronet_data = [] - regional_merra_data = [] - - if regional_mask.sum() > 0: - regional_stations = station_metrics[regional_mask] - - for _, station_row in regional_stations.iterrows(): - station_name = station_row['station'] - station_files = [f for f in csv_files if station_name in os.path.basename(f)] - - if station_files: - try: - station_df = pd.read_csv(station_files[0]) - - # Apply same quality filters as before - quality_mask = ( - (~station_df['aeronet_aod_550'].isna()) & - (~station_df['merra_aod_550'].isna()) & - (station_df['aeronet_aod_550'] >= 0) & (station_df['aeronet_aod_550'] < 10) & - (station_df['merra_aod_550'] >= 0) & (station_df['merra_aod_550'] < 10) & - (np.isfinite(station_df['aeronet_aod_550'])) & - (np.isfinite(station_df['merra_aod_550'])) - ) - - clean_data = station_df[quality_mask] - - # Filter by years if specified - if years is not None: - clean_data['datetime'] = pd.to_datetime(clean_data['datetime']) - clean_data = clean_data[clean_data['datetime'].dt.year.isin(years)] - - if len(clean_data) > 0: - regional_aeronet_data.extend(clean_data['aeronet_aod_550'].values) - regional_merra_data.extend(clean_data['merra_aod_550'].values) - - except Exception as e: - if debug: - print(f"Error reading data for {station_name}: {e}") - continue - - # Store regional data and add to global collection - all_regional_data[region_key] = { - 'aeronet': regional_aeronet_data, - 'merra': regional_merra_data, - 'mask': regional_mask - } - - if len(regional_aeronet_data) > 0: - all_aeronet_log.extend(np.log10(np.array(regional_aeronet_data) + 0.01)) - all_merra_log.extend(np.log10(np.array(regional_merra_data) + 0.01)) - - # Determine global axis ranges in log space - if len(all_aeronet_log) > 0: - global_x_min = min(all_aeronet_log) - global_x_max = max(all_aeronet_log) - global_y_min = min(all_merra_log) - global_y_max = max(all_merra_log) - - # Make ranges symmetric and add some padding - global_min = min(global_x_min, global_y_min) - global_max = max(global_x_max, global_y_max) - - # Add 10% padding - range_size = global_max - global_min - global_min -= 0.1 * range_size - global_max += 0.1 * range_size - else: - # Fallback ranges if no data - global_min = -2.5 - global_max = 0.5 - - # Custom formatter to convert log values back to AOD values - def log_to_aod_formatter(x, pos): - aod_val = 10**x - 0.01 - if aod_val < 0.001: - return f'{aod_val:.4f}' - elif aod_val < 0.01: - return f'{aod_val:.3f}' - elif aod_val < 0.1: - return f'{aod_val:.2f}' - else: - return f'{aod_val:.1f}' - - # Store all contourf objects and their density ranges for shared colorbar - all_contourfs = [] - all_densities = [] - - # Create plots for each region - for i, (region_key, ax) in enumerate(zip(region_keys, axes_map)): - region = regions[region_key] - regional_data = all_regional_data[region_key] - - # Initialize statistics variables - correlation = np.nan - bias = np.nan - n_points = len(regional_data['aeronet']) - n_stations = regional_data['mask'].sum() - - if len(regional_data['aeronet']) < 50: # Need minimum data for KDE - ax.text(0.5, 0.5, f'Insufficient data in\n{region["name"]}\n({n_points} points)', - transform=ax.transAxes, ha='center', va='center', fontsize=18) - all_contourfs.append(None) - else: - # Convert to log space - aeronet_log = np.log10(np.array(regional_data['aeronet']) + 0.01) - merra_log = np.log10(np.array(regional_data['merra']) + 0.01) - - # Calculate statistics in log space - try: - correlation, _ = pearsonr(aeronet_log, merra_log) - bias = np.mean(merra_log - aeronet_log) # Mean bias in log space - except Exception as e: - if debug: - print(f"Error calculating statistics for {region['name']}: {e}") - correlation = np.nan - bias = np.nan - - try: - # Create kernel density estimate - data_points = np.vstack([aeronet_log, merra_log]) - kde = gaussian_kde(data_points) - - # Create meshgrid using global ranges - xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] - positions = np.vstack([xx.ravel(), yy.ravel()]) - - # Evaluate KDE - density = kde(positions).reshape(xx.shape) - all_densities.append(density) - - # Plot KDE as contours (no individual colorbars) - contour = ax.contour(xx, yy, density, colors='black', alpha=0.6, linewidths=0.8) - - # Store contourf for shared colorbar (but don't create individual colorbars yet) - all_contourfs.append((xx, yy, density)) - - except Exception as e: - # Fallback to scatter plot if KDE fails - if debug: - print(f"KDE failed for {region['name']}, using scatter plot: {e}") - ax.scatter(aeronet_log, merra_log, alpha=0.5, s=1) - all_contourfs.append(None) - - # Set consistent axis ranges for all panels - ax.set_xlim(global_min, global_max) - ax.set_ylim(global_min, global_max) - - # Add 1:1 line - ax.plot([global_min, global_max], [global_min, global_max], 'r--', linewidth=2, alpha=0.8) - - # Set up custom tick formatting to show AOD values - ax.xaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) - ax.yaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) - - # Set appropriate tick locations - log_ticks = np.arange(np.ceil(global_min), np.floor(global_max) + 0.5, 0.5) - ax.set_xticks(log_ticks) - ax.set_yticks(log_ticks) - - # Set labels - ax.set_xlabel('AERONET AOD', fontsize=18) - ax.set_ylabel('MERRA-21C AOD', fontsize=18) - ax.tick_params(labelsize=16) - ax.grid(True, alpha=0.3) - - # Add panel label - ax.text(0.03, 0.95, f"({panel_labels[i]})", transform=ax.transAxes, - fontsize=18, fontweight='bold', ha='left', va='top', - bbox=dict(facecolor='white', alpha=0.8, pad=0.1, edgecolor='none')) - - # Add region name, data count, correlation, and bias - # Format statistics text - if not np.isnan(correlation): - corr_text = f"r = {correlation:.3f}" - else: - corr_text = "r = N/A" - - if not np.isnan(bias): - bias_text = f"bias = {bias:.3f}" - else: - bias_text = "bias = N/A" - - stats_text = f"{region['name']}\n{n_stations} stations\n{n_points:,} points\n{corr_text}\n{bias_text}" - - ax.text(0.97, 0.03, stats_text, - transform=ax.transAxes, ha='right', va='bottom', fontsize=14, - bbox=dict(facecolor='white', alpha=0.8, pad=0.1, edgecolor='none')) - - # Adjust layout to make room for shared colorbar - plt.tight_layout(rect=[0, 0.08, 1, 0.92]) - - # Create shared colorbar with white-viridis colormap - if any(cf is not None for cf in all_contourfs): - # Determine global density range for consistent colorbar - valid_densities = [density for density in all_densities if density is not None] - if valid_densities: - global_density_min = min(np.min(d) for d in valid_densities) - global_density_max = max(np.max(d) for d in valid_densities) - - # Create contourf plots with consistent density range using white-viridis colormap - for i, (cf, ax) in enumerate(zip(all_contourfs, axes_map)): - if cf is not None: - xx, yy, density = cf - # Create contourf with global density range and white-viridis colormap - contourf = ax.contourf(xx, yy, density, alpha=0.7, cmap=white_viridis, - levels=np.linspace(global_density_min, global_density_max, 20), - vmin=global_density_min, vmax=global_density_max) - - # Create single horizontal colorbar below the bottom row - # Position: [left, bottom, width, height] - cbar_ax = fig.add_axes([0.15, 0.02, 0.7, 0.03]) - cbar = plt.colorbar(contourf, cax=cbar_ax, orientation='horizontal') - cbar.set_label('Density', fontsize=16) - cbar.ax.tick_params(labelsize=14) - - # Set overall title with unique station count - title_year = f" ({years[0]})" if years and len(years) == 1 else f" ({min(years)}-{max(years)})" if years else "" - unique_stations = station_metrics['station'].nunique() - fig.suptitle(f'Regional AOD Density Distributions{title_year}\n({unique_stations} stations)', - fontsize=20, fontweight='bold', y=0.96) - - # Save the figure - plt.savefig(os.path.join(output_dir, f'regional_kde_plots{year_str}.png'), - dpi=300, bbox_inches='tight') - plt.close() - - print(f"Generated regional KDE plots: regional_kde_plots{year_str}.png") - - # Generate data coverage map (separate figure) - plt.figure(figsize=(15, 10)) - ax = plt.axes(projection=ccrs.PlateCarree()) - ax.add_feature(cfeature.COASTLINE) - ax.add_feature(cfeature.BORDERS, linestyle=':') - ax.add_feature(cfeature.LAND, alpha=0.5) - ax.add_feature(cfeature.OCEAN, alpha=0.5) - ax.set_global() - - # Add gridlines - gl = ax.gridlines(draw_labels=True, alpha=0.3) - gl.top_labels = False - gl.right_labels = False - - # Create scatter plot with point sizes and colors based on data count - data_counts_array = np.array(valid_data_counts) - min_count = np.min(data_counts_array) - max_count = np.max(data_counts_array) - - # Use different sizing strategies based on the range of data counts - if max_count > 10 * min_count and min_count > 0: - # Wide range - use log scale for sizing - sizes = 20 + 100 * (np.log10(data_counts_array) - np.log10(min_count)) / (np.log10(max_count) - np.log10(min_count)) - size_label = "Log-scaled by data count" - else: - # Narrow range - use linear scale - if max_count > min_count: - sizes = 20 + 100 * (data_counts_array - min_count) / (max_count - min_count) - else: - sizes = np.full_like(data_counts_array, 60) # Uniform size if all same - size_label = "Scaled by data count" - - sc = ax.scatter( - station_metrics['longitude'], - station_metrics['latitude'], - c=station_metrics['valid_data_points'], - s=sizes, - cmap=count_cmap, - alpha=0.8, - edgecolor='black', - linewidth=0.5, - transform=ccrs.PlateCarree() - ) - - # Add colorbar - cbar = plt.colorbar(sc, label='Number of Valid Data Points', shrink=0.8) - - title_year = f" ({years[0]})" if years and len(years) == 1 else f" ({min(years)}-{max(years)})" if years else "" - unique_stations = station_metrics['station'].nunique() - plt.title(f'Data Point Coverage at AERONET Stations{title_year}\n({unique_stations} stations, {size_label})', fontsize=14) - - # Add text annotation for size scale - plt.text(0.02, 0.02, f'Point size: {min_count}-{max_count} data points', - transform=ax.transAxes, fontsize=10, - bbox=dict(boxstyle="round,pad=0.3", facecolor="white", alpha=0.8)) - - plt.savefig(os.path.join(output_dir, f'data_coverage_map{year_str}.png'), dpi=300, bbox_inches='tight') - plt.close() - - print(f"Generated all maps in {output_dir}") - print(f"Main comparison figure: comparison_4panel{year_str}.png") - print(f"Data coverage figure: data_coverage_map{year_str}.png") - - # Generate scatter plots for overall comparison with unique station handling - fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(12, 10)) - - # Group by station and aggregate metrics for unique stations - unique_station_metrics = station_metrics.groupby('station').agg({ - 'mean_aeronet_aod': 'mean', - 'mean_merra_aod': 'mean', - 'mean_aod_bias': 'mean', - 'aod_correlation': 'mean', - 'valid_data_points': 'sum' # Sum data points across files for same station - }).reset_index() - - # AOD scatter plot - ax1.scatter(unique_station_metrics['mean_aeronet_aod'], unique_station_metrics['mean_merra_aod'], alpha=0.6) - max_aod = max(unique_station_metrics['mean_aeronet_aod'].max(), unique_station_metrics['mean_merra_aod'].max()) - ax1.plot([0, max_aod], [0, max_aod], 'k--', alpha=0.8) - ax1.set_xlabel('AERONET AOD 550nm') - ax1.set_ylabel('MERRA-21C AOD 550nm') - ax1.set_title('AOD Comparison') - ax1.grid(True, alpha=0.3) - - # AOD bias histogram - ax2.hist(unique_station_metrics['mean_aod_bias'], bins=20, alpha=0.7, edgecolor='black') - ax2.axvline(0, color='red', linestyle='--', alpha=0.8) - ax2.set_xlabel('AOD Bias (MERRA-21C - AERONET)') - ax2.set_ylabel('Number of Stations') - ax2.set_title('AOD Bias Distribution') - ax2.grid(True, alpha=0.3) - - # AOD correlation histogram (filter out NaN values) - valid_aod_correlations = unique_station_metrics['aod_correlation'][~np.isnan(unique_station_metrics['aod_correlation'])] - if len(valid_aod_correlations) > 0: - ax3.hist(valid_aod_correlations, bins=20, alpha=0.7, edgecolor='black') - ax3.set_xlabel('AOD Correlation') - ax3.set_ylabel('Number of Stations') - ax3.set_title('AOD Correlation Distribution') - ax3.grid(True, alpha=0.3) - - # Data points histogram - ax4.hist(unique_station_metrics['valid_data_points'], bins=20, alpha=0.7, edgecolor='black') - ax4.set_xlabel('Number of Valid Data Points') - ax4.set_ylabel('Number of Stations') - ax4.set_title('Data Points Distribution') - ax4.grid(True, alpha=0.3) - - plt.tight_layout() - plt.savefig(os.path.join(output_dir, f'comparison_summary{year_str}.png'), dpi=300, bbox_inches='tight') - plt.close() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Generate comparison maps between AERONET and MERRA-21C data.') - parser.add_argument('--data-dir', type=str, default="./aeronet_merra21c_comparison/", - help='Directory containing processed CSV files') - parser.add_argument('--output-dir', type=str, default="./comparison_figures/", - help='Directory to save output figures') - parser.add_argument('--min-points', type=int, default=30, - help='Minimum number of data points required for a station') - parser.add_argument('--years', nargs='+', type=int, default=None, - help='Years to include in analysis (e.g., --years 2018 2019)') - parser.add_argument('--file-pattern', type=str, default=None, - help='Custom file pattern to match CSV files') - parser.add_argument('--debug', action='store_true', - help='Print additional debugging information') - - args = parser.parse_args() - - generate_comparison_maps( - data_dir=args.data_dir, - output_dir=args.output_dir, - min_points=args.min_points, - years=args.years, - file_pattern=args.file_pattern, - debug=args.debug - ) diff --git a/src/pyobs/evaluation/AERONET/plotsinglestationonly.py b/src/pyobs/evaluation/AERONET/plotsinglestationonly.py deleted file mode 100644 index 0871260..0000000 --- a/src/pyobs/evaluation/AERONET/plotsinglestationonly.py +++ /dev/null @@ -1,320 +0,0 @@ -import os -import glob -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -import argparse -import warnings -import importlib - -warnings.filterwarnings('ignore') - -def generate_station_analysis_only(data_dir="./aeronet_merra21c_comparison/", - output_dir="./figures/", - min_points=30, - years=None, - file_pattern=None, - debug=False, - target_station="Mauna_Loa", - include_merra2=False, - merra2_dir="./aeronet_merra2_comparison/"): - """ - Generate only a single station analysis figure. - - Parameters: - ----------- - data_dir : str - Directory containing processed MERRA-21C CSV files - merra2_dir : str - Directory containing processed MERRA-2 CSV files - output_dir : str - Directory to save output figures - min_points : int - Minimum number of data points required for a station to be included - years : list or None - List of years to include in analysis. If None, uses all available data. - file_pattern : str or None - Custom file pattern to match CSV files. If None, uses default pattern. - debug : bool - If True, print additional debugging information - target_station : str - Name of the station to analyze - include_merra2 : bool - If True, use station_analysis_withm2.py to include MERRA-2 in plots - """ - # Create output directory - os.makedirs(output_dir, exist_ok=True) - - # Dynamically import the appropriate station analysis module - if include_merra2: - try: - station_analysis = importlib.import_module('station_analysis_withm2') - print("๐Ÿ”„ Using station_analysis_withm2.py (includes MERRA-2 data)") - except ImportError: - print("โŒ Error: station_analysis_withm2.py not found!") - print(" Falling back to station_analysis.py (MERRA-21C only)") - station_analysis = importlib.import_module('station_analysis') - include_merra2 = False - else: - station_analysis = importlib.import_module('station_analysis') - print("๐Ÿ”„ Using station_analysis.py (MERRA-21C only)") - - # Collect CSV files from both directories if MERRA-2 is included - all_csv_files = [] - - # Get MERRA-21C files - if os.path.exists(data_dir): - m21c_files = glob.glob(os.path.join(data_dir, "*.csv")) - all_csv_files.extend(m21c_files) - print(f"๐Ÿ“‚ Found {len(m21c_files)} MERRA-21C files in {data_dir}") - else: - print(f"โš ๏ธ MERRA-21C directory not found: {data_dir}") - - # Get MERRA-2 files if requested - if include_merra2: - if os.path.exists(merra2_dir): - m2_files = glob.glob(os.path.join(merra2_dir, "*.csv")) - all_csv_files.extend(m2_files) - print(f"๐Ÿ“‚ Found {len(m2_files)} MERRA-2 files in {merra2_dir}") - else: - print(f"โš ๏ธ MERRA-2 directory not found: {merra2_dir}") - - if not all_csv_files: - print("โŒ No CSV files found in any of the specified directories") - return - - # Filter files by station name and years if specified - station_files = [] - for csv_file in all_csv_files: - filename = os.path.basename(csv_file) - if target_station in filename: - # Check year filtering - if years is not None: - # Extract year from filename (assuming format like Station_YYYY.csv) - try: - year_in_filename = int(filename.split('_')[-1].replace('.csv', '')) - if year_in_filename in years: - station_files.append(csv_file) - except: - # If year extraction fails, include the file - station_files.append(csv_file) - else: - station_files.append(csv_file) - - if not station_files: - print(f"โŒ No CSV files found for station '{target_station}'") - available_stations = set() - for csv_file in all_csv_files[:20]: - filename = os.path.basename(csv_file) - station_name = filename.split('_')[0] if '_' in filename else filename.replace('.csv', '') - available_stations.add(station_name) - - if available_stations: - print("๐Ÿ“ Available stations:") - for station in sorted(available_stations): - print(f" ๐Ÿ“Š {station}") - return - - print(f"โœ… Found {len(station_files)} files for station '{target_station}'") - if debug: - for f in station_files: - print(f" ๐Ÿ“„ {f}") - - # If we're including MERRA-2, we need to merge the data - if include_merra2: - merged_files = merge_merra_datasets(station_files, target_station, debug) - if not merged_files: - print("โŒ Failed to merge MERRA-21C and MERRA-2 datasets") - return - csv_files_to_use = merged_files - else: - csv_files_to_use = station_files - - # Create a minimal station_metrics dataframe for the target station - station_metrics = [] - - for csv_file in csv_files_to_use: - try: - df = pd.read_csv(csv_file) - if len(df) > 0 and 'station' in df.columns and 'lat' in df.columns and 'lon' in df.columns: - station_info = { - 'station': df['station'].iloc[0], - 'latitude': df['lat'].iloc[0], - 'longitude': df['lon'].iloc[0] - } - station_metrics.append(station_info) - break # Found our station - except Exception as e: - if debug: - print(f"โš ๏ธ Error reading {csv_file}: {e}") - continue - - if not station_metrics: - print(f"โŒ Could not extract station metadata from CSV files") - return - - # Convert to DataFrame - station_metrics_df = pd.DataFrame(station_metrics) - - print(f"๐ŸŽฏ Analyzing station: {target_station}") - if include_merra2: - print("๐Ÿ“ˆ Analysis will include both MERRA-21C and MERRA-2 data") - else: - print("๐Ÿ“ˆ Analysis will include MERRA-21C data only") - - # Create station analysis using the dynamically imported module - analyzer = station_analysis.StationAnalyzer(station_metrics_df, csv_files_to_use, output_dir, years, debug) - - # Generate AOD figure - success = analyzer.create_station_figure(target_station) - - if success: - print(f"โœ… Successfully created AOD station analysis for {target_station}") - - # Generate Angstrom Exponent figure - print(f"๐Ÿ”„ Creating Angstrom Exponent analysis for {target_station}") - angstrom_success = analyzer.create_angstrom_figure(target_station) - - if angstrom_success: - print(f"โœ… Successfully created Angstrom Exponent analysis for {target_station}") - else: - print(f"โš ๏ธ Failed to create Angstrom Exponent figure for {target_station}") - else: - print(f"โŒ Failed to create AOD station figure for {target_station}") - -def merge_merra_datasets(station_files, target_station, debug=False): - """Merge MERRA-21C and MERRA-2 CSV files for the same station and years""" - import tempfile - - # Separate files by source - m21c_files = [f for f in station_files if 'aeronet_merra21c_comparison' in f] - m2_files = [f for f in station_files if 'aeronet_merra2_comparison' in f] - - if debug: - print(f"๐Ÿ”„ MERRA-21C files: {len(m21c_files)}") - print(f"๐Ÿ”„ MERRA-2 files: {len(m2_files)}") - - merged_files = [] - - # Process each year - years_processed = set() - - # Get all years from both datasets - all_years = set() - for f in m21c_files + m2_files: - try: - year = int(os.path.basename(f).split('_')[-1].replace('.csv', '')) - all_years.add(year) - except: - continue - - for year in sorted(all_years): - # Find corresponding files for this year - m21c_year_file = None - m2_year_file = None - - for f in m21c_files: - if f.endswith(f'{year}.csv'): - m21c_year_file = f - break - - for f in m2_files: - if f.endswith(f'{year}.csv'): - m2_year_file = f - break - - if debug: - print(f"๐Ÿ”„ Year {year}: M21C={m21c_year_file is not None}, M2={m2_year_file is not None}") - - # Merge data for this year - try: - merged_df = None - - if m21c_year_file and m2_year_file: - # Both datasets available - merge them - df_m21c = pd.read_csv(m21c_year_file) - df_m2 = pd.read_csv(m2_year_file) - - # Rename MERRA-2 columns to avoid conflicts - df_m2_renamed = df_m2.rename(columns={ - 'merra_aod_550': 'merra2_aod_550', - 'merra_angstrom': 'merra2_angstrom' - }) - - # Merge on datetime - merged_df = pd.merge(df_m21c, df_m2_renamed[['datetime', 'merra2_aod_550', 'merra2_angstrom']], - on='datetime', how='outer') - - print(f"โœ… Merged {year}: {len(merged_df)} combined records") - - elif m21c_year_file: - # Only MERRA-21C available - merged_df = pd.read_csv(m21c_year_file) - # Add empty MERRA-2 columns - merged_df['merra2_aod_550'] = np.nan - merged_df['merra2_angstrom'] = np.nan - - print(f"โœ… MERRA-21C only {year}: {len(merged_df)} records") - - elif m2_year_file: - # Only MERRA-2 available - df_m2 = pd.read_csv(m2_year_file) - # Rename and add missing columns - merged_df = df_m2.rename(columns={ - 'merra_aod_550': 'merra2_aod_550', - 'merra_angstrom': 'merra2_angstrom' - }) - # Add empty MERRA-21C columns - merged_df['merra_aod_550'] = np.nan - merged_df['merra_angstrom'] = np.nan - - print(f"โœ… MERRA-2 only {year}: {len(merged_df)} records") - - if merged_df is not None and len(merged_df) > 0: - # Save merged file to temporary location - temp_file = tempfile.NamedTemporaryFile(mode='w', suffix=f'_{target_station}_{year}_merged.csv', - delete=False) - merged_df.to_csv(temp_file.name, index=False) - merged_files.append(temp_file.name) - temp_file.close() - - except Exception as e: - print(f"โš ๏ธ Error merging data for year {year}: {e}") - continue - - return merged_files - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Generate single station analysis between AERONET and MERRA data.') - parser.add_argument('--data-dir', type=str, default="./aeronet_merra21c_comparison/", - help='Directory containing processed MERRA-21C CSV files') - parser.add_argument('--merra2-dir', type=str, default="./aeronet_merra2_comparison/", - help='Directory containing processed MERRA-2 CSV files') - parser.add_argument('--output-dir', type=str, default="./station_figures/", - help='Directory to save output figures') - parser.add_argument('--min-points', type=int, default=30, - help='Minimum number of data points required for a station') - parser.add_argument('--years', nargs='+', type=int, default=None, - help='Years to include in analysis (e.g., --years 2018 2019)') - parser.add_argument('--file-pattern', type=str, default=None, - help='Custom file pattern to match CSV files') - parser.add_argument('--station', type=str, default="Mauna_Loa", - help='Station name for analysis (default: Mauna_Loa)') - parser.add_argument('--debug', action='store_true', - help='Print additional debugging information') - parser.add_argument('--include-merra2', action='store_true', - help='Include MERRA-2 data in plots (uses station_analysis_withm2.py)') - - args = parser.parse_args() - - generate_station_analysis_only( - data_dir=args.data_dir, - merra2_dir=args.merra2_dir, - output_dir=args.output_dir, - min_points=args.min_points, - years=args.years, - file_pattern=args.file_pattern, - debug=args.debug, - target_station=args.station, - include_merra2=args.include_merra2 - ) diff --git a/src/pyobs/evaluation/AERONET/processaeronet.py b/src/pyobs/evaluation/AERONET/processaeronet.py deleted file mode 100644 index 3968df3..0000000 --- a/src/pyobs/evaluation/AERONET/processaeronet.py +++ /dev/null @@ -1,492 +0,0 @@ -import os -import glob -import numpy as np -import pandas as pd -import xarray as xr -from datetime import datetime, timedelta -import warnings -import multiprocessing as mp -from functools import partial -import time -import sys -warnings.filterwarnings('ignore') - -def process_station(station, years_to_process, aeronet_dir, merra_dir, output_dir): - """ - Process a single AERONET station and match with MERRA-21C data - - Parameters: - ----------- - station : str - Station name to process - years_to_process : list - List of years to process - aeronet_dir : str - Directory containing AERONET lunar AOD data files - merra_dir : str - Directory containing MERRA-21C data files - output_dir : str - Directory to save output CSV files - - Returns: - -------- - tuple - (station_name, status, message) where status is True if successful - """ - - try: - # Station name might contain underscores - station_file = os.path.join(aeronet_dir, f"20130101_20250920_{station}.lev20") - - if not os.path.exists(station_file): - return station, False, "File not found" - - # Read AERONET data with different encodings if needed - try: - # First try UTF-8 - with open(station_file, 'r', encoding='utf-8') as f: - lines = f.readlines() - header_line = lines[6].strip() - aeronet_data = pd.read_csv(station_file, skiprows=7, header=None, sep=',', encoding='utf-8') - except UnicodeDecodeError: - # If UTF-8 fails, try Latin-1 which is more permissive - with open(station_file, 'r', encoding='latin-1') as f: - lines = f.readlines() - header_line = lines[6].strip() - aeronet_data = pd.read_csv(station_file, skiprows=7, header=None, sep=',', encoding='latin-1') - - # Assign column names based on header line - column_names = header_line.split(',') - aeronet_data.columns = column_names - - # Convert date and time to datetime - aeronet_data['DateTime'] = pd.to_datetime( - aeronet_data['Date(dd:mm:yyyy)'] + ' ' + aeronet_data['Time(hh:mm:ss)'], - format='%d:%m:%Y %H:%M:%S' - ) - - # Filter for years we're interested in - year_mask = aeronet_data['DateTime'].dt.year.isin(years_to_process) - aeronet_data = aeronet_data[year_mask] - - if len(aeronet_data) == 0: - return station, False, f"No data for years {years_to_process}" - - # Extract lat/lon for the station - lat = float(aeronet_data['Site_Latitude(Degrees)'].iloc[0]) - lon = float(aeronet_data['Site_Longitude(Degrees)'].iloc[0]) - - # More robust approach to get AOD at 550nm - # Check all possible columns that could be used for 550nm AOD - aod_columns = ['AOD_551nm', 'AOD_550nm', 'AOD_532nm', 'AOD_531nm', 'AOD_555nm', 'AOD_560nm'] - aod_550 = None - used_column = None - - for col in aod_columns: - if col in aeronet_data.columns and not aeronet_data[col].isna().all(): - # Filter out negative values and very large values (likely fill values) - valid_aod = aeronet_data[col][(aeronet_data[col] >= 0) & (aeronet_data[col] < 10)] - if len(valid_aod) > 0: - aod_550 = aeronet_data[col].copy() - # Set negative and unreasonable values to NaN - aod_550[(aeronet_data[col] < 0) | (aeronet_data[col] >= 10)] = np.nan - used_column = col - break - - # If no direct measurement near 550nm, interpolate - if aod_550 is None: - # Try common wavelength pairs for interpolation - interpolation_pairs = [ - ('AOD_500nm', 'AOD_675nm'), - ('AOD_500nm', 'AOD_667nm'), - ('AOD_500nm', 'AOD_620nm'), - ('AOD_490nm', 'AOD_675nm'), - ('AOD_440nm', 'AOD_675nm') - ] - - for short_col, long_col in interpolation_pairs: - if (short_col in aeronet_data.columns and long_col in aeronet_data.columns and - not aeronet_data[short_col].isna().all() and not aeronet_data[long_col].isna().all()): - - # Extract wavelengths from column names - lambda1 = float(short_col.replace('AOD_', '').replace('nm', '')) / 1000.0 # Convert to ยตm - lambda2 = float(long_col.replace('AOD_', '').replace('nm', '')) / 1000.0 # Convert to ยตm - target_lambda = 0.550 # 550nm in ยตm - - # Apply quality filters: AOD >= 0, AOD < 10, both wavelengths valid - valid_mask = ((aeronet_data[short_col] >= 0) & (aeronet_data[short_col] < 10) & - (aeronet_data[long_col] >= 0) & (aeronet_data[long_col] < 10) & - (~aeronet_data[short_col].isna()) & (~aeronet_data[long_col].isna())) - - if valid_mask.sum() == 0: - continue - - aod_short = aeronet_data[short_col][valid_mask] - aod_long = aeronet_data[long_col][valid_mask] - - # Calculate Angstrom exponent - alpha = -np.log(aod_long/aod_short) / np.log(lambda2/lambda1) - - # Filter out unreasonable Angstrom exponents - reasonable_alpha_mask = (alpha >= -1) & (alpha <= 3) # Reasonable range for alpha - - if reasonable_alpha_mask.sum() == 0: - continue - - # Interpolate to get AOD at 550nm - aod_550_valid = aod_short[reasonable_alpha_mask] * (target_lambda/lambda1)**(-alpha[reasonable_alpha_mask]) - - # Create full array with NaNs for invalid points - aod_550 = pd.Series(np.nan, index=aeronet_data.index) - valid_indices = valid_mask[valid_mask].index[reasonable_alpha_mask] - aod_550[valid_indices] = aod_550_valid - - used_column = f"Interpolated from {short_col} and {long_col}" - break - - # If we still don't have AOD at 550nm, give up - if aod_550 is None: - return station, False, "Cannot obtain AOD at 550nm from available wavelengths" - - # Get Angstrom exponent with quality filtering - angstrom_columns = ['440-870_Angstrom_Exponent', '500-870_Angstrom_Exponent', '440-675_Angstrom_Exponent'] - ang_exponent = None - ang_source = None - - for col in angstrom_columns: - if col in aeronet_data.columns and not aeronet_data[col].isna().all(): - # Filter out unreasonable Angstrom values - valid_ang = aeronet_data[col][(aeronet_data[col] >= -1) & (aeronet_data[col] <= 3)] - if len(valid_ang) > 0: - ang_exponent = aeronet_data[col].copy() - # Set unreasonable values to NaN - ang_exponent[(aeronet_data[col] < -1) | (aeronet_data[col] > 3)] = np.nan - ang_source = col - break - - # If no direct Angstrom measurement, calculate it - if ang_exponent is None: - # Try common wavelength pairs for Angstrom calculation - angstrom_pairs = [ - ('AOD_440nm', 'AOD_870nm'), # Close to desired 470-870 - ('AOD_443nm', 'AOD_870nm'), - ('AOD_500nm', 'AOD_870nm'), - ('AOD_440nm', 'AOD_675nm') - ] - - for short_col, long_col in angstrom_pairs: - if (short_col in aeronet_data.columns and long_col in aeronet_data.columns and - not aeronet_data[short_col].isna().all() and not aeronet_data[long_col].isna().all()): - - # Extract wavelengths from column names - lambda1 = float(short_col.replace('AOD_', '').replace('nm', '')) / 1000.0 # Convert to ยตm - lambda2 = float(long_col.replace('AOD_', '').replace('nm', '')) / 1000.0 # Convert to ยตm - - # Apply quality filters - valid_mask = ((aeronet_data[short_col] >= 0) & (aeronet_data[short_col] < 10) & - (aeronet_data[long_col] >= 0) & (aeronet_data[long_col] < 10) & - (~aeronet_data[short_col].isna()) & (~aeronet_data[long_col].isna())) - - if valid_mask.sum() == 0: - continue - - aod_short = aeronet_data[short_col][valid_mask] - aod_long = aeronet_data[long_col][valid_mask] - - alpha_valid = -np.log(aod_long/aod_short) / np.log(lambda2/lambda1) - - # Filter reasonable Angstrom values - reasonable_mask = (alpha_valid >= -1) & (alpha_valid <= 3) - - if reasonable_mask.sum() == 0: - continue - - # Create full array with NaNs for invalid points - ang_exponent = pd.Series(np.nan, index=aeronet_data.index) - valid_indices = valid_mask[valid_mask].index[reasonable_mask] - ang_exponent[valid_indices] = alpha_valid[reasonable_mask] - - ang_source = f"Calculated from {short_col} and {long_col}" - break - - # If we still don't have Angstrom exponent, give up - if ang_exponent is None: - return station, False, "Cannot obtain Angstrom exponent from available wavelengths" - - # Create hourly means - aeronet_data['hour'] = aeronet_data['DateTime'].dt.floor('H') - - # Group by hour and calculate means (using nanmean to handle NaN values) - hourly_groups = aeronet_data.groupby('hour') - - results = pd.DataFrame({ - 'datetime': hourly_groups.groups.keys(), - 'aeronet_aod_550': hourly_groups.apply(lambda x: np.nanmean(aod_550.loc[x.index])), - 'aeronet_angstrom': hourly_groups.apply(lambda x: np.nanmean(ang_exponent.loc[x.index])), - 'station': station, - 'lat': lat, - 'lon': lon, - 'aod_source': used_column, - 'angstrom_source': ang_source - }) - - results = results.reset_index(drop=True) - - # Filter out hours where we couldn't calculate meaningful averages - results = results[~np.isnan(results['aeronet_aod_550']) & ~np.isnan(results['aeronet_angstrom'])] - - if len(results) == 0: - return station, False, "No valid hourly averages after quality filtering" - - # Add MERRA-21C data for each hourly point - merra_aod = [] - merra_angstrom = [] - merra_cache = {} - merra_file_found = 0 - merra_file_missing = 0 - - # Process MERRA data for each datetime in the AERONET dataset - for dt in results['datetime']: - year = dt.year - month = dt.month - day = dt.day - hour = dt.hour - - # Construct MERRA file path - merra_file = os.path.join( - merra_dir, - f"Y{year}", - f"M{month:02d}", - f"e5303_m21c_jan18.aer_inst_1hr_glo_L1152x721_slv.{year}-{month:02d}-{day:02d}T{hour:02d}00Z.nc4" - ) - - # Check if file exists - if not os.path.exists(merra_file): - merra_aod.append(np.nan) - merra_angstrom.append(np.nan) - merra_file_missing += 1 - continue - - merra_file_found += 1 - - try: - # Use cached dataset if available, otherwise open the file - if merra_file in merra_cache: - ds = merra_cache[merra_file] - else: - # Limit cache size to avoid memory issues - if len(merra_cache) > 10: - # Close oldest file and remove from cache - oldest_file = list(merra_cache.keys())[0] - merra_cache[oldest_file].close() - del merra_cache[oldest_file] - - ds = xr.open_dataset(merra_file) - merra_cache[merra_file] = ds - - # Find closest grid point to station location - # Convert longitude to 0-360 if MERRA uses that convention - merra_lon = lon - if ds.lon.min() >= 0 and lon < 0: - merra_lon = lon + 360 - - # Get MERRA-21C data at station location - aod_at_station = ds['TOTEXTTAU'].sel(lat=lat, lon=merra_lon, method='nearest').values - angstrom_at_station = ds['TOTANGSTR'].sel(lat=lat, lon=merra_lon, method='nearest').values - - # Apply quality filters to MERRA data too - if aod_at_station < 0 or aod_at_station >= 10: - aod_at_station = np.nan - if angstrom_at_station < -1 or angstrom_at_station > 3: - angstrom_at_station = np.nan - - merra_aod.append(float(aod_at_station)) - merra_angstrom.append(float(angstrom_at_station)) - - except Exception as e: - merra_aod.append(np.nan) - merra_angstrom.append(np.nan) - - # Close all open datasets - for ds in merra_cache.values(): - ds.close() - - # Add MERRA data to results - results['merra_aod_550'] = merra_aod - results['merra_angstrom'] = merra_angstrom - - # Calculate bias and other metrics - results['aod_bias'] = results['merra_aod_550'] - results['aeronet_aod_550'] - results['aod_rel_bias'] = (results['merra_aod_550'] / results['aeronet_aod_550']) * 100 - 100 - results['angstrom_bias'] = results['merra_angstrom'] - results['aeronet_angstrom'] - - # Final quality check - remove any remaining invalid data - valid_mask = (~np.isnan(results['aeronet_aod_550']) & - ~np.isnan(results['merra_aod_550']) & - ~np.isnan(results['aeronet_angstrom']) & - ~np.isnan(results['merra_angstrom']) & - (results['aeronet_aod_550'] >= 0) & - (results['merra_aod_550'] >= 0) & - (results['aeronet_angstrom'] >= -1) & (results['aeronet_angstrom'] <= 3) & - (results['merra_angstrom'] >= -1) & (results['merra_angstrom'] <= 3)) - - results_clean = results[valid_mask].copy() - - # Only proceed if we have enough valid data points - if len(results_clean) < 10: - return station, False, f"Insufficient matched data points ({len(results_clean)})" - - # Create a safe filename (replace characters that might be problematic in filenames) - safe_station_name = station.replace('/', '_').replace('\\', '_') - - # Save to CSV - year_str = f"{years_to_process[0]}_{years_to_process[-1]}" if len(years_to_process) > 1 else f"{years_to_process[0]}" - output_file = os.path.join(output_dir, f"{safe_station_name}_{year_str}.csv") - results.to_csv(output_file, index=False) - - return station, True, f"Processed successfully with {len(results_clean)} valid comparison points (total: {len(results)})" - - except Exception as e: - return station, False, f"Error: {str(e)}" - -def process_aeronet_merra_data_parallel(years_to_process=[2018, 2019, 2020], - station_names=None, - aeronet_dir="/discover/nobackup/acollow/aeroeval/aeronet_lunar/AOD_LUNAR/AOD20/ALL_POINTS/", - merra_dir="/discover/nobackup/projects/gmao/merra21c/archive/e5303_m21c_jan18/chem/", - output_dir="./processed_data/", - n_processes=None): - """ - Process AERONET lunar AOD data and MERRA-21C data for specified years and stations in parallel. - - Parameters: - ----------- - years_to_process : list - Years to process - station_names : list or None - List of station names to process. If None, process all stations. - aeronet_dir : str - Directory containing AERONET lunar AOD data files - merra_dir : str - Directory containing MERRA-21C data files - output_dir : str - Directory to save output CSV files - n_processes : int or None - Number of parallel processes to use. If None, will use CPU count - 1 - """ - start_time = time.time() - - # Create output directory if it doesn't exist - os.makedirs(output_dir, exist_ok=True) - - # Get list of all AERONET files - aeronet_files = glob.glob(os.path.join(aeronet_dir, "20130101_20250920_*.lev20")) - - if len(aeronet_files) == 0: - print(f"No AERONET files found in {aeronet_dir}") - print(f"Checking if directory exists: {os.path.exists(aeronet_dir)}") - if os.path.exists(aeronet_dir): - print(f"Directory contents: {os.listdir(aeronet_dir)[:10]} ...") - return - - # Extract station names from filenames if not specified - if station_names is None: - station_names = [] - for file_path in aeronet_files: - filename = os.path.basename(file_path) - # More robust station name extraction - # Format is: 20130101_20250920_STATIONNAME.lev20 - parts = filename.split("_", 2) # Split on first two underscores only - if len(parts) >= 3: - station_with_extension = parts[2] - station = station_with_extension.split(".")[0] # Remove .lev20 - station_names.append(station) - - station_names = list(set(station_names)) - - print(f"Processing {len(station_names)} stations for years {years_to_process}") - - # Determine number of processes to use - if n_processes is None: - n_processes = max(1, mp.cpu_count() - 1) # Leave one CPU free for system processes - - print(f"Using {n_processes} parallel processes") - - # Create a partial function with fixed parameters - process_station_partial = partial( - process_station, - years_to_process=years_to_process, - aeronet_dir=aeronet_dir, - merra_dir=merra_dir, - output_dir=output_dir - ) - - # Create a multiprocessing pool - with mp.Pool(processes=n_processes) as pool: - # Process stations in parallel with progress updates - results = [] - for i, result in enumerate(pool.imap_unordered(process_station_partial, station_names)): - results.append(result) - if (i+1) % 10 == 0 or (i+1) == len(station_names): - print(f"Progress: {i+1}/{len(station_names)} stations processed") - - # Summarize results - successful = 0 - failed = 0 - failure_reasons = {} - - for station, status, message in results: - if status: - successful += 1 - print(f"โœ“ {station}: {message}") - else: - failed += 1 - print(f"โœ— {station}: {message}") - - # Count failure reasons - reason = message.split(':')[0] if ':' in message else message - failure_reasons[reason] = failure_reasons.get(reason, 0) + 1 - - end_time = time.time() - elapsed_time = end_time - start_time - - print(f"\nProcessing complete in {elapsed_time:.1f} seconds") - print(f"Successfully processed {successful} stations") - print(f"Failed to process {failed} stations") - - # Print summary of failure reasons - if failure_reasons: - print("\nFailure reasons summary:") - for reason, count in sorted(failure_reasons.items(), key=lambda x: x[1], reverse=True): - print(f" {reason}: {count} stations") - -if __name__ == "__main__": - # Set up command line arguments - import argparse - - parser = argparse.ArgumentParser(description='Process AERONET lunar AOD data and match with MERRA-21C.') - parser.add_argument('--years', nargs='+', type=int, default=[2018, 2019, 2020], - help='Years to process') - parser.add_argument('--stations', nargs='+', type=str, default=None, - help='Specific stations to process (default: all stations)') - parser.add_argument('--aeronet-dir', type=str, - default="/discover/nobackup/acollow/aeroeval/aeronet_lunar/AOD_LUNAR/AOD20/ALL_POINTS/", - help='Directory containing AERONET lunar AOD data files') - parser.add_argument('--merra-dir', type=str, - default="/discover/nobackup/projects/gmao/merra21c/archive/e5303_m21c_jan18/chem/", - help='Directory containing MERRA-21C data files') - parser.add_argument('--output-dir', type=str, default="./aeronet_merra21c_comparison/", - help='Directory to save output CSV files') - parser.add_argument('--processes', type=int, default=None, - help='Number of parallel processes to use (default: CPU count - 1)') - - args = parser.parse_args() - - # Execute with command line arguments - process_aeronet_merra_data_parallel( - years_to_process=args.years, - station_names=args.stations, - aeronet_dir=args.aeronet_dir, - merra_dir=args.merra_dir, - output_dir=args.output_dir, - n_processes=args.processes - ) diff --git a/src/pyobs/evaluation/AERONET/station_analysis.py b/src/pyobs/evaluation/AERONET/station_analysis.py deleted file mode 100644 index 3e25a36..0000000 --- a/src/pyobs/evaluation/AERONET/station_analysis.py +++ /dev/null @@ -1,670 +0,0 @@ -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -import matplotlib.dates as mdates -import matplotlib.ticker as ticker -from matplotlib.colors import LinearSegmentedColormap -from scipy.stats import gaussian_kde, pearsonr -import os - -def create_white_viridis_cmap(): - """Create a custom colormap that starts with white and transitions to viridis""" - # Get the viridis colormap - viridis = plt.cm.get_cmap('viridis') - - # Create colors: more white values at the beginning for low densities - n_white = 50 # Number of white/near-white colors for low densities - n_viridis = 206 # Remaining colors for viridis - - # Create white to light colors transition - white_colors = [] - for i in range(n_white): - # Transition from pure white to very light viridis - alpha = i / n_white - viridis_light = viridis(0.1) # Very light viridis color - white_colors.append([ - 1 - alpha * (1 - viridis_light[0]), # R - 1 - alpha * (1 - viridis_light[1]), # G - 1 - alpha * (1 - viridis_light[2]), # B - 1.0 # Alpha - ]) - - # Add viridis colors for higher densities - viridis_colors = [viridis(i) for i in np.linspace(0.1, 1, n_viridis)] - - # Combine all colors - all_colors = white_colors + viridis_colors - - # Create the custom colormap - white_viridis = LinearSegmentedColormap.from_list('white_viridis', all_colors, N=256) - - return white_viridis - -class StationAnalyzer: - def __init__(self, station_metrics, csv_files, output_dir, years=None, debug=False): - self.station_metrics = station_metrics - self.csv_files = csv_files - self.output_dir = output_dir - self.years = years - self.debug = debug - self.white_viridis = create_white_viridis_cmap() - - def load_station_data(self, station_name): - """Load and combine all CSV files for a station""" - # Find all files for this station - station_files = [f for f in self.csv_files if station_name in os.path.basename(f)] - if not station_files: - return None, f"No CSV files found for {station_name}" - - # Read and combine all files - combined_data = [] - for file_path in station_files: - try: - df = pd.read_csv(file_path) - combined_data.append(df) - except Exception as e: - if self.debug: - print(f"Error reading {file_path}: {e}") - continue - - if not combined_data: - return None, "No valid data files" - - # Combine and clean data - data = pd.concat(combined_data, ignore_index=True) - data['datetime'] = pd.to_datetime(data['datetime']) - data = data.drop_duplicates(subset=['datetime']) - - # Filter by years if specified - if self.years is not None: - data = data[data['datetime'].dt.year.isin(self.years)] - - return data, "Success" - - def apply_quality_filters(self, data): - """Apply quality filters to the data""" - quality_mask = ( - (data['aeronet_aod_550'] >= 0) & (data['aeronet_aod_550'] < 10) & - (data['merra_aod_550'] >= 0) & (data['merra_aod_550'] < 10) & - (np.isfinite(data['aeronet_aod_550'])) & - (np.isfinite(data['merra_aod_550'])) & - (~data['aeronet_aod_550'].isna()) & - (~data['merra_aod_550'].isna()) - ) - return quality_mask - - def apply_angstrom_quality_filters(self, data): - """Apply quality filters to the Angstrom Exponent data""" - quality_mask = ( - (data['aeronet_angstrom'] >= -0.5) & (data['aeronet_angstrom'] <= 3.0) & - (data['merra_angstrom'] >= -0.5) & (data['merra_angstrom'] <= 3.0) & - (np.isfinite(data['aeronet_angstrom'])) & - (np.isfinite(data['merra_angstrom'])) & - (~data['aeronet_angstrom'].isna()) & - (~data['merra_angstrom'].isna()) - ) - return quality_mask - - def create_daily_timeseries(self, data): - """Create daily mean time series with proper gap handling""" - # Create daily means - data['date'] = data['datetime'].dt.date - daily_means = data.groupby('date').agg({ - 'aeronet_aod_550': 'mean', - 'merra_aod_550': 'mean' - }).reset_index() - - # Create complete date range - if len(daily_means) > 0: - start_date = daily_means['date'].min() - end_date = daily_means['date'].max() - complete_dates = pd.date_range(start=start_date, end=end_date, freq='D') - complete_df = pd.DataFrame({'date': complete_dates.date}) - daily_complete = complete_df.merge(daily_means, on='date', how='left') - daily_complete['date_dt'] = pd.to_datetime(daily_complete['date']) - else: - daily_complete = pd.DataFrame() - - return daily_complete - - def create_angstrom_daily_timeseries(self, data): - """Create daily mean time series for Angstrom Exponent with proper gap handling""" - # Create daily means - data['date'] = data['datetime'].dt.date - daily_means = data.groupby('date').agg({ - 'aeronet_angstrom': 'mean', - 'merra_angstrom': 'mean' - }).reset_index() - - # Create complete date range - if len(daily_means) > 0: - start_date = daily_means['date'].min() - end_date = daily_means['date'].max() - complete_dates = pd.date_range(start=start_date, end=end_date, freq='D') - complete_df = pd.DataFrame({'date': complete_dates.date}) - daily_complete = complete_df.merge(daily_means, on='date', how='left') - daily_complete['date_dt'] = pd.to_datetime(daily_complete['date']) - else: - daily_complete = pd.DataFrame() - - return daily_complete - - def calculate_seasonal_cycle(self, data): - """Calculate monthly seasonal cycle with percentiles""" - if data is None or len(data) == 0: - return pd.DataFrame() - - # Apply quality filters - quality_mask = self.apply_quality_filters(data) - valid_data = data[quality_mask].copy() - - if len(valid_data) == 0: - return pd.DataFrame() - - # Add month column - valid_data['month'] = valid_data['datetime'].dt.month - - # Calculate monthly statistics - monthly_stats = valid_data.groupby('month').agg({ - 'aeronet_aod_550': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'], - 'merra_aod_550': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] - }).reset_index() - - # Flatten column names - monthly_stats.columns = [ - 'month', - 'aeronet_median', 'aeronet_p25', 'aeronet_p75', 'aeronet_count', - 'merra_median', 'merra_p25', 'merra_p75', 'merra_count' - ] - - # Ensure all months are present (fill with NaN if missing) - all_months = pd.DataFrame({'month': range(1, 13)}) - monthly_stats = all_months.merge(monthly_stats, on='month', how='left') - - return monthly_stats - - def calculate_angstrom_seasonal_cycle(self, data): - """Calculate monthly seasonal cycle for Angstrom Exponent with percentiles""" - if data is None or len(data) == 0: - return pd.DataFrame() - - # Apply quality filters - quality_mask = self.apply_angstrom_quality_filters(data) - valid_data = data[quality_mask].copy() - - if len(valid_data) == 0: - return pd.DataFrame() - - # Add month column - valid_data['month'] = valid_data['datetime'].dt.month - - # Calculate monthly statistics - monthly_stats = valid_data.groupby('month').agg({ - 'aeronet_angstrom': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'], - 'merra_angstrom': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] - }).reset_index() - - # Flatten column names - monthly_stats.columns = [ - 'month', - 'aeronet_median', 'aeronet_p25', 'aeronet_p75', 'aeronet_count', - 'merra_median', 'merra_p25', 'merra_p75', 'merra_count' - ] - - # Ensure all months are present (fill with NaN if missing) - all_months = pd.DataFrame({'month': range(1, 13)}) - monthly_stats = all_months.merge(monthly_stats, on='month', how='left') - - return monthly_stats - - def calculate_statistics(self, aeronet_values, merra_values): - """Calculate comparison statistics""" - try: - correlation, _ = pearsonr(aeronet_values, merra_values) - bias = np.mean(merra_values - aeronet_values) - rmse = np.sqrt(np.mean((merra_values - aeronet_values)**2)) - return correlation, bias, rmse - except: - return np.nan, np.nan, np.nan - - def plot_timeseries(self, ax, daily_data, station_name): - """Plot the time series panel""" - ax.plot(daily_data['date_dt'], daily_data['merra_aod_550'], - 'k-', linewidth=1.5, label='MERRA-21C', alpha=0.8, marker='o', markersize=2) - ax.plot(daily_data['date_dt'], daily_data['aeronet_aod_550'], - 'r-', linewidth=1.5, label='AERONET', alpha=0.8, marker='o', markersize=2) - - # Format axes - if not daily_data.empty: - y_max = max( - daily_data['merra_aod_550'].max() if not daily_data['merra_aod_550'].isna().all() else 0, - daily_data['aeronet_aod_550'].max() if not daily_data['aeronet_aod_550'].isna().all() else 0 - ) - ax.set_ylim(0, y_max * 1.1) - - ax.set_xlabel('Date', fontsize=16) - ax.set_ylabel('AOD 550nm', fontsize=16) - ax.set_title('(a) Daily Mean AOD Time Series', fontsize=16, pad=10) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - ax.legend(fontsize=14) - - # Format dates - if len(daily_data) > 100: - ax.xaxis.set_major_locator(mdates.MonthLocator(interval=2)) - else: - ax.xaxis.set_major_locator(mdates.MonthLocator()) - ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) - plt.setp(ax.xaxis.get_majorticklabels(), rotation=45) - - def plot_angstrom_timeseries(self, ax, daily_data, station_name): - """Plot the Angstrom Exponent time series panel""" - ax.plot(daily_data['date_dt'], daily_data['merra_angstrom'], - 'k-', linewidth=1.5, label='MERRA-21C', alpha=0.8, marker='o', markersize=2) - ax.plot(daily_data['date_dt'], daily_data['aeronet_angstrom'], - 'r-', linewidth=1.5, label='AERONET', alpha=0.8, marker='o', markersize=2) - - # Format axes - if not daily_data.empty: - y_min = min( - daily_data['merra_angstrom'].min() if not daily_data['merra_angstrom'].isna().all() else 0, - daily_data['aeronet_angstrom'].min() if not daily_data['aeronet_angstrom'].isna().all() else 0 - ) - y_max = max( - daily_data['merra_angstrom'].max() if not daily_data['merra_angstrom'].isna().all() else 2, - daily_data['aeronet_angstrom'].max() if not daily_data['aeronet_angstrom'].isna().all() else 2 - ) - ax.set_ylim(y_min - 0.1, y_max + 0.1) - - ax.set_xlabel('Date', fontsize=16) - ax.set_ylabel('Angstrom Exponent', fontsize=16) - ax.set_title('(a) Daily Mean Angstrom Exponent Time Series', fontsize=16, pad=10) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - ax.legend(fontsize=14) - - # Format dates - if len(daily_data) > 100: - ax.xaxis.set_major_locator(mdates.MonthLocator(interval=2)) - else: - ax.xaxis.set_major_locator(mdates.MonthLocator()) - ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) - plt.setp(ax.xaxis.get_majorticklabels(), rotation=45) - - def plot_kde(self, ax, valid_data, station_info): - """Plot the KDE panel with white-to-viridis colormap using the proven approach""" - if len(valid_data) < 20: - ax.text(0.5, 0.5, f"Insufficient data\n({len(valid_data)} points)", - ha='center', va='center', fontsize=14, transform=ax.transAxes) - stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data)} points" - else: - # Log transform and calculate stats - aeronet_log = np.log10(valid_data['aeronet_aod_550'] + 0.01) - merra_log = np.log10(valid_data['merra_aod_550'] + 0.01) - correlation, bias, rmse = self.calculate_statistics(aeronet_log, merra_log) - - # Create KDE plot using the proven approach - try: - data_points = np.vstack([aeronet_log, merra_log]) - kde = gaussian_kde(data_points) - - x_min, x_max = aeronet_log.min(), aeronet_log.max() - y_min, y_max = merra_log.min(), merra_log.max() - global_min = min(x_min, y_min) - 0.1 * (max(x_max, y_max) - min(x_min, y_min)) - global_max = max(x_max, y_max) + 0.1 * (max(x_max, y_max) - min(x_min, y_min)) - - xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] - positions = np.vstack([xx.ravel(), yy.ravel()]) - density = kde(positions).reshape(xx.shape) - - # Apply the proven white-to-viridis approach - f_min = np.min(density) - f_max = np.max(density) - f_range = f_max - f_min - - # Set the minimum level higher to push low densities to white - # This makes the bottom 20% of the density range appear white/very light - threshold_factor = 0.2 # Adjust this to control how much appears white - adjusted_min = f_min + threshold_factor * f_range - - # Create levels starting from the adjusted minimum - levels = np.linspace(adjusted_min, f_max, 15) - - # Create filled contour plot with custom colormap - contour = ax.contour(xx, yy, density, colors='black', alpha=0.6, linewidths=0.8) - contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, - alpha=0.95, extend='min') - - cbar = plt.colorbar(contourf, ax=ax, shrink=0.8, extend='min') - cbar.set_label('Density', fontsize=14) - cbar.ax.tick_params(labelsize=12) - - ax.set_xlim(global_min, global_max) - ax.set_ylim(global_min, global_max) - except: - ax.scatter(aeronet_log, merra_log, alpha=0.6, s=20) - - # Add 1:1 line - xlim, ylim = ax.get_xlim(), ax.get_ylim() - min_lim, max_lim = min(xlim[0], ylim[0]), max(xlim[1], ylim[1]) - ax.plot([min_lim, max_lim], [min_lim, max_lim], 'r--', linewidth=2, alpha=0.8) - - # Format stats - corr_text = f"r = {correlation:.3f}" if not np.isnan(correlation) else "r = N/A" - bias_text = f"bias = {bias:.3f}" if not np.isnan(bias) else "bias = N/A" - rmse_text = f"RMSE = {rmse:.3f}" if not np.isnan(rmse) else "RMSE = N/A" - - stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data):,} points\n{corr_text}\n{bias_text}\n{rmse_text}" - - # Format axes - def log_to_aod_formatter(x, pos): - aod_val = 10**x - 0.01 - if aod_val < 0.001: return f'{aod_val:.4f}' - elif aod_val < 0.01: return f'{aod_val:.3f}' - elif aod_val < 0.1: return f'{aod_val:.2f}' - else: return f'{aod_val:.1f}' - - ax.xaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) - ax.yaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) - ax.set_xlabel('AERONET AOD', fontsize=16) - ax.set_ylabel('MERRA-21C AOD', fontsize=16) - ax.set_title('(b) AOD Density Distribution', fontsize=16, pad=10) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - - # Add stats box - ax.text(0.97, 0.03, stats_text, transform=ax.transAxes, ha='right', va='bottom', - fontsize=12, bbox=dict(facecolor='white', alpha=0.8, pad=0.5, edgecolor='black')) - - def plot_angstrom_kde(self, ax, valid_data, station_info): - """Plot the Angstrom Exponent KDE panel with white-to-viridis colormap using the proven approach""" - if len(valid_data) < 20: - ax.text(0.5, 0.5, f"Insufficient data\n({len(valid_data)} points)", - ha='center', va='center', fontsize=14, transform=ax.transAxes) - stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data)} points" - else: - # Calculate stats (no log transform needed for Angstrom) - aeronet_angstrom = valid_data['aeronet_angstrom'] - merra_angstrom = valid_data['merra_angstrom'] - correlation, bias, rmse = self.calculate_statistics(aeronet_angstrom, merra_angstrom) - - # Create KDE plot using the proven approach - try: - data_points = np.vstack([aeronet_angstrom, merra_angstrom]) - kde = gaussian_kde(data_points) - - x_min, x_max = aeronet_angstrom.min(), aeronet_angstrom.max() - y_min, y_max = merra_angstrom.min(), merra_angstrom.max() - global_min = min(x_min, y_min) - 0.1 * (max(x_max, y_max) - min(x_min, y_min)) - global_max = max(x_max, y_max) + 0.1 * (max(x_max, y_max) - min(x_min, y_min)) - - xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] - positions = np.vstack([xx.ravel(), yy.ravel()]) - density = kde(positions).reshape(xx.shape) - - # Apply the proven white-to-viridis approach - f_min = np.min(density) - f_max = np.max(density) - f_range = f_max - f_min - - # Set the minimum level higher to push low densities to white - # This makes the bottom 20% of the density range appear white/very light - threshold_factor = 0.2 # Adjust this to control how much appears white - adjusted_min = f_min + threshold_factor * f_range - - # Create levels starting from the adjusted minimum - levels = np.linspace(adjusted_min, f_max, 15) - - # Create filled contour plot with custom colormap - contour = ax.contour(xx, yy, density, colors='black', alpha=0.6, linewidths=0.8) - contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, - alpha=0.95, extend='min') - - cbar = plt.colorbar(contourf, ax=ax, shrink=0.8, extend='min') - cbar.set_label('Density', fontsize=14) - cbar.ax.tick_params(labelsize=12) - - ax.set_xlim(global_min, global_max) - ax.set_ylim(global_min, global_max) - except: - ax.scatter(aeronet_angstrom, merra_angstrom, alpha=0.6, s=20) - - # Add 1:1 line - xlim, ylim = ax.get_xlim(), ax.get_ylim() - min_lim, max_lim = min(xlim[0], ylim[0]), max(xlim[1], ylim[1]) - ax.plot([min_lim, max_lim], [min_lim, max_lim], 'r--', linewidth=2, alpha=0.8) - - # Format stats - corr_text = f"r = {correlation:.3f}" if not np.isnan(correlation) else "r = N/A" - bias_text = f"bias = {bias:.3f}" if not np.isnan(bias) else "bias = N/A" - rmse_text = f"RMSE = {rmse:.3f}" if not np.isnan(rmse) else "RMSE = N/A" - - stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data):,} points\n{corr_text}\n{bias_text}\n{rmse_text}" - - # Format axes - ax.set_xlabel('AERONET Angstrom Exponent', fontsize=16) - ax.set_ylabel('MERRA-21C Angstrom Exponent', fontsize=16) - ax.set_title('(b) Angstrom Exponent Density Distribution', fontsize=16, pad=10) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - - # Add stats box - ax.text(0.97, 0.03, stats_text, transform=ax.transAxes, ha='right', va='bottom', - fontsize=12, bbox=dict(facecolor='white', alpha=0.8, pad=0.5, edgecolor='black')) - - def plot_seasonal_cycle(self, ax, monthly_stats, station_info): - """Plot the seasonal cycle panel""" - if monthly_stats.empty or monthly_stats['aeronet_median'].isna().all(): - ax.text(0.5, 0.5, 'No seasonal data available', - ha='center', va='center', fontsize=14, transform=ax.transAxes) - ax.set_title('(c) Seasonal Cycle', fontsize=16, pad=10) - return - - months = monthly_stats['month'] - month_names = ['J', 'F', 'M', 'A', 'M', 'J', 'J', 'A', 'S', 'O', 'N', 'D'] - - # Plot AERONET seasonal cycle - aeronet_median = monthly_stats['aeronet_median'] - aeronet_p25 = monthly_stats['aeronet_p25'] - aeronet_p75 = monthly_stats['aeronet_p75'] - - # Only plot where we have data - valid_aeronet = ~aeronet_median.isna() - if valid_aeronet.any(): - ax.plot(months[valid_aeronet], aeronet_median[valid_aeronet], - 'ro-', linewidth=2, markersize=6, label='AERONET', alpha=0.8) - ax.fill_between(months[valid_aeronet], - aeronet_p25[valid_aeronet], - aeronet_p75[valid_aeronet], - alpha=0.3, color='red', label='AERONET 25-75%') - - # Plot MERRA seasonal cycle - merra_median = monthly_stats['merra_median'] - merra_p25 = monthly_stats['merra_p25'] - merra_p75 = monthly_stats['merra_p75'] - - # Only plot where we have data - valid_merra = ~merra_median.isna() - if valid_merra.any(): - ax.plot(months[valid_merra], merra_median[valid_merra], - 'ko-', linewidth=2, markersize=6, label='MERRA-21C', alpha=0.8) - ax.fill_between(months[valid_merra], - merra_p25[valid_merra], - merra_p75[valid_merra], - alpha=0.3, color='black', label='MERRA-21C 25-75%') - - # Format axes - ax.set_xlim(0.5, 12.5) - ax.set_xticks(range(1, 13)) - ax.set_xticklabels(month_names) - ax.set_xlabel('Month', fontsize=16) - ax.set_ylabel('AOD 550nm', fontsize=16) - ax.set_title('(c) Seasonal Cycle', fontsize=16, pad=10) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - ax.legend(fontsize=12, loc='upper right') - - # Set y-axis to start from 0 - current_ylim = ax.get_ylim() - ax.set_ylim(0, current_ylim[1]) - - - def plot_angstrom_seasonal_cycle(self, ax, monthly_stats, station_info): - """Plot the Angstrom Exponent seasonal cycle panel""" - if monthly_stats.empty or monthly_stats['aeronet_median'].isna().all(): - ax.text(0.5, 0.5, 'No seasonal data available', - ha='center', va='center', fontsize=14, transform=ax.transAxes) - ax.set_title('(c) Seasonal Cycle', fontsize=16, pad=10) - return - - months = monthly_stats['month'] - month_names = ['J', 'F', 'M', 'A', 'M', 'J', 'J', 'A', 'S', 'O', 'N', 'D'] - - # Plot AERONET seasonal cycle - aeronet_median = monthly_stats['aeronet_median'] - aeronet_p25 = monthly_stats['aeronet_p25'] - aeronet_p75 = monthly_stats['aeronet_p75'] - - # Only plot where we have data - valid_aeronet = ~aeronet_median.isna() - if valid_aeronet.any(): - ax.plot(months[valid_aeronet], aeronet_median[valid_aeronet], - 'ro-', linewidth=2, markersize=6, label='AERONET', alpha=0.8) - ax.fill_between(months[valid_aeronet], - aeronet_p25[valid_aeronet], - aeronet_p75[valid_aeronet], - alpha=0.3, color='red', label='AERONET 25-75%') - - # Plot MERRA seasonal cycle - merra_median = monthly_stats['merra_median'] - merra_p25 = monthly_stats['merra_p25'] - merra_p75 = monthly_stats['merra_p75'] - - # Only plot where we have data - valid_merra = ~merra_median.isna() - if valid_merra.any(): - ax.plot(months[valid_merra], merra_median[valid_merra], - 'ko-', linewidth=2, markersize=6, label='MERRA-21C', alpha=0.8) - ax.fill_between(months[valid_merra], - merra_p25[valid_merra], - merra_p75[valid_merra], - alpha=0.3, color='black', label='MERRA-21C 25-75%') - - # Format axes - ax.set_xlim(0.5, 12.5) - ax.set_xticks(range(1, 13)) - ax.set_xticklabels(month_names) - ax.set_xlabel('Month', fontsize=16) - ax.set_ylabel('Angstrom Exponent', fontsize=16) - ax.set_title('(c) Seasonal Cycle', fontsize=16, pad=10) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - ax.legend(fontsize=12, loc='upper right') - - - def create_station_figure(self, station_name): - """Main function to create station analysis figure with three panels""" - # Check if station exists - station_mask = self.station_metrics['station'] == station_name - if not station_mask.any(): - print(f"Station '{station_name}' not found") - return False - - # Load data - data, message = self.load_station_data(station_name) - if data is None: - print(f"Failed to load data for {station_name}: {message}") - return False - - # Get station info - station_info = { - 'name': station_name.replace('_', ' '), - 'lat': self.station_metrics[station_mask].iloc[0]['latitude'], - 'lon': self.station_metrics[station_mask].iloc[0]['longitude'] - } - - # Prepare data - quality_mask = self.apply_quality_filters(data) - daily_data = self.create_daily_timeseries(data) - valid_data = data[quality_mask] - monthly_stats = self.calculate_seasonal_cycle(data) - - # Create figure with three panels - fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(24, 8)) - - # Plot panels - self.plot_timeseries(ax1, daily_data, station_name) - self.plot_kde(ax2, valid_data, station_info) - self.plot_seasonal_cycle(ax3, monthly_stats, station_info) - - # Overall formatting - year_str = f" ({self.years[0]})" if self.years and len(self.years) == 1 else f" ({min(self.years)}-{max(self.years)})" if self.years else "" - fig.suptitle(f'Station Analysis: {station_info["name"]}{year_str}', fontsize=22, fontweight='bold', y=0.95) - plt.tight_layout(rect=[0, 0, 1, 0.92]) - - # Save - station_filename = station_name.replace('_', '-').lower() - year_suffix = f"_{self.years[0]}_{self.years[-1]}" if self.years and len(self.years) > 1 else f"_{self.years[0]}" if self.years else "" - plt.savefig(os.path.join(self.output_dir, f'station_analysis_{station_filename}{year_suffix}.png'), - dpi=300, bbox_inches='tight') - plt.close() - - print(f"Generated station analysis: station_analysis_{station_filename}{year_suffix}.png") - return True - - def create_angstrom_figure(self, station_name): - """Main function to create Angstrom Exponent analysis figure with three panels""" - # Check if station exists - station_mask = self.station_metrics['station'] == station_name - if not station_mask.any(): - print(f"Station '{station_name}' not found") - return False - - # Load data - data, message = self.load_station_data(station_name) - if data is None: - print(f"Failed to load data for {station_name}: {message}") - return False - - # Check if Angstrom columns exist - required_cols = ['aeronet_angstrom', 'merra_angstrom'] - missing_cols = [col for col in required_cols if col not in data.columns] - if missing_cols: - print(f"Missing Angstrom Exponent columns: {missing_cols}") - return False - - # Get station info - station_info = { - 'name': station_name.replace('_', ' '), - 'lat': self.station_metrics[station_mask].iloc[0]['latitude'], - 'lon': self.station_metrics[station_mask].iloc[0]['longitude'] - } - - # Prepare data - quality_mask = self.apply_angstrom_quality_filters(data) - daily_data = self.create_angstrom_daily_timeseries(data) - valid_data = data[quality_mask] - monthly_stats = self.calculate_angstrom_seasonal_cycle(data) - - # Create figure with three panels - fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(24, 8)) - - # Plot panels - self.plot_angstrom_timeseries(ax1, daily_data, station_name) - self.plot_angstrom_kde(ax2, valid_data, station_info) - self.plot_angstrom_seasonal_cycle(ax3, monthly_stats, station_info) - - # Overall formatting - year_str = f" ({self.years[0]})" if self.years and len(self.years) == 1 else f" ({min(self.years)}-{max(self.years)})" if self.years else "" - fig.suptitle(f'Angstrom Exponent Analysis: {station_info["name"]}{year_str}', fontsize=22, fontweight='bold', y=0.95) - plt.tight_layout(rect=[0, 0, 1, 0.92]) - - # Save - station_filename = station_name.replace('_', '-').lower() - year_suffix = f"_{self.years[0]}_{self.years[-1]}" if self.years and len(self.years) > 1 else f"_{self.years[0]}" if self.years else "" - plt.savefig(os.path.join(self.output_dir, f'angstrom_analysis_{station_filename}{year_suffix}.png'), - dpi=300, bbox_inches='tight') - plt.close() - - print(f"Generated Angstrom analysis: angstrom_analysis_{station_filename}{year_suffix}.png") - return True diff --git a/src/pyobs/evaluation/AERONET/station_analysis_withm2.py b/src/pyobs/evaluation/AERONET/station_analysis_withm2.py deleted file mode 100644 index 0bf2e5e..0000000 --- a/src/pyobs/evaluation/AERONET/station_analysis_withm2.py +++ /dev/null @@ -1,987 +0,0 @@ -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -import matplotlib.dates as mdates -import matplotlib.ticker as ticker -from matplotlib.colors import LinearSegmentedColormap -from scipy.stats import gaussian_kde, pearsonr -import os - -def create_white_viridis_cmap(): - """Create a custom colormap that starts with white and transitions to viridis""" - # Get the viridis colormap - viridis = plt.cm.get_cmap('viridis') - - # Create colors: more white values at the beginning for low densities - n_white = 50 # Number of white/near-white colors for low densities - n_viridis = 206 # Remaining colors for viridis - - # Create white to light colors transition - white_colors = [] - for i in range(n_white): - # Transition from pure white to very light viridis - alpha = i / n_white - viridis_light = viridis(0.1) # Very light viridis color - white_colors.append([ - 1 - alpha * (1 - viridis_light[0]), # R - 1 - alpha * (1 - viridis_light[1]), # G - 1 - alpha * (1 - viridis_light[2]), # B - 1.0 # Alpha - ]) - - # Add viridis colors for higher densities - viridis_colors = [viridis(i) for i in np.linspace(0.1, 1, n_viridis)] - - # Combine all colors - all_colors = white_colors + viridis_colors - - # Create the custom colormap - white_viridis = LinearSegmentedColormap.from_list('white_viridis', all_colors, N=256) - - return white_viridis - -class StationAnalyzer: - def __init__(self, station_metrics, csv_files, output_dir, years=None, debug=False): - self.station_metrics = station_metrics - self.csv_files = csv_files - self.output_dir = output_dir - self.years = years - self.debug = debug - self.white_viridis = create_white_viridis_cmap() - - def load_station_data(self, station_name): - """Load and combine all CSV files for a station""" - # Find all files for this station - station_files = [f for f in self.csv_files if station_name in os.path.basename(f)] - if not station_files: - return None, f"No CSV files found for {station_name}" - - # Read and combine all files - combined_data = [] - for file_path in station_files: - try: - df = pd.read_csv(file_path) - combined_data.append(df) - except Exception as e: - if self.debug: - print(f"Error reading {file_path}: {e}") - continue - - if not combined_data: - return None, "No valid data files" - - # Combine and clean data - data = pd.concat(combined_data, ignore_index=True) - data['datetime'] = pd.to_datetime(data['datetime']) - data = data.drop_duplicates(subset=['datetime']) - - # Filter by years if specified - if self.years is not None: - data = data[data['datetime'].dt.year.isin(self.years)] - - return data, "Success" - - def apply_quality_filters(self, data): - """Apply quality filters to the data - supports both MERRA-21C and MERRA-2""" - # Check which columns exist - has_m21c = 'merra_aod_550' in data.columns - has_m2 = 'merra2_aod_550' in data.columns - - base_mask = ( - (data['aeronet_aod_550'] >= 0) & (data['aeronet_aod_550'] < 10) & - (np.isfinite(data['aeronet_aod_550'])) & - (~data['aeronet_aod_550'].isna()) - ) - - if has_m21c and has_m2: - # Both datasets present - quality_mask = base_mask & ( - (data['merra_aod_550'] >= 0) & (data['merra_aod_550'] < 10) & - (data['merra2_aod_550'] >= 0) & (data['merra2_aod_550'] < 10) & - (np.isfinite(data['merra_aod_550'])) & - (np.isfinite(data['merra2_aod_550'])) & - (~data['merra_aod_550'].isna()) & - (~data['merra2_aod_550'].isna()) - ) - elif has_m21c: - # Only MERRA-21C - quality_mask = base_mask & ( - (data['merra_aod_550'] >= 0) & (data['merra_aod_550'] < 10) & - (np.isfinite(data['merra_aod_550'])) & - (~data['merra_aod_550'].isna()) - ) - elif has_m2: - # Only MERRA-2 - quality_mask = base_mask & ( - (data['merra2_aod_550'] >= 0) & (data['merra2_aod_550'] < 10) & - (np.isfinite(data['merra2_aod_550'])) & - (~data['merra2_aod_550'].isna()) - ) - else: - # No MERRA data - quality_mask = base_mask - - return quality_mask - - def apply_angstrom_quality_filters(self, data): - """Apply quality filters to the Angstrom Exponent data""" - # Check which columns exist - has_m21c = 'merra_angstrom' in data.columns - has_m2 = 'merra2_angstrom' in data.columns - - base_mask = ( - (data['aeronet_angstrom'] >= -0.5) & (data['aeronet_angstrom'] <= 3.0) & - (np.isfinite(data['aeronet_angstrom'])) & - (~data['aeronet_angstrom'].isna()) - ) - - if has_m21c and has_m2: - # Both datasets present - quality_mask = base_mask & ( - (data['merra_angstrom'] >= -0.5) & (data['merra_angstrom'] <= 3.0) & - (data['merra2_angstrom'] >= -0.5) & (data['merra2_angstrom'] <= 3.0) & - (np.isfinite(data['merra_angstrom'])) & - (np.isfinite(data['merra2_angstrom'])) & - (~data['merra_angstrom'].isna()) & - (~data['merra2_angstrom'].isna()) - ) - elif has_m21c: - # Only MERRA-21C - quality_mask = base_mask & ( - (data['merra_angstrom'] >= -0.5) & (data['merra_angstrom'] <= 3.0) & - (np.isfinite(data['merra_angstrom'])) & - (~data['merra_angstrom'].isna()) - ) - elif has_m2: - # Only MERRA-2 - quality_mask = base_mask & ( - (data['merra2_angstrom'] >= -0.5) & (data['merra2_angstrom'] <= 3.0) & - (np.isfinite(data['merra2_angstrom'])) & - (~data['merra2_angstrom'].isna()) - ) - else: - # No MERRA data - quality_mask = base_mask - - return quality_mask - - def create_daily_timeseries(self, data): - """Create daily mean time series with proper gap handling for all datasets""" - # Create daily means - data['date'] = data['datetime'].dt.date - - # Determine which columns to aggregate - agg_dict = {'aeronet_aod_550': 'mean'} - if 'merra_aod_550' in data.columns: - agg_dict['merra_aod_550'] = 'mean' - if 'merra2_aod_550' in data.columns: - agg_dict['merra2_aod_550'] = 'mean' - - daily_means = data.groupby('date').agg(agg_dict).reset_index() - - # Create complete date range - if len(daily_means) > 0: - start_date = daily_means['date'].min() - end_date = daily_means['date'].max() - complete_dates = pd.date_range(start=start_date, end=end_date, freq='D') - complete_df = pd.DataFrame({'date': complete_dates.date}) - daily_complete = complete_df.merge(daily_means, on='date', how='left') - daily_complete['date_dt'] = pd.to_datetime(daily_complete['date']) - else: - daily_complete = pd.DataFrame() - - return daily_complete - - def create_angstrom_daily_timeseries(self, data): - """Create daily mean time series for Angstrom Exponent with proper gap handling""" - # Create daily means - data['date'] = data['datetime'].dt.date - - # Determine which columns to aggregate - agg_dict = {'aeronet_angstrom': 'mean'} - if 'merra_angstrom' in data.columns: - agg_dict['merra_angstrom'] = 'mean' - if 'merra2_angstrom' in data.columns: - agg_dict['merra2_angstrom'] = 'mean' - - daily_means = data.groupby('date').agg(agg_dict).reset_index() - - # Create complete date range - if len(daily_means) > 0: - start_date = daily_means['date'].min() - end_date = daily_means['date'].max() - complete_dates = pd.date_range(start=start_date, end=end_date, freq='D') - complete_df = pd.DataFrame({'date': complete_dates.date}) - daily_complete = complete_df.merge(daily_means, on='date', how='left') - daily_complete['date_dt'] = pd.to_datetime(daily_complete['date']) - else: - daily_complete = pd.DataFrame() - - return daily_complete - - def calculate_seasonal_cycle(self, data): - """Calculate monthly seasonal cycle with percentiles for all datasets""" - if data is None or len(data) == 0: - return pd.DataFrame() - - # Apply quality filters - quality_mask = self.apply_quality_filters(data) - valid_data = data[quality_mask].copy() - - if len(valid_data) == 0: - return pd.DataFrame() - - # Add month column - valid_data['month'] = valid_data['datetime'].dt.month - - # Determine which columns to aggregate - agg_dict = { - 'aeronet_aod_550': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] - } - - if 'merra_aod_550' in valid_data.columns: - agg_dict['merra_aod_550'] = ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] - - if 'merra2_aod_550' in valid_data.columns: - agg_dict['merra2_aod_550'] = ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] - - # Calculate monthly statistics - monthly_stats = valid_data.groupby('month').agg(agg_dict).reset_index() - - # Flatten column names - flattened_columns = ['month'] - if 'aeronet_aod_550' in agg_dict: - flattened_columns.extend(['aeronet_median', 'aeronet_p25', 'aeronet_p75', 'aeronet_count']) - if 'merra_aod_550' in agg_dict: - flattened_columns.extend(['merra_median', 'merra_p25', 'merra_p75', 'merra_count']) - if 'merra2_aod_550' in agg_dict: - flattened_columns.extend(['merra2_median', 'merra2_p25', 'merra2_p75', 'merra2_count']) - - monthly_stats.columns = flattened_columns - - # Ensure all months are present (fill with NaN if missing) - all_months = pd.DataFrame({'month': range(1, 13)}) - monthly_stats = all_months.merge(monthly_stats, on='month', how='left') - - return monthly_stats - - def calculate_angstrom_seasonal_cycle(self, data): - """Calculate monthly seasonal cycle for Angstrom Exponent with percentiles for all datasets""" - if data is None or len(data) == 0: - return pd.DataFrame() - - # Apply quality filters - quality_mask = self.apply_angstrom_quality_filters(data) - valid_data = data[quality_mask].copy() - - if len(valid_data) == 0: - return pd.DataFrame() - - # Add month column - valid_data['month'] = valid_data['datetime'].dt.month - - # Determine which columns to aggregate - agg_dict = { - 'aeronet_angstrom': ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] - } - - if 'merra_angstrom' in valid_data.columns: - agg_dict['merra_angstrom'] = ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] - - if 'merra2_angstrom' in valid_data.columns: - agg_dict['merra2_angstrom'] = ['median', lambda x: np.percentile(x, 25), lambda x: np.percentile(x, 75), 'count'] - - # Calculate monthly statistics - monthly_stats = valid_data.groupby('month').agg(agg_dict).reset_index() - - # Flatten column names - flattened_columns = ['month'] - if 'aeronet_angstrom' in agg_dict: - flattened_columns.extend(['aeronet_median', 'aeronet_p25', 'aeronet_p75', 'aeronet_count']) - if 'merra_angstrom' in agg_dict: - flattened_columns.extend(['merra_median', 'merra_p25', 'merra_p75', 'merra_count']) - if 'merra2_angstrom' in agg_dict: - flattened_columns.extend(['merra2_median', 'merra2_p25', 'merra2_p75', 'merra2_count']) - - monthly_stats.columns = flattened_columns - - # Ensure all months are present (fill with NaN if missing) - all_months = pd.DataFrame({'month': range(1, 13)}) - monthly_stats = all_months.merge(monthly_stats, on='month', how='left') - - return monthly_stats - - def calculate_statistics(self, aeronet_values, model_values): - """Calculate comparison statistics""" - try: - correlation, _ = pearsonr(aeronet_values, model_values) - bias = np.mean(model_values - aeronet_values) - rmse = np.sqrt(np.mean((model_values - aeronet_values)**2)) - return correlation, bias, rmse - except: - return np.nan, np.nan, np.nan - - def get_axis_limits(self, valid_data): - """Get consistent axis limits for KDE plots""" - # Log transform all data to determine global limits - aeronet_log = np.log10(valid_data['aeronet_aod_550'] + 0.01) - - all_model_values = [] - if 'merra_aod_550' in valid_data.columns: - merra21c_log = np.log10(valid_data['merra_aod_550'] + 0.01) - all_model_values.extend(merra21c_log) - if 'merra2_aod_550' in valid_data.columns: - merra2_log = np.log10(valid_data['merra2_aod_550'] + 0.01) - all_model_values.extend(merra2_log) - - if all_model_values: - all_values = np.concatenate([aeronet_log, all_model_values]) - else: - all_values = aeronet_log - - # Calculate global limits with some padding - global_min = np.min(all_values) - global_max = np.max(all_values) - data_range = global_max - global_min - - # Add padding - padded_min = global_min - 0.1 * data_range - padded_max = global_max + 0.1 * data_range - - return padded_min, padded_max - - def plot_timeseries(self, ax, daily_data, station_name): - """Plot the time series panel with all three datasets""" - # Plot AERONET first (red) - ax.plot(daily_data['date_dt'], daily_data['aeronet_aod_550'], - 'r-', linewidth=1.5, label='AERONET', alpha=0.8, marker='o', markersize=2) - - # Plot MERRA-21C (black) - if 'merra_aod_550' in daily_data.columns: - ax.plot(daily_data['date_dt'], daily_data['merra_aod_550'], - 'k-', linewidth=1.5, label='MERRA-21C', alpha=0.8, marker='s', markersize=2) - - # Plot MERRA-2 (blue) - if 'merra2_aod_550' in daily_data.columns: - ax.plot(daily_data['date_dt'], daily_data['merra2_aod_550'], - 'b-', linewidth=1.5, label='MERRA-2', alpha=0.8, marker='^', markersize=2) - - # Format axes - if not daily_data.empty: - y_values = [] - for col in ['aeronet_aod_550', 'merra_aod_550', 'merra2_aod_550']: - if col in daily_data.columns and not daily_data[col].isna().all(): - y_values.extend(daily_data[col].dropna()) - - if y_values: - y_max = max(y_values) - ax.set_ylim(0, y_max * 1.1) - - ax.set_xlabel('Date', fontsize=16) - ax.set_ylabel('AOD 550nm', fontsize=16) - ax.set_title('(a) Daily Mean AOD Time Series', fontsize=16, pad=10) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - ax.legend(fontsize=14) - - # Format dates - if len(daily_data) > 100: - ax.xaxis.set_major_locator(mdates.MonthLocator(interval=2)) - else: - ax.xaxis.set_major_locator(mdates.MonthLocator()) - ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) - plt.setp(ax.xaxis.get_majorticklabels(), rotation=45) - - def plot_angstrom_timeseries(self, ax, daily_data, station_name): - """Plot the Angstrom Exponent time series panel with all three datasets""" - # Plot AERONET first (red) - ax.plot(daily_data['date_dt'], daily_data['aeronet_angstrom'], - 'r-', linewidth=1.5, label='AERONET', alpha=0.8, marker='o', markersize=2) - - # Plot MERRA-21C (black) - if 'merra_angstrom' in daily_data.columns: - ax.plot(daily_data['date_dt'], daily_data['merra_angstrom'], - 'k-', linewidth=1.5, label='MERRA-21C', alpha=0.8, marker='s', markersize=2) - - # Plot MERRA-2 (blue) - if 'merra2_angstrom' in daily_data.columns: - ax.plot(daily_data['date_dt'], daily_data['merra2_angstrom'], - 'b-', linewidth=1.5, label='MERRA-2', alpha=0.8, marker='^', markersize=2) - - # Format axes - if not daily_data.empty: - y_values = [] - for col in ['aeronet_angstrom', 'merra_angstrom', 'merra2_angstrom']: - if col in daily_data.columns and not daily_data[col].isna().all(): - y_values.extend(daily_data[col].dropna()) - - if y_values: - y_min = min(y_values) - y_max = max(y_values) - ax.set_ylim(y_min - 0.1, y_max + 0.1) - - ax.set_xlabel('Date', fontsize=16) - ax.set_ylabel('Angstrom Exponent', fontsize=16) - ax.set_title('(a) Daily Mean Angstrom Exponent Time Series', fontsize=16, pad=10) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - ax.legend(fontsize=14) - - # Format dates - if len(daily_data) > 100: - ax.xaxis.set_major_locator(mdates.MonthLocator(interval=2)) - else: - ax.xaxis.set_major_locator(mdates.MonthLocator()) - ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) - plt.setp(ax.xaxis.get_majorticklabels(), rotation=45) - - def plot_seasonal_cycle(self, ax, monthly_stats, station_info): - """Plot the seasonal cycle panel with all three datasets""" - if monthly_stats.empty or monthly_stats['aeronet_median'].isna().all(): - ax.text(0.5, 0.5, 'No seasonal data available', - ha='center', va='center', fontsize=14, transform=ax.transAxes) - ax.set_title('(b) Seasonal Cycle', fontsize=16, pad=10) - return - - months = monthly_stats['month'] - month_names = ['J', 'F', 'M', 'A', 'M', 'J', 'J', 'A', 'S', 'O', 'N', 'D'] - - # Plot AERONET seasonal cycle (red) - aeronet_median = monthly_stats['aeronet_median'] - aeronet_p25 = monthly_stats['aeronet_p25'] - aeronet_p75 = monthly_stats['aeronet_p75'] - - valid_aeronet = ~aeronet_median.isna() - if valid_aeronet.any(): - ax.plot(months[valid_aeronet], aeronet_median[valid_aeronet], - 'ro-', linewidth=2, markersize=6, label='AERONET', alpha=0.8) - ax.fill_between(months[valid_aeronet], - aeronet_p25[valid_aeronet], - aeronet_p75[valid_aeronet], - alpha=0.3, color='red') - - # Plot MERRA-21C seasonal cycle (black) - if 'merra_median' in monthly_stats.columns: - merra_median = monthly_stats['merra_median'] - merra_p25 = monthly_stats['merra_p25'] - merra_p75 = monthly_stats['merra_p75'] - - valid_merra = ~merra_median.isna() - if valid_merra.any(): - ax.plot(months[valid_merra], merra_median[valid_merra], - 'ko-', linewidth=2, markersize=6, label='MERRA-21C', alpha=0.8) - ax.fill_between(months[valid_merra], - merra_p25[valid_merra], - merra_p75[valid_merra], - alpha=0.3, color='black') - - # Plot MERRA-2 seasonal cycle (blue) - if 'merra2_median' in monthly_stats.columns: - merra2_median = monthly_stats['merra2_median'] - merra2_p25 = monthly_stats['merra2_p25'] - merra2_p75 = monthly_stats['merra2_p75'] - - valid_merra2 = ~merra2_median.isna() - if valid_merra2.any(): - ax.plot(months[valid_merra2], merra2_median[valid_merra2], - 'bo-', linewidth=2, markersize=6, label='MERRA-2', alpha=0.8) - ax.fill_between(months[valid_merra2], - merra2_p25[valid_merra2], - merra2_p75[valid_merra2], - alpha=0.3, color='blue') - - # Format axes - ax.set_xlim(0.5, 12.5) - ax.set_xticks(range(1, 13)) - ax.set_xticklabels(month_names) - ax.set_xlabel('Month', fontsize=16) - ax.set_ylabel('AOD 550nm', fontsize=16) - ax.set_title('(b) Seasonal Cycle', fontsize=16, pad=10) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - ax.legend(fontsize=12, loc='upper right') - - # Set y-axis to start from 0 - current_ylim = ax.get_ylim() - ax.set_ylim(0, current_ylim[1]) - - def plot_angstrom_seasonal_cycle(self, ax, monthly_stats, station_info): - """Plot the Angstrom Exponent seasonal cycle panel with all three datasets""" - if monthly_stats.empty or monthly_stats['aeronet_median'].isna().all(): - ax.text(0.5, 0.5, 'No seasonal data available', - ha='center', va='center', fontsize=14, transform=ax.transAxes) - ax.set_title('(b) Seasonal Cycle', fontsize=16, pad=10) - return - - months = monthly_stats['month'] - month_names = ['J', 'F', 'M', 'A', 'M', 'J', 'J', 'A', 'S', 'O', 'N', 'D'] - - # Plot AERONET seasonal cycle (red) - aeronet_median = monthly_stats['aeronet_median'] - aeronet_p25 = monthly_stats['aeronet_p25'] - aeronet_p75 = monthly_stats['aeronet_p75'] - - valid_aeronet = ~aeronet_median.isna() - if valid_aeronet.any(): - ax.plot(months[valid_aeronet], aeronet_median[valid_aeronet], - 'ro-', linewidth=2, markersize=6, label='AERONET', alpha=0.8) - ax.fill_between(months[valid_aeronet], - aeronet_p25[valid_aeronet], - aeronet_p75[valid_aeronet], - alpha=0.3, color='red') - - # Plot MERRA-21C seasonal cycle (black) - if 'merra_median' in monthly_stats.columns: - merra_median = monthly_stats['merra_median'] - merra_p25 = monthly_stats['merra_p25'] - merra_p75 = monthly_stats['merra_p75'] - - valid_merra = ~merra_median.isna() - if valid_merra.any(): - ax.plot(months[valid_merra], merra_median[valid_merra], - 'ko-', linewidth=2, markersize=6, label='MERRA-21C', alpha=0.8) - ax.fill_between(months[valid_merra], - merra_p25[valid_merra], - merra_p75[valid_merra], - alpha=0.3, color='black') - - # Plot MERRA-2 seasonal cycle (blue) - if 'merra2_median' in monthly_stats.columns: - merra2_median = monthly_stats['merra2_median'] - merra2_p25 = monthly_stats['merra2_p25'] - merra2_p75 = monthly_stats['merra2_p75'] - - valid_merra2 = ~merra2_median.isna() - if valid_merra2.any(): - ax.plot(months[valid_merra2], merra2_median[valid_merra2], - 'bo-', linewidth=2, markersize=6, label='MERRA-2', alpha=0.8) - ax.fill_between(months[valid_merra2], - merra2_p25[valid_merra2], - merra2_p75[valid_merra2], - alpha=0.3, color='blue') - - # Format axes - ax.set_xlim(0.5, 12.5) - ax.set_xticks(range(1, 13)) - ax.set_xticklabels(month_names) - ax.set_xlabel('Month', fontsize=16) - ax.set_ylabel('Angstrom Exponent', fontsize=16) - ax.set_title('(b) Seasonal Cycle', fontsize=16, pad=10) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - ax.legend(fontsize=12, loc='upper right') - - def plot_kde_panel(self, ax, valid_data, station_info, model_col, model_name, global_min, global_max, vmin=None, vmax=None): - """Plot a single KDE panel for model vs AERONET comparison""" - if len(valid_data) < 20: - ax.text(0.5, 0.5, f"Insufficient data\n({len(valid_data)} points)", - ha='center', va='center', fontsize=14, transform=ax.transAxes) - stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data)} points" - else: - # Log transform and calculate stats - aeronet_log = np.log10(valid_data['aeronet_aod_550'] + 0.01) - model_log = np.log10(valid_data[model_col] + 0.01) - correlation, bias, rmse = self.calculate_statistics(aeronet_log, model_log) - - # Create KDE plot - try: - data_points = np.vstack([aeronet_log, model_log]) - kde = gaussian_kde(data_points) - - xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] - positions = np.vstack([xx.ravel(), yy.ravel()]) - density = kde(positions).reshape(xx.shape) - - # Apply consistent density scaling if provided - if vmin is not None and vmax is not None: - # Use global levels - f_range = vmax - vmin - threshold_factor = 0.2 - adjusted_min = vmin + threshold_factor * f_range - levels = np.linspace(adjusted_min, vmax, 15) - contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, - alpha=0.95, extend='min', vmin=adjusted_min, vmax=vmax) - else: - f_min = np.min(density) - f_max = np.max(density) - f_range = f_max - f_min - threshold_factor = 0.2 - adjusted_min = f_min + threshold_factor * f_range - levels = np.linspace(adjusted_min, f_max, 15) - contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, - alpha=0.95, extend='min') - - contour = ax.contour(xx, yy, density, colors='black', alpha=0.6, linewidths=0.8) - - cbar = plt.colorbar(contourf, ax=ax, shrink=0.8, extend='min') - cbar.set_label('Density', fontsize=14) - cbar.ax.tick_params(labelsize=12) - - ax.set_xlim(global_min, global_max) - ax.set_ylim(global_min, global_max) - except: - ax.scatter(aeronet_log, model_log, alpha=0.6, s=20) - - # Add 1:1 line - ax.plot([global_min, global_max], [global_min, global_max], 'r--', linewidth=2, alpha=0.8) - - # Format stats - corr_text = f"r = {correlation:.3f}" if not np.isnan(correlation) else "r = N/A" - bias_text = f"bias = {bias:.3f}" if not np.isnan(bias) else "bias = N/A" - rmse_text = f"RMSE = {rmse:.3f}" if not np.isnan(rmse) else "RMSE = N/A" - - stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data):,} points\n{corr_text}\n{bias_text}\n{rmse_text}" - - # Format axes - def log_to_aod_formatter(x, pos): - aod_val = 10**x - 0.01 - if aod_val < 0.001: return f'{aod_val:.4f}' - elif aod_val < 0.01: return f'{aod_val:.3f}' - elif aod_val < 0.1: return f'{aod_val:.2f}' - else: return f'{aod_val:.1f}' - - ax.xaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) - ax.yaxis.set_major_formatter(ticker.FuncFormatter(log_to_aod_formatter)) - ax.set_xlabel('AERONET AOD', fontsize=16) - ax.set_ylabel(f'{model_name} AOD', fontsize=16) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - - # Add stats box - ax.text(0.97, 0.03, stats_text, transform=ax.transAxes, ha='right', va='bottom', - fontsize=12, bbox=dict(facecolor='white', alpha=0.8, pad=0.5, edgecolor='black')) - - return density if 'density' in locals() else None - - def plot_angstrom_kde_panel(self, ax, valid_data, station_info, model_col, model_name, global_min, global_max, vmin=None, vmax=None): - """Plot a single Angstrom KDE panel for model vs AERONET comparison""" - if len(valid_data) < 20: - ax.text(0.5, 0.5, f"Insufficient data\n({len(valid_data)} points)", - ha='center', va='center', fontsize=14, transform=ax.transAxes) - stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data)} points" - else: - # No log transform for Angstrom data - aeronet_angstrom = valid_data['aeronet_angstrom'] - model_angstrom = valid_data[model_col] - correlation, bias, rmse = self.calculate_statistics(aeronet_angstrom, model_angstrom) - - # Create KDE plot - try: - data_points = np.vstack([aeronet_angstrom, model_angstrom]) - kde = gaussian_kde(data_points) - - xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] - positions = np.vstack([xx.ravel(), yy.ravel()]) - density = kde(positions).reshape(xx.shape) - - # Apply consistent density scaling if provided - if vmin is not None and vmax is not None: - f_range = vmax - vmin - threshold_factor = 0.2 - adjusted_min = vmin + threshold_factor * f_range - levels = np.linspace(adjusted_min, vmax, 15) - contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, - alpha=0.95, extend='min', vmin=adjusted_min, vmax=vmax) - else: - f_min = np.min(density) - f_max = np.max(density) - f_range = f_max - f_min - threshold_factor = 0.2 - adjusted_min = f_min + threshold_factor * f_range - levels = np.linspace(adjusted_min, f_max, 15) - contourf = ax.contourf(xx, yy, density, levels=levels, cmap=self.white_viridis, - alpha=0.95, extend='min') - - contour = ax.contour(xx, yy, density, colors='black', alpha=0.6, linewidths=0.8) - - cbar = plt.colorbar(contourf, ax=ax, shrink=0.8, extend='min') - cbar.set_label('Density', fontsize=14) - cbar.ax.tick_params(labelsize=12) - - ax.set_xlim(global_min, global_max) - ax.set_ylim(global_min, global_max) - except: - ax.scatter(aeronet_angstrom, model_angstrom, alpha=0.6, s=20) - - # Add 1:1 line - ax.plot([global_min, global_max], [global_min, global_max], 'r--', linewidth=2, alpha=0.8) - - # Format stats - corr_text = f"r = {correlation:.3f}" if not np.isnan(correlation) else "r = N/A" - bias_text = f"bias = {bias:.3f}" if not np.isnan(bias) else "bias = N/A" - rmse_text = f"RMSE = {rmse:.3f}" if not np.isnan(rmse) else "RMSE = N/A" - - stats_text = f"{station_info['name']}\n{station_info['lat']:.2f}ยฐN, {station_info['lon']:.2f}ยฐE\n{len(valid_data):,} points\n{corr_text}\n{bias_text}\n{rmse_text}" - - # Format axes - ax.set_xlabel('AERONET Angstrom Exponent', fontsize=16) - ax.set_ylabel(f'{model_name} Angstrom Exponent', fontsize=16) - ax.tick_params(labelsize=14) - ax.grid(True, alpha=0.3) - - # Add stats box - ax.text(0.97, 0.03, stats_text, transform=ax.transAxes, ha='right', va='bottom', - fontsize=12, bbox=dict(facecolor='white', alpha=0.8, pad=0.5, edgecolor='black')) - - return density if 'density' in locals() else None - - def create_station_figure(self, station_name): - """Main function to create 4-panel station analysis figure""" - # Check if station exists - station_mask = self.station_metrics['station'] == station_name - if not station_mask.any(): - print(f"Station '{station_name}' not found") - return False - - # Load data - data, message = self.load_station_data(station_name) - if data is None: - print(f"Failed to load data for {station_name}: {message}") - return False - - # Check which model datasets are available - has_m21c = 'merra_aod_550' in data.columns - has_m2 = 'merra2_aod_550' in data.columns - - if not (has_m21c or has_m2): - print(f"No MERRA data found for {station_name}") - return False - - # Get station info - station_info = { - 'name': station_name.replace('_', ' '), - 'lat': self.station_metrics[station_mask].iloc[0]['latitude'], - 'lon': self.station_metrics[station_mask].iloc[0]['longitude'] - } - - # Prepare data - quality_mask = self.apply_quality_filters(data) - daily_data = self.create_daily_timeseries(data) - valid_data = data[quality_mask] - monthly_stats = self.calculate_seasonal_cycle(data) - - if len(valid_data) == 0: - print(f"No valid data after quality filtering for {station_name}") - return False - - # Get consistent axis limits for KDE plots - global_min, global_max = self.get_axis_limits(valid_data) - - # Calculate global density range for consistent colorbars - all_densities = [] - if has_m21c and len(valid_data) >= 20: - try: - aeronet_log = np.log10(valid_data['aeronet_aod_550'] + 0.01) - m21c_log = np.log10(valid_data['merra_aod_550'] + 0.01) - data_points = np.vstack([aeronet_log, m21c_log]) - kde = gaussian_kde(data_points) - xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] - positions = np.vstack([xx.ravel(), yy.ravel()]) - density = kde(positions).reshape(xx.shape) - all_densities.extend(density.flatten()) - except: - pass - - if has_m2 and len(valid_data) >= 20: - try: - aeronet_log = np.log10(valid_data['aeronet_aod_550'] + 0.01) - m2_log = np.log10(valid_data['merra2_aod_550'] + 0.01) - data_points = np.vstack([aeronet_log, m2_log]) - kde = gaussian_kde(data_points) - xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] - positions = np.vstack([xx.ravel(), yy.ravel()]) - density = kde(positions).reshape(xx.shape) - all_densities.extend(density.flatten()) - except: - pass - - # Calculate global density limits - if all_densities: - global_vmin = np.min(all_densities) - global_vmax = np.max(all_densities) - else: - global_vmin = global_vmax = None - - # Create figure with 2x2 layout - fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(20, 16)) - - # Top row: Time series (left) and Seasonal cycle (right) - self.plot_timeseries(ax1, daily_data, station_name) - self.plot_seasonal_cycle(ax2, monthly_stats, station_info) - - # Bottom row: KDE plots - if has_m21c and has_m2: - # Both datasets available - self.plot_kde_panel(ax3, valid_data, station_info, 'merra_aod_550', 'MERRA-21C', - global_min, global_max, global_vmin, global_vmax) - self.plot_kde_panel(ax4, valid_data, station_info, 'merra2_aod_550', 'MERRA-2', - global_min, global_max, global_vmin, global_vmax) - ax3.set_title('(c) MERRA-21C vs AERONET AOD', fontsize=16, pad=10) - ax4.set_title('(d) MERRA-2 vs AERONET AOD', fontsize=16, pad=10) - elif has_m21c: - # Only MERRA-21C available - self.plot_kde_panel(ax3, valid_data, station_info, 'merra_aod_550', 'MERRA-21C', - global_min, global_max) - ax4.axis('off') - ax4.text(0.5, 0.5, 'MERRA-2 data\nnot available', ha='center', va='center', - fontsize=16, transform=ax4.transAxes) - ax3.set_title('(c) MERRA-21C vs AERONET AOD', fontsize=16, pad=10) - elif has_m2: - # Only MERRA-2 available - ax3.axis('off') - ax3.text(0.5, 0.5, 'MERRA-21C data\nnot available', ha='center', va='center', - fontsize=16, transform=ax3.transAxes) - self.plot_kde_panel(ax4, valid_data, station_info, 'merra2_aod_550', 'MERRA-2', - global_min, global_max) - ax4.set_title('(d) MERRA-2 vs AERONET AOD', fontsize=16, pad=10) - - # Overall formatting - year_str = f" ({self.years[0]})" if self.years and len(self.years) == 1 else f" ({min(self.years)}-{max(self.years)})" if self.years else "" - fig.suptitle(f'Station Analysis: {station_info["name"]}{year_str}', fontsize=22, fontweight='bold', y=0.95) - plt.tight_layout(rect=[0, 0, 1, 0.92]) - - # Save - station_filename = station_name.replace('_', '-').lower() - year_suffix = f"_{self.years[0]}_{self.years[-1]}" if self.years and len(self.years) > 1 else f"_{self.years[0]}" if self.years else "" - plt.savefig(os.path.join(self.output_dir, f'station_analysis_{station_filename}{year_suffix}_withm2.png'), - dpi=300, bbox_inches='tight') - plt.close() - - print(f"Generated 4-panel station analysis: station_analysis_{station_filename}{year_suffix}_withm2.png") - return True - - def create_angstrom_figure(self, station_name): - """Main function to create 4-panel Angstrom Exponent analysis figure""" - # Check if station exists - station_mask = self.station_metrics['station'] == station_name - if not station_mask.any(): - print(f"Station '{station_name}' not found") - return False - - # Load data - data, message = self.load_station_data(station_name) - if data is None: - print(f"Failed to load data for {station_name}: {message}") - return False - - # Check if Angstrom columns exist - has_aeronet = 'aeronet_angstrom' in data.columns - has_m21c = 'merra_angstrom' in data.columns - has_m2 = 'merra2_angstrom' in data.columns - - if not has_aeronet: - print(f"Missing AERONET Angstrom Exponent data for {station_name}") - return False - - if not (has_m21c or has_m2): - print(f"No MERRA Angstrom Exponent data found for {station_name}") - return False - - # Get station info - station_info = { - 'name': station_name.replace('_', ' '), - 'lat': self.station_metrics[station_mask].iloc[0]['latitude'], - 'lon': self.station_metrics[station_mask].iloc[0]['longitude'] - } - - # Prepare data - quality_mask = self.apply_angstrom_quality_filters(data) - daily_data = self.create_angstrom_daily_timeseries(data) - valid_data = data[quality_mask] - monthly_stats = self.calculate_angstrom_seasonal_cycle(data) - - if len(valid_data) == 0: - print(f"No valid Angstrom data after quality filtering for {station_name}") - return False - - # Get consistent axis limits for KDE plots (no log transform for Angstrom) - aeronet_vals = valid_data['aeronet_angstrom'] - all_model_values = [] - - if has_m21c: - all_model_values.extend(valid_data['merra_angstrom']) - if has_m2: - all_model_values.extend(valid_data['merra2_angstrom']) - - all_values = np.concatenate([aeronet_vals, all_model_values]) - global_min = np.min(all_values) - 0.1 * (np.max(all_values) - np.min(all_values)) - global_max = np.max(all_values) + 0.1 * (np.max(all_values) - np.min(all_values)) - - # Calculate global density range for consistent colorbars - all_densities = [] - if has_m21c and len(valid_data) >= 20: - try: - data_points = np.vstack([valid_data['aeronet_angstrom'], valid_data['merra_angstrom']]) - kde = gaussian_kde(data_points) - xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] - positions = np.vstack([xx.ravel(), yy.ravel()]) - density = kde(positions).reshape(xx.shape) - all_densities.extend(density.flatten()) - except: - pass - - if has_m2 and len(valid_data) >= 20: - try: - data_points = np.vstack([valid_data['aeronet_angstrom'], valid_data['merra2_angstrom']]) - kde = gaussian_kde(data_points) - xx, yy = np.mgrid[global_min:global_max:50j, global_min:global_max:50j] - positions = np.vstack([xx.ravel(), yy.ravel()]) - density = kde(positions).reshape(xx.shape) - all_densities.extend(density.flatten()) - except: - pass - - # Calculate global density limits - if all_densities: - global_vmin = np.min(all_densities) - global_vmax = np.max(all_densities) - else: - global_vmin = global_vmax = None - - # Create figure with 2x2 layout - fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(20, 16)) - - # Top row: Time series (left) and Seasonal cycle (right) - self.plot_angstrom_timeseries(ax1, daily_data, station_name) - self.plot_angstrom_seasonal_cycle(ax2, monthly_stats, station_info) - - # Bottom row: KDE plots for Angstrom - if has_m21c and has_m2: - # Both datasets available - self.plot_angstrom_kde_panel(ax3, valid_data, station_info, 'merra_angstrom', 'MERRA-21C', - global_min, global_max, global_vmin, global_vmax) - self.plot_angstrom_kde_panel(ax4, valid_data, station_info, 'merra2_angstrom', 'MERRA-2', - global_min, global_max, global_vmin, global_vmax) - ax3.set_title('(c) MERRA-21C vs AERONET Angstrom', fontsize=16, pad=10) - ax4.set_title('(d) MERRA-2 vs AERONET Angstrom', fontsize=16, pad=10) - elif has_m21c: - # Only MERRA-21C available - self.plot_angstrom_kde_panel(ax3, valid_data, station_info, 'merra_angstrom', 'MERRA-21C', - global_min, global_max) - ax4.axis('off') - ax4.text(0.5, 0.5, 'MERRA-2 Angstrom data\nnot available', ha='center', va='center', - fontsize=16, transform=ax4.transAxes) - ax3.set_title('(c) MERRA-21C vs AERONET Angstrom', fontsize=16, pad=10) - elif has_m2: - # Only MERRA-2 available - ax3.axis('off') - ax3.text(0.5, 0.5, 'MERRA-21C Angstrom data\nnot available', ha='center', va='center', - fontsize=16, transform=ax3.transAxes) - self.plot_angstrom_kde_panel(ax4, valid_data, station_info, 'merra2_angstrom', 'MERRA-2', - global_min, global_max) - ax4.set_title('(d) MERRA-2 vs AERONET Angstrom', fontsize=16, pad=10) - - # Overall formatting - year_str = f" ({self.years[0]})" if self.years and len(self.years) == 1 else f" ({min(self.years)}-{max(self.years)})" if self.years else "" - fig.suptitle(f'Angstrom Exponent Analysis: {station_info["name"]}{year_str}', fontsize=22, fontweight='bold', y=0.95) - plt.tight_layout(rect=[0, 0, 1, 0.92]) - - # Save - station_filename = station_name.replace('_', '-').lower() - year_suffix = f"_{self.years[0]}_{self.years[-1]}" if self.years and len(self.years) > 1 else f"_{self.years[0]}" if self.years else "" - plt.savefig(os.path.join(self.output_dir, f'angstrom_analysis_{station_filename}{year_suffix}_withm2.png'), - dpi=300, bbox_inches='tight') - plt.close() - - print(f"Generated 4-panel Angstrom analysis: angstrom_analysis_{station_filename}{year_suffix}_withm2.png") - return True diff --git a/src/pyobs/evaluation/MODIS_NNR/blendmodisstreams.py b/src/pyobs/evaluation/MODIS_NNR/blendmodisstreams.py deleted file mode 100644 index 8c22955..0000000 --- a/src/pyobs/evaluation/MODIS_NNR/blendmodisstreams.py +++ /dev/null @@ -1,320 +0,0 @@ -#!/usr/bin/env python3 -""" -Processes MODIS satellite AOD data by blending deep blue, land, and ocean retrievals based on observation count. -Example usage: python blendmodisstreams.py -y 2024 -m 1 -s MOD04 -""" - -import argparse -import calendar -import os -import sys -from pathlib import Path -import numpy as np -import netCDF4 as nc -from datetime import datetime -from concurrent.futures import ProcessPoolExecutor, as_completed -import multiprocessing as mp -from functools import partial - -def get_coordinates(satellite='MOD04'): - """ - Get longitude and latitude coordinates from reference file. - - Args: - satellite (str): Satellite identifier (MOD04 or MYD04) - - Returns: - tuple: (longitude, latitude) arrays - """ - ref_file = f'/css/gmao/dp/gds/AeroObs/nnr_003_{satellite}_061/Level3/Y2020/M02/nnr_003.{satellite}_L3a.ocean.20200224_2100z.nc4' - - try: - with nc.Dataset(ref_file, 'r') as ncfile: - lon = ncfile.variables['lon'][:] - lat = ncfile.variables['lat'][:] - return lon, lat - except (FileNotFoundError, KeyError) as e: - print(f"Error reading coordinates from {ref_file}: {e}") - sys.exit(1) - -def process_single_timestep(args_tuple): - """ - Process single timestep: - 1. Create weighted average of land+deep where they overlap - 2. Add in ocean data - """ - year, month, day, hour, satellite, output_dir, lon, lat = args_tuple - - # Format strings - year_str = f"{year:04d}" - month_str = f"{month:02d}" - day_str = f"{day:02d}" - hour_str = f"{hour:02d}" - - # Create output directory structure - output_path = Path(output_dir) / f"nnr_003_blend/{satellite}/Y{year_str}/M{month_str}" - output_path.mkdir(parents=True, exist_ok=True) - - # Construct file paths - base_path = f"/css/gmao/dp/gds/AeroObs/nnr_003_{satellite}_061/Level3/Y{year_str}/M{month_str}" - - deep_file = f"{base_path}/nnr_003.{satellite}_L3a.deep.{year_str}{month_str}{day_str}_{hour_str}00z.nc4" - land_file = f"{base_path}/nnr_003.{satellite}_L3a.land.{year_str}{month_str}{day_str}_{hour_str}00z.nc4" - ocean_file = f"{base_path}/nnr_003.{satellite}_L3a.ocean.{year_str}{month_str}{day_str}_{hour_str}00z.nc4" - - try: - # Read data - with nc.Dataset(deep_file, 'r') as ncfile: - deep = ncfile.variables['tau_'][:] - deep_nobs = ncfile.variables['count_tau_'][:] - - with nc.Dataset(land_file, 'r') as ncfile: - land = ncfile.variables['tau_'][:] - land_nobs = ncfile.variables['count_tau_'][:] - - with nc.Dataset(ocean_file, 'r') as ncfile: - ocean = ncfile.variables['tau_'][:] - - # Step 1: Create weighted average of land and deep (following MATLAB logic) - deep_processed = deep.copy() - land_processed = land.copy() - - # Set to 0 where no observations (MATLAB: deep(deep_nobs==0)=0) - deep_processed[deep_nobs == 0] = 0 - land_processed[land_nobs == 0] = 0 - - # Calculate weighted blend of land and deep - total_land_deep_obs = deep_nobs + land_nobs - land_deep_blend = np.full_like(deep, np.nan) - - # Only calculate where we have observations - mask = total_land_deep_obs > 0 - if np.sum(mask) > 0: - land_deep_blend[mask] = ((deep_processed[mask] * deep_nobs[mask]) + - (land_processed[mask] * land_nobs[mask])) / total_land_deep_obs[mask] - - # Set calculated zeros to NaN (MATLAB: blend(blend==0)=nan) - land_deep_blend[land_deep_blend == 0] = np.nan - - # Step 2: Start with land+deep blend as the foundation - final_blend = land_deep_blend.copy() - - # Step 3: Add ocean data ONLY where land+deep is NaN (no land/deep data available) - land_deep_missing = np.isnan(land_deep_blend) - ocean_available = ~np.isnan(ocean) - use_ocean = land_deep_missing & ocean_available - - final_blend[use_ocean] = ocean[use_ocean] - - # Debug output - print(f"=== BLEND {year_str}{month_str}{day_str}_{hour_str} ===") - land_deep_valid = ~np.isnan(land_deep_blend) - print(f"Land+Deep blend: {np.sum(land_deep_valid)} pixels") - print(f"Ocean fills gaps: {np.sum(use_ocean)} pixels") - print(f"Total combined: {np.sum(~np.isnan(final_blend))}") - - if np.sum(~np.isnan(final_blend)) > 0: - print(f"Final range: {np.nanmin(final_blend):.6f} to {np.nanmax(final_blend):.6f}") - - # Create output filename - output_file = output_path / f"nnr_003.{satellite}_L3a.blend.{year_str}{month_str}{day_str}_{hour_str}00z.nc4" - - if output_file.exists(): - output_file.unlink() - - # Write NetCDF file - with nc.Dataset(output_file, 'w') as ncfile: - ncfile.createDimension('lon', len(lon)) - ncfile.createDimension('lat', len(lat)) - - tau_var = ncfile.createVariable('tau', 'f4', ('lat', 'lon'), fill_value=np.nan) - lon_var = ncfile.createVariable('lon', 'f4', ('lon',)) - lat_var = ncfile.createVariable('lat', 'f4', ('lat',)) - - tau_var[:] = final_blend - lon_var[:] = lon - lat_var[:] = lat - - tau_var.long_name = "Aerosol Optical Depth at 550nm (blended)" - tau_var.units = "1" - lon_var.long_name = "Longitude" - lon_var.units = "degrees_east" - lat_var.long_name = "Latitude" - lat_var.units = "degrees_north" - - ncfile.title = f"Blended AOD from {satellite}" - ncfile.source = "Weighted land+deep blend with ocean gap-filling" - ncfile.created = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - - return True, f"Processed: {output_file.name}" - - except FileNotFoundError as e: - return False, f"Missing input file: {str(e)}" - except Exception as e: - return False, f"Error processing: {str(e)}" - -def process_aod_blend_parallel(year, month, satellite, output_dir, max_workers=None): - """ - Process AOD blending for a given year, month, and satellite using parallel processing. - - Args: - year (int): Year to process - month (int): Month to process (1-12) - satellite (str): Satellite identifier (MOD04 or MYD04) - output_dir (str): Output directory path - max_workers (int): Maximum number of parallel workers (None for auto) - """ - # Get month length - month_length = calendar.monthrange(year, month)[1] - - # Get coordinates (only once) - lon, lat = get_coordinates(satellite) - - # Create list of all timesteps to process - timesteps = [] - for day in range(1, month_length + 1): - for hour in range(0, 24, 3): # 0, 3, 6, 9, 12, 15, 18, 21 - timesteps.append((year, month, day, hour, satellite, output_dir, lon, lat)) - - # Determine number of workers - if max_workers is None: - max_workers = min(mp.cpu_count(), len(timesteps)) - - print(f"Processing {len(timesteps)} timesteps using {max_workers} parallel workers...") - - # Process in parallel - successful = 0 - failed = 0 - - with ProcessPoolExecutor(max_workers=max_workers) as executor: - # Submit all jobs - future_to_timestep = { - executor.submit(process_single_timestep, timestep): timestep - for timestep in timesteps - } - - # Process completed jobs - for future in as_completed(future_to_timestep): - timestep = future_to_timestep[future] - try: - success, message = future.result() - if success: - successful += 1 - print(f"โœ“ {message}") - else: - failed += 1 - print(f"โœ— {message}") - except Exception as e: - failed += 1 - year, month, day, hour = timestep[:4] - print(f"โœ— Unexpected error for {year:04d}{month:02d}{day:02d}_{hour:02d}: {e}") - - print(f"\nProcessing summary:") - print(f" Successful: {successful}") - print(f" Failed: {failed}") - print(f" Total: {len(timesteps)}") - -def main(): - """Main function with command-line argument parsing.""" - parser = argparse.ArgumentParser( - description="Blend satellite AOD retrievals from deep blue, land, and ocean products (parallel version)", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - - parser.add_argument( - '--year', '-y', - type=int, - required=True, - help='Year to process (e.g., 2024)' - ) - - parser.add_argument( - '--month', '-m', - type=int, - required=True, - choices=range(1, 13), - help='Month to process (1-12)' - ) - - parser.add_argument( - '--satellite', '-s', - type=str, - required=True, - choices=['MOD04', 'MYD04'], - help='Satellite identifier (MOD04 for Terra, MYD04 for Aqua)' - ) - - parser.add_argument( - '--output', '-o', - type=str, - default='reprocessedblend', - help='Base output directory path (default: ./reprocessedblend/)' - ) - - parser.add_argument( - '--workers', '-w', - type=int, - default=None, - help='Number of parallel workers (default: auto-detect based on CPU count)' - ) - - args = parser.parse_args() - - # Create base output directory if it doesn't exist - output_path = Path(args.output) - if not output_path.exists(): - try: - output_path.mkdir(parents=True) - except Exception as e: - print(f"Error creating output directory {output_path}: {e}") - sys.exit(1) - - # Satellite info - sat_info = { - 'MOD04': 'Terra', - 'MYD04': 'Aqua' - } - - # Determine worker count - if args.workers is None: - workers = mp.cpu_count() - worker_text = f"{workers} (auto-detected)" - else: - workers = args.workers - worker_text = f"{workers} (specified)" - - # Show full output path - year_str = f"{args.year:04d}" - month_str = f"{args.month:02d}" - full_output_path = Path(args.output) / f"{args.satellite}/Y{year_str}/M{month_str}" - - print(f"Processing AOD blending for:") - print(f" Year: {args.year}") - print(f" Month: {args.month}") - print(f" Satellite: {args.satellite} ({sat_info[args.satellite]})") - print(f" Base output: {args.output}") - print(f" Full output path: {full_output_path}") - print(f" Workers: {worker_text}") - print() - - # Process the data - try: - start_time = datetime.now() - - # Pass the base directory; the function will create the full structure - base_output = str(output_path.parent) if args.output == 'reprocessedblend' else args.output - process_aod_blend_parallel(args.year, args.month, args.satellite, base_output, args.workers) - - end_time = datetime.now() - duration = end_time - start_time - - print(f"\nCompleted processing for {args.year}-{args.month:02d} {args.satellite} ({sat_info[args.satellite]})") - print(f"Total processing time: {duration}") - print(f"Output files saved to: {full_output_path}") - - except Exception as e: - print(f"Error during processing: {e}") - sys.exit(1) - -if __name__ == "__main__": - main() diff --git a/src/pyobs/evaluation/MODIS_NNR/monthlygeossample_speciated.py b/src/pyobs/evaluation/MODIS_NNR/monthlygeossample_speciated.py deleted file mode 100755 index fc70d48..0000000 --- a/src/pyobs/evaluation/MODIS_NNR/monthlygeossample_speciated.py +++ /dev/null @@ -1,474 +0,0 @@ -#!/usr/bin/env python -""" -Monthly AOD Model-Observation Comparison Tool -A script to compare GEOS model AOD to monthly mean MODIS NNR Retrievals -Original by Sampa Das (May 2020), Refactored -""" - -import numpy as np -import os -import logging -from datetime import datetime -from scipy.interpolate import RegularGridInterpolator -from netCDF4 import Dataset - -# Constants -MODEL_LAT_SIZE = 361 -MODEL_LON_SIZE = 720 -DAILY_FILES = 8 # number of MODIS files per day -TIME_STEPS = range(0, 24, 3) -FILL_VALUE = 999.0 -AOD_THRESHOLD = 100.0 - -# Set up logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger(__name__) - - -class AODProcessor: - """Class to handle AOD processing operations""" - - def __init__(self): - self.variable_names = ['tot', 'bc', 'oc', 'br', 'ss', 'su', 'du', 'ni'] - self.netcdf_mapping = { - 'tot': 'TOTEXTTAU', - 'bc': 'BCEXTTAU', - 'oc': 'OCEXTTAU', - 'br': 'BREXTTAU', - 'ss': 'SSEXTTAU', - 'su': 'SUEXTTAU', - 'du': 'DUEXTTAU', - 'ni': 'NIEXTTAU' - } - - def initialize_arrays(self, shape): - """Initialize arrays with NaN values""" - arrays = {} - for name in self.variable_names: - arrays[name] = np.full(shape, np.nan, dtype=np.float32) - return arrays - - def validate_inputs(self, yy, mm, EXPID, sat): - """Validate input parameters""" - if not (1 <= mm <= 12): - raise ValueError(f"Month must be between 1-12, got {mm}") - if yy < 1900 or yy > 2100: - raise ValueError(f"Year seems unrealistic: {yy}") - if not EXPID.strip(): - raise ValueError("EXPID cannot be empty") - if not sat.strip(): - raise ValueError("Satellite identifier cannot be empty") - - def check_directories(self, dir_obs, dirm): - """Check if required directories exist""" - if not os.path.exists(dir_obs): - raise FileNotFoundError(f"Observation directory not found: {dir_obs}") - if not os.path.exists(dirm): - raise FileNotFoundError(f"Model directory not found: {dirm}") - - def parse_file_dates(self, MOD_files): - """Parse start and end dates from file list""" - if len(MOD_files) < 2: - raise ValueError("Insufficient files found") - - try: - dds = int(MOD_files[0][30:32]) - dde = int(MOD_files[-1][30:32]) # Use last file instead of [-2] - return dds, dde - except (IndexError, ValueError) as e: - raise ValueError(f"Error parsing file names: {e}") - - def interpolate_observations(self, lons, lats, tau_nnr_L, lonm, latm): - """Interpolate observation data to model grid""" - # Replace NaN with fill value for interpolation - tau_for_interp = tau_nnr_L.copy() - tau_for_interp[np.isnan(tau_for_interp)] = FILL_VALUE - - # Create interpolator - interpolator = RegularGridInterpolator( - (lats, lons), tau_for_interp, - method='linear', - bounds_error=False, - fill_value=np.nan - ) - - # Create coordinate arrays for interpolation - lonm_grid, latm_grid = np.meshgrid(lonm, latm) - points = np.column_stack([latm_grid.ravel(), lonm_grid.ravel()]) - - # Perform interpolation - tau_obs_modelres = interpolator(points).reshape(latm_grid.shape) - - # Clean up unreasonable values - tau_obs_modelres[tau_obs_modelres >= AOD_THRESHOLD] = np.nan - - return tau_obs_modelres - - def process_single_timestep(self, nc_fileL, nc_fileM, lonm, latm): - """Process a single timestep of data""" - result = { - 'success': False, - 'tau_obs': None, - 'model_data': {} - } - - if not os.path.isfile(nc_fileL): - return result - - if not os.path.isfile(nc_fileM): - logger.warning(f"Model file missing: {nc_fileM}") - return result - - try: - # Read observation data - with Dataset(nc_fileL, 'r') as ncid: - lons = ncid.variables['lon'][:] - lats = ncid.variables['lat'][:] - tau_nnr_L = np.squeeze(ncid.variables['tau'][:]) - - # Read model data - with Dataset(nc_fileM, 'r') as ncid: - lonm = ncid.variables['lon'][:] - latm = ncid.variables['lat'][:] - - model_data = {} - for var_name in self.variable_names: - nc_var_name = self.netcdf_mapping[var_name] - model_data[var_name] = np.squeeze(ncid.variables[nc_var_name][:]) - - # Interpolate observations to model grid - tau_obs_modelres = self.interpolate_observations(lons, lats, tau_nnr_L, lonm, latm) - - # Apply observation mask to model data - for var_name in self.variable_names: - model_data[var_name][np.isnan(tau_obs_modelres)] = np.nan - - result.update({ - 'success': True, - 'tau_obs': tau_obs_modelres, - 'model_data': model_data - }) - - except Exception as e: - logger.error(f"Error processing files {nc_fileL}, {nc_fileM}: {e}") - - return result - - def process_daily_data(self, dd, dds, dir_obs, dirm, yy, mm, EXPID, sat, lonm, latm): - """Process data for a single day""" - hourly_shape = (MODEL_LAT_SIZE, MODEL_LON_SIZE, DAILY_FILES) - - # Initialize arrays - mod_arrays = self.initialize_arrays(hourly_shape) - tau_nnrLOD = np.full(hourly_shape, np.nan, dtype=np.float32) - - valid_timesteps = 0 - - for i, t in enumerate(TIME_STEPS): - nc_fileL = (f"{dir_obs}nnr_003.{sat}_L3a.blend." - f"{yy:04d}{mm:02d}{dd:02d}_{t:02d}00z.nc4") - nc_fileM = (f"{dirm}{EXPID}.inst2d_hwl_x." - f"{yy:04d}{mm:02d}{dd:02d}_{t:02d}00z.nc4") - - result = self.process_single_timestep(nc_fileL, nc_fileM, lonm, latm) - - if result['success']: - tau_nnrLOD[:, :, i] = result['tau_obs'] - for var_name in self.variable_names: - mod_arrays[var_name][:, :, i] = result['model_data'][var_name] - valid_timesteps += 1 - - if valid_timesteps == 0: - logger.warning(f"No valid data found for day {dd}") - - return mod_arrays, tau_nnrLOD - - def write_netcdf_output(self, filename, data_dict, lonm, latm, yy, mm, EXPID, sat): - """Write data to NetCDF file with proper metadata""" - try: - with Dataset(filename, mode='w', format='NETCDF4_CLASSIC') as ncfile: - # Create dimensions - ncfile.createDimension('lat', MODEL_LAT_SIZE) - ncfile.createDimension('lon', MODEL_LON_SIZE) - ncfile.createDimension('time', 1) - - # Variable definitions with metadata - var_info = { - 'MODtau': { - 'data': data_dict['obs'], - 'long_name': 'MODIS AOD', - 'units': '1', - 'description': 'Monthly mean MODIS Neural Network Retrieval AOD' - }, - 'GEOStau': { - 'data': data_dict['tot'], - 'long_name': 'GEOS Total AOD', - 'units': '1', - 'description': 'GEOS model total aerosol optical depth' - }, - 'bcexttau': { - 'data': data_dict['bc'], - 'long_name': 'Black Carbon AOD', - 'units': '1' - }, - 'ocexttau': { - 'data': data_dict['oc'], - 'long_name': 'Organic Carbon AOD', - 'units': '1' - }, - 'brexttau': { - 'data': data_dict['br'], - 'long_name': 'Brown Carbon AOD', - 'units': '1' - }, - 'ssexttau': { - 'data': data_dict['ss'], - 'long_name': 'Sea Salt AOD', - 'units': '1' - }, - 'duexttau': { - 'data': data_dict['du'], - 'long_name': 'Dust AOD', - 'units': '1' - }, - 'suexttau': { - 'data': data_dict['su'], - 'long_name': 'Sulfate AOD', - 'units': '1' - }, - 'niexttau': { - 'data': data_dict['ni'], - 'long_name': 'Nitrate AOD', - 'units': '1' - } - } - - # Create data variables - for var_name, info in var_info.items(): - var = ncfile.createVariable( - var_name, 'f', ('lat', 'lon'), - compression='zlib', complevel=4, - fill_value=np.nan - ) - var[:] = info['data'] - var.long_name = info['long_name'] - var.units = info['units'] - if 'description' in info: - var.description = info['description'] - - # Coordinate variables - lat_var = ncfile.createVariable('lat', np.float32, ('lat',)) - lat_var.units = 'degrees_north' - lat_var.long_name = 'latitude' - lat_var.standard_name = 'latitude' - lat_var[:] = latm - - lon_var = ncfile.createVariable('lon', np.float32, ('lon',)) - lon_var.units = 'degrees_east' - lon_var.long_name = 'longitude' - lon_var.standard_name = 'longitude' - lon_var[:] = lonm - - time_var = ncfile.createVariable('time', np.float64, ('time',)) - time_var.units = f'hours since {yy}-{mm:02d}-01' - time_var.long_name = 'time' - time_var.standard_name = 'time' - time_var[:] = [0] - - # Global attributes - ncfile.title = f'Monthly AOD comparison for {yy}-{mm:02d}' - ncfile.institution = 'NASA GSFC' - ncfile.source = f'GEOS model experiment {EXPID}' - ncfile.satellite_data = f'{sat} MODIS Neural Network Retrievals' - ncfile.history = f'Created on {datetime.now().isoformat()}' - ncfile.conventions = 'CF-1.6' - ncfile.contact = 'geosaerosols@lists.nasa.gov' - - logger.info(f"Successfully wrote: {filename}") - - except Exception as e: - logger.error(f"Error writing NetCDF file {filename}: {e}") - raise - - -def monthlyAOD_mod_obs(yy, mm, EXPID, sat, output_dir="./"): - """ - Main function to process monthly AOD model-observation comparison - - Parameters: - ----------- - yy : int - Year - mm : int - Month (1-12) - EXPID : str - Model experiment ID - sat : str - Satellite identifier (e.g., 'MYD04', 'MOD04') - output_dir : str - Output directory for results - - Returns: - -------- - tuple : (tau_nnrLOD_mm, Mod_TotAOD_mm, lonm, latm) - Monthly mean observations, model total AOD, and coordinates - """ - - processor = AODProcessor() - - # Validate inputs - processor.validate_inputs(yy, mm, EXPID, sat) - - # Set up directories - dir_obs = f"/css/gmao/dp/gds/AeroObs/nnr_003_blend/{sat}/Y{yy:04d}/M{mm:02d}/" - dirm = f"/discover/nobackup/acollow/geos_aerosols/acollow/{EXPID}/holding/inst2d_hwl_x/{yy:04d}{mm:02d}/" - - # Check directories exist - processor.check_directories(dir_obs, dirm) - - # Get file list and parse dates - try: - MOD_files = sorted([f for f in os.listdir(dir_obs) if f.endswith('.nc4')]) - dds, dde = processor.parse_file_dates(MOD_files) - logger.info(f"Processing {yy}-{mm:02d}: Days {dds} to {dde}") - except Exception as e: - logger.error(f"Error processing file list: {e}") - raise - - # Initialize arrays - num_days = dde - dds + 1 - daily_shape = (MODEL_LAT_SIZE, MODEL_LON_SIZE, num_days) - - mod_arrays_dd = processor.initialize_arrays(daily_shape) - tau_nnrLOD_dd = np.full(daily_shape, np.nan, dtype=np.float32) - - # Get coordinate arrays from first available file - lonm, latm = None, None - for dd in range(dds, dde + 1): - for t in TIME_STEPS: - nc_fileM = (f"{dirm}{EXPID}.inst2d_hwl_x." - f"{yy:04d}{mm:02d}{dd:02d}_{t:02d}00z.nc4") - if os.path.isfile(nc_fileM): - try: - with Dataset(nc_fileM, 'r') as ncid: - lonm = ncid.variables['lon'][:] - latm = ncid.variables['lat'][:] - break - except Exception as e: - logger.warning(f"Error reading coordinates from {nc_fileM}: {e}") - continue - if lonm is not None: - break - - if lonm is None: - raise FileNotFoundError("Could not find valid model file to read coordinates") - - # Process each day - valid_days = 0 - for dd in range(dds, dde + 1): - try: - day_index = dd - dds - mod_arrays, tau_nnrLOD = processor.process_daily_data( - dd, dds, dir_obs, dirm, yy, mm, EXPID, sat, lonm, latm - ) - - # Store daily means - for var_name in processor.variable_names: - mod_arrays_dd[var_name][:, :, day_index] = np.nanmean(mod_arrays[var_name], axis=2) - tau_nnrLOD_dd[:, :, day_index] = np.nanmean(tau_nnrLOD, axis=2) - - valid_days += 1 - logger.info(f"Processed day {dd}") - - except Exception as e: - logger.error(f"Error processing day {dd}: {e}") - continue - - if valid_days == 0: - raise ValueError(f"No valid days processed for {yy}-{mm:02d}") - - logger.info(f"Successfully processed {valid_days} out of {num_days} days") - - # Calculate monthly means - tau_nnrLOD_mm = np.nanmean(tau_nnrLOD_dd, axis=2) - mod_monthly = {} - for var_name in processor.variable_names: - mod_monthly[var_name] = np.nanmean(mod_arrays_dd[var_name], axis=2) - - # Prepare data for NetCDF output - output_data = { - 'obs': tau_nnrLOD_mm, - 'tot': mod_monthly['tot'], - 'bc': mod_monthly['bc'], - 'oc': mod_monthly['oc'], - 'br': mod_monthly['br'], - 'ss': mod_monthly['ss'], - 'du': mod_monthly['du'], - 'su': mod_monthly['su'], - 'ni': mod_monthly['ni'] - } - - # Write NetCDF output - output_filename = f"{output_dir}/{EXPID}.tavgM_aod_{sat}filtered.{yy}{mm:02d}.nc4" - processor.write_netcdf_output(output_filename, output_data, lonm, latm, yy, mm, EXPID, sat) - - return tau_nnrLOD_mm, mod_monthly['tot'], lonm, latm - - -def main(): - """Main execution function""" - # Configuration - config = { - 'year': 2024, - 'experiment_id': "c180R_qfed3igbp_xf", - 'satellite': "MYD04", - 'months': range(1, 13), - 'output_dir': './sampledGEOS/c180R_qfed3igbp_xf/' - } - - # Create output directory - os.makedirs(config['output_dir'], exist_ok=True) - - logger.info(f"Starting AOD processing for {config['year']}") - logger.info(f"Experiment: {config['experiment_id']}") - logger.info(f"Satellite: {config['satellite']}") - logger.info(f"Months: {list(config['months'])}") - - successful_months = [] - failed_months = [] - - for mm in config['months']: - try: - logger.info(f"Processing {config['year']}-{mm:02d}") - start_time = datetime.now() - - result = monthlyAOD_mod_obs( - config['year'], mm, - config['experiment_id'], - config['satellite'], - config['output_dir'] - ) - - end_time = datetime.now() - duration = (end_time - start_time).total_seconds() - - logger.info(f"Successfully completed {config['year']}-{mm:02d} in {duration:.1f} seconds") - successful_months.append(mm) - - except Exception as e: - logger.error(f"Failed to process {config['year']}-{mm:02d}: {e}") - failed_months.append(mm) - continue - - # Summary - logger.info(f"Processing complete!") - logger.info(f"Successful months: {successful_months}") - if failed_months: - logger.warning(f"Failed months: {failed_months}") - - -if __name__ == "__main__": - main() diff --git a/src/pyobs/evaluation/MODIS_NNR/plotregionalcomparison.py b/src/pyobs/evaluation/MODIS_NNR/plotregionalcomparison.py deleted file mode 100755 index 79d7e44..0000000 --- a/src/pyobs/evaluation/MODIS_NNR/plotregionalcomparison.py +++ /dev/null @@ -1,1238 +0,0 @@ -#!/usr/bin/env python -""" -This code uses the output generated by monthlygeossample_speciated.py to -create figures comparing GEOS with MODIS NNR data. Plots include global maps -of regional summaries (bias, correlation, scaling), and individual figures -showing the annual cycle of total AOD and how that is broken down by species in GEOS. -""" - -import numpy as np -import xarray as xr -import matplotlib.pyplot as plt -import matplotlib.colors as mcolors -import matplotlib.patches as patches -from matplotlib.patches import Rectangle -from matplotlib.colors import TwoSlopeNorm -import cartopy.crs as ccrs -import cartopy.feature as cfeature -from pathlib import Path -import pandas as pd -from datetime import datetime -import seaborn as sns -import argparse -import sys -from scipy.stats import pearsonr - -# Define regions (including global) -REGIONS = { - 0: {'name': 'Global', 'lon': [-180, 180], 'lat': [-90, 90]}, - 1: {'name': 'Alaska', 'lon': [-170, -140], 'lat': [50, 70]}, - 2: {'name': 'Canada', 'lon': [-140, -80], 'lat': [50, 70]}, - 3: {'name': 'Quebec', 'lon': [-80, -55], 'lat': [45, 65]}, - 4: {'name': 'USWest', 'lon': [-130, -105], 'lat': [30, 50]}, - 5: {'name': 'USCentral', 'lon': [-105, -90], 'lat': [30, 50]}, - 6: {'name': 'USEast', 'lon': [-90, -70], 'lat': [25, 45]}, - 7: {'name': 'Mexico', 'lon': [-120, -85], 'lat': [10, 30]}, - 8: {'name': 'BrazilFor', 'lon': [-75, -50], 'lat': [-15, 5]}, - 9: {'name': 'BrazilCer', 'lon': [-50, -30], 'lat': [-20, 0]}, - 10: {'name': 'Argentina', 'lon': [-75, -50], 'lat': [-60, -15]}, - 11: {'name': 'AfricaWest', 'lon': [-20, 15], 'lat': [0, 15]}, - 12: {'name': 'AfricaCent', 'lon': [15, 30], 'lat': [5, 15]}, - 13: {'name': 'AfricaEast', 'lon': [30, 50], 'lat': [-10, 15]}, - 14: {'name': 'Congo', 'lon': [10, 30], 'lat': [-10, 5]}, - 15: {'name': 'Zambia', 'lon': [22, 35], 'lat': [-18, -8]}, - 16: {'name': 'AfricaSouth', 'lon': [10, 35], 'lat': [-35, -20]}, - 17: {'name': 'Madagascar', 'lon': [42, 50], 'lat': [-25, -12]}, - 18: {'name': 'Scandinavia', 'lon': [0, 35], 'lat': [55, 75]}, - 19: {'name': 'Moscow', 'lon': [30, 60], 'lat': [45, 60]}, - 20: {'name': 'SiberiaWest', 'lon': [35, 90], 'lat': [60, 75]}, - 21: {'name': 'SiberiaEast', 'lon': [90, 140], 'lat': [60, 75]}, - 22: {'name': 'EuropeWest', 'lon': [-10, 30], 'lat': [35, 55]}, - 23: {'name': 'MiddleEast', 'lon': [30, 60], 'lat': [30, 45]}, - 24: {'name': 'AsiaCent', 'lon': [60, 110], 'lat': [35, 50]}, - 25: {'name': 'ChinaEast', 'lon': [110, 150], 'lat': [35, 60]}, - 26: {'name': 'Nepal', 'lon': [65, 95], 'lat': [25, 35]}, - 27: {'name': 'India', 'lon': [70, 90], 'lat': [5, 25]}, - 28: {'name': 'ChinaSouth', 'lon': [100, 125], 'lat': [20, 40]}, - 29: {'name': 'Indochina', 'lon': [90, 110], 'lat': [10, 25]}, - 30: {'name': 'Philippines', 'lon': [115, 130], 'lat': [5, 20]}, - 31: {'name': 'Sumatra', 'lon': [95, 110], 'lat': [-10, 10]}, - 32: {'name': 'Borneo', 'lon': [110, 120], 'lat': [-5, 8]}, - 33: {'name': 'Indonesia', 'lon': [120, 160], 'lat': [-10, 5]}, - 34: {'name': 'AustraliaN', 'lon': [120, 150], 'lat': [-20, -10]}, - 35: {'name': 'AustraliaW', 'lon': [110, 130], 'lat': [-35, -20]}, - 36: {'name': 'AustraliaE', 'lon': [135, 155], 'lat': [-45, -20]}, - 37: {'name': 'SiberiaFE', 'lon': [140, 170], 'lat': [60, 75]}, - 38: {'name': 'Sahara', 'lon': [-15, 30], 'lat': [13, 35]}, - 39: {'name': 'Sahel', 'lon': [-15, 35], 'lat': [12, 13]}, - 40: {'name': 'CapeVerde', 'lon': [-26, -20], 'lat': [10, 20]}, - 41: {'name': 'RedSea', 'lon': [30, 45], 'lat': [10, 30]}, - 42: {'name': 'PersianGulf', 'lon': [45, 60], 'lat': [20, 30]}, - 43: {'name': 'ArabSea', 'lon': [60, 70], 'lat': [10, 20]}, - 44: {'name': 'Caribbean', 'lon': [-80, -60], 'lat': [13, 23]}, - 45: {'name': 'SALDust', 'lon': [-60, -26], 'lat': [13, 30]}, - 46: {'name': 'SAmerBB', 'lon': [-45, -20], 'lat': [-45, -25]} -} - -def load_monthly_data(base_path='sampledGEOS/c180R_qfed3igbp_allviirs', sensor='both'): - """Load all monthly data files and combine into datasets.""" - base_path = Path(base_path) - sensor_lower = sensor.lower() - - # Get all MOD and MYD files - mod_files = sorted(base_path.glob('*MOD04filtered*.nc4')) - myd_files = sorted(base_path.glob('*MYD04filtered*.nc4')) - - # Filter based on sensor selection - if sensor_lower in ['terra', 'mod']: - myd_files = [] - print(f"Using Terra (MOD) data only") - elif sensor_lower in ['aqua', 'myd']: - mod_files = [] - print(f"Using Aqua (MYD) data only") - elif sensor_lower == 'both': - print(f"Using both Terra (MOD) and Aqua (MYD) data separately") - else: - raise ValueError(f"Invalid sensor option: {sensor}. Use 'terra', 'aqua', or 'both'") - - print(f"Found {len(mod_files)} MOD files and {len(myd_files)} MYD files") - - # Load and combine data - mod_data = [] - myd_data = [] - - for f in mod_files: - try: - ds = xr.open_dataset(f) - # Extract month from filename - month_str = f.name.split('.')[-2] # e.g., '202401' - month = int(month_str[-2:]) - ds = ds.assign_coords(month=month) - mod_data.append(ds) - print(f" Loaded Terra month {month}") - except Exception as e: - print(f"Error loading {f}: {e}") - - for f in myd_files: - try: - ds = xr.open_dataset(f) - month_str = f.name.split('.')[-2] - month = int(month_str[-2:]) - ds = ds.assign_coords(month=month) - myd_data.append(ds) - print(f" Loaded Aqua month {month}") - except Exception as e: - print(f"Error loading {f}: {e}") - - # Combine datasets - if mod_data: - mod_combined = xr.concat(mod_data, dim='month') - mod_combined.attrs['sensor'] = 'Terra' - else: - mod_combined = None - - if myd_data: - myd_combined = xr.concat(myd_data, dim='month') - myd_combined.attrs['sensor'] = 'Aqua' - else: - myd_combined = None - - return mod_combined, myd_combined - -def calculate_regional_means(data, region_id): - """Calculate regional means for a specific region.""" - region = REGIONS[region_id] - lon_min, lon_max = region['lon'] - lat_min, lat_max = region['lat'] - - sensor_name = data.attrs.get('sensor', 'MODIS') - print(f"Calculating means for {region['name']} using {sensor_name}") - - if region_id == 0: # Global region - region_data = data - print(" Using global data (no spatial subsetting)") - else: - # Handle longitude conversion for regional data - data_lon = data.lon.values - if np.any(data_lon > 180): - # Data is in 0-360 format, convert region bounds - if lon_min < 0: - lon_min += 360 - if lon_max < 0: - lon_max += 360 - - # Select region - if lon_min > lon_max: # Crossing dateline - region_data = data.where( - ((data.lon >= lon_min) | (data.lon <= lon_max)) & - (data.lat >= lat_min) & (data.lat <= lat_max) - ) - else: - region_data = data.where( - (data.lon >= lon_min) & (data.lon <= lon_max) & - (data.lat >= lat_min) & (data.lat <= lat_max) - ) - - # Calculate means - means = {} - for var in ['MODtau', 'GEOStau', 'bcexttau', 'ocexttau', 'brexttau', - 'ssexttau', 'duexttau', 'suexttau', 'niexttau']: - if var in region_data.data_vars: - var_mean = region_data[var].mean(dim=['lat', 'lon'], skipna=True) - means[var] = var_mean - - return means - -def calculate_regional_statistics(mod_data, myd_data, sensor='both'): - """Calculate correlation, bias, and scaling factor statistics for all regions.""" - print("Calculating regional statistics...") - - stats_data = [] - - for region_id, region_info in REGIONS.items(): - region_name = region_info['name'] - print(f" Processing {region_name}...") - - # Process each sensor - sensors_to_process = [] - if sensor.lower() in ['terra', 'both'] and mod_data is not None: - sensors_to_process.append(('Terra', mod_data)) - if sensor.lower() in ['aqua', 'both'] and myd_data is not None: - sensors_to_process.append(('Aqua', myd_data)) - - for sensor_name, data in sensors_to_process: - try: - means = calculate_regional_means(data, region_id) - - if not means or all(var.isnull().all() for var in means.values()): - continue - - # Extract valid data - mod_vals = means['MODtau'].values - geos_vals = means['GEOStau'].values - valid_mask = ~(np.isnan(mod_vals) | np.isnan(geos_vals)) - - if np.sum(valid_mask) < 2: # Need at least 2 points for correlation - continue - - valid_mod = mod_vals[valid_mask] - valid_geos = geos_vals[valid_mask] - - # Calculate statistics - corr, p_value = pearsonr(valid_mod, valid_geos) - bias = np.mean(valid_geos - valid_mod) - rmse = np.sqrt(np.mean((valid_geos - valid_mod)**2)) - mean_obs = np.mean(valid_mod) - mean_model = np.mean(valid_geos) - n_points = len(valid_mod) - - # Calculate scaling factor chi = exp(log(MODtau+0.01) - log(GEOStau+0.01)) - # This is equivalent to (MODtau+0.01)/(GEOStau+0.01) - mod_adjusted = valid_mod + 0.01 - geos_adjusted = valid_geos + 0.01 - - # Calculate chi for each time point - chi_values = np.exp(np.log(mod_adjusted) - np.log(geos_adjusted)) - - # Calculate statistics of chi - chi_mean = np.mean(chi_values) - chi_median = np.median(chi_values) - chi_std = np.std(chi_values) - chi_min = np.min(chi_values) - chi_max = np.max(chi_values) - - stats_data.append({ - 'region_id': region_id, - 'region_name': region_name, - 'sensor': sensor_name, - 'correlation': corr, - 'p_value': p_value, - 'bias': bias, - 'rmse': rmse, - 'mean_obs': mean_obs, - 'mean_model': mean_model, - 'n_points': n_points, - 'chi_mean': chi_mean, - 'chi_median': chi_median, - 'chi_std': chi_std, - 'chi_min': chi_min, - 'chi_max': chi_max, - 'lon_min': region_info['lon'][0], - 'lon_max': region_info['lon'][1], - 'lat_min': region_info['lat'][0], - 'lat_max': region_info['lat'][1] - }) - - except Exception as e: - print(f" Error processing {sensor_name} for {region_name}: {e}") - continue - - return pd.DataFrame(stats_data) - -def create_map_plot(stats_df, metric='correlation', sensor='both', vmin=None, vmax=None, - title_suffix='', output_dir='regional_analysis', experiment_name='', - show_values=False): - """Create map visualization of regional statistics with optional value annotations.""" - - # Filter data by sensor if needed - if sensor.lower() in ['terra', 'aqua']: - plot_data = stats_df[stats_df['sensor'].str.lower() == sensor.lower()].copy() - sensor_title = sensor.capitalize() - else: - # For 'both', average the metrics across sensors for each region - if len(stats_df) == 0: - print("No data available for mapping") - return None - - # Group by region and average metrics - numeric_cols = ['correlation', 'bias', 'rmse', 'mean_obs', 'mean_model', - 'chi_mean', 'chi_median', 'chi_std', 'chi_min', 'chi_max'] - plot_data = stats_df.groupby('region_id').agg({ - 'region_name': 'first', - 'lon_min': 'first', 'lon_max': 'first', - 'lat_min': 'first', 'lat_max': 'first', - 'n_points': 'sum', - **{col: 'mean' for col in numeric_cols if col in stats_df.columns} - }).reset_index() - sensor_title = 'Combined (Terra+Aqua)' - - if len(plot_data) == 0: - print(f"No data available for sensor: {sensor}") - return None - - # Set up the plot - fig = plt.figure(figsize=(18, 12)) # Slightly larger for text annotations - ax = plt.axes(projection=ccrs.PlateCarree()) - - # Add map features - ax.add_feature(cfeature.COASTLINE, alpha=0.5) - ax.add_feature(cfeature.BORDERS, alpha=0.3) - ax.add_feature(cfeature.OCEAN, color='lightblue', alpha=0.3) - ax.add_feature(cfeature.LAND, color='lightgray', alpha=0.3) - - # Set global extent - ax.set_global() - - # Define color mapping based on metric - if metric == 'correlation': - if vmin is None or vmax is None: - vmin, vmax = -1, 1 - cmap = plt.cm.RdBu_r # Red for negative, blue for positive - cmap_label = 'Correlation Coefficient' - title_metric = 'Correlation' - elif metric == 'bias': - if vmin is None or vmax is None: - abs_max = max(abs(plot_data[metric].min()), abs(plot_data[metric].max())) - vmin, vmax = -abs_max, abs_max - cmap = plt.cm.RdBu # Blue for negative bias, red for positive - cmap_label = 'Bias (GEOS - MODIS)' - title_metric = 'Bias' - elif metric == 'rmse': - if vmin is None or vmax is None: - vmin, vmax = 0, plot_data[metric].max() - cmap = plt.cm.Reds # White to red for RMSE - cmap_label = 'Root Mean Square Error' - title_metric = 'RMSE' - elif metric == 'chi_mean': - if vmin is None or vmax is None: - vmin, vmax = 0.5, 2.0 # Reasonable default range for scaling factors - cmap = plt.cm.RdYlBu_r # Red for high scaling, blue for low scaling - cmap_label = 'Scaling Factor (ฯ‡)' - title_metric = 'Scaling Factor (ฯ‡)' - show_values = True # Always show values for chi - else: - # Default settings - if vmin is None or vmax is None: - vmin, vmax = plot_data[metric].min(), plot_data[metric].max() - cmap = plt.cm.viridis - cmap_label = metric.replace('_', ' ').title() - title_metric = metric.replace('_', ' ').title() - - # Create color normalization - norm = plt.Normalize(vmin=vmin, vmax=vmax) - - # Plot regions with different styles for better visibility - for idx, row in plot_data.iterrows(): - # Skip global region (region_id = 0) for mapping - if row['region_id'] == 0: - continue - - lon_min, lon_max = row['lon_min'], row['lon_max'] - lat_min, lat_max = row['lat_min'], row['lat_max'] - - # Handle longitude wrapping - if lon_min > lon_max: # Crosses dateline - # Split into two rectangles - rect1 = Rectangle((lon_min, lat_min), 180 - lon_min, lat_max - lat_min, - transform=ccrs.PlateCarree(), alpha=0.7, - edgecolor='black', linewidth=1.5) - rect2 = Rectangle((-180, lat_min), lon_max + 180, lat_max - lat_min, - transform=ccrs.PlateCarree(), alpha=0.7, - edgecolor='black', linewidth=1.5) - - color = cmap(norm(row[metric])) - rect1.set_facecolor(color) - rect2.set_facecolor(color) - ax.add_patch(rect1) - ax.add_patch(rect2) - else: - # Regular rectangle - rect = Rectangle((lon_min, lat_min), lon_max - lon_min, lat_max - lat_min, - transform=ccrs.PlateCarree(), alpha=0.7, - edgecolor='black', linewidth=1.5) - - color = cmap(norm(row[metric])) - rect.set_facecolor(color) - ax.add_patch(rect) - - # Calculate center for text placement - center_lon = (lon_min + lon_max) / 2 - center_lat = (lat_min + lat_max) / 2 - - # Adjust for dateline crossing - if lon_min > lon_max: - if center_lon < 0: - center_lon += 180 - else: - center_lon -= 180 - - if show_values: - # Add both region name and metric value - value_text = f"{row['region_name']}\nฯ‡ = {row[metric]:.3f}" - fontsize = 9 - bbox_props = dict(boxstyle='round,pad=0.4', facecolor='white', - alpha=0.9, edgecolor='black', linewidth=0.5) - else: - # Add just region name - value_text = row['region_name'] - fontsize = 8 - bbox_props = dict(boxstyle='round,pad=0.3', facecolor='white', - alpha=0.8, edgecolor='none') - - ax.text(center_lon, center_lat, value_text, - transform=ccrs.PlateCarree(), - ha='center', va='center', fontsize=fontsize, fontweight='bold', - bbox=bbox_props) - - # Add colorbar - sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm) - sm.set_array([]) - cbar = plt.colorbar(sm, ax=ax, shrink=0.6, aspect=20, pad=0.02) - cbar.set_label(cmap_label, fontsize=12) - - # Add gridlines - gl = ax.gridlines(draw_labels=True, alpha=0.5) - gl.top_labels = False - gl.right_labels = False - - # Set title - title = f'{title_metric} - {sensor_title}' - if title_suffix: - title += f' - {title_suffix}' - if experiment_name: - title += f' ({experiment_name})' - - plt.title(title, fontsize=14, fontweight='bold', pad=20) - - # Save the plot - output_dir = Path(output_dir) - output_dir.mkdir(exist_ok=True) - - if show_values and metric == 'chi_mean': - filename = f"{experiment_name}_{metric}_{sensor.lower()}_map_with_values.png" - else: - filename = f"{experiment_name}_{metric}_{sensor.lower()}_map.png" - - save_path = output_dir / filename - plt.savefig(save_path, dpi=300, bbox_inches='tight', facecolor='white') - print(f"Map saved to: {save_path}") - - return fig, save_path - -def create_chi_value_map(stats_df, sensor='both', output_dir='regional_analysis', - experiment_name='', title_suffix=''): - """Create a dedicated map showing chi values with annotations.""" - - print("Creating chi scaling factor map with values...") - - # Filter data by sensor if needed - if sensor.lower() in ['terra', 'aqua']: - plot_data = stats_df[stats_df['sensor'].str.lower() == sensor.lower()].copy() - sensor_title = sensor.capitalize() - else: - # For 'both', average the metrics across sensors for each region - if len(stats_df) == 0: - print("No data available for mapping") - return None - - # Group by region and average chi values - numeric_cols = ['chi_mean', 'chi_median', 'chi_std', 'chi_min', 'chi_max'] - plot_data = stats_df.groupby('region_id').agg({ - 'region_name': 'first', - 'lon_min': 'first', 'lon_max': 'first', - 'lat_min': 'first', 'lat_max': 'first', - 'n_points': 'sum', - **{col: 'mean' for col in numeric_cols if col in stats_df.columns} - }).reset_index() - sensor_title = 'Combined (Terra+Aqua)' - - if len(plot_data) == 0 or 'chi_mean' not in plot_data.columns: - print(f"No chi data available for sensor: {sensor}") - return None - - # Set up the plot with larger size for better text visibility - fig = plt.figure(figsize=(20, 14)) - ax = plt.axes(projection=ccrs.PlateCarree()) - - # Add map features - ax.add_feature(cfeature.COASTLINE, alpha=0.5) - ax.add_feature(cfeature.BORDERS, alpha=0.3) - ax.add_feature(cfeature.OCEAN, color='lightblue', alpha=0.3) - ax.add_feature(cfeature.LAND, color='lightgray', alpha=0.3) - - # Set global extent - ax.set_global() - - # Define chi-specific color mapping - chi_min = plot_data['chi_mean'].min() - chi_max = plot_data['chi_mean'].max() - - # Set reasonable limits for chi visualization - vmin = max(0.2, chi_min * 0.9) # Don't go below 0.2 - vmax = min(3.0, chi_max * 1.1) # Don't go above 3.0 - - # Use a diverging colormap centered at 1.0 (perfect scaling) - norm = TwoSlopeNorm(vmin=vmin, vcenter=1.0, vmax=vmax) - cmap = plt.cm.RdYlBu_r # Red for overestimate (chi > 1), blue for underestimate (chi < 1) - - # Plot regions - for idx, row in plot_data.iterrows(): - # Skip global region for mapping - if row['region_id'] == 0: - continue - - lon_min, lon_max = row['lon_min'], row['lon_max'] - lat_min, lat_max = row['lat_min'], row['lat_max'] - - # Check if chi_mean is valid, skip region if not - if 'chi_mean' not in row or pd.isna(row['chi_mean']): - print(f"Warning: No valid chi value for region {row['region_name']}") - continue - - chi_value = row['chi_mean'] - - # Handle longitude wrapping - if lon_min > lon_max: # Crosses dateline - # Split into two rectangles - rect1 = Rectangle((lon_min, lat_min), 180 - lon_min, lat_max - lat_min, - transform=ccrs.PlateCarree(), alpha=0.8, - edgecolor='black', linewidth=2) - rect2 = Rectangle((-180, lat_min), lon_max + 180, lat_max - lat_min, - transform=ccrs.PlateCarree(), alpha=0.8, - edgecolor='black', linewidth=2) - - color = cmap(norm(chi_value)) - rect1.set_facecolor(color) - rect2.set_facecolor(color) - ax.add_patch(rect1) - ax.add_patch(rect2) - else: - # Regular rectangle - rect = Rectangle((lon_min, lat_min), lon_max - lon_min, lat_max - lat_min, - transform=ccrs.PlateCarree(), alpha=0.8, - edgecolor='black', linewidth=2) - - color = cmap(norm(chi_value)) - rect.set_facecolor(color) - ax.add_patch(rect) - - # Calculate center for text placement - center_lon = (lon_min + lon_max) / 2 - center_lat = (lat_min + lat_max) / 2 - - # Adjust for dateline crossing - if lon_min > lon_max: - if center_lon < 0: - center_lon += 180 - else: - center_lon -= 180 - - # Create text with region name and chi value - # Ensure chi value is properly formatted to handle extreme values - chi_text = f"{row['region_name']}\nฯ‡ = {chi_value:.3f}" - - # Choose text color based on background - if chi_value > 1.5: # Red background - text_color = 'white' - elif chi_value < 0.7: # Blue background - text_color = 'white' - else: # Light background - text_color = 'black' - - ax.text(center_lon, center_lat, chi_text, - transform=ccrs.PlateCarree(), - ha='center', va='center', fontsize=10, fontweight='bold', - color=text_color, - bbox=dict(boxstyle='round,pad=0.4', facecolor='white', - alpha=0.9, edgecolor='black', linewidth=0.8)) - - # Add colorbar with custom ticks - sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm) - sm.set_array([]) - cbar = plt.colorbar(sm, ax=ax, shrink=0.7, aspect=25, pad=0.02) - - # Use the exact requested formula in the colorbar label - cbar.set_label('Scaling Factor (ฯ‡ = exp(log(MODtau+0.01)-log(GEOStau+0.01)))', - fontsize=12, fontweight='bold') - - # Add interpretation text to colorbar - cbar.ax.text(1.15, 1.02, 'GEOS underestimates', transform=cbar.ax.transAxes, - rotation=0, ha='left', va='bottom', fontsize=10, color='red', fontweight='bold') - cbar.ax.text(1.15, -0.02, 'GEOS overestimates', transform=cbar.ax.transAxes, - rotation=0, ha='left', va='top', fontsize=10, color='blue', fontweight='bold') - - # Add reference line at chi=1 - cbar.ax.axhline(y=norm(1.0), color='black', linestyle='--', linewidth=2, alpha=0.7) - cbar.ax.text(1.05, norm(1.0), 'Perfect match (ฯ‡=1)', transform=cbar.ax.get_yaxis_transform(), - ha='left', va='center', fontsize=10, fontweight='bold') - - # Add gridlines - gl = ax.gridlines(draw_labels=True, alpha=0.5) - gl.top_labels = False - gl.right_labels = False - - # Set title - title = f'Scaling Factor (ฯ‡) - {sensor_title}' - if title_suffix: - title += f' - {title_suffix}' - if experiment_name: - title += f' ({experiment_name})' - - plt.title(title, fontsize=16, fontweight='bold', pad=25) - - # Add subtitle with interpretation - plt.figtext(0.5, 0.02, 'ฯ‡ > 1: GEOS underestimates AOD relative to MODIS | ฯ‡ < 1: GEOS overestimates AOD relative to MODIS', - ha='center', va='bottom', fontsize=12, style='italic') - - # Save the plot - output_dir = Path(output_dir) - output_dir.mkdir(exist_ok=True) - - filename = f"{experiment_name}_chi_scaling_factor_{sensor.lower()}_map_with_values.png" - save_path = output_dir / filename - plt.savefig(save_path, dpi=300, bbox_inches='tight', facecolor='white') - print(f"Chi scaling factor map with values saved to: {save_path}") - - return fig, save_path - -def create_statistical_summary_table(stats_df, output_dir='regional_analysis', experiment_name=''): - """Create a comprehensive statistical summary table including scaling factors.""" - - if len(stats_df) == 0: - print("No data available for summary table") - return None - - # Create output directory - output_dir = Path(output_dir) - output_dir.mkdir(exist_ok=True) - - # Prepare data for table - summary_data = [] - - for sensor in stats_df['sensor'].unique(): - sensor_data = stats_df[stats_df['sensor'] == sensor].copy() - - for idx, row in sensor_data.iterrows(): - summary_data.append({ - 'Region ID': row['region_id'], - 'Region Name': row['region_name'], - 'Sensor': row['sensor'], - 'Correlation': f"{row['correlation']:.3f}", - 'P-value': f"{row['p_value']:.4f}" if not np.isnan(row['p_value']) else 'N/A', - 'Bias': f"{row['bias']:.4f}", - 'RMSE': f"{row['rmse']:.4f}", - 'Mean MODIS': f"{row['mean_obs']:.4f}", - 'Mean GEOS': f"{row['mean_model']:.4f}", - 'Chi Mean': f"{row['chi_mean']:.4f}", - 'Chi Median': f"{row['chi_median']:.4f}", - 'Chi Std': f"{row['chi_std']:.4f}", - 'Chi Min': f"{row['chi_min']:.4f}", - 'Chi Max': f"{row['chi_max']:.4f}", - 'N Points': int(row['n_points']) - }) - - # Convert to DataFrame and save - summary_df = pd.DataFrame(summary_data) - - # Save as CSV - csv_path = output_dir / f"{experiment_name}_regional_statistics_with_chi.csv" - summary_df.to_csv(csv_path, index=False) - print(f"Statistical summary with scaling factors saved to: {csv_path}") - - # Create a separate chi-focused table - chi_data = [] - for sensor in stats_df['sensor'].unique(): - sensor_data = stats_df[stats_df['sensor'] == sensor].copy() - - for idx, row in sensor_data.iterrows(): - chi_data.append({ - 'Region ID': row['region_id'], - 'Region Name': row['region_name'], - 'Sensor': row['sensor'], - 'Chi Mean': f"{row['chi_mean']:.4f}", - 'Chi Median': f"{row['chi_median']:.4f}", - 'Chi Std': f"{row['chi_std']:.4f}", - 'Chi Range': f"[{row['chi_min']:.4f}, {row['chi_max']:.4f}]", - 'N Points': int(row['n_points']) - }) - - chi_df = pd.DataFrame(chi_data) - chi_csv_path = output_dir / f"{experiment_name}_scaling_factors_chi.csv" - chi_df.to_csv(chi_csv_path, index=False) - print(f"Scaling factor (chi) table saved to: {chi_csv_path}") - - # Create a formatted version for display - print("\n" + "="*120) - print("REGIONAL STATISTICS SUMMARY WITH SCALING FACTORS") - print("="*120) - print(summary_df.to_string(index=False)) - - print("\n" + "="*80) - print("SCALING FACTOR (CHI) SUMMARY") - print("="*80) - print(chi_df.to_string(index=False)) - - return summary_df, csv_path, chi_df, chi_csv_path - -def create_all_maps(mod_data, myd_data, sensor='both', output_dir='regional_analysis', - experiment_name='', title_suffix=''): - """Create maps for all key metrics including chi values.""" - - # Calculate statistics - stats_df = calculate_regional_statistics(mod_data, myd_data, sensor=sensor) - - if len(stats_df) == 0: - print("No statistics calculated - no valid data found") - return None, None - - # Create statistical summary - summary_result = create_statistical_summary_table(stats_df, output_dir, experiment_name) - - # Handle both old and new return formats - if len(summary_result) == 4: - summary_df, csv_path, chi_df, chi_csv_path = summary_result - else: - summary_df, csv_path = summary_result - - # Create maps for different metrics - metrics = ['correlation', 'bias', 'rmse', 'chi_mean'] - figures = {} - - for metric in metrics: - if metric in stats_df.columns: - print(f"\nCreating {metric} map...") - try: - fig, save_path = create_map_plot(stats_df, metric=metric, sensor=sensor, - title_suffix=title_suffix, - output_dir=output_dir, - experiment_name=experiment_name) - if fig: - figures[metric] = (fig, save_path) - plt.close(fig) # Close to save memory - except Exception as e: - print(f"Error creating {metric} map: {e}") - - # Create special chi value map with annotations - if 'chi_mean' in stats_df.columns: - print(f"\nCreating detailed chi scaling factor map with values...") - try: - chi_fig, chi_save_path = create_chi_value_map(stats_df, sensor=sensor, - output_dir=output_dir, - experiment_name=experiment_name, - title_suffix=title_suffix) - if chi_fig: - figures['chi_detailed'] = (chi_fig, chi_save_path) - plt.close(chi_fig) - except Exception as e: - print(f"Error creating detailed chi map: {e}") - - return figures, stats_df - -def plot_single_sensor_panels(data, region_id, axes, sensor_name, shared_limits=None, shared_bc_max=None, shared_y_limits=None, show_legend=True, legend_position='upper left'): - """Plot time series and scatter plot for a single sensor with shared scaling.""" - means = calculate_regional_means(data, region_id) - - if not means or all(var.isnull().all() for var in means.values()): - axes[0].text(0.5, 0.5, f'No valid data for {sensor_name}', - transform=axes[0].transAxes, ha='center', va='center', fontsize=12) - axes[1].text(0.5, 0.5, f'No valid data for {sensor_name}', - transform=axes[1].transAxes, ha='center', va='center', fontsize=12) - return False, None, None, None - - # Extract data - months = data.month.values - mod_vals = means['MODtau'].values - geos_vals = means['GEOStau'].values - - # Aerosol components with BETTER color separation - components = { - 'Black Carbon': means.get('bcexttau', xr.DataArray(np.zeros_like(months))).values, - 'Organic Carbon': means.get('ocexttau', xr.DataArray(np.zeros_like(months))).values, - 'Brown Carbon': means.get('brexttau', xr.DataArray(np.zeros_like(months))).values, - 'Sea Salt': means.get('ssexttau', xr.DataArray(np.zeros_like(months))).values, - 'Dust': means.get('duexttau', xr.DataArray(np.zeros_like(months))).values, - 'Sulfate': means.get('suexttau', xr.DataArray(np.zeros_like(months))).values, - 'Nitrate': means.get('niexttau', xr.DataArray(np.zeros_like(months))).values - } - - valid_mask = ~(np.isnan(mod_vals) | np.isnan(geos_vals)) - - if np.sum(valid_mask) > 0: - valid_months = months[valid_mask] - valid_mod = mod_vals[valid_mask] - valid_geos = geos_vals[valid_mask] - - # Time series with BETTER color separation - bottom = np.zeros(len(valid_months)) - component_colors = { - 'Black Carbon': '#2C2C2C', # Dark gray/black - 'Organic Carbon': '#228B22', # Forest green (changed from brown) - 'Brown Carbon': '#8B4513', # Saddle brown - 'Sea Salt': '#4682B4', # Steel blue - 'Dust': '#DAA520', # Goldenrod - 'Sulfate': '#FF6347', # Tomato red - 'Nitrate': '#9370DB' # Medium purple - } - - # Store the middle y-position of each component for labeling - component_middles = {} - - for comp_name, comp_vals in components.items(): - valid_comp = comp_vals[valid_mask] - valid_comp = np.nan_to_num(valid_comp, nan=0.0) - if np.any(valid_comp > 0): - axes[0].fill_between(valid_months, bottom, bottom + valid_comp, - alpha=0.8, color=component_colors[comp_name], - edgecolor='white', linewidth=0.5) - # Calculate middle position for this component - component_middles[comp_name] = np.mean(bottom + valid_comp/2) - bottom += valid_comp - - axes[0].plot(valid_months, valid_mod, 'ko-', linewidth=2, markersize=6, - label=f'MODIS AOD ({sensor_name})', zorder=10) - axes[0].plot(valid_months, valid_geos, 'r--', linewidth=2, - label='GEOS Total AOD', zorder=9) - - # Set shared y-axis limits for time series if provided - if shared_y_limits is not None: - axes[0].set_ylim(shared_y_limits) - - axes[0].set_xlabel('Month') - axes[0].set_ylabel('AOD') - axes[0].set_title(f'{sensor_name} - AOD Time Series') - axes[0].grid(True, alpha=0.3) - axes[0].set_xticks(valid_months) - - # Add legend conditionally and with specified position - if show_legend: - axes[0].legend(loc=legend_position, frameon=True, fancybox=True, shadow=True) - - # Add component labels directly on the plot - x_center = np.mean(valid_months) # Center x-position - for comp_name, y_middle in component_middles.items(): - # Only label if the component has significant contribution - if y_middle > 0.01: # Only label components with meaningful contribution - axes[0].text(x_center, y_middle, comp_name, - ha='center', va='center', fontsize=10, - fontweight='bold', color='white', - bbox=dict(boxstyle='round,pad=0.3', - facecolor='black', alpha=0.7, edgecolor='none')) - - # Scatter plot with SHARED sizing and limits - bc_fraction = means.get('brexttau', xr.DataArray(np.zeros_like(months))).values / np.maximum(means['GEOStau'].values, 1e-10) - bc_fraction = np.nan_to_num(bc_fraction, nan=0.0) - - # Use shared brown carbon maximum for consistent sizing - min_size = 30 - max_size = 200 - bc_frac_valid = bc_fraction[valid_mask] - - if shared_bc_max is not None and shared_bc_max > 0: - sizes = min_size + (bc_frac_valid / shared_bc_max) * (max_size - min_size) - elif np.max(bc_frac_valid) > 0: - sizes = min_size + (bc_frac_valid / np.max(bc_frac_valid)) * (max_size - min_size) - else: - sizes = np.full(len(bc_frac_valid), min_size) - - scatter = axes[1].scatter(valid_mod, valid_geos, c=valid_months, s=sizes, - alpha=0.7, cmap='viridis', edgecolors='black', linewidth=0.5) - - # Use shared axis limits if provided with EXPANDED RANGE (scatter plot only) - if shared_limits is not None: - # EXPAND the axis limits by 10% for scatter plot only - range_val = shared_limits[1] - shared_limits[0] - expanded_limits = [shared_limits[0] - range_val * 0.1, - shared_limits[1] + range_val * 0.1] - axes[1].set_xlim(expanded_limits) - axes[1].set_ylim(expanded_limits) - - # 1:1 line using expanded limits - axes[1].plot(expanded_limits, expanded_limits, 'k--', alpha=0.5, label='1:1 line') - else: - # Calculate expanded limits for this panel - max_val = max(np.max(valid_mod), np.max(valid_geos)) - min_val = min(np.min(valid_mod), np.min(valid_geos)) - range_val = max_val - min_val - expanded_limits = [min_val - range_val * 0.1, max_val + range_val * 0.1] - axes[1].set_xlim(expanded_limits) - axes[1].set_ylim(expanded_limits) - axes[1].plot(expanded_limits, expanded_limits, 'k--', alpha=0.5, label='1:1 line') - - axes[1].set_xlabel(f'MODIS AOD ({sensor_name})') - axes[1].set_ylabel('GEOS AOD') - axes[1].set_title(f'{sensor_name} - MODIS vs GEOS AOD') - axes[1].grid(True, alpha=0.3) - axes[1].legend() - - # Size legend for brown carbon fraction (using shared scale) - bc_max_for_legend = shared_bc_max if shared_bc_max is not None else np.max(bc_frac_valid) - if bc_max_for_legend > 0: - size_legend_values = [0, bc_max_for_legend * 0.5, bc_max_for_legend] - size_legend_sizes = [min_size, (min_size + max_size) / 2, max_size] - size_legend_labels = [f'{val:.3f}' for val in size_legend_values] - - legend_elements = [] - for size, label in zip(size_legend_sizes, size_legend_labels): - legend_elements.append(plt.scatter([], [], s=size, c='gray', alpha=0.7, - edgecolors='black', linewidth=0.5)) - - size_legend = axes[1].legend(legend_elements, size_legend_labels, - title='Brown Carbon\nFraction', - loc='upper left', bbox_to_anchor=(0.02, 0.98), - frameon=True, fancybox=True, shadow=True) - axes[1].add_artist(size_legend) - - # Calculate and display chi scaling factor on the scatter plot - valid_mod_adj = valid_mod + 0.01 - valid_geos_adj = valid_geos + 0.01 - chi_values = np.exp(np.log(valid_mod_adj) - np.log(valid_geos_adj)) - chi_mean = np.mean(chi_values) - - # Statistics - try: - from scipy.stats import pearsonr - if len(valid_mod) > 1: - corr, _ = pearsonr(valid_mod, valid_geos) - bias = np.mean(valid_geos - valid_mod) - rmse = np.sqrt(np.mean((valid_geos - valid_mod)**2)) - - stats_text = f'R = {corr:.3f}\nBias = {bias:.3f}\nRMSE = {rmse:.3f}\nฯ‡ = {chi_mean:.3f}\nN = {len(valid_mod)}' - axes[1].text(0.98, 0.02, stats_text, transform=axes[1].transAxes, - verticalalignment='bottom', horizontalalignment='right', - bbox=dict(boxstyle='round', facecolor='white', alpha=0.8)) - except ImportError: - pass - - # Calculate y-axis limits for time series (including stacked components) - # Find maximum values including stacked components - total_vals = np.zeros_like(valid_geos) - for comp_vals in components.values(): - valid_comp = comp_vals[valid_mask] - valid_comp = np.nan_to_num(valid_comp, nan=0.0) - total_vals += valid_comp - - # Include MODIS and GEOS values in y-range calculation - all_y_values = np.concatenate([valid_mod, valid_geos, total_vals]) - current_y_limits = [0, np.max(all_y_values) * 1.05] # Start from 0, add 5% padding at top - - # Return original data limits (not expanded) for proper shared scaling calculation - current_limits = [min(np.min(valid_mod), np.min(valid_geos)), - max(np.max(valid_mod), np.max(valid_geos))] - current_bc_max = np.max(bc_frac_valid) if len(bc_frac_valid) > 0 else 0 - - return True, current_limits, current_bc_max, current_y_limits - else: - axes[0].text(0.5, 0.5, f'No valid data for {sensor_name}', - transform=axes[0].transAxes, ha='center', va='center', fontsize=12) - axes[1].text(0.5, 0.5, f'No valid data for {sensor_name}', - transform=axes[1].transAxes, ha='center', va='center', fontsize=12) - return False, None, None, None - -def create_and_save_plot(mod_data, myd_data, region_id, sensor='both', output_dir='regional_analysis', experiment_name='c180R_qfed3igbp_allviirs'): - """Create and automatically save the analysis plot.""" - region = REGIONS[region_id] - region_name = region['name'] - - # Create output directory - output_dir = Path(output_dir) - output_dir.mkdir(exist_ok=True) - - print(f"Creating plot for {region_name}...") - - has_mod = mod_data is not None - has_myd = myd_data is not None - - if sensor.lower() == 'both' and has_mod and has_myd: - # 4-panel figure with IMPROVED layout and panel labels - fig = plt.figure(figsize=(20, 14)) # Increased width - - # Better layout: 2 rows, 2 real columns with uniform spacing - # Time series and scatter plot are treated as main columns - # Colorbar will be added later to use proper positioning - gs = fig.add_gridspec(2, 2, width_ratios=[1.2, 1], height_ratios=[1, 1], - left=0.06, right=0.9, top=0.90, bottom=0.08, - hspace=0.3, wspace=0.25) # Proper spacing between main columns - - # Create main axes - ax_terra_ts = fig.add_subplot(gs[0, 0]) - ax_terra_scatter = fig.add_subplot(gs[0, 1]) - ax_aqua_ts = fig.add_subplot(gs[1, 0]) - ax_aqua_scatter = fig.add_subplot(gs[1, 1]) - - axes_terra = [ax_terra_ts, ax_terra_scatter] - axes_aqua = [ax_aqua_ts, ax_aqua_scatter] - - # First pass: get data ranges for shared scaling (without labels) - terra_success, terra_limits, terra_bc_max, terra_y_limits = plot_single_sensor_panels( - mod_data, region_id, axes_terra, 'Terra') - aqua_success, aqua_limits, aqua_bc_max, aqua_y_limits = plot_single_sensor_panels( - myd_data, region_id, axes_aqua, 'Aqua') - - # Calculate shared limits and brown carbon maximum - shared_limits = None - shared_bc_max = None - shared_y_limits = None - - if terra_success and aqua_success: - if terra_limits and aqua_limits: - shared_limits = [min(terra_limits[0], aqua_limits[0]), - max(terra_limits[1], aqua_limits[1])] - if terra_bc_max and aqua_bc_max: - shared_bc_max = max(terra_bc_max, aqua_bc_max) - if terra_y_limits and aqua_y_limits: - shared_y_limits = [min(terra_y_limits[0], aqua_y_limits[0]), - max(terra_y_limits[1], aqua_y_limits[1])] - elif terra_success and terra_limits: - shared_limits = terra_limits - shared_bc_max = terra_bc_max - shared_y_limits = terra_y_limits - elif aqua_success and aqua_limits: - shared_limits = aqua_limits - shared_bc_max = aqua_bc_max - shared_y_limits = aqua_y_limits - - # Second pass: plot with shared scaling - if terra_success: - ax_terra_ts.clear() - ax_terra_scatter.clear() - - plot_single_sensor_panels(mod_data, region_id, axes_terra, 'Terra', - shared_limits, shared_bc_max, shared_y_limits, - show_legend=True, legend_position='upper left') - - if aqua_success: - ax_aqua_ts.clear() - ax_aqua_scatter.clear() - - plot_single_sensor_panels(myd_data, region_id, axes_aqua, 'Aqua', - shared_limits, shared_bc_max, shared_y_limits, - show_legend=False) - - # ADD CLEAN PANEL LABELS OUTSIDE the panels (after final plotting) - ax_terra_ts.text(-0.1, 1.02, 'a)', transform=ax_terra_ts.transAxes, - fontsize=12, fontweight='bold', va='bottom', ha='right') - ax_terra_scatter.text(-0.1, 1.02, 'b)', transform=ax_terra_scatter.transAxes, - fontsize=12, fontweight='bold', va='bottom', ha='right') - ax_aqua_ts.text(-0.1, 1.02, 'c)', transform=ax_aqua_ts.transAxes, - fontsize=12, fontweight='bold', va='bottom', ha='right') - ax_aqua_scatter.text(-0.1, 1.02, 'd)', transform=ax_aqua_scatter.transAxes, - fontsize=12, fontweight='bold', va='bottom', ha='right') - - # Add colorbar with proper positioning (minimal gap) - if aqua_success and len(ax_aqua_scatter.collections) > 0: - scatter = ax_aqua_scatter.collections[0] - cbar_ax = fig.add_axes([0.92, 0.15, 0.02, 0.7]) # [left, bottom, width, height] - cbar = fig.colorbar(scatter, cax=cbar_ax) - cbar.set_label('Month', rotation=270, labelpad=15) - elif terra_success and len(ax_terra_scatter.collections) > 0: - scatter = ax_terra_scatter.collections[0] - cbar_ax = fig.add_axes([0.92, 0.15, 0.02, 0.7]) # [left, bottom, width, height] - cbar = fig.colorbar(scatter, cax=cbar_ax) - cbar.set_label('Month', rotation=270, labelpad=15) - - fig.suptitle(f'{region_name} - Terra vs Aqua Comparison ({experiment_name})', fontsize=18) - sensor_str = 'terra_aqua' - - else: - # 2-panel figure with panel labels - fig, axes = plt.subplots(2, 1, figsize=(14, 12)) - - if sensor.lower() in ['terra', 'mod'] or (sensor.lower() == 'both' and has_mod and not has_myd): - if not has_mod: - print("No Terra data available") - return None, None - success, _, _, _ = plot_single_sensor_panels(mod_data, region_id, axes, 'Terra') - sensor_name = 'Terra' - sensor_str = 'terra' - elif sensor.lower() in ['aqua', 'myd'] or (sensor.lower() == 'both' and has_myd and not has_mod): - if not has_myd: - print("No Aqua data available") - return None, None - success, _, _, _ = plot_single_sensor_panels(myd_data, region_id, axes, 'Aqua') - sensor_name = 'Aqua' - sensor_str = 'aqua' - else: - print("No data available") - return None, None - - # ADD CLEAN PANEL LABELS for single sensor (after plotting, outside panels) - axes[0].text(-0.1, 1.02, 'a)', transform=axes[0].transAxes, - fontsize=12, fontweight='bold', va='bottom', ha='right') - axes[1].text(-0.1, 1.02, 'b)', transform=axes[1].transAxes, - fontsize=12, fontweight='bold', va='bottom', ha='right') - - if success: - # For single sensor plots, we don't need to remove the legend positioning - # since it's handled within the plotting function - if len(axes[1].collections) > 0: - cbar = plt.colorbar(axes[1].collections[0], ax=axes[1], shrink=0.8, aspect=20) - cbar.set_label('Month', rotation=270, labelpad=15) - - fig.suptitle(f'{region_name} - {sensor_name} Analysis ({experiment_name})', fontsize=18) - plt.subplots_adjust(left=0.08, right=0.85, top=0.92, bottom=0.08, hspace=0.3) - - # Save with experiment name in filename (NO region number) - filename = f"{experiment_name}_{region_name}_{sensor_str}.png" - save_path = output_dir / filename - plt.savefig(save_path, dpi=300, bbox_inches='tight', facecolor='white') - print(f"Plot saved to: {save_path}") - - return fig, save_path - -def main(): - """Enhanced main function with mapping capability.""" - parser = argparse.ArgumentParser(description='AOD Regional Analysis Tool') - - parser.add_argument('--region', '-r', type=int, - help='Single region ID to analyze. Use --list-regions to see options.') - parser.add_argument('--list-regions', action='store_true', - help='List all available regions and exit') - parser.add_argument('--sensor', '-s', choices=['terra', 'aqua', 'both'], default='both', - help='Which MODIS sensor(s) to use (default: both)') - parser.add_argument('--data-path', '-d', type=str, - default='sampledGEOS/c180R_qfed3igbp_allviirs', - help='Path to data directory') - parser.add_argument('--output', '-o', type=str, default='regional_analysis', - help='Output directory for plots') - parser.add_argument('--experiment', '-e', type=str, default='c180R_qfed3igbp_allviirs', - help='Experiment name for output filenames') - parser.add_argument('--no-display', action='store_true', - help='Do not display plots interactively') - - # Add mapping arguments - parser.add_argument('--create-maps', action='store_true', - help='Create regional statistics maps') - parser.add_argument('--maps-only', action='store_true', - help='Only create maps, skip individual region plots') - parser.add_argument('--chi-map', action='store_true', - help='Create dedicated chi scaling factor map with values') - - args = parser.parse_args() - - if args.list_regions: - print("Available regions:") - print("ID | Name") - print("----|" + "-"*30) - for region_id, region_info in REGIONS.items(): - print(f"{region_id:2d} | {region_info['name']}") - return - - if args.no_display: - import matplotlib - matplotlib.use('Agg') - - print(f"AOD Regional Analysis Tool") - print(f"=" * 50) - print(f"Experiment: {args.experiment}") - print(f"Data path: {args.data_path}") - print(f"Sensor: {args.sensor}") - print(f"Output directory: {args.output}") - - try: - # Load data - print(f"\nLoading data...") - mod_data, myd_data = load_monthly_data(base_path=args.data_path, sensor=args.sensor) - - if mod_data is None and myd_data is None: - print("ERROR: No data loaded!") - return 1 - - # Create maps if requested - if args.create_maps or args.maps_only or args.chi_map: - print("\nCreating regional statistics maps...") - figures, stats_df = create_all_maps(mod_data, myd_data, - sensor=args.sensor, - output_dir=args.output, - experiment_name=args.experiment) - - # If only chi map is requested, create it specifically - if args.chi_map and not args.create_maps: - if 'chi_mean' in stats_df.columns: - print("\nCreating dedicated chi scaling factor map...") - chi_fig, chi_save_path = create_chi_value_map( - stats_df, sensor=args.sensor, - output_dir=args.output, - experiment_name=args.experiment) - - if not args.no_display and chi_fig: - plt.figure(chi_fig.number) - plt.show() - elif not args.no_display and figures: - # Display one of the maps - correlation_fig = figures.get('correlation') - if correlation_fig: - plt.figure(correlation_fig[0].number) - plt.show() - - # Skip individual plots if maps-only is specified - if args.maps_only: - print(f"\nMaps created! Check {args.output}/ for results.") - return 0 - - if args.region is not None: - # Single region - if args.region not in REGIONS: - print(f"ERROR: Invalid region ID {args.region}") - return 1 - - fig, save_path = create_and_save_plot(mod_data, myd_data, args.region, - args.sensor, args.output, args.experiment) - - if fig and not args.no_display: - plt.show() - - else: - # All regions (only if not maps-only) - if not args.create_maps and not args.chi_map: # Avoid double processing - print(f"\nProcessing all regions...") - for region_id in REGIONS.keys(): - try: - fig, save_path = create_and_save_plot(mod_data, myd_data, region_id, - args.sensor, args.output, args.experiment) - if fig: - plt.close(fig) - except Exception as e: - print(f"Error processing region {region_id}: {e}") - - print(f"\nCompleted! Check {args.output}/ for results.") - - return 0 - - except Exception as e: - print(f"ERROR: {e}") - import traceback - traceback.print_exc() - return 1 - -if __name__ == "__main__": - sys.exit(main()) From f7232809ddfbf33e5ca785b2ea2afa0405252bfa Mon Sep 17 00:00:00 2001 From: Patricia Castellanos <69310046+patricia-nasa@users.noreply.github.com> Date: Fri, 29 May 2026 12:20:37 -0400 Subject: [PATCH 08/10] Update hsrl.py --- src/pyobs/hsrl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyobs/hsrl.py b/src/pyobs/hsrl.py index d412cbc..f8396a6 100644 --- a/src/pyobs/hsrl.py +++ b/src/pyobs/hsrl.py @@ -178,7 +178,7 @@ def __init__ (self,hsrl_filename,Nav_only=False,verbose=True,freq=3.0, # ----------------------------------------------- if getattr(self, 'date', None) is None: import re - match = re.search(r'_(\d{8})_', hsrl_filename) + match = re.search(r'_(\d{8})_', os.path.basename(hsrl_filename)) if match: dt_str = match.group(1) # Format as MM/DD/YYYY to match HSRL convention From 77181f0384bf5f8b08193b75f4ee55e988377b38 Mon Sep 17 00:00:00 2001 From: Patricia Castellanos <69310046+patricia-nasa@users.noreply.github.com> Date: Fri, 29 May 2026 12:21:09 -0400 Subject: [PATCH 09/10] Update hsrl.py --- src/pyobs/hsrl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyobs/hsrl.py b/src/pyobs/hsrl.py index f8396a6..7797256 100644 --- a/src/pyobs/hsrl.py +++ b/src/pyobs/hsrl.py @@ -2,7 +2,7 @@ """ Implements Python interface to the HSRL L2 data. """ - +import os import h5py import numpy as np from datetime import datetime, timedelta From 4330b4c066f62474762cf0dc38f3640b2852ed5e Mon Sep 17 00:00:00 2001 From: Patricia Castellanos <69310046+patricia-nasa@users.noreply.github.com> Date: Fri, 29 May 2026 12:25:39 -0400 Subject: [PATCH 10/10] Update CHANGELOG.md --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f78cd5d..c5befb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 # [Unreleased] - yyyy-mm-dd ### Fixed +- dial.py to accomodate python change for nan - update hsrl.py to use py3 integer divide. fixes date parsing. - units for backscatter coeffiecient to km-1 sr-1 in output files generated with aop.py - conversion from km to m in icartt.py @@ -16,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - add configuration file for GEOS-5430 (current FP) - add a buddy check code for station observation QC - add reader for GloSSAC data +- support for HALO in hsrl.py ### Changed - use xesmf regridder to station sampling. this is more efficient that using xarray native interp @@ -105,14 +107,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Added list parsing for variables in trajectory sampler -- dial.py to accomodate python change for nan - fixed byte string bug in aeronet.py -- - use a local copy of RH in aop calculator. otherwise it overwrites when fixRH is used +- use a local copy of RH in aop calculator. otherwise it overwrites when fixRH is used ### Added - MPL reader and plot curtain - calculation of total backscatter coefficient in aop.py - xrctl supports providing a list of control files -- support for HALO in hsrl.py - parse time in MPL reader to return datetimes - sampler notebook that uses station sampler at an MPL - add option for vacuum aerodynamic size cutoff