Skip to content

View or edit on GitHub

This page is synchronized from trase/models/deduce/v1/notebooks/comparison.ipynb. Last modified on 2026-11-02 10:41 CET by dependabot[bot]. Please view or edit the original file there; changes should be reflected here after a midnight build (CET time), or manually triggering it with a GitHub action (link).

import pandas as pd

from trase.tools.aws.aws_helpers_cached import get_pandas_df_once
from trase.tools.aws.aws_helpers import read_s3_parquet

Config Session

PATH_REFERENCE_DEDUCE = (
    "data/Forest_loss_to_CLASSIFICATION_United States_PythonAPI_2024-06-03T170956.csv"
)

PATH_DEDUCE_NEW = "deduce/tables/DeDuCE_area_statistics_commodity_year_GLOBAL_names.csv"

PATH_FAO_COUNTRY_CODE = "deduce/tables/country_codes.csv"

KEY_COLUMNS = [
    #'country_code',
    "country_name",
    "year",
    "class_code_str",
    "area_hectares",
]

KEY_COLS_REF = [
    "country_name",
    "class_code",
] + ["loss_" + str(year) for year in list(range(2001, 2023))]

Load Inputs

df_deduce_new = get_pandas_df_once(key=PATH_DEDUCE_NEW, sep=",")

df_deduce_ref = pd.read_csv(PATH_REFERENCE_DEDUCE)

df_fao_country_code = get_pandas_df_once(key=PATH_FAO_COUNTRY_CODE, sep=",")

Pre-processing

df_deduce_new["year"] = (
    df_deduce_new["year"]
    .astype(str)
    .str.replace(".", "", regex=False)
    .str.replace(",", "", regex=False)
    .astype(int)
)

df_deduce_new_pivot = (
    df_deduce_new[KEY_COLUMNS]
    .groupby(by=["country_name", "year", "class_code_str"], as_index=False)[
        "area_hectares"
    ]
    .sum()
)

df_pivot = (
    df_deduce_new_pivot.pivot_table(
        index=["country_name", "class_code_str"],
        columns="year",
        values="area_hectares",
        aggfunc="sum",
    )
    .reset_index()
    .fillna(0)
)


# Rename year columns
df_pivot.columns = [
    f"loss_{col}" if str(col).isdigit() else col for col in df_pivot.columns
]

df_deduce_ref = df_deduce_ref.rename(
    columns={"COUNTRY": "country_name", "Class": "class_code"}
)

df_pivot = df_pivot.rename(columns={"class_code_str": "class_code"})


# replace country name
df_deduce_ref = df_deduce_ref.replace(
    {"country_name": {"United States": "United States of America"}}
)

Display Charts

df_deduce_country_new = df_pivot.query(
    'country_name.str.startswith("United States of Am")'
)

df_deduce_country_ref = (
    df_deduce_ref[KEY_COLS_REF]
    .groupby(by=["country_name", "class_code"], as_index=False)
    .sum()
)

# Set version
df_deduce_country_new["version"] = "New"
df_deduce_country_ref["version"] = "Ref"

df_comp = pd.concat([df_deduce_country_new, df_deduce_country_ref]).fillna(0)

/var/folders/42/_z7t3f3n2b74bggwwctg77z80000gn/T/ipykernel_36379/3274307982.py:10: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  df_deduce_country_new['version'] = 'New'

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Patch

# 1. Reshape (Melt) the dataframe from wide to long format
loss_cols = [col for col in df_comp.columns if str(col).startswith("loss_")]

df_melt = df_comp.melt(
    id_vars=["version", "class_code"],
    value_vars=loss_cols,
    var_name="year_col",
    value_name="loss",
)

# Extract the integer year from the column names
df_melt["year"] = df_melt["year_col"].str.replace("loss_", "").astype(int)

# 1. Force class_code to be a string so matrix alignment never misses a column
df_melt["class_code"] = df_melt["class_code"].astype(str)

# 2. Strip commas and spaces from the loss values, then convert to strict numbers
df_melt["loss"] = (
    df_melt["loss"]
    .astype(str)
    .str.replace(",", "", regex=False)
    .str.replace(" ", "", regex=False)
)
df_melt["loss"] = pd.to_numeric(df_melt["loss"], errors="coerce").fillna(0)

# 2. Group the data by year, version, and class_code
grouped = df_melt.groupby(["year", "version", "class_code"])["loss"].sum().reset_index()

unique_years = sorted(grouped["year"].unique())
class_codes = sorted(grouped["class_code"].unique())
versions = ["Ref", "New"]

# --- NOVO: Limite dinâmico para anotações ---
# Calcula o valor máximo de um bloco individual para definir o que é "alto volume".
# Ajuste o multiplicador (ex: 0.10 para 10%) para mostrar mais ou menos anotações.
volume_threshold = grouped["loss"].abs().max() * 0.10

# Estilo da caixa de texto para garantir leitura fácil sobre qualquer cor
bbox_props = dict(
    boxstyle="round,pad=0.2", facecolor="white", alpha=0.7, edgecolor="none"
)

# 3. Setup the Matplotlib Plot
fig, ax = plt.subplots(figsize=(18, 9))
width = 0.35  # Set the width of the bars

# Create a colormap dictionary for consistent coloring of each class_code
colors = plt.cm.get_cmap("tab20")
color_dict = {cc: colors(i % 20) for i, cc in enumerate(class_codes)}

legend_dict = {}

# 4. Plot the stacked bars
for version in versions:
    offset = -width / 2 if version == "Ref" else width / 2

    edge_color = "black" if version == "New" else "white"
    line_width = 1.2 if version == "New" else 0.5

    v_data = grouped[grouped["version"] == version]

    # Force strict matrix alignment
    pivot = v_data.pivot(index="year", columns="class_code", values="loss")
    pivot = pivot.reindex(index=unique_years, columns=class_codes, fill_value=0)

    # Keep track of the bottom edge to appropriately stack the segments
    bottom = np.zeros(len(unique_years), dtype="float64")

    # Looping through the strictly sorted class_codes
    for cc in class_codes:
        # Explicitly extract values as floats
        values = pivot[cc].astype(float).values

        bars = ax.bar(
            np.array(unique_years) + offset,
            values,
            width,
            bottom=bottom,
            color=color_dict[cc],
            edgecolor=edge_color,
            linewidth=line_width,
        )

        # --- NOVO: Adiciona o rótulo dentro dos maiores blocos ---
        for bar in bars:
            height = bar.get_height()
            if abs(height) >= volume_threshold:
                ax.text(
                    bar.get_x() + bar.get_width() / 2,
                    bar.get_y() + height / 2,
                    str(cc),
                    ha="center",
                    va="center",
                    fontsize=8,
                    fontweight="bold",
                    color="black",
                    bbox=bbox_props,
                )

        bottom += values

        if cc not in legend_dict:
            legend_dict[cc] = bars[0]

    # Add totals on top of the completed stacked bars
    for i, year in enumerate(unique_years):
        total_val = bottom[i]
        if total_val > 0:
            ax.text(
                year + offset,
                total_val + (total_val * 0.01),
                f"{total_val:,.0f}",
                ha="center",
                va="bottom",
                rotation=90,
                fontsize=9,
            )

# 5. Format the axes and labels
ax.set_xticks(unique_years)
ax.set_xticklabels(unique_years, rotation=45, fontsize=11)

# Increase the upper y-limit slightly to make room for the text totals
current_ylim = ax.get_ylim()
ax.set_ylim(current_ylim[0], current_ylim[1] * 1.15)

ax.set_ylabel("Total Loss", fontweight="bold", fontsize=12)
ax.set_title("Yearly Loss Comparison: Ref vs New - USA", fontweight="bold", fontsize=14)

# 6. Customize the legend
handles = [legend_dict[cc] for cc in sorted(legend_dict.keys())]
labels = [f"Class {cc}" for cc in sorted(legend_dict.keys())]

ref_marker = Patch(
    facecolor="white", edgecolor="gray", label="Left Bar: Ref (No Border)"
)
new_marker = Patch(
    facecolor="white",
    edgecolor="black",
    linewidth=1.5,
    label="Right Bar: New (Black Border)",
)
handles.extend([ref_marker, new_marker])
labels.extend(["Left Bar: Ref", "Right Bar: New"])

ax.legend(
    handles=handles,
    labels=labels,
    title="Legend",
    bbox_to_anchor=(1.02, 1),
    loc="upper left",
    ncol=2,
    fontsize=10,
)

plt.subplots_adjust(right=0.8, top=0.9)

plt.tight_layout()
plt.show()

/var/folders/42/_z7t3f3n2b74bggwwctg77z80000gn/T/ipykernel_36379/1245224827.py:46: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed in 3.11. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap()`` or ``pyplot.get_cmap()`` instead.
  colors = plt.cm.get_cmap('tab20')

png

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Patch

df_postive = df_comp.query("class_code > 0")

# 1. Reshape (Melt) the dataframe from wide to long format
loss_cols = [col for col in df_postive.columns if str(col).startswith("loss_")]

df_melt = df_postive.melt(
    id_vars=["version", "class_code"],
    value_vars=loss_cols,
    var_name="year_col",
    value_name="loss",
)

# Extract the integer year from the column names
df_melt["year"] = df_melt["year_col"].str.replace("loss_", "").astype(int)

# 1. Force class_code to be a string so matrix alignment never misses a column
df_melt["class_code"] = df_melt["class_code"].astype(str)

# 2. Strip commas and spaces from the loss values, then convert to strict numbers
df_melt["loss"] = (
    df_melt["loss"]
    .astype(str)
    .str.replace(",", "", regex=False)
    .str.replace(" ", "", regex=False)
)
df_melt["loss"] = pd.to_numeric(df_melt["loss"], errors="coerce").fillna(0)

# 2. Group the data by year, version, and class_code
grouped = df_melt.groupby(["year", "version", "class_code"])["loss"].sum().reset_index()

unique_years = sorted(grouped["year"].unique())
class_codes = sorted(grouped["class_code"].unique())
versions = ["Ref", "New"]

# 3. Setup the Matplotlib Plot
fig, ax = plt.subplots(figsize=(18, 9))
width = 0.35  # Set the width of the bars

# Create a colormap dictionary for consistent coloring of each class_code
colors = plt.cm.get_cmap("tab20")
color_dict = {cc: colors(i % 20) for i, cc in enumerate(class_codes)}

legend_dict = {}

# 4. Plot the stacked bars
for version in versions:
    offset = -width / 2 if version == "Ref" else width / 2

    edge_color = "black" if version == "New" else "white"
    line_width = 1.2 if version == "New" else 0.5

    v_data = grouped[grouped["version"] == version]

    # Force strict matrix alignment
    pivot = v_data.pivot(index="year", columns="class_code", values="loss")
    pivot = pivot.reindex(index=unique_years, columns=class_codes, fill_value=0)

    # Keep track of the bottom edge to appropriately stack the segments
    bottom = np.zeros(len(unique_years), dtype="float64")

    # Looping through the strictly sorted class_codes
    for cc in class_codes:
        # Explicitly extract values as floats
        values = pivot[cc].astype(float).values

        bars = ax.bar(
            np.array(unique_years) + offset,
            values,
            width,
            bottom=bottom,
            color=color_dict[cc],
            edgecolor=edge_color,
            linewidth=line_width,
        )
        bottom += values

        if cc not in legend_dict:
            legend_dict[cc] = bars[0]

    # Add totals on top of the completed stacked bars
    for i, year in enumerate(unique_years):
        total_val = bottom[i]
        if total_val > 0:
            ax.text(
                year + offset,
                total_val + (total_val * 0.01),
                f"{total_val:,.0f}",
                ha="center",
                va="bottom",
                rotation=90,
                fontsize=9,
            )

# 5. Format the axes and labels
ax.set_xticks(unique_years)
ax.set_xticklabels(unique_years, rotation=45, fontsize=11)

# Increase the upper y-limit slightly to make room for the text totals
current_ylim = ax.get_ylim()
ax.set_ylim(current_ylim[0], current_ylim[1] * 1.15)

ax.set_ylabel("Total Loss", fontweight="bold", fontsize=12)
ax.set_title(
    "Yearly Loss Comparison: Ref vs New - USA (positive codes)",
    fontweight="bold",
    fontsize=14,
)

# 6. Customize the legend
handles = [legend_dict[cc] for cc in sorted(legend_dict.keys())]
labels = [f"Class {cc}" for cc in sorted(legend_dict.keys())]

ref_marker = Patch(
    facecolor="white", edgecolor="gray", label="Left Bar: Ref (No Border)"
)
new_marker = Patch(
    facecolor="white",
    edgecolor="black",
    linewidth=1.5,
    label="Right Bar: New (Black Border)",
)
handles.extend([ref_marker, new_marker])
labels.extend(["Left Bar: Ref", "Right Bar: New"])

ax.legend(
    handles=handles,
    labels=labels,
    title="Legend",
    bbox_to_anchor=(1.02, 1),
    loc="upper left",
    ncol=2,
    fontsize=10,
)

plt.subplots_adjust(right=0.8, top=0.9)

plt.tight_layout()
plt.show()

/var/folders/42/_z7t3f3n2b74bggwwctg77z80000gn/T/ipykernel_36379/1459930092.py:40: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed in 3.11. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap()`` or ``pyplot.get_cmap()`` instead.
  colors = plt.cm.get_cmap('tab20')

png

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Patch

# 1. Reshape (Melt) the dataframe from wide to long format
loss_cols = [col for col in df_comp.columns if str(col).startswith("loss_")]

df_melt = df_comp.melt(
    id_vars=["version", "class_code"],
    value_vars=loss_cols,
    var_name="year_col",
    value_name="loss",
)

# Extract the integer year
df_melt["year"] = df_melt["year_col"].str.replace("loss_", "").astype(int)

# Clean the data to ensure no strings/commas break the math
df_melt["class_code"] = df_melt["class_code"].astype(str)
df_melt["loss"] = (
    df_melt["loss"]
    .astype(str)
    .str.replace(",", "", regex=False)
    .str.replace(" ", "", regex=False)
)
df_melt["loss"] = pd.to_numeric(df_melt["loss"], errors="coerce").fillna(0)

# 2. Group the data by year, version, and class_code
grouped = df_melt.groupby(["year", "version", "class_code"])["loss"].sum().reset_index()

# 3. Calculate the Difference (New - Ref) per year and class code
diff_df = grouped.pivot_table(
    index=["year", "class_code"], columns="version", values="loss", fill_value=0
).reset_index()

if "New" not in diff_df.columns:
    diff_df["New"] = 0
if "Ref" not in diff_df.columns:
    diff_df["Ref"] = 0
diff_df["difference"] = diff_df["New"] - diff_df["Ref"]

chart_data = diff_df.pivot(
    index="year", columns="class_code", values="difference"
).fillna(0)

unique_years = sorted(chart_data.index)
class_codes = sorted(chart_data.columns)

# 4. Setup the Matplotlib Plot
fig, ax = plt.subplots(figsize=(18, 9))
width = 0.6

colors = plt.cm.get_cmap("tab20")
color_dict = {cc: colors(i % 20) for i, cc in enumerate(class_codes)}

legend_dict = {}

pos_bottom = np.zeros(len(unique_years))
neg_bottom = np.zeros(len(unique_years))

# --- DYNAMIC THRESHOLD ---
# Calculate a threshold to define "high volume" (e.g., 10% of the largest single block).
# Adjust this decimal to show MORE labels (e.g., 0.05) or FEWER labels (e.g., 0.20).
volume_threshold = diff_df["difference"].abs().max() * 0.10

# Helper dictionary for the text box styling to guarantee readability
bbox_props = dict(
    boxstyle="round,pad=0.2", facecolor="white", alpha=0.7, edgecolor="none"
)

for cc in class_codes:
    values = chart_data[cc].values

    pos_values = np.where(values > 0, values, 0)
    neg_values = np.where(values < 0, values, 0)

    pos_bars = ax.bar(
        unique_years,
        pos_values,
        width,
        bottom=pos_bottom,
        color=color_dict[cc],
        edgecolor="white",
        linewidth=0.5,
    )
    neg_bars = ax.bar(
        unique_years,
        neg_values,
        width,
        bottom=neg_bottom,
        color=color_dict[cc],
        edgecolor="white",
        linewidth=0.5,
    )

    # --- ADD LABELS TO POSITIVE BARS ---
    for bar in pos_bars:
        height = bar.get_height()
        if height >= volume_threshold:
            ax.text(
                bar.get_x() + bar.get_width() / 2,
                bar.get_y() + height / 2,
                str(cc),
                ha="center",
                va="center",
                fontsize=8,
                fontweight="bold",
                color="black",
                bbox=bbox_props,
            )

    # --- ADD LABELS TO NEGATIVE BARS ---
    for bar in neg_bars:
        height = bar.get_height()
        if abs(height) >= volume_threshold:
            ax.text(
                bar.get_x() + bar.get_width() / 2,
                bar.get_y() + height / 2,
                str(cc),
                ha="center",
                va="center",
                fontsize=8,
                fontweight="bold",
                color="black",
                bbox=bbox_props,
            )

    pos_bottom += pos_values
    neg_bottom += neg_values

    if cc not in legend_dict:
        legend_dict[cc] = pos_bars[0] if sum(pos_values) > 0 else neg_bars[0]

ax.axhline(0, color="black", linewidth=1.5, zorder=0)

# 5. Format the axes and labels
ax.set_xticks(unique_years)
ax.set_xticklabels(unique_years, rotation=45, fontsize=11)
ax.set_ylabel("Loss Difference (New - Ref)", fontweight="bold", fontsize=12)
ax.set_title(
    "Yearly Loss Difference per Class Code (New minus Ref)",
    fontweight="bold",
    fontsize=14,
)

# 6. Customize the legend
handles = [legend_dict[cc] for cc in class_codes]
labels = [f"Class {cc}" for cc in class_codes]

ax.legend(
    handles=handles,
    labels=labels,
    title="Class Codes",
    bbox_to_anchor=(1.02, 1),
    loc="upper left",
    ncol=2,
    fontsize=10,
)

plt.subplots_adjust(right=0.8)
plt.tight_layout()
plt.show()

/var/folders/42/_z7t3f3n2b74bggwwctg77z80000gn/T/ipykernel_36379/311525573.py:48: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed in 3.11. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap()`` or ``pyplot.get_cmap()`` instead.
  colors = plt.cm.get_cmap('tab20')

png