# ============================================================================
# DATA INSTANTIATION CELL
# ============================================================================
#
# INSTRUCTIONS FOR NOTEBOOK REORGANIZATION:
# This cell contains ALL data collection, API calls, and heavy computations.
# All Collections and Series objects are instantiated here to separate
# time-consuming data operations from visualization logic.
#
# PATTERN TO FOLLOW:
# 1. This second cell (after imports) contains ALL data fetching:
#    - API calls (Bloomberg, Haver, Goldman Sachs)
#    - Collection and Series instantiation
#    - Data transformations and calculations
#    - DataFrame processing and preparation
#
# 2. Subsequent cells contain ONLY visualization code:
#    - plot_lines(), plot_line(), plot_area() calls
#    - .dashboard.table(), .dashboard.plots() calls
#    - iris.summarize() calls
#    - DataFrame display operations (df.iloc[:, x:y])
#
# 3. Variable names created here are referenced in later cells
# 4. This enables quick re-rendering of visualizations without re-fetching data
# 5. All intermediate calculations and data prep happen here
#
# TO REORGANIZE A NOTEBOOK IN THIS MANNER:
# - Identify all lines that fetch data or create Collections/Series
# - Move those lines to this second cell, maintaining logical groupings
# - Keep variable assignments that will be used in visualization cells
# - Leave only plot/display commands in subsequent cells
# - Preserve the order of variable creation to maintain dependencies
# ============================================================================


# ----------------------------------------------------------------------------
# Activity Indicators
# ----------------------------------------------------------------------------
# CAI (Current Activity Indicator)
cai_series_soft_vs_hard = gs.get_CAI_series(
    geographyId=ISO_2,
    metricName=[
        "CAI_HEADLINE",
        "CAI_CONTRIBUTION_TYPE_HARD",
        "CAI_CONTRIBUTION_TYPE_SOFT",
    ],
    startDate="1980-01-01",
)
cai_series_soft_vs_hard = cai_series_soft_vs_hard.set_index("metricName", append=True)[
    "metricValue"
].unstack("metricName")
cai_series_soft_vs_hard.columns = ["Hard", "Soft", "Headline"]

# YHEI (Yicai High Frequency Economic Activity Index)
yicai = get_series_from_haver("V924YCI@INTWKLY")

# PMI Data
pmi_series = {
    "S&P/RatingDog PMI Mfg+Srvs": "S924VPTG@INTSRVYS",
    "S&P/RatingDog PMI Mfg": "S924VPMS@INTSRVYS",
    "S&P/RatingDog PMI Srvcs": "S924VPMM@INTSRVYS",
    "Composite PMI": "S924VPMI@EMERGEPR",
    "Mfg PMI": "S924VM@EMERGEPR",
    "Mfg Employment PMI": "S924VME@EMERGEPR",
    "Non-Mfg PMI": "S924VNG@EMERGEPR",
    "Services PMI": "S924VNGS@EMERGEPR",
}
pmi_collection = get_collection_from_haver(pmi_series)
pmi_df = pmi_collection.ts.rename(columns=pmi_collection.titles)

# Industrial Production
china_ip = get_series_from_haver("H924D@EMERGE").transform(
    func=lambda x: x.pct_change(12) * 100
)

# Daily Coal Consumption
daily_coal_consumption = get_series_from_haver("N924CS@INTDAILY")

# ----------------------------------------------------------------------------
# Surveys
# ----------------------------------------------------------------------------
# Consumer Confidence
consumer_confidence_series = {
    "Consumer Confidence": "H924VCC@EMERGEPR",
    "Future Price Expectations": "H924VFP@EMERGEPR",
    "Current Income Sentiment": "H924VCY@EMERGEPR",
    "Future Income Confidence": "H924VFY@EMERGEPR",
    "Current Employment Sentiment": "H924VCL@EMERGEPR",
    "Future Employment Expectations": "H924VCEL@EMERGEPR",
}
consumer_confidence_collection = get_collection_from_haver(consumer_confidence_series)

# Business Surveys
business_surveys = get_collection_from_haver(
    {
        "Industrial Enterprises Macro Performance": "N924VIE@EMERGEPR",
        "Industrial Enterprises Overseas Orders": "N924VIX@EMERGEPR",
        "Logistics Prosperity Index": "N924VTLP@EMERGEPR",
        "Economic Policy Uncertainty (News-Based)": "N924VIUP@EMERGEPR",
        "Trade Policy Uncertainty": "N924VIUT@EMERGEPR",
        "Yicai Chief Economists Confidence": "N924VECI@EMERGEPR",
        "CKGSB Business Conditions Index": "N924VVC@EMERGEPR",
        "Small & Medium Enterprises Development Index": "N924SMD@EMERGEPR",
        "Industrial Enterprises Survey: Sales": "N924VIP@EMERGEPR",
    }
)
chinese_surveys = consumer_confidence_collection.combine(business_surveys)

# Macroeconomic Climate Index
macroeconomic_climate_index = get_collection_from_haver(
    {
        "MC Leading Index": "N924VLD@EMERGEPR",
        "MC Coincident Index": "N924VLCC@EMERGEPR",
        "MC Lagging Index": "N924VLAG@EMERGEPR",
    }
)
climate_indexes_df = macroeconomic_climate_index.ts.rename(
    columns=macroeconomic_climate_index.titles
)

# ----------------------------------------------------------------------------
# Fixed Investment
# ----------------------------------------------------------------------------
fixed_investment_collection = get_collection_from_haver(
    {
        "Investment in Fixed Assets, ex Rural Households": "N924VU@EMERGEPR",
        "Cement Production": "N924OMNE@EMERGEPR",
        "Rebar Prices": "P924FR@INTDAILY",
    }
)

# ----------------------------------------------------------------------------
# GDP Data
# ----------------------------------------------------------------------------
# GDP by Supply Side
gdp_by_supply = get_collection_from_haver(
    {
        "Gross domestic product": "H924NGPC@EMERGEPR",
        "Primary industry": "H924G1C@EMERGEPR",
        "Secondary industry": "H924G2C@EMERGEPR",
        "Tertibary industry (services)": "H924G3C@EMERGEPR",
    }
).apply("yoy")

# GDP by Expenditure
real_gdp_series = {
    "Real GDP (YoY % Change)": "N924GY@EMERGE",
    "Final consumption expenditure": "Y924NCT@EMERGE",
    "Gross capital formation": "Y924NFBT@EMERGE",
    "Net exports of goods and services": "Y924NXNT@EMERGE",
    "Total domestic demand": "Y924NDTT@EMERGE",
}
real_gdp_collection = get_collection_from_haver(real_gdp_series)

# ----------------------------------------------------------------------------
# Inflation Data
# ----------------------------------------------------------------------------
price_series_collection = get_collection_from_haver(
    {
        "CPI": "H924PC@EMERGE",
        "CPI Core": "H924PCXG@EMERGE",
        "CPI Food": "H924PJFB@EMERGE",
        "CPI Housing": "H924PCH@EMERGE",
        "PPI": "H924PP@EMERGE",
        "PPI Mfg": "H924PPM@EMERGE",
        "PPI Consumer goods": "H924PPC@EMERGE",
    }
).apply("yoy")

price_leading_indicators = get_collection_from_haver(
    {
        "Lean Meat Pork Price (RMB/kg)": "P924HCN@INTDAILY",
        "PMI Nonmanufacturing Input Price (SA, 50+=Expansion)": "S924VNPI@INTSRVYS",
        "Wholesale Price 200 Index - Agricultural Products (EOP,2015=100)": "P924WAPE@INTWKLY",
    }
).apply("yoy")

inflation_collections = merge_collections(
    [price_series_collection, price_leading_indicators]
)

price_series_collection_df = price_series_collection.ts.rename(
    columns=price_series_collection.titles
)
cpi_df = price_series_collection_df.filter(regex="CPI", axis=1)
ppi_df = price_series_collection_df.filter(regex="PPI", axis=1)

# ----------------------------------------------------------------------------
# Labor Market Data
# ----------------------------------------------------------------------------
employment = {
    "Surveyed Urban Unemployment Rate": "N924ELUR@EMERGE",
    "Youth Unemployment (Age 16-24)": "N924EUR1@EMERGEPR",
    "Weekly Hours Worked (Avg)": "N924EH@EMERGEPR",
}
employment_collection = get_collection_from_haver(employment)
for i in range(3):
    employment_collection[i].good_is = -1

unemp_df = employment_collection.ts.rename(columns=employment_collection.titles)
unemp_df = unemp_df[[c for c in unemp_df.columns if "Unemployment" in c]]

# Capacity Utilization
china_capacity = get_series_from_haver("N924DUM@EMERGEPR")

# ----------------------------------------------------------------------------
# Trade & External Balance
# ----------------------------------------------------------------------------
freight_cols = bb.create_collection(["CNIFSCTT Index", "SHSPSCFI Index"])
freight_df = freight_cols.ts.ffill()
freight_df.columns = [
    "China Port Cargo Throughput",
    "Shanghai Containerized Freight Index",
]

current_account_balance = get_series_from_haver("H924BCPG@EMERGE")

# ----------------------------------------------------------------------------
# Credit & Financial Conditions
# ----------------------------------------------------------------------------
aggregate_social_financing = get_series_from_haver("P924FCZM@EMERGEPR")

loans_to_the_economy_collection = get_collection_from_haver(
    {
        "Total Loans": "N924FC@EMERGE",
        "Loans to Households": "N924FCH@EMERGE",
    }
).apply("yoy")

bberg_eco_china_credit_impulse_12_month_net_change = bb.get_series("CHBGREVA Index")

# ----------------------------------------------------------------------------
# Property Sector
# ----------------------------------------------------------------------------
home_prices = get_collection_from_haver(
    {
        "Existing 70 cities": "N924HK@EMERGEPR",
        "Existing Tier 1": "N924HK1@EMERGEPR",
        "Existing Tier 2": "N924HK2@EMERGEPR",
        "Existing Tier 3": "N924HK3@EMERGEPR",
        "New 70 cities": "N924HWM@EMERGEPR",
        "New Tier 1": "N924HWM1@EMERGEPR",
        "New Tier 2": "N924HWM2@EMERGEPR",
        "New Tier 3": "N924HWM3@EMERGEPR",
    }
)
home_prices_df = home_prices.ts.rename(columns=home_prices.titles)
buying_a_house = get_series_from_haver("N924VFSM@EMERGEPR")

# ============================================================================
# MAIN COLLECTIONS - Merge Most Important Collections
# ============================================================================
main_collections = merge_collections(
    [
        pmi_collection,  # PMI indicators (manufacturing & services activity)
        macroeconomic_climate_index,  # NBS climate index (leading, coincident, lagging)
        fixed_investment_collection,  # Fixed asset investment indicators
        gdp_by_supply,  # GDP by industry sectors
        real_gdp_collection,  # GDP by expenditure components
        price_series_collection,  # Core inflation measures (CPI, PPI)
        employment_collection,  # Labor market indicators
        loans_to_the_economy_collection,  # Credit growth metrics
        home_prices,  # Property price indices
    ]
)

print("✓ All data instantiated successfully")
print(f"✓ Main collection contains {len(main_collections)} key economic indicators")

overall_chinese_summary = iris.summarize(
    main_collections,
    "what trends on growth, inflation, employment and credit do you see in the chinese economy? Think of the flows, what is changing over time.",
    model="claude-sonnet-4-5-20250929",
)

overall_chinese_summary.html()

cai_plot = plot_lines(
    cai_series_soft_vs_hard,
    show_0=True,
    title=f"<b>{pyctry.name} Current Activity Indicator</b> Updated: {pd.Timestamp.today().strftime('%Y-%m-%d')}",
    years_limit=4,
    tick_suffix=" %",
    source="GS",
)

cai_plot

yicai.plot(
    title="China Weekly High Frequency Indicator (Yicai, 2019 = 1)", mma=12, hlines=[1]
)

pmi_response = iris.summarize(pmi_collection, bullets=3)
pmi_response.html()

pmi_collection.dashboard.table()

govt_pmis = plot_lines(
    pmi_df.filter(regex=r"^(?!.*S&P)").dropna(),
    title="<b>China Government PMIs</b>",
)
govt_pmis.add_hline(50, line_width=0.5)
govt_pmis.show()

sp_cmi = plot_lines(
    pmi_df.filter(regex="S&P").dropna(), title="<b>China S&P/RatingDog PMIs</b>"
)
sp_cmi.add_hline(50, line_width=0.5)
sp_cmi.show()

china_ip.plot(
    years_limit=5,
    title="China Industrial Production (Value Added ex Construction) - YoY % Change",
    source="Haver Analytics",
    tick_suffix=" %",
)

daily_coal_consumption.plot(
    title="China Daily Coal Consumption (Metric Tons per Day)",
    mma=250,
    source="Haver Analytics",
)

chinese_surveys_state = iris.summarize(chinese_surveys, bullets=3)
chinese_surveys_state.html()

consumer_confidence_collection.dashboard.table()

consumer_confidence_collection.dashboard.plots(hlines=[100, 50])

business_surveys.dashboard.table()

business_surveys.dashboard.plots(hlines=[50])

fig = plot_lines(
    climate_indexes_df,
    title="Macroeconomic Cycle Index",
    source="National Bureau of Statistics (NBS) of China",
)
fig.add_hline(y=100, line_width=1, line_color="gray")
for trace in fig.data:
    if trace.name == "MC Lagging Index":
        trace.update(line=dict(width=1))

fig

fixed_investment_collection[0].plot(
    mma=12, title="China Investment in Fixed Assets, ex Rural Households"
)

fixed_investment_collection[1].plot(mma=12, title="China Cement Production")

# **TODO**: Key growth indicators
# - Industrial Production YoY
# - Fixed Asset Investment YoY (total and ex-rural)
# - Retail Sales YoY
# - Manufacturing Investment
# - Infrastructure Investment

gdp_by_supply.dashboard.table()

plot_lines(
    gdp_by_supply.ts.rename(columns=gdp_by_supply.titles),
    title="GDP by Supply Side Growth Rates",
    tick_suffix=" %",
    show_0=True,
    top_margin=130,
    years_limit=10,
    source="Haver Analytics",
)

real_gdp_collection.dashboard.table()

real_gdp_collection.dashboard.plots(
    show_0=True, tick_suffix=" %", years_limit=10, source="Haver Analytics"
)

inflation_state = iris.summarize(inflation_collections, bullets=3)
inflation_state.html()

price_series_collection.dashboard.table()

cpi_df = price_series_collection_df.filter(regex="CPI", axis=1)

cpi_fig_inflation = plot_lines(
    cpi_df,
    title=f"{pyctry.name} CPI Inflation Measures (YoY)",
    tick_suffix=" %",
    show_0=True,
    years_limit=10,
    top_margin=120,
    source="Haver Analytics",
)

cpi_fig_inflation.update_layout(legend=dict(orientation="h", x=0, y=1.02))
cpi_fig_inflation

ppi_df = price_series_collection_df.filter(regex="PPI", axis=1)

ppi_fig_inflation = plot_lines(
    ppi_df,
    title=f"{pyctry.name} PPI Inflation Measures (YoY)",
    tick_suffix=" %",
    show_0=True,
    years_limit=10,
    #   top_margin=120,
    source="Haver Analytics",
)

ppi_fig_inflation.update_layout(legend=dict(orientation="h", x=0, y=1.02))
ppi_fig_inflation

price_leading_indicators.dashboard.plots(
    years_limit=2, show_0=True, tick_suffix=" %", connect_gaps=True
)

# **TODO**: Employment and slack indicators
# - Surveyed Unemployment Rate (urban)
# - Youth Unemployment (if available - note recent data suspension)
# - Manufacturing employment indices from PMI surveys
# - Wage growth indicators
# - Note: Set `good_is = -1` for unemployment metrics

employment_collection.dashboard.table()

unemp_df = employment_collection.ts.rename(columns=employment_collection.titles)
unemp_df = unemp_df[[c for c in unemp_df.columns if "Unemployment" in c]]

unemp_fig = plot_lines(
    unemp_df,
    title=f"{pyctry.name} Unemployment Rates",
    tick_suffix=" %",
    years_limit=2,
    # top_margin=120,
    show_0=True,
    source="Haver Analytics",
    default_y_range=[0, 25],
)

unemp_fig.update_layout(legend=dict(orientation="h", x=0, y=1.02))
unemp_fig

employment_collection[-1].plot(connect_gaps=True)

plots = [
    f"{pyctry.name} Employment Gap, Current Rate vs 5y Median (rolling)",
    f"{pyctry.name} Employment Gap in 12-24yr, Current Rate vs 5y Median (rolling)",
]
gap_df = 100 - unemp_df
figs = []
for i, title in enumerate(plots):
    s = 100 - unemp_df.iloc[:, i].dropna()
    rolling_median_5y = s.rolling(window=60, min_periods=12).median()
    gap_vs_5y_median = (s - rolling_median_5y).to_frame(name="Gap vs 5y median (pp)")
    fig = plot_lines(
        gap_vs_5y_median,
        title=title,
        tick_suffix=" pp",
        show_0=True,
        years_limit=5,
        source="Kate Capital, Haver",
    )
    figs.append(fig)

with columns(2, gap="30px", vertical_alignment="center") as cols:
    cols[0].plotly(figs[0])
    cols[1].plotly(figs[1])

china_capacity = get_series_from_haver("N924DUM@EMERGEPR")
fig = china_capacity.plot(show_0=True, tick_suffix="%")
fig.add_hline(
    y=china_capacity.ts.mean(),
    line_width=1,
    line_color="gray",
    annotation_text="<i>Mean</i>",
    annotation_position="bottom left",
    annotation_font_color="grey",
)
fig.show()

# **TODO**: Trade statistics - critical for China
# - Exports YoY (USD and local currency terms)
# - Imports YoY
# - Trade Balance
# - Current Account Balance % GDP
# - FX Reserves
# - Consider breaking down exports by destination (US, EU, etc.) if available

freight_cols = bb.create_collection(["CNIFSCTT Index", "SHSPSCFI Index"])
freight_cols.dashboard.table()

freight_df = freight_cols.ts.ffill()
freight_df.columns = [
    "China Port Cargo Throughput",
    "Shanghai Containerized Freight Index",
]

plot_line(
    blue=freight_df.iloc[:, 0],  # .cumsum().pct_change(52),
    red=freight_df.iloc[:, 1],
    blue_name="China Port Cargo Throughput (Containers)",
    red_name="Shanghai Containerized Freight Index",
    title="China Freight Indicators",
    years_limit=3,
    tick_suffix="",
    source="Bloomberg",
)

current_account_balance = get_series_from_haver("H924BCPG@EMERGE")
current_account_balance.plot(
    title="China Current Account Balance",
    tick_suffix="%",
    years_limit=10,
    show_0=True,
)

# trade_stats = {
#     "Exports YoY USD": "...",
#     "Imports YoY USD": "...",
#     "Trade Balance": "...",
#     "Current Account % GDP": "...",
# }
# trade_collection = hc.create_collection(trade_stats)

# Missing Stuff
# done - Total Social Financing (TSF) - broadest credit measure
# done - New Yuan Loans
# - M2 Money Supply growth
# - Shadow banking components (trust loans, entrusted loans)
# - Corporate debt % GDP
# - Household debt % GDP
# - Local Government Financing Vehicle (LGFV) debt if available

aggregate_social_financing = get_series_from_haver("P924FCZM@EMERGEPR")
aggregate_social_financing.plot(
    title="China Aggregate Social Financing to the Real Economy - YoY % Change",
    tick_suffix=" %",
    years_limit=10,
)

loans_to_the_economy_collection = get_collection_from_haver(
    {
        "Total Loans": "N924FC@EMERGE",
        "Loans to Households": "N924FCH@EMERGE",
    }
).apply("yoy")
loans_to_the_economy_collection.dashboard.table()

plot_lines(
    loans_to_the_economy_collection.ts.rename(
        columns=loans_to_the_economy_collection.titles
    ),
    tick_suffix=" %",
    title="China: Total Loans and Loans to Households - YoY % Change",
)

bberg_eco_china_credit_impulse_12_month_net_change = bb.get_series("CHBGREVA Index")
bberg_eco_china_credit_impulse_12_month_net_change.plot(
    title="Bloomberg China Credit Impulse 12 Month Net Change", tick_suffix=" %"
)

# credit_stats = {
#     "Total Social Financing YoY": "...",
#     "New Yuan Loans": "...",
#     "M2 YoY": "...",
#     "Corporate Debt % GDP": "...",
#     "Household Debt % GDP": "...",
# }
# credit_collection = hc.create_collection(credit_stats)

## Other Ideas
# - Property sales (floor space sold)
# - Property investment YoY
# - Housing starts
# - Developer funding sources
# - Inventory levels (months to clear)

home_prices = get_collection_from_haver(
    {
        "Existing 70 cities": "N924HK@EMERGEPR",
        "Existing Tier 1": "N924HK1@EMERGEPR",
        "Existing Tier 2": "N924HK2@EMERGEPR",
        "Existing Tier 3": "N924HK3@EMERGEPR",
        "New 70 cities": "N924HWM@EMERGEPR",
        "New Tier 1": "N924HWM1@EMERGEPR",
        "New Tier 2": "N924HWM2@EMERGEPR",
        "New Tier 3": "N924HWM3@EMERGEPR",
    }
)

home_prices_df = home_prices.ts.rename(columns=home_prices.titles)

home_prices_plot = {}
for k, v in {
    "New Residential Price Index": home_prices_df.filter(like="New"),
    "Existing Residential Price Index": home_prices_df.filter(like="Existing"),
}.items():
    fig = plot_lines(
        v,
        title=f"China {k}",
        # tick_suffix="",
        top_margin=120,
        years_limit=5,
        figsize=(800, 400),
        source="Haver Analytics",
    )

    fig.add_hline(y=100, line_width=1, line_color="gray")

    for trace in fig.data:
        name = getattr(trace, "name", "") or ""
        if "tier" in name.lower():
            trace.update(line=dict(width=0.5))

    home_prices_plot[k] = fig


with columns(2, gap="30px", vertical_alignment="center") as cols:
    cols[0].plotly(home_prices_plot["New Residential Price Index"])
    cols[1].plotly(home_prices_plot["Existing Residential Price Index"])

buying_a_house = get_series_from_haver("N924VFSM@EMERGEPR")
buying_a_house.plot(
    title="Residents spending to increase the next three months on buying a house",
    tick_suffix="%",
)

# **TODO**: Government finance
# - Fiscal balance (augmented deficit including off-budget)
# - Government debt % GDP
# - Local government debt
# - Fiscal revenue and expenditure growth

# fiscal_stats = {
#     "Fiscal Balance % GDP": "...",
#     "Government Debt % GDP": "...",
# }

# **TODO**: China-specific metrics to consider:
# - Li Keqiang Index (electricity consumption, rail freight, loans)
# - Commodity imports (iron ore, copper - proxy for construction activity)
# - Car sales
# - Semiconductor sales
# - Tech sector indicators
# - Capital flows (FDI, portfolio flows)
# - RMB exchange rate and volatility
# China Net monthly changes in Position f... CNNMFXPG Index

# **TODO**: Use `iris.summarize()` on key collections to get AI-generated insights
# - Example: `iris.summarize(credit_collection, "What trends do you see in China's credit creation?")`
# - Can also reference external sources: `iris.summarize(collection, "Summary with context from: https://...")`
# iris_analysis = iris.summarize(some_collection, "What trends are you seeing?")
# iris_analysis.html()

from tulip_mania.notebook_related import notebook_updated

notebook_updated()