Halloween Lasts One Night. The Business Behind It Runs All Year.

import os
import warnings

# keep the output to results: no download progress bars or library notices
os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
warnings.filterwarnings("ignore")

import numpy as np
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
import plotly.io as pio
from huggingface_hub import hf_hub_download
from huggingface_hub.utils import logging as hf_logging

hf_logging.set_verbosity_error()

# one look for every chart: dark, transparent, and colorblind-friendly
COLORS = {"Halloween": "#fb923c", "Christmas": "#34d399", "Everything else": "#94a3b8", "Last year": "#a78bfa", "Sales pace": "#38bdf8"}
pio.templates["post"] = go.layout.Template(
    layout=dict(
        paper_bgcolor="rgba(0,0,0,0)",
        plot_bgcolor="rgba(0,0,0,0)",
        font=dict(family="Inter, system-ui, sans-serif", color="#cbd5e1", size=13),
        title=dict(font=dict(size=17, color="#f1f5f9"), x=0, xanchor="left"),
        xaxis=dict(gridcolor="rgba(148,163,184,0.15)", zerolinecolor="rgba(148,163,184,0.3)"),
        yaxis=dict(gridcolor="rgba(148,163,184,0.15)", zerolinecolor="rgba(148,163,184,0.3)"),
        colorway=list(COLORS.values()),
        margin=dict(l=10, r=10, t=60, b=10),
        legend=dict(bgcolor="rgba(0,0,0,0)"),
    )
)
pio.templates.default = "plotly_dark+post"
# show charts in JupyterLab and VS Code, and keep their data in the saved notebook
pio.renderers.default = "plotly_mimetype+notebook_connected"

# NRF annual Halloween surveys (Prosper Insights & Analytics); 2026 is a forecast
nrf = pd.DataFrame({
    "year": ["2021", "2022", "2023", "2024", "2025", "2026"],
    "billions": [10.14, 10.6, 12.2, 11.6, 13.1, 13.5],
})
nrf["kind"] = np.where(nrf["year"] == "2026", "Forecast", "Survey")

nrf_fig = px.bar(
    nrf, x="year", y="billions", color="kind", text="billions",
    color_discrete_map={"Survey": COLORS["Everything else"], "Forecast": COLORS["Halloween"]},
    title="U.S. Halloween spending keeps setting records",
    labels={"year": "", "billions": "Billions of dollars", "kind": ""},
)
nrf_fig.update_traces(texttemplate="$%{text:.1f}B", textposition="outside", hovertemplate="%{x}: $%{y:.1f} billion<extra></extra>")
nrf_fig.update_layout(yaxis_range=[0, 15.5], height=380)
nrf_fig.show()

steps = pd.DataFrame([
    ("Plan", "2025-11-01", "2026-02-28", "Review what sold and what was left over; set next year's budget and product mix"),
    ("Buy", "2026-01-15", "2026-05-31", "Trade shows and purchase orders; TransWorld's Halloween & Attractions Show ran March 26-29 in St. Louis"),
    ("Make and ship", "2026-03-15", "2026-08-31", "Factories produce and goods move to warehouses; wholesale shoppers stock up in late summer"),
    ("Hire and open", "2026-06-11", "2026-09-07", "Spirit Halloween announced 52,000 seasonal hires on June 11 and opened about 1,575 pop-up stores, most in August"),
    ("Sell", "2026-09-01", "2026-10-31", "About half of shoppers start in September or earlier, but sales pile up in the final weeks"),
    ("Clear", "2026-11-01", "2026-11-30", "Markdowns; leftovers sold off or stored; next year's plan starts"),
], columns=["step", "start", "end", "detail"])

timeline = px.timeline(
    steps, x_start="start", x_end="end", y="step", custom_data=["detail"],
    title="One night of sales, a year of work",
    color_discrete_sequence=[COLORS["Halloween"]],
)
timeline.update_traces(hovertemplate="<b>%{y}</b><br>%{customdata[0]}<extra></extra>")
timeline.update_yaxes(autorange="reversed", title="")
timeline.add_vline(x=pd.Timestamp("2026-10-31").timestamp() * 1000, line_dash="dot", line_color="#f1f5f9")
timeline.add_annotation(x="2026-10-31", y=1.06, yref="paper", text="Oct 31", showarrow=False, font=dict(color="#f1f5f9"))
timeline.update_layout(height=360, xaxis_title="")
timeline.show()

path = hf_hub_download("michaelmallari/online-retail-ii", "online-retail-ii.xlsx", repo_type="dataset")
sheets = pd.read_excel(path, sheet_name=None, dtype={"Invoice": str, "StockCode": str, "Description": str})
orders = pd.concat(sheets.values(), ignore_index=True)

# keep real sales: drop returns (invoice numbers starting with C), zero prices, and negative quantities
orders = orders[(orders["Quantity"] > 0) & (orders["Price"] > 0) & ~orders["Invoice"].str.startswith("C")].copy()
orders["revenue"] = orders["Quantity"] * orders["Price"]
print(f"{len(orders):,} order lines, {orders['InvoiceDate'].min():%b %Y} to {orders['InvoiceDate'].max():%b %Y}")
print(f"{orders['Customer ID'].nunique():,} customers in {orders['Country'].nunique()} countries")

desc = orders["Description"].fillna("").str.upper()
for name, pattern in {"Christmas": "CHRISTMAS|XMAS|SANTA|REINDEER|ADVENT", "Halloween": "HALLOWEEN"}.items():
    match = desc.str.contains(pattern)
    print(f"{name}: {desc[match].nunique()} products, £{orders.loc[match, 'revenue'].sum():,.0f} in sales")

christmas = orders[desc.str.contains("CHRISTMAS|XMAS|SANTA|REINDEER|ADVENT")].copy()
# days from each order to the next Christmas
target = pd.to_datetime(christmas["InvoiceDate"].dt.year.astype(str) + "-12-25")
target = target.where(christmas["InvoiceDate"] <= target, target + pd.DateOffset(years=1))
christmas["days_before"] = (target - christmas["InvoiceDate"]).dt.days

lead = (
    christmas.assign(months_before=christmas["days_before"] // 30)
    .groupby("months_before")["revenue"].sum()
    .pipe(lambda s: s / s.sum())
    .loc[:6]
)
lead_fig = px.bar(
    x=lead.index, y=lead.values,
    color_discrete_sequence=[COLORS["Christmas"]],
    title="Shops order Christmas stock one to three months ahead",
    labels={"x": "Months before Christmas", "y": "Share of Christmas orders"},
)
lead_fig.update_traces(hovertemplate="%{x} to %{x}+1 months before: %{y:.0%} of orders<extra></extra>")
lead_fig.update_layout(xaxis_autorange="reversed", yaxis_tickformat=".0%", xaxis_dtick=1, height=400)
lead_fig.show()

share = lambda days: christmas.loc[christmas["days_before"] >= days, "revenue"].sum() / christmas["revenue"].sum()
print(f"Ordered at least 1 month ahead: {share(30):.0%}   2 months: {share(60):.0%}   3 months: {share(90):.0%}")

calendar = pd.read_csv(hf_hub_download("denephew/M5_Forecasting", "calendar.csv", repo_type="dataset"))
sales = pd.read_csv(hf_hub_download("denephew/M5_Forecasting", "sales_train_evaluation.csv", repo_type="dataset"))

day_cols = [c for c in sales.columns if c.startswith("d_")]
dates = pd.DatetimeIndex(pd.to_datetime(calendar.set_index("d").loc[day_cols, "date"]))

# add up all 10 stores: one row per product, one column per day
items = sales.groupby(["item_id", "dept_id"])[day_cols].sum()
items.columns = dates
print(f"{len(items):,} products in {items.index.get_level_values('dept_id').nunique()} departments, {items.shape[1]:,} days ({dates[0]:%b %d, %Y} to {dates[-1]:%b %d, %Y})")
print("Halloween in the calendar:", ", ".join(calendar.loc[calendar["event_name_1"] == "Halloween", "date"]))

YEARS = range(2011, 2016)
cols = items.columns

def daily_rate(year, start, end):
    """Average units per day for every product, from `start` to `end` days around Halloween."""
    halloween = pd.Timestamp(year, 10, 31)
    window = (cols >= halloween + pd.Timedelta(days=start)) & (cols <= halloween + pd.Timedelta(days=end))
    return items.loc[:, window].mean(axis=1)

# a Halloween product sells at least 1.8x faster in the 3 weeks before Oct 31
# than in late summer AND than in the 2 weeks after, in at least 3 of the 5 years
hits = pd.DataFrame({
    year: (daily_rate(year, -20, 0) > 1.8 * daily_rate(year, 1, 14) + 0.05)
    & (daily_rate(year, -20, 0) > 1.8 * daily_rate(year, -90, -50) + 0.05)
    for year in YEARS
})
is_halloween = hits.sum(axis=1) >= 3
halloween_items = items[is_halloween]
print(f"{is_halloween.sum()} of {len(items):,} products pass the test")
halloween_items.index.get_level_values("dept_id").value_counts().rename("products").to_frame()

depts = items.groupby(level="dept_id").sum()
ratio = pd.DataFrame({
    year: depts.loc[:, (cols >= f"{year}-10-25") & (cols <= f"{year}-10-31")].mean(axis=1)
    / depts.loc[:, (cols >= f"{year}-11-01") & (cols <= f"{year}-11-07")].mean(axis=1)
    for year in YEARS
}).mean(axis=1).sort_values()

dept_fig = px.bar(
    x=ratio.values, y=ratio.index, orientation="h",
    color=np.where(ratio.index == "HOBBIES_2", "Halloween", "Everything else"),
    color_discrete_map=COLORS,
    title="One department sells almost twice as much the week before Halloween",
    labels={"x": "Sales in the week before Oct 31 vs. the week after", "y": ""},
)
dept_fig.add_vline(x=1, line_dash="dot", line_color="#f1f5f9")
dept_fig.update_traces(hovertemplate="%{y}: %{x:.2f}x<extra></extra>")
dept_fig.update_layout(showlegend=False, height=380, xaxis_ticksuffix="x")
dept_fig.show()

# average each group's daily sales around Halloween across 2011-2015, scaled so late summer = 1
def profile(rows):
    curves = []
    for year in YEARS:
        halloween = pd.Timestamp(year, 10, 31)
        window = (cols >= halloween - pd.Timedelta(days=90)) & (cols <= halloween + pd.Timedelta(days=20))
        daily = rows.loc[:, window].sum(axis=0).values
        curves.append(daily / daily[:40].mean())
    return np.mean(curves, axis=0)

offsets = np.arange(-90, 21)
shape = pd.DataFrame({
    "days": np.tile(offsets, 2),
    "index": np.concatenate([profile(halloween_items), profile(items[~is_halloween])]),
    "group": np.repeat([f"The {len(halloween_items)} Halloween products", "Everything else"], len(offsets)),
})
shape_fig = px.line(
    shape, x="days", y="index", color="group",
    color_discrete_sequence=[COLORS["Halloween"], COLORS["Everything else"]],
    title="Halloween products build for weeks, then fall off a cliff on Nov 1",
    labels={"days": "Days from Halloween", "index": "Daily sales (late summer = 1)", "group": ""},
)
shape_fig.add_vline(x=0, line_dash="dot", line_color="#f1f5f9")
shape_fig.update_traces(hovertemplate="%{x} days: %{y:.1f}x<extra></extra>")
shape_fig.update_layout(height=420, legend=dict(orientation="h", y=1.02, x=0, yanchor="bottom"))
shape_fig.show()

# what share of the Sep 1 to Oct 31 season has sold by each day?
def season_mask(year):
    return (cols >= f"{year}-09-01") & (cols <= f"{year}-10-31")

curves = []
for year in YEARS:
    daily = halloween_items.loc[:, season_mask(year)].sum(axis=0).values
    curves.append(np.cumsum(daily) / daily.sum())
sold = np.mean(curves, axis=0)
days_left = np.arange(len(sold) - 1, -1, -1)
half_point = days_left[np.argmax(sold >= 0.5)]
print(f"Half of the season's units sell in the last {half_point} days before Halloween")
print(f"Sold by Oct 1: {sold[30]:.0%}   by Oct 15: {sold[44]:.0%}   by Oct 24: {sold[53]:.0%}")

season = pd.DataFrame({year: halloween_items.loc[:, season_mask(year)].sum(axis=1) for year in YEARS})

def wape(forecast, actual):
    """Total miss across products, as a share of what actually sold."""
    return np.abs(forecast - actual).sum() / actual.sum()

rows = []
for days_out in range(60, 0, -1):
    for year in range(2012, 2016):
        this_cut = pd.Timestamp(year, 10, 31) - pd.Timedelta(days=days_out)
        last_cut = pd.Timestamp(year - 1, 10, 31) - pd.Timedelta(days=days_out)
        so_far = halloween_items.loc[:, (cols >= f"{year}-09-01") & (cols <= this_cut)].sum(axis=1)
        last_so_far = halloween_items.loc[:, (cols >= f"{year - 1}-09-01") & (cols <= last_cut)].sum(axis=1)
        share_sold_by_now = last_so_far.sum() / season[year - 1].sum()
        rows.append({
            "days": days_out, "year": year,
            "Last year": wape(season[year - 1], season[year]),
            "Sales pace": wape(so_far / share_sold_by_now, season[year]),
        })
errors = pd.DataFrame(rows).groupby("days")[["Last year", "Sales pace"]].mean()

err_fig = px.line(
    errors.reset_index().melt(id_vars="days", var_name="method", value_name="miss"),
    x="days", y="miss", color="method", color_discrete_map=COLORS,
    title="A good forecast shows up about three weeks before Halloween: too late to reorder",
    labels={"days": "Days before Halloween", "miss": "Average miss per product", "method": "Forecast"},
)
err_fig.update_traces(hovertemplate="%{x} days out: off by %{y:.0%}<extra></extra>")
err_fig.update_layout(xaxis_autorange="reversed", yaxis_tickformat=".0%", height=420, yaxis_rangemode="tozero")
err_fig.show()

crossover = errors.index[errors["Sales pace"] < errors["Last year"]].max()
print(f"Last year's sales, used as the forecast, miss by {errors['Last year'].iloc[0]:.0%} per product")
print(f"Tracking this year's sales pace beats that only from {crossover} days out")

# how this year's season compared with last year's, product by product
ratios = pd.concat([
    season[year] / season[year - 1] for year in range(2012, 2016)
])[pd.concat([season[year - 1] for year in range(2012, 2016)]) >= 20].values
print(f"{len(ratios)} product-years. This year as a share of last year: "
      f"10th percentile {np.percentile(ratios, 10):.0%}, median {np.median(ratios):.0%}, 90th percentile {np.percentile(ratios, 90):.0%}")

# two products, per unit: what it costs, what it sells for, what's left is worth after Nov 1
PRODUCTS = {
    "Costume: cost $10, price $30, clears at $5": (10, 30, 5),
    "Candy: cost $3, price $4, clears at $2": (3, 4, 2),
}
buy = np.linspace(0.4, 2.0, 161)  # order size as a multiple of last year's sales
curves, best = [], {}
for name, (cost, price, clearance) in PRODUCTS.items():
    profit = np.array([
        np.mean(price * np.minimum(q, ratios) + clearance * np.maximum(q - ratios, 0) - cost * q)
        for q in buy
    ])
    profit = profit / profit.max()
    best[name] = buy[profit.argmax()]
    curves.append(pd.DataFrame({"buy": buy, "profit": profit, "product": name}))
    critical = (price - cost) / (price - clearance)
    print(f"{name}: short costs ${price - cost:.2f}, over costs ${cost - clearance:.2f}, "
          f"so aim to cover demand {critical:.0%} of the time -> order {best[name]:.0%} of last year "
          f"(ordering exactly last year's amount gives up {1 - profit[np.argmin(abs(buy - 1))]:.0%} of the best profit)")

nv_fig = px.line(
    pd.concat(curves), x="buy", y="profit", color="product",
    color_discrete_sequence=[COLORS["Halloween"], COLORS["Last year"]],
    title="A fat margin justifies a bigger order. A thin one does not.",
    labels={"buy": "Order size (last year's sales = 100%)", "profit": "Expected profit (best = 1)", "product": ""},
)
for (name, q), color in zip(best.items(), [COLORS["Halloween"], COLORS["Last year"]]):
    nv_fig.add_vline(x=q, line_dash="dot", line_color=color)
nv_fig.update_traces(hovertemplate="Order %{x:.0%} of last year: %{y:.2f}<extra></extra>")
nv_fig.update_layout(xaxis_tickformat=".0%", yaxis_range=[0, 1.05], height=440,
                     legend=dict(orientation="h", y=1.02, x=0, yanchor="bottom"))
nv_fig.show()