diff --git a/arxiv-style/equations.tex b/arxiv-style/equations.tex index 7b64203..c63fa43 100644 --- a/arxiv-style/equations.tex +++ b/arxiv-style/equations.tex @@ -1,23 +1,9 @@ \documentclass[10pt, twocolumn]{article} \usepackage{amsmath, amssymb} \usepackage{bm} -\usepackage{booktabs} \usepackage[margin=1in]{geometry} \usepackage{microtype} -% Custom operators -\DeclareMathOperator*{\argmin}{arg\,min} -\DeclareMathOperator*{\argmax}{arg\,max} -\DeclareMathOperator{\CE}{CE} -\DeclareMathOperator{\SNR}{SNR} - -% Bold math symbols -\newcommand{\bX}{\bm{X}} -\newcommand{\bS}{\bm{S}} -\newcommand{\bR}{\bm{R}} -\newcommand{\br}{\bm{r}} -\newcommand{\by}{\bm{y}} - \title{Equations: Mask-DDPM Methodology} \author{} \date{} @@ -26,53 +12,53 @@ \maketitle \section{Problem Formulation} -Each training instance is a fixed-length window of length $L$, comprising continuous channels $\bX \in \mathbb{R}^{L \times d_c}$ and discrete channels $\bY = \{y^{(j)}_{1:L}\}_{j=1}^{d_d}$, where each discrete variable satisfies $y^{(j)}_t \in \mathcal{V}_j$ for a finite vocabulary $\mathcal{V}_j$. +Each training instance is a fixed-length window of length $L$, comprising continuous channels $\bm{X} \in \mathbb{R}^{L \times d_c}$ and discrete channels $\bm{Y} = \{y^{(j)}_{1:L}\}_{j=1}^{d_d}$, where each discrete variable satisfies $y^{(j)}_t \in \mathcal{V}_j$ for a finite vocabulary $\mathcal{V}_j$. \section{Transformer Trend Module for Continuous Dynamics} We posit an additive decomposition of the continuous signal: \begin{equation} -\bX = \bS + \bR, +\bm{X} = \bm{S} + \bm{R}, \label{eq:additive_decomp} \end{equation} -where $\bS \in \mathbb{R}^{L \times d_c}$ captures the smooth temporal trend and $\bR \in \mathbb{R}^{L \times d_c}$ represents distributional residuals. +where $\bm{S} \in \mathbb{R}^{L \times d_c}$ captures the smooth temporal trend and $\bm{R} \in \mathbb{R}^{L \times d_c}$ represents distributional residuals. The causal Transformer trend extractor $f_{\phi}$ predicts the next-step trend via: \begin{equation} -\hat{\bS}_{t+1} = f_{\phi}(\bX_{1:t}), \quad t = 1, \dots, L-1. +\hat{\bm{S}}_{t+1} = f_{\phi}(\bm{X}_{1:t}), \quad t = 1, \dots, L-1. \label{eq:trend_prediction} \end{equation} Training minimizes the mean-squared error: \begin{equation} -\mathcal{L}_{\text{trend}}(\phi) = \frac{1}{(L-1)d_c} \sum_{t=1}^{L-1} \bigl\| \hat{\bS}_{t+1} - \bX_{t+1} \bigr\|_2^2. +\mathcal{L}_{\text{trend}}(\phi) = \frac{1}{(L-1)d_c} \sum_{t=1}^{L-1} \bigl\| \hat{\bm{S}}_{t+1} - \bm{X}_{t+1} \bigr\|_2^2. \label{eq:trend_loss} \end{equation} -At inference, the residual target is defined as $\bR = \bX - \hat{\bS}$. +At inference, the residual target is defined as $\bm{R} = \bm{X} - \hat{\bm{S}}$. \section{DDPM for Continuous Residual Generation} Let $K$ denote diffusion steps with noise schedule $\{\beta_k\}_{k=1}^K$, $\alpha_k = 1 - \beta_k$, and $\bar{\alpha}_k = \prod_{i=1}^k \alpha_i$. The forward corruption process is: \begin{align} -q(\br_k \mid \br_0) &= \mathcal{N}\bigl( \sqrt{\bar{\alpha}_k}\,\br_0,\; (1 - \bar{\alpha}_k)\mathbf{I} \bigr), \\ -\br_k &= \sqrt{\bar{\alpha}_k}\,\br_0 + \sqrt{1 - \bar{\alpha}_k}\,\boldsymbol{\epsilon}, \quad \boldsymbol{\epsilon} \sim \mathcal{N}(\mathbf{0}, \mathbf{I}), +q(\bm{r}_k \mid \bm{r}_0) &= \mathcal{N}\bigl( \sqrt{\bar{\alpha}_k}\,\bm{r}_0,\; (1 - \bar{\alpha}_k)\mathbf{I} \bigr), \\ +\bm{r}_k &= \sqrt{\bar{\alpha}_k}\,\bm{r}_0 + \sqrt{1 - \bar{\alpha}_k}\,\boldsymbol{\epsilon}, \quad \boldsymbol{\epsilon} \sim \mathcal{N}(\mathbf{0}, \mathbf{I}), \label{eq:forward_process} \end{align} -where $\br_0 \equiv \bR$. +where $\bm{r}_0 \equiv \bm{R}$. The reverse process is parameterized as: \begin{equation} -p_{\theta}(\br_{k-1} \mid \br_k, \hat{\bS}) = \mathcal{N}\bigl( \boldsymbol{\mu}_{\theta}(\br_k, k, \hat{\bS}),\; \boldsymbol{\Sigma}(k) \bigr). +p_{\theta}(\bm{r}_{k-1} \mid \bm{r}_k, \hat{\bm{S}}) = \mathcal{N}\bigl( \boldsymbol{\mu}_{\theta}(\bm{r}_k, k, \hat{\bm{S}}),\; \boldsymbol{\Sigma}(k) \bigr). \label{eq:reverse_process} \end{equation} Training employs the $\epsilon$-prediction objective: \begin{equation} -\mathcal{L}_{\text{cont}}(\theta) = \mathbb{E}_{k,\br_0,\boldsymbol{\epsilon}} \left[ \bigl\| \boldsymbol{\epsilon} - \boldsymbol{\epsilon}_{\theta}(\br_k, k, \hat{\bS}) \bigr\|_2^2 \right]. +\mathcal{L}_{\text{cont}}(\theta) = \mathbb{E}_{k,\bm{r}_0,\boldsymbol{\epsilon}} \left[ \bigl\| \boldsymbol{\epsilon} - \boldsymbol{\epsilon}_{\theta}(\bm{r}_k, k, \hat{\bm{S}}) \bigr\|_2^2 \right]. \label{eq:ddpm_loss} \end{equation} Optionally, SNR-based reweighting yields: \begin{equation} -\mathcal{L}^{\text{snr}}_{\text{cont}}(\theta) = \mathbb{E}_{k,\br_0,\boldsymbol{\epsilon}} \left[ w_k \bigl\| \boldsymbol{\epsilon} - \boldsymbol{\epsilon}_{\theta}(\br_k, k, \hat{\bS}) \bigr\|_2^2 \right], +\mathcal{L}^{\text{snr}}_{\text{cont}}(\theta) = \mathbb{E}_{k,\bm{r}_0,\boldsymbol{\epsilon}} \left[ w_k \bigl\| \boldsymbol{\epsilon} - \boldsymbol{\epsilon}_{\theta}(\bm{r}_k, k, \hat{\bm{S}}) \bigr\|_2^2 \right], \label{eq:snr_loss} \end{equation} -where $w_k = \min(\SNR_k, \gamma) / \SNR_k$ and $\SNR_k = \bar{\alpha}_k / (1 - \bar{\alpha}_k)$. The final continuous output is reconstructed as $\hat{\bX} = \hat{\bS} + \hat{\bR}$. +where $w_k = \min(\mathrm{SNR}_k, \gamma) / \mathrm{SNR}_k$ and $\mathrm{SNR}_k = \bar{\alpha}_k / (1 - \bar{\alpha}_k)$. The final continuous output is reconstructed as $\hat{\bm{X}} = \hat{\bm{S}} + \hat{\bm{R}}$. \section{Masked Diffusion for Discrete Variables} For discrete channel $j$, the forward masking process follows schedule $\{m_k\}_{k=1}^K$: @@ -88,12 +74,12 @@ applied independently across variables and timesteps. The denoiser $h_{\psi}$ predicts categorical distributions conditioned on continuous context: \begin{equation} -p_{\psi}\bigl( y^{(j)}_0 \mid y_k, k, \hat{\bS}, \hat{\bX} \bigr) = h_{\psi}(y_k, k, \hat{\bS}, \hat{\bX}). +p_{\psi}\bigl( y^{(j)}_0 \mid y_k, k, \hat{\bm{S}}, \hat{\bm{X}} \bigr) = h_{\psi}(y_k, k, \hat{\bm{S}}, \hat{\bm{X}}). \label{eq:discrete_denoising} \end{equation} Training minimizes the categorical cross-entropy: \begin{equation} -\mathcal{L}_{\text{disc}}(\psi) = \mathbb{E}_{k} \left[ \frac{1}{|\mathcal{M}|} \sum_{(j,t) \in \mathcal{M}} \CE\bigl( h_{\psi}(y_k, k, \hat{\bS}, \hat{\bX})_{j,t},\; y^{(j)}_{0,t} \bigr) \right], +\mathcal{L}_{\text{disc}}(\psi) = \mathbb{E}_{k} \left[ \frac{1}{|\mathcal{M}|} \sum_{(j,t) \in \mathcal{M}} \mathrm{CE}\bigl( h_{\psi}(y_k, k, \hat{\bm{S}}, \hat{\bm{X}})_{j,t},\; y^{(j)}_{0,t} \bigr) \right], \label{eq:discrete_loss} \end{equation} where $\mathcal{M}$ denotes masked positions at step $k$. @@ -104,6 +90,6 @@ The combined objective balances continuous and discrete learning: \mathcal{L} = \lambda \, \mathcal{L}_{\text{cont}} + (1 - \lambda) \, \mathcal{L}_{\text{disc}}, \quad \lambda \in [0,1]. \label{eq:joint_objective} \end{equation} -Type-aware routing enforces deterministic reconstruction $\hat{x}^{(i)} = g_i(\hat{\bX}, \hat{\bY})$ for derived variables. +Type-aware routing enforces deterministic reconstruction $\hat{x}^{(i)} = g_i(\hat{\bm{X}}, \hat{\bm{Y}})$ for derived variables. \end{document} \ No newline at end of file