2026
Ashraf, Tajamul; Saqib, Amal; Gani, Hanan; AlMahri, Muhra; Li, Yuhao; Ahsan, Noor; Nawaz, Umair; Lahoud, Jean; Cholakkal, Hisham; Shah, Mubarak; Torr, Philip; Khan, Fahad; Anwer, Rao; Khan, Salman
Agent-X: Evaluating Deep Multimodal Reasoning in Vision-Centric Agentic Tasks Conference
Fourteenth International Conference on Learning Representations (ICLR), 2026., 2026.
Abstract | Tags: ICLR | Links:
@conference{Ashraf2026,
title = {Agent-X: Evaluating Deep Multimodal Reasoning in Vision-Centric Agentic Tasks},
author = {Tajamul Ashraf and Amal Saqib and Hanan Gani and Muhra AlMahri and Yuhao Li and Noor Ahsan and Umair Nawaz and Jean Lahoud and Hisham Cholakkal and Mubarak Shah and Philip Torr and Fahad Khan and Rao Anwer and Salman Khan},
url = {https://openreview.net/pdf?id=Vjruxvp1Xd},
year = {2026},
date = {2026-01-26},
publisher = {Fourteenth International Conference on Learning Representations (ICLR), 2026.},
abstract = {Deep reasoning is fundamental for solving complex tasks, especially in vision-centric scenarios that demand sequential, multimodal understanding. However, existing benchmarks typically evaluate agents with fully synthetic, single-turn queries, limited visual modalities, and lack a framework to assess reasoning quality over multiple steps as required in real-world settings. To address this, we introduce Agent-X, a large-scale benchmark for evaluating vision-centric agents’ multistep and deep reasoning capabilities in real-world, multimodal settings. AgentX features 828 agentic tasks with authentic visual contexts, including images, multi-image comparisons, videos, and instructional text. These tasks span six major agentic environments: general visual reasoning, web browsing, security and surveillance, autonomous driving, sports, and math reasoning. Our benchmark requires agents to integrate tool use with explicit, stepwise decision-making in these diverse settings. In addition, we propose a fine-grained, step-level evaluation framework that assesses the correctness and logical coherence of each reasoning step and the effectiveness of tool usage throughout the task. Our results reveal that even the best-performing models, including GPT, Gemini, and Qwen families, struggle to solve multi-step vision tasks, achieving less than 50% full-chain success. These findings highlight key bottlenecks in current LMM reasoning and tool-use capabilities and identify future research directions in vision-centric agentic reasoning models},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Deep reasoning is fundamental for solving complex tasks, especially in vision-centric scenarios that demand sequential, multimodal understanding. However, existing benchmarks typically evaluate agents with fully synthetic, single-turn queries, limited visual modalities, and lack a framework to assess reasoning quality over multiple steps as required in real-world settings. To address this, we introduce Agent-X, a large-scale benchmark for evaluating vision-centric agents’ multistep and deep reasoning capabilities in real-world, multimodal settings. AgentX features 828 agentic tasks with authentic visual contexts, including images, multi-image comparisons, videos, and instructional text. These tasks span six major agentic environments: general visual reasoning, web browsing, security and surveillance, autonomous driving, sports, and math reasoning. Our benchmark requires agents to integrate tool use with explicit, stepwise decision-making in these diverse settings. In addition, we propose a fine-grained, step-level evaluation framework that assesses the correctness and logical coherence of each reasoning step and the effectiveness of tool usage throughout the task. Our results reveal that even the best-performing models, including GPT, Gemini, and Qwen families, struggle to solve multi-step vision tasks, achieving less than 50% full-chain success. These findings highlight key bottlenecks in current LMM reasoning and tool-use capabilities and identify future research directions in vision-centric agentic reasoning models
Singh, Utsav; Chakraborty, Souradip; Suttle, Wesley; Sadler, Brian; Asher, Derrik; Sahu, Anit Kumar; Shah, Mubarak; Namboodiri, Vinay Purushothaman; Bedi, Amrit
Direct Preference Optimization for Primitive-Enabled Hierarchical RL: A Bilevel Approach Conference
Fourteenth International Conference on Learning Representations (ICLR), 2026., 2026.
Abstract | Tags: ICLR | Links:
@conference{Singh2026,
title = { Direct Preference Optimization for Primitive-Enabled Hierarchical RL: A Bilevel Approach},
author = {Utsav Singh and Souradip Chakraborty and Wesley Suttle and Brian Sadler and Derrik Asher and Anit Kumar Sahu and Mubarak Shah and Vinay Purushothaman Namboodiri and Amrit Bedi
},
url = {https://arxiv.org/html/2411.00361v3},
year = {2026},
date = {2026-01-26},
urldate = {2026-01-26},
publisher = {Fourteenth International Conference on Learning Representations (ICLR), 2026.},
abstract = {Hierarchical reinforcement learning (HRL) enables agents to solve complex, long-horizon tasks by decomposing them into manageable sub-tasks. However, HRL methods face two fundamental challenges: (i) non-stationarity caused by the evolving lower-level policy during training, which destabilizes higher-level learning, and (ii) the generation of infeasible subgoals that lower-level policies cannot achieve. To address these challenges, we introduce DIPPER, a novel HRL framework that formulates goal-conditioned HRL as a bi-level optimization problem and leverages direct preference optimization (DPO) to train the higher-level policy. By learning from preference comparisons over subgoal sequences rather than rewards that depend on the evolving lower-level policy, DIPPER mitigates the impact of non-stationarity on higher-level learning. To address infeasible subgoals, DIPPER incorporates lower-level value function regularization that encourages the higher-level policy to propose achievable subgoals. We introduce two novel metrics to quantitatively verify that DIPPER mitigates non-stationarity and infeasible subgoal generation issues in HRL. Empirical evaluation on challenging robotic navigation and manipulation benchmarks shows that DIPPER achieves upto 40% improvements over state-of-the-art baselines on challenging sparse-reward scenarios, highlighting the potential of preference-based learning for addressing longstanding HRL limitations.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Hierarchical reinforcement learning (HRL) enables agents to solve complex, long-horizon tasks by decomposing them into manageable sub-tasks. However, HRL methods face two fundamental challenges: (i) non-stationarity caused by the evolving lower-level policy during training, which destabilizes higher-level learning, and (ii) the generation of infeasible subgoals that lower-level policies cannot achieve. To address these challenges, we introduce DIPPER, a novel HRL framework that formulates goal-conditioned HRL as a bi-level optimization problem and leverages direct preference optimization (DPO) to train the higher-level policy. By learning from preference comparisons over subgoal sequences rather than rewards that depend on the evolving lower-level policy, DIPPER mitigates the impact of non-stationarity on higher-level learning. To address infeasible subgoals, DIPPER incorporates lower-level value function regularization that encourages the higher-level policy to propose achievable subgoals. We introduce two novel metrics to quantitatively verify that DIPPER mitigates non-stationarity and infeasible subgoal generation issues in HRL. Empirical evaluation on challenging robotic navigation and manipulation benchmarks shows that DIPPER achieves upto 40% improvements over state-of-the-art baselines on challenging sparse-reward scenarios, highlighting the potential of preference-based learning for addressing longstanding HRL limitations.
Fioresi, Joseph; Dave, Ishan Rajendrakumar; Shah, Mubarak
Privacy Beyond Pixels: Latent Anonymization for Privacy-Preserving Video Understanding Conference
Fourteenth International Conference on Learning Representations (ICLR), 2026., 2026.
Abstract | Tags: ICLR | Links:
@conference{Fioresi2026b,
title = {Privacy Beyond Pixels: Latent Anonymization for Privacy-Preserving Video Understanding},
author = {Joseph Fioresi and Ishan Rajendrakumar Dave and Mubarak Shah},
url = {https://arxiv.org/abs/2511.08666},
year = {2026},
date = {2026-01-26},
urldate = {2026-01-26},
publisher = {Fourteenth International Conference on Learning Representations (ICLR), 2026.},
abstract = {We introduce a novel formulation of visual privacy preservation for video foundation models that operates entirely in the latent space. While spatio-temporal features learned by foundation models have deepened general understanding of video content, sharing or storing these extracted visual features for downstream tasks inadvertently reveals sensitive personal information like skin color, gender, or clothing. Current privacy preservation methods focus on input-pixel level anonymization, which requires retraining the entire utility video model and results in task-specific anonymization, making them unsuitable for recent video foundational models. To address these challenges, we introduce a lightweight Anonymizing Adapter Module (AAM) that removes private information from video features while retaining general task utility. AAM can be applied in a plug and play fashion to frozen video encoders, minimizing the computational burden of finetuning and re-extracting features. Our framework employs three newly designed training objectives: (1) a clip-level self-supervised privacy objective to reduce mutual information between static clips, (2) a co-training objective to retain utility across seen tasks, and (3) a latent consistency loss for generalization on unseen tasks. Our extensive evaluations demonstrate a significant 35% reduction in privacy leakage while maintaining near-baseline utility performance across various downstream tasks: Action Recognition (Kinetics400, UCF101, HMDB51), Temporal Action Detection (THUMOS14), and Anomaly Detection (UCF-Crime). We also provide an....},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
We introduce a novel formulation of visual privacy preservation for video foundation models that operates entirely in the latent space. While spatio-temporal features learned by foundation models have deepened general understanding of video content, sharing or storing these extracted visual features for downstream tasks inadvertently reveals sensitive personal information like skin color, gender, or clothing. Current privacy preservation methods focus on input-pixel level anonymization, which requires retraining the entire utility video model and results in task-specific anonymization, making them unsuitable for recent video foundational models. To address these challenges, we introduce a lightweight Anonymizing Adapter Module (AAM) that removes private information from video features while retaining general task utility. AAM can be applied in a plug and play fashion to frozen video encoders, minimizing the computational burden of finetuning and re-extracting features. Our framework employs three newly designed training objectives: (1) a clip-level self-supervised privacy objective to reduce mutual information between static clips, (2) a co-training objective to retain utility across seen tasks, and (3) a latent consistency loss for generalization on unseen tasks. Our extensive evaluations demonstrate a significant 35% reduction in privacy leakage while maintaining near-baseline utility performance across various downstream tasks: Action Recognition (Kinetics400, UCF101, HMDB51), Temporal Action Detection (THUMOS14), and Anomaly Detection (UCF-Crime). We also provide an....
2025
Kang, Weitai; Qu, Mengxue; Kini, Jyoti; Wei, Yunchao; Shah, Mubarak; Yan, Yan
Intent3D: 3D Object Detection in RGB-D Scans Based on Human Intention Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{nokey,
title = {Intent3D: 3D Object Detection in RGB-D Scans Based on Human Intention},
author = {Weitai Kang and Mengxue Qu and Jyoti Kini and Yunchao Wei and Mubarak Shah and Yan Yan},
url = {https://openreview.net/forum?id=5GgjiRzYp3&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/3450_Intent3D_3D_Object_Detect.pdf.pdf},
year = {2025},
date = {2025-04-24},
urldate = {2025-04-24},
booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {In real-life scenarios, humans seek out objects in the 3D world to fulfill their daily needs or intentions. This inspires us to introduce 3D intention grounding, a new task in 3D object detection employing RGB-D, based on human intention, such as “I want something to support my back.” Closely related, 3D visual grounding focuses on understanding human reference. To achieve detection based on human intention, it relies on humans to observe the scene, reason out the target that aligns with their intention (“pillow” in this case), and finally provide a reference to the AI system, such as “A pillow on the couch”. Instead, 3D intention grounding challenges AI agents to automatically observe, reason and detect the desired target solely based on human intention. To tackle this challenge, we introduce the new Intent3D dataset, consisting of 44,990 intention texts associated with 209 f ine-grained classes from 1,042 scenes of the ScanNet [Dai et al., 2017] dataset. We also establish several baselines based on different language-based 3D object detection models on our benchmark. Finally, we propose IntentNet, our unique approach, designed to tackle this intention-based detection problem. It focuses on three key aspects: intention understanding, reasoning to identify object candidates, and cascaded adaptive learning that leverages the intrinsic priority logic of different losses for multiple objective optimization.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
In real-life scenarios, humans seek out objects in the 3D world to fulfill their daily needs or intentions. This inspires us to introduce 3D intention grounding, a new task in 3D object detection employing RGB-D, based on human intention, such as “I want something to support my back.” Closely related, 3D visual grounding focuses on understanding human reference. To achieve detection based on human intention, it relies on humans to observe the scene, reason out the target that aligns with their intention (“pillow” in this case), and finally provide a reference to the AI system, such as “A pillow on the couch”. Instead, 3D intention grounding challenges AI agents to automatically observe, reason and detect the desired target solely based on human intention. To tackle this challenge, we introduce the new Intent3D dataset, consisting of 44,990 intention texts associated with 209 f ine-grained classes from 1,042 scenes of the ScanNet [Dai et al., 2017] dataset. We also establish several baselines based on different language-based 3D object detection models on our benchmark. Finally, we propose IntentNet, our unique approach, designed to tackle this intention-based detection problem. It focuses on three key aspects: intention understanding, reasoning to identify object candidates, and cascaded adaptive learning that leverages the intrinsic priority logic of different losses for multiple objective optimization.
Chhipa, Prakash Chandra; Vashishtha, Gautam; Jithamanyu, Settur; Saini, Rajkumar; Shah, Mubarak; Liwicki, Marcus
ASTrA: Adversarial Self-supervised Training with Adaptive-Attacks Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Chhipa2025,
title = {ASTrA: Adversarial Self-supervised Training with Adaptive-Attacks},
author = {Prakash Chandra Chhipa and Gautam Vashishtha and Settur Jithamanyu and Rajkumar Saini and Mubarak Shah and Marcus Liwicki},
url = {https://prakashchhipa.github.io/projects/ASTrA
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7338_ASTrA_Adversarial_Self_su.pdf.pdf},
year = {2025},
date = {2025-04-24},
booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {Existing self-supervised adversarial training (self-AT) methods rely on handcrafted adversarial attack strategies for PGD attacks, which fail to adapt to the evolving learning dynamics of the model and do not account for instancespecific characteristics of images. This results in sub-optimal adversarial robustness and limits the alignment between clean and adversarial data distributions. To address this, we propose ASTrA (Adversarial Self-supervised Training with Adaptive-Attacks), a novel framework introducing a learnable, self-supervised attack strategy network that autonomously discovers optimal attack parameters through exploration-exploitation in a single training episode. ASTrA leverages a reward mechanism based on contrastive loss, optimized with REINFORCE, enabling adaptive attack strategies without labeled data or additional hyperparameters. We further introduce a mixed contrastive objective to align the distribution of clean and adversarial examples in representation space. ASTrA achieves state-of-the-art results on CIFAR10, CIFAR100, and STL10 while integrating seamlessly as a plug-and-play module for other self-AT methods. ASTrAshows scalability to larger datasets, demonstrates strong semi-supervised performance, and is resilient to robust overfitting, backed by explainability analysis on optimal attack strategies. Project page for source code and other details at https://prakashchhipa.github.io/projects/ASTrA.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Existing self-supervised adversarial training (self-AT) methods rely on handcrafted adversarial attack strategies for PGD attacks, which fail to adapt to the evolving learning dynamics of the model and do not account for instancespecific characteristics of images. This results in sub-optimal adversarial robustness and limits the alignment between clean and adversarial data distributions. To address this, we propose ASTrA (Adversarial Self-supervised Training with Adaptive-Attacks), a novel framework introducing a learnable, self-supervised attack strategy network that autonomously discovers optimal attack parameters through exploration-exploitation in a single training episode. ASTrA leverages a reward mechanism based on contrastive loss, optimized with REINFORCE, enabling adaptive attack strategies without labeled data or additional hyperparameters. We further introduce a mixed contrastive objective to align the distribution of clean and adversarial examples in representation space. ASTrA achieves state-of-the-art results on CIFAR10, CIFAR100, and STL10 while integrating seamlessly as a plug-and-play module for other self-AT methods. ASTrAshows scalability to larger datasets, demonstrates strong semi-supervised performance, and is resilient to robust overfitting, backed by explainability analysis on optimal attack strategies. Project page for source code and other details at https://prakashchhipa.github.io/projects/ASTrA.
Fioresi, Joseph; Dave, Ishan Rajendrakumar; Shah, Mubarak
ALBAR: Adversarial Learning approach to mitigate Biases in Action Recognition Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Fioresi2025,
title = {ALBAR: Adversarial Learning approach to mitigate Biases in Action Recognition},
author = {Joseph Fioresi and Ishan Rajendrakumar Dave and Mubarak Shah},
url = {https://openreview.net/forum?id=9KiE3t6CsL&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7609_ALBAR_Adversarial_Learnin.pdf},
year = {2025},
date = {2025-04-24},
urldate = {2025-04-24},
booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {Bias in machine learning models can lead to unfair decision making, and while it has been well-studied in the image and text domains, it remains underexplored in action recognition. Action recognition models often suffer from background bias (i.e., inferring actions based on background cues) and foreground bias (i.e., relying on subject appearance), which can be detrimental to real-life applications such as autonomous vehicles or assisted living monitoring. While prior approaches have mainly focused on mitigating background bias using specialized augmentations, we thoroughly study both biases. We propose approachname, a novel adversarial training method that mitigates foreground and background biases without requiring specialized knowledge of the bias attributes. Our framework applies an adversarial cross-entropy loss to the sampled static clip (where all the frames are the same) and aims to make its class probabilities uniform using a proposed textit{entropy maximization} loss. Additionally, we introduce a textit{gradient penalty} loss for regularization against the debiasing process. We evaluate our method on established background and foreground bias protocols, setting a new state-of-the-art and strongly improving combined debiasing performance by over textbf{12%} on HMDB51. Furthermore, we identify an issue of background leakage in the existing UCF101 protocol for bias evaluation which provides a shortcut to predict actions and does not provide an accurate measure of the debiasing capability of a model. We address this issue by proposing more fine-grained segmentation boundaries for the actor, where our method also outperforms existing approaches.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Bias in machine learning models can lead to unfair decision making, and while it has been well-studied in the image and text domains, it remains underexplored in action recognition. Action recognition models often suffer from background bias (i.e., inferring actions based on background cues) and foreground bias (i.e., relying on subject appearance), which can be detrimental to real-life applications such as autonomous vehicles or assisted living monitoring. While prior approaches have mainly focused on mitigating background bias using specialized augmentations, we thoroughly study both biases. We propose approachname, a novel adversarial training method that mitigates foreground and background biases without requiring specialized knowledge of the bias attributes. Our framework applies an adversarial cross-entropy loss to the sampled static clip (where all the frames are the same) and aims to make its class probabilities uniform using a proposed textit{entropy maximization} loss. Additionally, we introduce a textit{gradient penalty} loss for regularization against the debiasing process. We evaluate our method on established background and foreground bias protocols, setting a new state-of-the-art and strongly improving combined debiasing performance by over textbf{12%} on HMDB51. Furthermore, we identify an issue of background leakage in the existing UCF101 protocol for bias evaluation which provides a shortcut to predict actions and does not provide an accurate measure of the debiasing capability of a model. We address this issue by proposing more fine-grained segmentation boundaries for the actor, where our method also outperforms existing approaches.
Cui, Yuning; Zamir, Syed Waqas; Khan, Salman; Knoll, Alois; Shah, Mubarak; Khan, Fahad Shahbaz
AdaIR: Adaptive All-in-One Image Restoration via Frequency Mining and Modulation Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Cui2025,
title = {AdaIR: Adaptive All-in-One Image Restoration via Frequency Mining and Modulation},
author = {Yuning Cui and Syed Waqas Zamir and Salman Khan and Alois Knoll and Mubarak Shah and Fahad Shahbaz Khan},
url = {https://openreview.net/forum?id=M5t0WvjfCg&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/8311_AdaIR_Adaptive_All_in_One.pdf.pdf},
year = {2025},
date = {2025-04-24},
urldate = {2025-04-24},
booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {In the image acquisition process, various forms of degradation, including noise, blur, haze, and rain, are frequently introduced. These degradations typically arise from the inherent limitations of cameras or unfavorable ambient conditions. To recover clean images from their degraded versions, numerous specialized restoration methods have been developed, each targeting a specific type of degradation. Recently, all-in-one algorithms have garnered significant attention by addressing different types of degradations within a single model without requiring the prior information of the input degradation type. However, most methods purely operate in the spatial domain and do not delve into the distinct frequency variations inherent to different degradation types. To address this gap, we propose an adaptive all-in-one image restoration network based on frequency mining and modulation. Our approach is motivated by the observation that different degradation types impact the image content on different frequency subbands, thereby requiring different treatments for each restoration task. Specifically, we first mine low- and high-frequency information from the input features, guided by the adaptively decoupled spectra of the degraded image. The extracted features are then modulated by a bidirectional operator to facilitate interactions between different frequency components. Finally, the modulated features are merged into the original input for a progressively guided restoration. With this approach, the model achieves adaptive reconstruction by accentuating the informative frequency subbands according to different input degradations. Extensive experiments demonstrate that the proposed method, named AdaIR, achieves state-of-the-art performance on different image restoration tasks, including image denoising, dehazing, deraining, motion deblurring, and low-light image enhancement. Our code and models will be made publicly available.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
In the image acquisition process, various forms of degradation, including noise, blur, haze, and rain, are frequently introduced. These degradations typically arise from the inherent limitations of cameras or unfavorable ambient conditions. To recover clean images from their degraded versions, numerous specialized restoration methods have been developed, each targeting a specific type of degradation. Recently, all-in-one algorithms have garnered significant attention by addressing different types of degradations within a single model without requiring the prior information of the input degradation type. However, most methods purely operate in the spatial domain and do not delve into the distinct frequency variations inherent to different degradation types. To address this gap, we propose an adaptive all-in-one image restoration network based on frequency mining and modulation. Our approach is motivated by the observation that different degradation types impact the image content on different frequency subbands, thereby requiring different treatments for each restoration task. Specifically, we first mine low- and high-frequency information from the input features, guided by the adaptively decoupled spectra of the degraded image. The extracted features are then modulated by a bidirectional operator to facilitate interactions between different frequency components. Finally, the modulated features are merged into the original input for a progressively guided restoration. With this approach, the model achieves adaptive reconstruction by accentuating the informative frequency subbands according to different input degradations. Extensive experiments demonstrate that the proposed method, named AdaIR, achieves state-of-the-art performance on different image restoration tasks, including image denoising, dehazing, deraining, motion deblurring, and low-light image enhancement. Our code and models will be made publicly available.
Chen, Chen; Liu, Daochang; Shah, Mubarak; Xu, Chang
Exploring Local Memorization in Diffusion Models via Bright Ending Attention Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Chen2025,
title = {Exploring Local Memorization in Diffusion Models via Bright Ending Attention},
author = {Chen Chen and Daochang Liu and Mubarak Shah and Chang Xu},
url = {https://openreview.net/forum?id=p4cLtzk4oe&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/9283_Exploring_Local_Memorizat.pdf},
year = {2025},
date = {2025-04-24},
urldate = {2025-04-24},
booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {In this paper, we identify and leverage a novel `bright ending' (BE) anomaly in diffusion models prone to memorizing training images to address a new task: locating localized memorization regions within these models. BE refers to a distinct cross-attention pattern observed in text-to-image generations using diffusion models. Specifically, memorized image patches exhibit significantly greater attention to the end token during the final inference step compared to non-memorized patches. This attention map effectively highlights regions where the generated image replicates training data. Furthermore, driven by our observation that local memorization significantly underperforms in existing tasks of measuring, detecting, and mitigating memorization in diffusion models compared to global memorization, we propose a simple yet effective method to integrate BE and the results of the new localization task into these existing frameworks. This integration effectively improves their performances by narrowing the performance gap caused by local memorization. Our results not only demonstrate the successful execution of the new localization task but also establish new state-of-the-art performance across all existing tasks, underscoring the significance of the BE phenomenon.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
In this paper, we identify and leverage a novel `bright ending' (BE) anomaly in diffusion models prone to memorizing training images to address a new task: locating localized memorization regions within these models. BE refers to a distinct cross-attention pattern observed in text-to-image generations using diffusion models. Specifically, memorized image patches exhibit significantly greater attention to the end token during the final inference step compared to non-memorized patches. This attention map effectively highlights regions where the generated image replicates training data. Furthermore, driven by our observation that local memorization significantly underperforms in existing tasks of measuring, detecting, and mitigating memorization in diffusion models compared to global memorization, we propose a simple yet effective method to integrate BE and the results of the new localization task into these existing frameworks. This integration effectively improves their performances by narrowing the performance gap caused by local memorization. Our results not only demonstrate the successful execution of the new localization task but also establish new state-of-the-art performance across all existing tasks, underscoring the significance of the BE phenomenon.
Monsefi, Amin Karimi; Zhou, Mengxi; Monsefi, Nastaran Karimi; Lim, Ser-Nam; Chao, Wei-Lun; Ramnath, Rajiv
Frequency-Guided Masking for Enhanced Vision Self-Supervised Learning Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Monsefi2025,
title = {Frequency-Guided Masking for Enhanced Vision Self-Supervised Learning },
author = {Amin Karimi Monsefi and Mengxi Zhou and Nastaran Karimi Monsefi and Ser-Nam Lim and Wei-Lun Chao and Rajiv Ramnath},
url = {https://arxiv.org/abs/2409.10362
https://arxiv.org/pdf/2409.10362},
year = {2025},
date = {2025-04-24},
urldate = {2025-04-24},
booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {We present a novel frequency-based Self-Supervised Learning (SSL) approach that significantly enhances its efficacy for pre-training. Prior work in this direction masks out pre-defined frequencies in the input image and employs a reconstruction loss to pre-train the model. While achieving promising results, such an implementation has two fundamental limitations as identified in our paper. First, using pre-defined frequencies overlooks the variability of image frequency responses. Second, pre-trained with frequency-filtered images, the resulting model needs relatively more data to adapt to naturally looking images during fine-tuning. To address these drawbacks, we propose FOurier transform compression with seLf-Knowledge distillation (FOLK), integrating two dedicated ideas. First, inspired by image compression, we adaptively select the masked-out frequencies based on image frequency responses, creating more suitable SSL tasks for pre-training. Second, we employ a two-branch framework empowered by knowledge distillation, enabling the model to take both the filtered and original images as input, largely reducing the burden of downstream tasks. Our experimental results demonstrate the effectiveness of FOLK in achieving competitive performance to many state-of-the-art SSL methods across various downstream tasks, including image classification, few-shot learning, and semantic segmentation.
},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
We present a novel frequency-based Self-Supervised Learning (SSL) approach that significantly enhances its efficacy for pre-training. Prior work in this direction masks out pre-defined frequencies in the input image and employs a reconstruction loss to pre-train the model. While achieving promising results, such an implementation has two fundamental limitations as identified in our paper. First, using pre-defined frequencies overlooks the variability of image frequency responses. Second, pre-trained with frequency-filtered images, the resulting model needs relatively more data to adapt to naturally looking images during fine-tuning. To address these drawbacks, we propose FOurier transform compression with seLf-Knowledge distillation (FOLK), integrating two dedicated ideas. First, inspired by image compression, we adaptively select the masked-out frequencies based on image frequency responses, creating more suitable SSL tasks for pre-training. Second, we employ a two-branch framework empowered by knowledge distillation, enabling the model to take both the filtered and original images as input, largely reducing the burden of downstream tasks. Our experimental results demonstrate the effectiveness of FOLK in achieving competitive performance to many state-of-the-art SSL methods across various downstream tasks, including image classification, few-shot learning, and semantic segmentation.
Kumar, Akash; Kira, Zsolt; Rawat, Yogesh Singh
Contextual Self-paced Learning for Weakly Supervised Spatio-Temporal Video Grounding Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Kumar2025,
title = {Contextual Self-paced Learning for Weakly Supervised Spatio-Temporal Video Grounding},
author = {Akash Kumar and Zsolt Kira and Yogesh Singh Rawat},
url = {https://arxiv.org/pdf/2501.17053
https://github.com/AKASH2907/copsal-weakly-stvg
https://akash2907.github.io/cospal_webpage/
},
year = {2025},
date = {2025-04-24},
urldate = {2025-04-24},
booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {In this work, we focus on Weakly Supervised Spatio-Temporal Video Grounding (WSTVG). It is a multimodal task aimed at localizing specific subjects spatiotemporally based on textual queries without bounding box supervision. Motivated by recent advancements in multi-modal foundation models for grounding tasks, we f irst explore the potential of state-of-the-art object detection models for WSTVG. Despite their robust zero-shot capabilities, our adaptation reveals significant limitations, including inconsistent temporal predictions, inadequate understanding of complex queries, and challenges in adapting to difficult scenarios. We propose CoSPaL (Contextual Self-Paced Learning), a novel approach which is designed to overcome these limitations. CoSPaL integrates three core components: (1) Tubelet Phrase Grounding (TPG), which introduces spatio-temporal prediction by linking textual queries to tubelets; (2) Contextual Referral Grounding (CRG), which improves comprehension of complex queries by extracting contextual information to refine object identification over time; and (3) Self-Paced Scene Understanding (SPS), a training paradigm that progressively increases task difficulty, enabling the model to adapt to complex scenarios by transitioning from coarse to fine-grained understanding. We demonstrate the effectiveness of CoSPaL on three benchmark WSTVGdatasets, achieving a 3.9% absolute improvement on VidSTG and a 7.9% improvement on HCSTVG-v1.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
In this work, we focus on Weakly Supervised Spatio-Temporal Video Grounding (WSTVG). It is a multimodal task aimed at localizing specific subjects spatiotemporally based on textual queries without bounding box supervision. Motivated by recent advancements in multi-modal foundation models for grounding tasks, we f irst explore the potential of state-of-the-art object detection models for WSTVG. Despite their robust zero-shot capabilities, our adaptation reveals significant limitations, including inconsistent temporal predictions, inadequate understanding of complex queries, and challenges in adapting to difficult scenarios. We propose CoSPaL (Contextual Self-Paced Learning), a novel approach which is designed to overcome these limitations. CoSPaL integrates three core components: (1) Tubelet Phrase Grounding (TPG), which introduces spatio-temporal prediction by linking textual queries to tubelets; (2) Contextual Referral Grounding (CRG), which improves comprehension of complex queries by extracting contextual information to refine object identification over time; and (3) Self-Paced Scene Understanding (SPS), a training paradigm that progressively increases task difficulty, enabling the model to adapt to complex scenarios by transitioning from coarse to fine-grained understanding. We demonstrate the effectiveness of CoSPaL on three benchmark WSTVGdatasets, achieving a 3.9% absolute improvement on VidSTG and a 7.9% improvement on HCSTVG-v1.
Tang, Feilong; Huang, Zile; Liu, Chengzhi; Sun, Qiang; Yang, Harry; Lim, Ser-Nam
Intervening Anchor Token: Decoding Strategy in Alleviating Hallucinations for MLLMs Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Tang2025,
title = {Intervening Anchor Token: Decoding Strategy in Alleviating Hallucinations for MLLMs},
author = {Feilong Tang and Zile Huang and Chengzhi Liu and Qiang Sun and Harry Yang and Ser-Nam Lim},
url = {https://openreview.net/forum?id=zGb4WgCW5i
https://openreview.net/pdf?id=zGb4WgCW5i},
year = {2025},
date = {2025-04-24},
booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {Multimodal large language models (MLLMs) offer a powerful mechanism for interpreting visual information. However, they often suffer from hallucinations, which impede the real-world usage of these models. Existing methods attempt to alleviate this issue by designing special decoding strategies that penalize the summary tokens. However, these methods lack analysis of the relationship between hallucination and summarization mechanism of LLMs. Interestingly, we find that penalizing summary tokens is not necessary: merely intervening the query-key parameters variance, without costing extra inference time, still alleviates hallucinations. Specifically, we explore the causes of hallucinations by analyzing localized self-attention patterns called ``anchor" tokens and define the attention localization degree of the model as token propagation probabilities. Our analysis reveals that over-propagation of anchor tokens occurs when the distribution of eigenvalues of the query and key matrices has a non-zero mean and a polarized variance, leading to excessive dependence on anchor tokens while neglecting vision information and describes the image content with hallucination. Based on the observation, we propose a versatile plug-and-play decoding strategy, Dynamic Token Propagation Mechanism (TAME), to alleviate excessive propagation by dynamically intervening the eigenspectrum variance of the attention weight, thereby alleviating hallucinations without relying on complex decoding strategies. Extensive experiments reveal a correlation between the eigenspectrum and hallucinations across various MLLMs, and show that TAME reduces the percentage of hallucinated objects.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Multimodal large language models (MLLMs) offer a powerful mechanism for interpreting visual information. However, they often suffer from hallucinations, which impede the real-world usage of these models. Existing methods attempt to alleviate this issue by designing special decoding strategies that penalize the summary tokens. However, these methods lack analysis of the relationship between hallucination and summarization mechanism of LLMs. Interestingly, we find that penalizing summary tokens is not necessary: merely intervening the query-key parameters variance, without costing extra inference time, still alleviates hallucinations. Specifically, we explore the causes of hallucinations by analyzing localized self-attention patterns called ``anchor" tokens and define the attention localization degree of the model as token propagation probabilities. Our analysis reveals that over-propagation of anchor tokens occurs when the distribution of eigenvalues of the query and key matrices has a non-zero mean and a polarized variance, leading to excessive dependence on anchor tokens while neglecting vision information and describes the image content with hallucination. Based on the observation, we propose a versatile plug-and-play decoding strategy, Dynamic Token Propagation Mechanism (TAME), to alleviate excessive propagation by dynamically intervening the eigenspectrum variance of the attention weight, thereby alleviating hallucinations without relying on complex decoding strategies. Extensive experiments reveal a correlation between the eigenspectrum and hallucinations across various MLLMs, and show that TAME reduces the percentage of hallucinated objects.
Huang, Jiani; Li, Ziyang; Nail, Mayur; Lim, Ser-Nam
LASER: A Neuro-Symbolic Framework for Learning Spatio-Temporal Scene Graphs with Weak Supervision Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Huang2025,
title = {LASER: A Neuro-Symbolic Framework for Learning Spatio-Temporal Scene Graphs with Weak Supervision},
author = {Jiani Huang and Ziyang Li and Mayur Nail and Ser-Nam Lim},
url = {https://arxiv.org/abs/2304.07647
https://arxiv.org/pdf/2304.07647},
year = {2025},
date = {2025-04-24},
booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {We propose LASER, a neuro-symbolic approach to learn semantic video representations that capture rich spatial and temporal properties in video data by leveraging high-level logic specifications. In particular, we formulate the problem in terms of alignment between raw videos and spatio-temporal logic specifications. The alignment algorithm leverages a differentiable symbolic reasoner and a combination of contrastive, temporal, and semantics losses. It effectively and efficiently trains low-level perception models to extract a fine-grained video representation in the form of a spatio-temporal scene graph that conforms to the desired high-level specification. To practically reduce the manual effort of obtaining ground truth labels, we derive logic specifications from captions by employing a large language model with a generic prompting template. In doing so, we explore a novel methodology that weakly supervises the learning of spatio-temporal scene graphs with widely accessible video-caption data. We evaluate our method on three datasets with rich spatial and temporal specifications: 20BN-Something-Something, MUGEN, and OpenPVSG. We demonstrate that our method learns better fine-grained video semantics than existing baselines.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
We propose LASER, a neuro-symbolic approach to learn semantic video representations that capture rich spatial and temporal properties in video data by leveraging high-level logic specifications. In particular, we formulate the problem in terms of alignment between raw videos and spatio-temporal logic specifications. The alignment algorithm leverages a differentiable symbolic reasoner and a combination of contrastive, temporal, and semantics losses. It effectively and efficiently trains low-level perception models to extract a fine-grained video representation in the form of a spatio-temporal scene graph that conforms to the desired high-level specification. To practically reduce the manual effort of obtaining ground truth labels, we derive logic specifications from captions by employing a large language model with a generic prompting template. In doing so, we explore a novel methodology that weakly supervises the learning of spatio-temporal scene graphs with widely accessible video-caption data. We evaluate our method on three datasets with rich spatial and temporal specifications: 20BN-Something-Something, MUGEN, and OpenPVSG. We demonstrate that our method learns better fine-grained video semantics than existing baselines.
Pathak, Priyank; Marjit, Shyam; Vyas, Shruti; Rawat, Yogesh Singh
LR0.FM: Low-Resolution Zero-Shot Classification Benchmark for Foundation Models Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Pathak2025,
title = {LR0.FM: Low-Resolution Zero-Shot Classification Benchmark for Foundation Models},
author = {Priyank Pathak and Shyam Marjit and Shruti Vyas and Yogesh Singh Rawat},
url = {https://arxiv.org/pdf/2502.03950
https://ucf-crcv.github.io/lr0.fm/
https://github.com/shyammarjit/LR0.FM
},
year = {2025},
date = {2025-04-24},
urldate = {2025-04-24},
booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {Visual-language foundation Models (FMs) exhibit remarkable zero-shot generalization across diverse tasks, largely attributed to extensive pre-training on largescale datasets. However, their robustness on low-resolution/pixelated (LR) images, a common challenge in real-world scenarios, remains underexplored. We introduce this http URL, a comprehensive benchmark evaluating the impact of low resolution on the zero-shot classification performance of 10 FM(s) across 66 backbones and 15 datasets. We propose a novel metric, Weighted Aggregated Robustness, to address the limitations of existing metrics and better evaluate model performance across resolutions and datasets. Our key findings show that: (i) model size positively correlates with robustness to resolution degradation, (ii) pre-training dataset quality is more important than its size, and (iii) fine-tuned and higher resolution models are less robust against LR. Our analysis further reveals that the model makes semantically reasonable predictions at LR, and the lack of fine-grained details in input adversely impacts the model's initial layers more than the deeper layers. We use these insights and introduce a simple strategy, LR-TK0, to enhance the robustness of models without compromising their pre-trained weights. We demonstrate the effectiveness of LR-TK0 for robustness against low-resolution across several datasets and its generalization capability across backbones and other approaches. },
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Visual-language foundation Models (FMs) exhibit remarkable zero-shot generalization across diverse tasks, largely attributed to extensive pre-training on largescale datasets. However, their robustness on low-resolution/pixelated (LR) images, a common challenge in real-world scenarios, remains underexplored. We introduce this http URL, a comprehensive benchmark evaluating the impact of low resolution on the zero-shot classification performance of 10 FM(s) across 66 backbones and 15 datasets. We propose a novel metric, Weighted Aggregated Robustness, to address the limitations of existing metrics and better evaluate model performance across resolutions and datasets. Our key findings show that: (i) model size positively correlates with robustness to resolution degradation, (ii) pre-training dataset quality is more important than its size, and (iii) fine-tuned and higher resolution models are less robust against LR. Our analysis further reveals that the model makes semantically reasonable predictions at LR, and the lack of fine-grained details in input adversely impacts the model's initial layers more than the deeper layers. We use these insights and introduce a simple strategy, LR-TK0, to enhance the robustness of models without compromising their pre-trained weights. We demonstrate the effectiveness of LR-TK0 for robustness against low-resolution across several datasets and its generalization capability across backbones and other approaches.
Zhang, Chi; Farhat, Zain Ulabedeen; Atia, George K.; Wang, Yue
Model-Free Offline Reinforcement Learning with Enhanced Robustness Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Zhang2025,
title = {Model-Free Offline Reinforcement Learning with Enhanced Robustness},
author = {Chi Zhang and Zain Ulabedeen Farhat and George K. Atia and Yue Wang},
url = {https://openreview.net/forum?id=QyVLJ7EnAC
https://openreview.net/pdf?id=QyVLJ7EnAC},
year = {2025},
date = {2025-04-24},
booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {Offline reinforcement learning (RL) has gained considerable attention for its ability to learn policies from pre-collected data without real-time interaction, which makes it particularly useful for high-risk applications. However, due to its reliance on offline datasets, existing works inevitably introduce assumptions to ensure effective learning, which, however, often lead to a trade-off between robustness to model mismatch and scalability to large environments. In this paper, we enhance both aspects with a novel double-pessimism principle, which conservatively estimates performance and accounts for both limited data and potential model mismatches, two major reasons for the previous trade-off. We then propose a universal, model-free algorithm to learn an optimal policy that is robust to potential environment mismatches, which enhances robustness in a scalable manner. Furthermore, we provide a sample complexity analysis of our algorithm when the mismatch is modeled by the
-norm, which also theoretically demonstrates the efficiency of our method. Extensive experiments further demonstrate that our approach significantly improves robustness in a more scalable manner than existing methods.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Offline reinforcement learning (RL) has gained considerable attention for its ability to learn policies from pre-collected data without real-time interaction, which makes it particularly useful for high-risk applications. However, due to its reliance on offline datasets, existing works inevitably introduce assumptions to ensure effective learning, which, however, often lead to a trade-off between robustness to model mismatch and scalability to large environments. In this paper, we enhance both aspects with a novel double-pessimism principle, which conservatively estimates performance and accounts for both limited data and potential model mismatches, two major reasons for the previous trade-off. We then propose a universal, model-free algorithm to learn an optimal policy that is robust to potential environment mismatches, which enhances robustness in a scalable manner. Furthermore, we provide a sample complexity analysis of our algorithm when the mismatch is modeled by the
-norm, which also theoretically demonstrates the efficiency of our method. Extensive experiments further demonstrate that our approach significantly improves robustness in a more scalable manner than existing methods.
-norm, which also theoretically demonstrates the efficiency of our method. Extensive experiments further demonstrate that our approach significantly improves robustness in a more scalable manner than existing methods.
Alom, Zulfikar; Ngo, Tran Gia Bao; Kantarcioglu, Murat; Akcora, Cuneyt Gurcan
GOttack: Universal Adversarial Attacks on Graph Neural Networks via Graph Orbits Learning Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{nokey,
title = {GOttack: Universal Adversarial Attacks on Graph Neural Networks via Graph Orbits Learning},
author = {Zulfikar Alom and Tran Gia Bao Ngo and Murat Kantarcioglu and Cuneyt Gurcan Akcora},
url = {https://openreview.net/forum?id=YbURbViE7l
https://openreview.net/pdf?id=YbURbViE7l
https://github.com/cakcora/GOttack},
year = {2025},
date = {2025-04-24},
urldate = {2025-04-24},
publisher = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {Graph Neural Networks (GNNs) have demonstrated superior performance in node classification tasks across diverse applications. However, their vulnerability to adversarial attacks, where minor perturbations can mislead model predictions, poses significant challenges. This study introduces GOttack, a novel adversarial attack framework that exploits the topological structure of graphs to undermine the integrity of GNN predictions systematically.
By defining a topology-aware method to manipulate graph orbits, our approach generates adversarial modifications that are both subtle and effective, posing a severe test to the robustness of GNNs. We evaluate the efficacy of GOttack across multiple prominent GNN architectures using standard benchmark datasets. Our results show that GOttack outperforms existing state-of-the-art adversarial techniques and completes training in approximately 55% of the time required by the fastest competing model, achieving the highest average misclassification rate in 155 tasks. This work not only sheds light on the susceptibility of GNNs to structured adversarial attacks but also shows that certain topological patterns may play a significant role in the underlying robustness of the GNNs. Our Python implementation is shared at https://github.com/cakcora/GOttack.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Graph Neural Networks (GNNs) have demonstrated superior performance in node classification tasks across diverse applications. However, their vulnerability to adversarial attacks, where minor perturbations can mislead model predictions, poses significant challenges. This study introduces GOttack, a novel adversarial attack framework that exploits the topological structure of graphs to undermine the integrity of GNN predictions systematically.
By defining a topology-aware method to manipulate graph orbits, our approach generates adversarial modifications that are both subtle and effective, posing a severe test to the robustness of GNNs. We evaluate the efficacy of GOttack across multiple prominent GNN architectures using standard benchmark datasets. Our results show that GOttack outperforms existing state-of-the-art adversarial techniques and completes training in approximately 55% of the time required by the fastest competing model, achieving the highest average misclassification rate in 155 tasks. This work not only sheds light on the susceptibility of GNNs to structured adversarial attacks but also shows that certain topological patterns may play a significant role in the underlying robustness of the GNNs. Our Python implementation is shared at https://github.com/cakcora/GOttack.
By defining a topology-aware method to manipulate graph orbits, our approach generates adversarial modifications that are both subtle and effective, posing a severe test to the robustness of GNNs. We evaluate the efficacy of GOttack across multiple prominent GNN architectures using standard benchmark datasets. Our results show that GOttack outperforms existing state-of-the-art adversarial techniques and completes training in approximately 55% of the time required by the fastest competing model, achieving the highest average misclassification rate in 155 tasks. This work not only sheds light on the susceptibility of GNNs to structured adversarial attacks but also shows that certain topological patterns may play a significant role in the underlying robustness of the GNNs. Our Python implementation is shared at https://github.com/cakcora/GOttack.
Peng, Qucheng; Planche, Benjamin; Gao, Zhongpai; Zheng, Meng; Choudhuri, Anwesa; Chen, Terrence; Chen, Chen; Wu, Ziyan
3D Vision-Language Gaussian Splatting Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Peng2025,
title = {3D Vision-Language Gaussian Splatting},
author = {Qucheng Peng and Benjamin Planche and Zhongpai Gao and Meng Zheng and Anwesa Choudhuri and Terrence Chen and Chen Chen and Ziyan Wu},
url = {https://iclr.cc/virtual/2025/poster/29604},
year = {2025},
date = {2025-04-24},
urldate = {2025-04-04},
publisher = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {Recent advancements in 3D reconstruction methods and vision-language models have propelled the development of multi-modal 3D scene understanding, which has vital applications in robotics, autonomous driving, and virtual/augmented reality. However, current multi-modal scene understanding approaches have naively embedded semantic representations into 3D reconstruction methods without striking a balance between visual and language modalities, which leads to unsatisfying semantic rasterization of translucent or reflective objects, as well as over-fitting on color modality. To alleviate these limitations, we propose a solution that adequately handles the distinct visual and semantic modalities, i.e., a 3D vision-language Gaussian splatting model for scene understanding, to put emphasis on the representation learning of language modality. We propose a novel cross-modal rasterizer, using modality fusion along with a smoothed semantic indicator for enhancing semantic rasterization. We also employ a camera-view blending technique to improve semantic consistency between existing and synthesized views, thereby effectively mitigating over-fitting. Extensive experiments demonstrate that our method achieves state-of-the-art performance in open-vocabulary semantic segmentation, surpassing existing methods by a significant margin.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Recent advancements in 3D reconstruction methods and vision-language models have propelled the development of multi-modal 3D scene understanding, which has vital applications in robotics, autonomous driving, and virtual/augmented reality. However, current multi-modal scene understanding approaches have naively embedded semantic representations into 3D reconstruction methods without striking a balance between visual and language modalities, which leads to unsatisfying semantic rasterization of translucent or reflective objects, as well as over-fitting on color modality. To alleviate these limitations, we propose a solution that adequately handles the distinct visual and semantic modalities, i.e., a 3D vision-language Gaussian splatting model for scene understanding, to put emphasis on the representation learning of language modality. We propose a novel cross-modal rasterizer, using modality fusion along with a smoothed semantic indicator for enhancing semantic rasterization. We also employ a camera-view blending technique to improve semantic consistency between existing and synthesized views, thereby effectively mitigating over-fitting. Extensive experiments demonstrate that our method achieves state-of-the-art performance in open-vocabulary semantic segmentation, surpassing existing methods by a significant margin.
Lai, Zhengfeng; Saveris, Vasileios; Chen, Chen; Chen, Hong-You; Zhang, Haotian; Zhang, Bowen; Hu, Wenze; Tebar, Juan; Gan, Zhe; Grasch, Peter; Cao, Meng; Yang, Yinfei
Revisit Large-Scale Image-Caption Data in Pre-training Multimodal Foundation Models Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Lai2025,
title = {Revisit Large-Scale Image-Caption Data in Pre-training Multimodal Foundation Models},
author = {Zhengfeng Lai and Vasileios Saveris and Chen Chen and Hong-You Chen and Haotian Zhang and Bowen Zhang and Wenze Hu and Juan Tebar and Zhe Gan and Peter Grasch and Meng Cao and Yinfei Yang},
url = {https://iclr.cc/virtual/2025/poster/29536},
year = {2025},
date = {2025-04-24},
publisher = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {Recent advancements in multimodal models highlight the value of rewritten captions for improving performance, yet key challenges remain. For example, while synthetic captions often provide superior quality and image-text alignment, it is not clear whether they can fully replace AltTexts: the role of synthetic captions and their interaction with original web-crawled AltTexts in pre-training is still not well understood. Moreover, different multimodal foundation models may have unique preferences for specific caption formats, but efforts to identify the optimal captions for each model remain limited. In this work, we propose a novel, controllable, and scalable captioning pipeline designed to generate diverse caption formats tailored to various multimodal models. By examining short synthetic captions (SSC) and descriptive synthetic captions (DSC) as case studies, we systematically explore their effects and interactions with AltTexts across models such as CLIP, multimodal LLMs, and diffusion models. Our findings reveal that a hybrid approach that keeps both synthetic captions and AltTexts can outperform the use of synthetic captions alone, improving both alignment and performance, with each model demonstrating preferences for particular caption formats. This comprehensive analysis provides valuable insights into optimizing captioning strategies, thereby advancing the pre-training of multimodal foundation models.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Recent advancements in multimodal models highlight the value of rewritten captions for improving performance, yet key challenges remain. For example, while synthetic captions often provide superior quality and image-text alignment, it is not clear whether they can fully replace AltTexts: the role of synthetic captions and their interaction with original web-crawled AltTexts in pre-training is still not well understood. Moreover, different multimodal foundation models may have unique preferences for specific caption formats, but efforts to identify the optimal captions for each model remain limited. In this work, we propose a novel, controllable, and scalable captioning pipeline designed to generate diverse caption formats tailored to various multimodal models. By examining short synthetic captions (SSC) and descriptive synthetic captions (DSC) as case studies, we systematically explore their effects and interactions with AltTexts across models such as CLIP, multimodal LLMs, and diffusion models. Our findings reveal that a hybrid approach that keeps both synthetic captions and AltTexts can outperform the use of synthetic captions alone, improving both alignment and performance, with each model demonstrating preferences for particular caption formats. This comprehensive analysis provides valuable insights into optimizing captioning strategies, thereby advancing the pre-training of multimodal foundation models.
Zhang, Yancheng; Xue, Jiaqi; Zheng, Mengxin; Xie, Mimi; Zhang, Mingzhe; Jiang, Lei; Lou, Qian
CipherPrune: Efficient and Scalable Private Transformer Inference Conference
Thirteenth International Conference on Learning Representations (ICLR), 2025.
Abstract | Tags: ICLR | Links:
@conference{Zhang2025b,
title = {CipherPrune: Efficient and Scalable Private Transformer Inference},
author = {Yancheng Zhang and Jiaqi Xue and Mengxin Zheng and Mimi Xie and Mingzhe Zhang and Lei Jiang and Qian Lou},
url = { https://openreview.net/pdf?id=mUMvr33FTu
https://github.com/UCF-Lou-Lab-PET/cipher-prune-inference},
year = {2025},
date = {2025-04-24},
publisher = {Thirteenth International Conference on Learning Representations (ICLR)},
abstract = {Private Transformer inference using cryptographic protocols offers promising solutions for privacy-preserving machine learning; however, it still faces significant runtime overhead (efficiency issues) and challenges in handling long-token inputs (scalability issues). We observe that the Transformer’s operational complexity scales quadratically with the number of input tokens, making it essential to reduce the input token length. Notably, each token varies in importance, and many inputs contain redundant tokens. Additionally, prior private inference methods that rely on high-degree polynomial approximations for non-linear activations are computationally expensive. Therefore, reducing the polynomial degree for less important tokens can significantly accelerate private inference. Building on these observations, we propose CipherPrune, an efficient and scalable private inference framework that includes a secure encrypted token pruning protocol, a polynomial reduction protocol, and corresponding Transformer network optimizations. At the protocol level, encrypted token pruning adaptively removes unimportant tokens from encrypted inputs in a progressive, layer-wise manner. Additionally, encrypted polynomial reduction assigns lower-degree polynomials to less important tokens after pruning, enhancing efficiency without decryption. At the network level, we introduce protocol-aware network optimization via a gradient-based search to maximize pruning thresholds and polynomial reduction conditions while maintaining the desired accuracy. Our experiments demonstrate that CipherPrune reduces the execution overhead of private Transformer inference by approximately 6.1× for 128-token inputs and 10.6× for 512-token inputs, compared to previous methods, with only a marginal drop in accuracy. The code is publicly available at https://github.com/UCF-Lou-Lab-PET/cipher-prune-inference.},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Private Transformer inference using cryptographic protocols offers promising solutions for privacy-preserving machine learning; however, it still faces significant runtime overhead (efficiency issues) and challenges in handling long-token inputs (scalability issues). We observe that the Transformer’s operational complexity scales quadratically with the number of input tokens, making it essential to reduce the input token length. Notably, each token varies in importance, and many inputs contain redundant tokens. Additionally, prior private inference methods that rely on high-degree polynomial approximations for non-linear activations are computationally expensive. Therefore, reducing the polynomial degree for less important tokens can significantly accelerate private inference. Building on these observations, we propose CipherPrune, an efficient and scalable private inference framework that includes a secure encrypted token pruning protocol, a polynomial reduction protocol, and corresponding Transformer network optimizations. At the protocol level, encrypted token pruning adaptively removes unimportant tokens from encrypted inputs in a progressive, layer-wise manner. Additionally, encrypted polynomial reduction assigns lower-degree polynomials to less important tokens after pruning, enhancing efficiency without decryption. At the network level, we introduce protocol-aware network optimization via a gradient-based search to maximize pruning thresholds and polynomial reduction conditions while maintaining the desired accuracy. Our experiments demonstrate that CipherPrune reduces the execution overhead of private Transformer inference by approximately 6.1× for 128-token inputs and 10.6× for 512-token inputs, compared to previous methods, with only a marginal drop in accuracy. The code is publicly available at https://github.com/UCF-Lou-Lab-PET/cipher-prune-inference.
2023
Yang, Taojiannan; Zhu, Yi; Xie, Yusheng; Zhang, Aston; Chen, Chen; Li, Mu
AIM: Adapting Image Models for Efficient Video Understanding Conference
AIM: Adapting Image Models for Efficient Video Understanding, Eleventh International Conference on Learning Representations (ICLR), 2023.
@conference{Yang2023,
title = {AIM: Adapting Image Models for Efficient Video Understanding},
author = {Taojiannan Yang and Yi Zhu and Yusheng Xie and Aston Zhang and Chen Chen and Mu Li},
year = {2023},
date = {2023-05-01},
urldate = {2023-05-01},
booktitle = {AIM: Adapting Image Models for Efficient Video Understanding},
publisher = {Eleventh International Conference on Learning Representations (ICLR)},
abstract = {Recent vision transformer based video models mostly follow the ``image pre-training then finetuning" paradigm and have achieved great success on multiple video benchmarks. However, full finetuning such a video model could be computationally expensive and unnecessary, given the pre-trained image transformer models have demonstrated exceptional transferability. In this work, we propose a novel method to Adapt pre-trained Image Models (AIM) for efficient video understanding. By freezing the pre-trained image model and adding a few lightweight Adapters, we introduce spatial adaptation, temporal adaptation and joint adaptation to gradually equip an image model with spatiotemporal reasoning capability. We show that our proposed AIM can achieve competitive or even better performance than prior arts with substantially fewer tunable parameters on four video action recognition benchmarks. Thanks to its simplicity, our method is also generally applicable to different image pre-trained models, which has the potential to leverage more powerful image foundation models in the future. },
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Recent vision transformer based video models mostly follow the ``image pre-training then finetuning" paradigm and have achieved great success on multiple video benchmarks. However, full finetuning such a video model could be computationally expensive and unnecessary, given the pre-trained image transformer models have demonstrated exceptional transferability. In this work, we propose a novel method to Adapt pre-trained Image Models (AIM) for efficient video understanding. By freezing the pre-trained image model and adding a few lightweight Adapters, we introduce spatial adaptation, temporal adaptation and joint adaptation to gradually equip an image model with spatiotemporal reasoning capability. We show that our proposed AIM can achieve competitive or even better performance than prior arts with substantially fewer tunable parameters on four video action recognition benchmarks. Thanks to its simplicity, our method is also generally applicable to different image pre-trained models, which has the potential to leverage more powerful image foundation models in the future.
Yang, Peiyu; Akhtar, Naveed; Wen, Zeyi; Shah, Mubarak; Mian, Ajmal
Re-calibrating Feature Attributions for Model Interpretation Conference
Re-calibrating Feature Attributions for Model Interpretation, Eleventh International Conference on Learning Representations (ICLR), notable top 25%, 2023.
Tags: ICLR
@conference{nokey,
title = {Re-calibrating Feature Attributions for Model Interpretation},
author = {Peiyu Yang and Naveed Akhtar and Zeyi Wen and Mubarak Shah and Ajmal Mian},
year = {2023},
date = {2023-05-01},
urldate = {2023-05-01},
booktitle = {Re-calibrating Feature Attributions for Model Interpretation},
publisher = {Eleventh International Conference on Learning Representations (ICLR), notable top 25%},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Beetham, James; Kardan, Navid; Mian, Ajmal; Shah, Mubarak
Dual Student Networks for Data-Free Model Stealing Conference
Eleventh International Conference on Learning Representations, 2023.
Tags: ICLR | Links:
@conference{Beetham2023b,
title = {Dual Student Networks for Data-Free Model Stealing},
author = {James Beetham and Navid Kardan and Ajmal Mian and Mubarak Shah},
url = {https://arxiv.org/abs/2309.10058},
year = {2023},
date = {2023-05-01},
booktitle = {Eleventh International Conference on Learning Representations},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
2022
Kardan, Navid; Hill, Mitchell; Shah, Mubarak
Self-Joint Supervised Learning Conference
International Conference on Learning Representations (ICLR), 2022.
Abstract | Tags: ICLR | Links:
@conference{Kardan2022,
title = {Self-Joint Supervised Learning},
author = {Navid Kardan and Mitchell Hill and Mubarak Shah},
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Self_joint_ICLR-002.pdf
https://github.com/ndkn/Self-joint-Learning},
year = {2022},
date = {2022-01-20},
urldate = {2022-01-20},
publisher = {International Conference on Learning Representations (ICLR)},
abstract = {Supervised learning is a fundamental framework used to train machine learning systems. A supervised learning problem is often formulated using an i.i.d. as-sumption that restricts model attention to a single relevant signal at a time when predicting. This contrasts with the human ability to actively use related samples as reference when making decisions. We hypothesize that the restriction to a single signal for each prediction in the standard i.i.d. framework contributes to well-known drawbacks of supervised learning: making overconfident predictions and vulnerability to overfitting, adversarial attacks, and out-of-distribution data. To address these limitations, we propose a new supervised learning paradigm called self-joint learning that generalizes the standard approach by modeling the joint conditional distribution of two observed samples, where each sample is an im-age and its label. Rather than assuming samples are independent, our models explicitly learn the sample-to-sample relation of conditional independence. Our framework can naturally incorporate auxiliary unlabeled data to further improve the performance. Experiments on benchmark image datasets show our method offers significant improvement over standard supervised learning in terms of ac-curacy, robustness against adversarial attacks, out-of-distribution detection, and overconfidence mitigation. Code: github.com/ndkn/Self-joint-Learning},
keywords = {ICLR},
pubstate = {published},
tppubtype = {conference}
}
Supervised learning is a fundamental framework used to train machine learning systems. A supervised learning problem is often formulated using an i.i.d. as-sumption that restricts model attention to a single relevant signal at a time when predicting. This contrasts with the human ability to actively use related samples as reference when making decisions. We hypothesize that the restriction to a single signal for each prediction in the standard i.i.d. framework contributes to well-known drawbacks of supervised learning: making overconfident predictions and vulnerability to overfitting, adversarial attacks, and out-of-distribution data. To address these limitations, we propose a new supervised learning paradigm called self-joint learning that generalizes the standard approach by modeling the joint conditional distribution of two observed samples, where each sample is an im-age and its label. Rather than assuming samples are independent, our models explicitly learn the sample-to-sample relation of conditional independence. Our framework can naturally incorporate auxiliary unlabeled data to further improve the performance. Experiments on benchmark image datasets show our method offers significant improvement over standard supervised learning in terms of ac-curacy, robustness against adversarial attacks, out-of-distribution detection, and overconfidence mitigation. Code: github.com/ndkn/Self-joint-Learning
2021
Rizve, Mamshad Nayeem; Duarte, Kevin; Rawat, Yogesh Singh; Shah, Mubarak
In Defense of Pseudo-Labeling: An Uncertainty-Aware Pseudo-label Selection Framework for Semi-Supervised Learning Conference
Ninth International Conference on Learning Representations (ICLR), 2021.
Tags: ICLR, Network Calibration, Pseudo-Labeling, Semi-supervised learning | Links:
@conference{Rizve2021,
title = {In Defense of Pseudo-Labeling: An Uncertainty-Aware Pseudo-label Selection Framework for Semi-Supervised Learning},
author = {Mamshad Nayeem Rizve and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah },
url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/In-Defense-Of-Pseudo-Labeling.pdf
https://github.com/nayeemrizve/ups
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/UPS_ICLR2021_Slides.pdf
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/UPS_ICLR2021_Slides.pptx
https://www.crcv.ucf.edu/wp-content/uploads/2018/11/UPS_Poster_ICLR2021.png},
year = {2021},
date = {2021-05-04},
urldate = {2021-05-04},
booktitle = {Ninth International Conference on Learning Representations (ICLR)},
keywords = {ICLR, Network Calibration, Pseudo-Labeling, Semi-supervised learning},
pubstate = {published},
tppubtype = {conference}
}