Mubarak Shah

@conference{nokey,

title = {Evaluating Constrained Iterative Refinement forScalable Vector Graphics Generation with Off-the-Shelf VLM},

author = {Matthew Perlman and James Beethman and Niels Da Vitoria Lobo and Amrit Singh Bedi and Mubarak Shah},

url = {https://www.researchgate.net/publication/413847591_Evaluating_Constrained_Iterative_Refinement_for_Scalable_Vector_Graphics_Generation_with_Off-the-Shelf_VLMs},

year  = {2026},

date = {2026-10-06},

urldate = {2026-10-06},

publisher = {34th Pacific Conference on Computer Graphics and Applications},

abstract = {Scalable Vector Graphics (SVGs) power much of the modern visual ecosystem, yet state-of-the-art generative models focusalmost entirely on rasterized images. We explore whether inference-time methods can unlock SVG generation capabilities in off-the-shelf vision-language models (VLMs). We systematically evaluate a constrained iterative refinement harness that combinesvisual feedback, structured editing, and constrained decoding to characterize the capabilities and limitations of current VLMsfor SVG generation. Across multiple VLMs and generation settings, we find that constrained decoding improves compilationsuccess rates, while iterative refinement reveals a deficit in visual reasoning and self-correction. Our results highlight both thepromise and current limitations of using inference-time methods to adapt general-purpose VLMs for SVG generation.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ding2026,

title = {CLARITY: Medical World Model for Guiding Treatment Decisions by Modeling Context-Aware Disease Trajectories in Latent Space},

author = {Tianxingjian Ding and Yuanhao Zou and Chen Chen and Mubarak Shah and Yu Tian},

url = {https://arxiv.org/abs/2512.08029},

year  = {2026},

date = {2026-09-08},

publisher = {European Conference on Computer Vision -- ECCV 2026},

abstract = {Clinical decision-making in oncology requires predicting dynamic disease evolution, a task current static AI predictors cannot perform. While world models (WMs) offer a paradigm for generative prediction, existing medical applications remain limited. Existing methods often rely on stochastic diffusion models, focusing on visual reconstruction rather than causal, physiological transitions. Furthermore, in medical domain, models like MeWM typically ignore patient-specific temporal and clinical contexts and lack a feedback mechanism to link predictions to treatment decisions. To address these gaps, we introduce CLARITY, a medical world model that forecasts disease evolution directly within a structured latent space. It explicitly integrates time intervals (temporal context) and patient-specific data (clinical context) to model treatment-conditioned progression as a smooth, interpretable trajectory, and thus generate physiologically faithful, individualized treatment plans. Finally, CLARITY introduces a novel prediction-to-decision framework, translating latent rollouts into transparent, actionable recommendations. CLARITY demonstrates state-of-the-art performance in treatment planning. On the MU-Glioma-Post dataset, our approach outperforms recent MeWM by 12%, and significantly surpasses all other medical-specific large language models.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fioresi2026,

title = {Learning to Share: Selective Memory for Efficient Parallel Agentic Systems},

author = {Joseph Fioresi and Ashmal Vayani and Parth Parag Kulkarni and Song Wang and Mubarak Shah},

url = {https://arxiv.org/abs/2602.05965},

year  = {2026},

date = {2026-06-14},

publisher = {Forty-Third International Conference on Machine Learning (ICML) 2026},

abstract = {Agentic systems solve complex tasks by coordinating multiple agents that iteratively reason, invoke tools, and exchange intermediate results. To improve robustness and solution quality, recent approaches deploy multiple agent teams running in parallel to explore diverse reasoning trajectories. However, parallel execution comes at a significant computational cost: when different teams independently reason about similar sub-problems or execute analogous steps, they repeatedly perform substantial overlapping computation. To address these limitations, in this paper, we propose Learning to Share (LTS), a learned shared-memory mechanism for parallel agentic frameworks that enables selective cross-team information reuse while controlling context growth. LTS introduces a global memory bank accessible to all teams and a lightweight controller that decides whether intermediate agent steps should be added to memory or not. The controller is trained using stepwise reinforcement learning with usage-aware credit assignment, allowing it to identify information that is globally useful across parallel executions. Experiments on the AssistantBench and GAIA benchmarks show that LTS significantly reduces overall runtime while matching or improving task performance compared to memory-free parallel baselines, demonstrating that learned memory admission is an effective strategy for improving the efficiency of parallel agentic systems.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Luo2026,

title = {Dictionary-Aligned Concept Control for Safeguarding Multimodal LLMs},

author = {Jinqi Luo and Jinyu Yang and Tal Neiman and Lei Fan and Bing Yina and Son Tran and Mubarak Shah and Rene Vidal},

url = {https://arxiv.org/abs/2604.08846},

year  = {2026},

date = {2026-04-10},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {Multimodal Large Language Models (MLLMs) have been shown to be vulnerable to malicious queries that can elicit unsafe responses. Recent work uses prompt engineering, response classification, or finetuning to improve MLLM safety. Nevertheless, such approaches are often ineffective against evolving malicious patterns, may require rerunning the query, or demand heavy computational resources. Steering the activations of a frozen model at inference time has recently emerged as a flexible and effective solution. However, existing steering methods for MLLMs typically handle only a narrow set of safety-related concepts or struggle to adjust specific concepts without affecting others. To address these challenges, we introduce Dictionary-Aligned Concept Control (DACO), a framework that utilizes a curated concept dictionary and a Sparse Autoencoder (SAE) to provide granular control over MLLM activations. First, we curate a dictionary of 15,000 multimodal concepts by retrieving over 400,000 caption-image stimuli and summarizing their activations into concept directions. We name the dataset DACO-400K. Second, we show that the curated dictionary can be used to intervene activations via sparse coding. Third, we propose a new steering approach that uses our dictionary to initialize the training of an SAE and automatically annotate the semantics of the SAE atoms for safeguarding MLLMs. Experiments on multiple MLLMs (e.g., QwenVL, LLaVA, InternVL) across safety benchmarks (e.g., MM-SafetyBench, JailBreakV) show that DACO significantly improves MLLM safety while maintaining general-purpose capabilities.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kulkarni2026,

title = {VidTAG: Temporally Aligned Video to GPS Geolocalization with Denoising Sequence Prediction at a Global Scale},

author = {Parth Parag Kulkarni and Rohit Gupta and Prakash Chandra Chhipa and Mubarak Shah},

url = {https://arxiv.org/abs/2604.12159},

year  = {2026},

date = {2026-04-10},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {The task of video geolocalization aims to determine the precise GPS coordinates of a video’s origin and map its trajectory; with applications in forensics, social media, and exploration. Existing classification-based approaches operate at a coarse city-level granularity and fail to capture fine-grained details, while image retrieval methods are impractical on a global scale due to the need for extensive image galleries which are infeasible to compile. Comparatively, constructing a gallery of GPS coordinates is straightforward and inexpensive. We propose VidTAG, a dual-encoder framework that performs frameto-GPS retrieval using both self-supervised and languagealigned features. To address temporal inconsistencies in video predictions, we introduce the TempGeo module, which aligns frame embeddings, and the GeoRefiner module, an encoder-decoder architecture that refines GPS features using the aligned frame embeddings. Evaluations on Mapillary (MSLS) and GAMa datasets demonstrate our model’s ability to generate temporally consistent trajectories and outperform baselines, achieving a 20% improvement at the 1 km threshold over GeoCLIP. We also beat current Stateof-the-Art by 25% on global coarse grained video geolocalization (CityGuessr68k). Our approach enables finegrained video geolocalization and lays a strong foundation for future research. More details on the project webpage: parthpk.github.io/vidtag webpage.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chan-Santiago2026,

title = {Learnability-Guided Diffusion for Dataset Distillation},

author = {Jeffrey A. Chan-Santiago and Mubarak Shah},

url = {https://arxiv.org/abs/2604.00519},

year  = {2026},

date = {2026-04-01},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {Training machine learning models on massive datasets is expensive and time-consuming. Dataset distillation addresses this by creating a small synthetic dataset that achieves the same performance as the full dataset. Recent methods use diffusion models to generate distilled data, either by promoting diversity or matching training gradients. However, existing approaches produce redundant training signals, where samples convey overlapping information. Empirically, disjoint subsets of distilled datasets capture 80-90% overlapping signals. This redundancy stems from optimizing visual diversity or average training dynamics without accounting for similarity across samples, leading to datasets where multiple samples share similar information rather than complementary knowledge. We propose learnability-driven dataset distillation, which constructs synthetic datasets incrementally through successive stages. Starting from a small set, we train a model and generate new samples guided by learnability scores that identify what the current model can learn from, creating an adaptive curriculum. We introduce Learnability-Guided Diffusion (LGD), which balances training utility for the current model with validity under a reference model to generate curriculum-aligned samples. Our approach reduces redundancy by 39.1%, promotes specialization across training stages, and achieves state-of-the-art results on ImageNet-1K (60.1%), ImageNette (87.2%), and ImageWoof (72.9%). Our code is available on our project page this https URL.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Swetha2026,

title = {VRR-QA: Visual Relational Reasoning in Videos Beyond Explicit Cues},

author = {Sirnam Swetha and Rohit Gupta and Parth Parag Kulkarni and David G Shatwell and Jeffrey A Chan Santiago and Nyle Siddiqui and Joseph Fioresi and Mubarak Shah},

url = {https://arxiv.org/abs/2506.21742},

year  = {2026},

date = {2026-03-29},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {Video Question Answering (VideoQA) has made significant strides by leveraging multimodal learning to align visual and textual modalities. However, current benchmarks overwhelmingly focus on questions answerable through explicit visual content - actions, objects, and events - directly observable within individual frames or short clips. To truly understand videos as humans do, models must go beyond what is directly shown, inferring hidden relationships and contextual cues that are only implied across frames. Current benchmarks fail to capture this essential aspect of video understanding. To address this gap, we introduce VRR-QA, a benchmark for Visual Relational Reasoning Beyond Explicit Cues. We curate our benchmark from creative and cinematic videos such as movies, that deliberately employ storytelling techniques which omit direct depictions of certain events or relations, requiring viewers to infer them. VRR-QA comprises 1K meticulously expert-annotated QA pairs drawn from 1K creative video clips covering 15 genres across 7 decades of content, from both live-action and animated titles. Our extensive evaluations on 14 leading VideoQA models reveals consistent and significant performance degradation, underscoring their reliance on surface-level visual cues and highlighting the difficulty of implicit reasoning. Even the best model substantially underperforms human baselines with only 64% accuracy. Performance variations across models further illustrate the complexity and diversity of the challenges presented by VRR-QA. By releasing both dataset and data collection framework, VRR-QA establishes a rigorous, diverse, and reproducible testbed for advancing VideoQA: this https URL.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Shatwell2026,

title = {TIGeR: A Unified Framework for Time, Images and Geo-location Retrieval},

author = {David G. Shatwell and Sirnam Swetha and Mubarak Shah},

url = {https://arxiv.org/abs/2603.24749},

year  = {2026},

date = {2026-03-28},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {Many real-world applications in digital forensics, urban monitoring, and environmental analysis require jointly reasoning about visual appearance, location, and time. Beyond standard geo-localization and time-of-capture prediction, these applications increasingly demand more complex capabilities, such as retrieving an image captured at the same location as a query image but at a specified target time. We formalize this problem as Geo-Time Aware Image Retrieval and propose TIGeR, a unified framework for Time, Images and Geo-location Retrieval. TIGeR supports flexible input configurations (single-modality and multi-modality queries) and uses the same representation to perform (i) geo-localization, (ii) time-of-capture prediction, and (iii) geo-time-aware retrieval. By preserving the underlying location identity despite large appearance changes, TIGeR enables retrieval based on where and when a scene was captured, rather than purely on visual similarity. To support this task, we design a multistage data curation pipeline and propose a new diverse dataset of 4.5M paired image-location-time triplets for training and 86k high-quality triplets for evaluation. Extensive experiments show that TIGeR consistently outperforms strong baselines and state-of-the-art methods by up to 16% on time-of-year, 8% time-of-day prediction, and 14% in geo-time aware retrieval recall, highlighting the benefits of unified geo-temporal modeling.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhai2026,

title = {CoRe: Context-Robust Remasking for Diffusion Language Models},

author = {Kevin Zhai and Sabbir Mollah and Zhenyi Wang and Mubarak Shah},

url = {https://arxiv.org/abs/2602.04096},

year  = {2026},

date = {2026-03-13},

publisher = {Forty-Third International Conference on Machine Learning (ICML) 2026},

abstract = {Standard decoding in Masked Diffusion Models (MDMs) is hindered by context rigidity: tokens are retained based on transient high confidence, often ignoring that early predictions lack full context. This creates cascade effects where initial inconsistencies misguide the remaining generation. Existing revision strategies attempt to mitigate this by relying on static confidence scores, but these signals are inherently myopic; inconsistent tokens can appear confident to the model itself. We propose Context-Robust Remasking (CORE), a training-free framework for inference-time revision. Rather than trusting static token probabilities, CORE identifies context-brittle tokens by probing their sensitivity to targeted masked-context perturbations. We formalize revision as a robust optimization objective over context shifts and efficiently approximate this objective to prioritize unstable tokens for revision. On LLaDA-8B-Base, CORE delivers consistent improvements across reasoning and code benchmarks, outperforming compute-matched baselines and improving MBPP by up to 9.2 percentage points.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Campos2026,

title = {GAEA: A Geolocation Aware Conversational Model},

author = {Ron Campos and Ashmal Vayani and Parth Parag Kulkarni and Rohit Gupta and Aizan Zafar and Aritra Dutta and Mubarak Shah},

url = {https://ucf-crcv.github.io/GAEA/#bibtex

https://arxiv.org/abs/2503.16423

https://huggingface.co/collections/ucf-crcv/gaea-67d514a61d48eb1708b13a08

https://github.com/UCF-CRCV/GAEA},

year  = {2026},

date = {2026-03-06},

urldate = {2026-03-06},

publisher = {The IEEE/CVF Winter conference on Applications of Computer Vision},

abstract = {Image geolocalization, in which an AI model traditionally predicts the precise GPS coordinates of an image, is a challenging task with many downstream applications. However, the user cannot utilize the model to further their knowledge beyond the GPS coordinates; the model lacks an understanding of the location and the conversational ability to communicate with the user. In recent days, with the tremendous progress of large multimodal models (LMMs)---proprietary and open-source---researchers have attempted to geolocalize images via LMMs. However, the issues remain unaddressed; beyond general tasks, for more specialized downstream tasks, such as geolocalization, LMMs struggle. In this work, we propose solving this problem by introducing a conversational model, GAEA, that provides information regarding the location of an image as the user requires. No large-scale dataset enabling the training of such a model exists. Thus, we propose GAEA-1.4M, a comprehensive dataset comprising over 800k images and approximately 1.4M question-answer pairs, constructed by leveraging OpenStreetMap (OSM) attributes and geographical context clues. For quantitative evaluation, we propose a diverse benchmark, GAEA-Bench, comprising 3.5k image-text pairs to evaluate conversational capabilities equipped with diverse question types. We consider 11 state-of-the-art open-source and proprietary LMMs and demonstrate that GAEA significantly outperforms the best open-source model, LLaVA-OneVision, by 18.2% and the best proprietary model, GPT-4o, by 7.2%.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ashraf2026,

title = {Agent-X: Evaluating Deep Multimodal Reasoning in Vision-Centric Agentic Tasks},

author = {Tajamul Ashraf and Amal Saqib and Hanan Gani and Muhra AlMahri and Yuhao Li and Noor Ahsan and Umair Nawaz and Jean Lahoud and Hisham Cholakkal and Mubarak Shah and Philip Torr and Fahad Khan and Rao Anwer and Salman Khan},

url = {https://openreview.net/pdf?id=Vjruxvp1Xd},

year  = {2026},

date = {2026-01-26},

publisher = {Fourteenth International Conference on Learning Representations (ICLR), 2026.},

abstract = {Deep reasoning is fundamental for solving complex tasks, especially in vision-centric scenarios that demand sequential, multimodal understanding. However, existing benchmarks typically evaluate agents with fully synthetic, single-turn queries, limited visual modalities, and lack a framework to assess reasoning quality over multiple steps as required in real-world settings. To address this, we introduce Agent-X, a large-scale benchmark for evaluating vision-centric agents’ multistep and deep reasoning capabilities in real-world, multimodal settings. AgentX features 828 agentic tasks with authentic visual contexts, including images, multi-image comparisons, videos, and instructional text. These tasks span six major agentic environments: general visual reasoning, web browsing, security and surveillance, autonomous driving, sports, and math reasoning. Our benchmark requires agents to integrate tool use with explicit, stepwise decision-making in these diverse settings. In addition, we propose a fine-grained, step-level evaluation framework that assesses the correctness and logical coherence of each reasoning step and the effectiveness of tool usage throughout the task. Our results reveal that even the best-performing models, including GPT, Gemini, and Qwen families, struggle to solve multi-step vision tasks, achieving less than 50% full-chain success. These findings highlight key bottlenecks in current LMM reasoning and tool-use capabilities and identify future research directions in vision-centric agentic reasoning models},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Singh2026,

title = { Direct Preference Optimization for Primitive-Enabled Hierarchical RL: A Bilevel Approach},

author = {Utsav Singh and Souradip Chakraborty and Wesley Suttle and Brian Sadler and Derrik Asher and Anit Kumar Sahu and Mubarak Shah and Vinay Purushothaman Namboodiri and Amrit Bedi

},

url = {https://arxiv.org/html/2411.00361v3},

year  = {2026},

date = {2026-01-26},

urldate = {2026-01-26},

publisher = {Fourteenth International Conference on Learning Representations (ICLR), 2026.},

abstract = {Hierarchical reinforcement learning (HRL) enables agents to solve complex, long-horizon tasks by decomposing them into manageable sub-tasks. However, HRL methods face two fundamental challenges: (i) non-stationarity caused by the evolving lower-level policy during training, which destabilizes higher-level learning, and (ii) the generation of infeasible subgoals that lower-level policies cannot achieve. To address these challenges, we introduce DIPPER, a novel HRL framework that formulates goal-conditioned HRL as a bi-level optimization problem and leverages direct preference optimization (DPO) to train the higher-level policy. By learning from preference comparisons over subgoal sequences rather than rewards that depend on the evolving lower-level policy, DIPPER mitigates the impact of non-stationarity on higher-level learning. To address infeasible subgoals, DIPPER incorporates lower-level value function regularization that encourages the higher-level policy to propose achievable subgoals. We introduce two novel metrics to quantitatively verify that DIPPER mitigates non-stationarity and infeasible subgoal generation issues in HRL. Empirical evaluation on challenging robotic navigation and manipulation benchmarks shows that DIPPER achieves upto 40% improvements over state-of-the-art baselines on challenging sparse-reward scenarios, highlighting the potential of preference-based learning for addressing longstanding HRL limitations.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fioresi2026b,

title = {Privacy Beyond Pixels: Latent Anonymization for Privacy-Preserving Video Understanding},

author = {Joseph Fioresi and Ishan Rajendrakumar Dave and Mubarak Shah},

url = {https://arxiv.org/abs/2511.08666},

year  = {2026},

date = {2026-01-26},

urldate = {2026-01-26},

publisher = {Fourteenth International Conference on Learning Representations (ICLR), 2026.},

abstract = {We introduce a novel formulation of visual privacy preservation for video foundation models that operates entirely in the latent space. While spatio-temporal features learned by foundation models have deepened general understanding of video content, sharing or storing these extracted visual features for downstream tasks inadvertently reveals sensitive personal information like skin color, gender, or clothing. Current privacy preservation methods focus on input-pixel level anonymization, which requires retraining the entire utility video model and results in task-specific anonymization, making them unsuitable for recent video foundational models. To address these challenges, we introduce a lightweight Anonymizing Adapter Module (AAM) that removes private information from video features while retaining general task utility. AAM can be applied in a plug and play fashion to frozen video encoders, minimizing the computational burden of finetuning and re-extracting features. Our framework employs three newly designed training objectives: (1) a clip-level self-supervised privacy objective to reduce mutual information between static clips, (2) a co-training objective to retain utility across seen tasks, and (3) a latent consistency loss for generalization on unseen tasks. Our extensive evaluations demonstrate a significant 35% reduction in privacy leakage while maintaining near-baseline utility performance across various downstream tasks: Action Recognition (Kinetics400, UCF101, HMDB51), Temporal Action Detection (THUMOS14), and Anomaly Detection (UCF-Crime). We also provide an....},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2025,

title = {From Play to Replay: Composed Video Retrieval for Sports Highlights},

author = {Animesh Gupta and Jay Parmar and Ishan Rajendrakumar Dave and Mubarak Shah},

url = {https://neurips.cc/virtual/2025/poster/121717

https://animesh-007.github.io/TF-CoVR-WEBSITE/},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Composed Video Retrieval (CoVR) retrieves a target video given a query video and a modification text describing the intended change. Existing CoVR benchmarks emphasize appearance shifts or coarse event changes and therefore do not test the ability to capture subtle, fast-paced temporal differences. We introduce TF-CoVR, the first large-scale benchmark dedicated to temporally fine-grained CoVR. TF-CoVR focuses on gymnastics and diving and provides 1.8 M triplets drawn from FineGym and FineDiving. Previous CoVR benchmarks focusing on temporal aspect, link each query to a single target segment taken from the same video, limiting practical usefulness. In TF-CoVR, we instead construct each pair by prompting an LLM with the label differences between clips drawn from different videos; every pair is thus associated with multiple valid target videos (3.9 on average), reflecting real-world tasks such as sports-highlight generation. To model these temporal dynamics we propose TF-CoVR-Base, a concise two-stage training framework: (i) pre-train a video encoder on fine-grained action classification to obtain temporally discriminative embeddings; (ii) align the composed query with candidate videos using contrastive learning. We conduct the first comprehensive study of image, video, and general multimodal embedding (GME) models on temporally fine-grained composed retrieval in both zero-shot and fine-tuning regimes. On TF-CoVR, TF-CoVR-Base improves zero-shot mAP@50 from 5.92 (LanguageBind) to 7.51, and after fine-tuning raises the state of the art from 19.83 to 25.82.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {DEXTER: Diffusion-Guided EXplanations with TExtual Reasoning for Vision Models},

author = {Simone Carnemolla and Matteo Pennisi and Sarinda Samarasinghe and Giovanni Bellitto and Simone Palazzo and Daniela Giordano and Mubarak Shah and Concetto Spampinato},

url = {https://neurips.cc/virtual/2025/poster/117167},

year  = {2025},

date = {2025-11-30},

publisher = {The Thirty-Ninth Annual Conference on Neural Information Processing Systems},

abstract = {Understanding and explaining the behavior of machine learning models is essential for building transparent and trustworthy AI systems. We introduce DEXTER, a data-free framework that combines diffusion models and large language models to generate global, textual explanations of visual classifiers. DEXTER operates by optimizing text prompts to synthesize class-conditional images that strongly activate a target classifier. These synthetic samples are then used to elicit detailed natural language reports that describe class-specific decision patterns and biases. Unlike prior work, DEXTER enables natural language reasoning about a classifier's decision process without access to training data or ground-truth labels. We demonstrate DEXTER's flexibility across three tasks—activation maximization, slice discovery and debiasing, and bias explanation—each illustrating its ability to uncover the internal mechanisms of visual classifiers. Quantitative and qualitative evaluations, including a user study, show that DEXTER produces accurate, interpretable outputs. Experiments on ImageNet, Waterbirds, CelebA, and FairFaces confirm that DEXTER outperforms existing approaches in global model explanation and class-level bias reporting.



},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Shafique2025,

title = {A Culturally-diverse Multilingual Multimodal Video Benchmark & Model},

author = {Bhuiyan Sanjid Shafique and Ashmal Vayani and Muhammad Maaz and Hanoona Abdul Rasheed and Dinura Dissanayake and Mohammed Irfan Kurpath and Yahya Hmaiti and Go Inoue and Jean Lahoud and Md. Safirur Rashid and Shadid Intisar Quasem and Maheen Fatima and Franco Vidal and Mykola Maslych and Ketan Pravin More and Sanoojan Baliah and Hasindri Watawana and Yuhao Li and Fabian Farestam and Leon Schaller and Roman Tymtsiv and Simon Weber and Hisham Cholakkal and Ivan Laptev and Shin'ichi Satoh and Michael Felsberg and Mubarak Shah and Salman Khan and Fahad Shahbaz Khan},

url = {https://mbzuai-oryx.github.io/ViMUL/

https://arxiv.org/abs/2506.07032

https://huggingface.co/datasets/MBZUAI/ViMUL-Bench},

year  = {2025},

date = {2025-11-04},

urldate = {2025-11-04},

publisher = {Empirical Methods in Natural Language Processing},

abstract = {Large multimodal models (LMMs) have recently gained attention due to their effectiveness to understand and generate descriptions of visual content. Most existing LMMs are in English language. While few recent works explore multilingual image LMMs, to the best of our knowledge, moving beyond the English language for cultural and linguistic inclusivity is yet to be investigated in the context of video LMMs. In pursuit of more inclusive video LMMs, we introduce a multilingual Video LMM benchmark, named ViMUL-Bench, to evaluate Video LMMs across 14 languages, including both low and high-resource languages: English, Chinese, Spanish, French, German, Hindi, Arabic, Russian, Bengali, Urdu, Sinhala, Tamil, Swedish, and Japanese. Our ViMUL-Bench is designed to rigorously test video LMMs across 15 categories including eight culturally diverse categories, ranging from lifestyles and festivals to foods and rituals and from local landmarks to prominent cultural personalities. ViMUL-Bench comprises both open-ended (short and long-form) and multiple-choice questions spanning various video durations (short, medium, and long) with 8k samples that are manually verified by native language speakers. In addition, we also introduce a machine translated multilingual video training set comprising 1.2 million samples and develop a simple multilingual video LMM, named ViMUL, that is shown to provide a better tradeoff between high-and low-resource languages for video understanding. We hope our ViMUL-Bench and multilingual video LMM along with a large-scale multilingual video training set will help ease future research in developing cultural and linguistic inclusive multilingual video LMMs. Our proposed benchmark, video LMM and training data will be publicly released.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Shatwell2025,

title = {GT-Loc: Unifying When and Where in Images through a Joint Embedding Space},

author = {David Shatwell and Ishan Dave and Swetha Sirnam and Mubarak Shah},

url = {https://chatpaper.com/fr/chatpaper/paper/163729

https://arxiv.org/abs/2507.10473#:~:text=14%20Jul%202025%5D-,GT%2DLoc%3A%20Unifying%20When%20and%20Where%20in%20Images,Through%20a%20Joint%20Embedding%20Space&text=Abstract%3ATimestamp%20prediction%20aims%20to,%2C%20retrieval%2C%20and%20digital%20forensics.},

year  = {2025},

date = {2025-10-19},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Timestamp prediction aims to determine when an image was captured using only visual information, supporting applications such as metadata correction, retrieval, and digital forensics. In outdoor scenarios, hourly estimates rely on cues like brightness, hue, and shadow positioning, while seasonal changes and weather inform date estimation. However, these visual cues significantly depend on geographic context, closely linking timestamp prediction to geo-localization. To address this interdependence, we introduce GT-Loc, a novel retrieval-based method that jointly predicts the capture time (hour and month) and geo-location (GPS coordinates) of an image. Our approach employs separate encoders for images, time, and location, aligning their embeddings within a shared high-dimensional feature space. Recognizing the cyclical nature of time, instead of conventional contrastive learning with hard positives and negatives, we propose a temporal metric-learning objective providing soft targets by modeling pairwise time differences over a cyclical toroidal surface. We present new benchmarks demonstrating that our joint optimization surpasses previous time prediction methods, even those using the ground-truth geo-location as an input during inference. Additionally, our approach achieves competitive results on standard geo-localization tasks, and the unified embedding space facilitates compositional and text-based image retrieval.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kang2025,

title = {Robin3D: Improving 3D Large Language Model via Robust Instruction Tuning},

author = {Weitai Kang and Haifeng Huang and Yuzhang Shang and Mubarak Shah and Yan Yan},

url = {https://arxiv.org/abs/2410.00255

https://github.com/WeitaiKang/Robin3D},

year  = {2025},

date = {2025-10-19},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Recent advancements in 3D Large Language Models (3DLLMs) have highlighted their potential in building general-purpose agents in the 3D real world, yet challenges remain due to the lack of high-quality robust instruction-following data, leading to limited discriminative power and generalization of 3DLLMs. In this paper, we introduce Robin3D, a powerful 3DLLM trained on large-scale instruction-following data generated by our novel data engine, Robust Instruction Generation (RIG) engine. RIG generates two key instruction data: 1) the Adversarial Instruction-following data, which features mixed negative and positive samples to enhance the model's discriminative understanding. 2) the Diverse Instruction-following data, which contains various instruction styles to enhance model's generalization. As a result, we construct 1 million instruction-following data, consisting of 344K Adversarial samples, 508K Diverse samples, and 165K benchmark training set samples. To better handle these complex instructions, Robin3D first incorporates Relation-Augmented Projector to enhance spatial understanding, and then strengthens the object referring and grounding ability through ID-Feature Bonding. Robin3D consistently outperforms previous methods across five widely-used 3D multimodal learning benchmarks, without the need for task-specific fine-tuning. Notably, we achieve a 7.8% improvement in the grounding task (Multi3DRefer) and a 6.9% improvement in the captioning task (Scan2Cap).},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Thawakar2025,

title = {Beyond Simple Edits: Composed Video Retrieval with Dense Modifications},

author = {Omkar Thawakar and Dmitry Demidov and Ritesh Thawkar and Rao Anwer and Mubarak Shah and Fahad Khan and Salman Khan},

url = {https://iccv.thecvf.com/virtual/2025/poster/1966},

year  = {2025},

date = {2025-10-19},

urldate = {2025-10-19},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Composed video retrieval is a challenging task that strives to retrieve a target video based on a query video and a textual description detailing specific modifications. Standard retrieval frameworks typically struggle to handle the complexity of fine-grained compositional queries and variations in temporal understanding limiting their retrieval ability in the fine-grained setting. To address this issue, we introduce a novel dataset that captures both fine-grained and composed actions across diverse video segments, enabling more detailed compositional changes in retrieved video content.The proposed dataset, named Dense-WebVid-CoVR, consists of 1.6 million samples with dense modification text that is around seven times more than its existing counterpart. We further develop a new model that integrates visual and textual information through Cross-Attention (CA) fusion using grounded text encoder, enabling precise alignment between dense query modifications and target videos. The proposed model achieves state-of-the-art results surpassing existing methods on all metrics. Notably, it achieves 71.3% Recall@1 in visual+text setting and outperforms the state-of-the-art by 3.4%, highlighting its efficacy in terms of leveraging detailed video descriptions and dense modification texts. Our proposed dataset, code, and model will be publicly released.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fan2025,

title = {Test-Time Retrieval-Augmented Adaptation for Vision-Language Models},

author = {Xinqi Fan and Xueli Chen and Luoxiao Yang and Chuin Hong Yap and Rizwan Qureshi and Qi Dou and Moi Hoon Yap and Mubarak Shah},

url = {https://iccv.thecvf.com/virtual/2025/poster/2327},

year  = {2025},

date = {2025-10-19},

publisher = {IEEE/CVF International Conference on Computer Vision},

abstract = {Vision-language models (VLMs) have shown promise in test-time adaptation tasks due to their remarkable capabilities in understanding and reasoning about visual content through natural language descriptions. However, training VLMs typically demands substantial computational resources, and they often struggle to adapt efficiently to new domains or tasks. Additionally, dynamically estimating the test distribution from streaming data at test time remains a significant challenge. In this work, we propose a novel test-time retrieval-augmented adaption (TT-RAA) method that enables VLMs to maintain high performance across diverse visual recognition tasks without the need for task-specific training or large computational overhead. During inference, TT-RAA employs a streaming mixture of Gaussian database (SMGD) to continuously estimate test distributions, requiring minimal storage. Then, TT-RAA retrieves the most relevant information from the SMGD, enhancing the original VLM outputs. A key limitation of CLIP-based VLMs is their inter-modal vision-language optimization, which does not optimize vision-space similarity, leading to larger intra-modal variance. To address this, we propose a multimodal retrieval augmentation module that transforms the SMGD into a unified multimodal space, enabling retrieval that aligns both vision and language modalities. Extensive experiments across both cross-domain and out-of-distribution benchmarks comprising fourteen datasets demonstrate TT-RAA’s superior performance compared to state-of-the-art methods. Ablation studies and hyperparameter analyses further validate the effectiveness of the proposed modules.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chan-Santiago2025,

title = {MGD^3: Mode-Guided Dataset Distillation using Diffusion Models},

author = {Jeffrey A. Chan-Santiago and Praveen Tirupattur and Gaurav Kumar Nayak and Gaowen Liu and Mubarak Shah},

url = {https://icml.cc/virtual/2025/poster/45507},

year  = {2025},

date = {2025-07-13},

publisher = {Forty-Second International Conference on Machine Learning},

abstract = {Dataset distillation has emerged as an effective strategy, significantly reducing training costs and facilitating more efficient model deployment.Recent advances have leveraged generative models to distill datasets by capturing the underlying data distribution. Unfortunately, existing methods require model fine-tuning with distillation losses to encourage diversity and representativeness. However, these methods do not guarantee sample diversity, limiting their performance.We propose a mode-guided diffusion model leveraging a pre-trained diffusion model without the need to fine-tune with distillation losses. Our approach addresses dataset diversity in three stages: Mode Discovery to identify distinct data modes, Mode Guidance to enhance intra-class diversity, and Stop Guidance to mitigate artifacts in synthetic samples that affect performance.We evaluate our approach on ImageNette, ImageIDC, ImageNet-100, and ImageNet-1K, achieving accuracy improvements of 4.4%, 2.9%, 1.6%, and 1.6%, respectively, over state-of-the-art methods. Our method eliminates the need for fine-tuning diffusion models with distillation losses, significantly reducing computational costs.



},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vayani2025,

title = {All Languages Matter: Evaluating LMMs on Culturally Diverse 100 Languages},

author = {Ashmal Vayani and Dinura Dissanayake and Hasindri Watawana and Noor Ahsan and Nevasini Sasikumar and Omkar Thawakar and Henok Biadglign Ademtew and Yahya Hmaiti and Amandeep Kumar and Kartik Kuckreja and Mykola Maslych and Wafa Al Ghallabi and Mihail Mihaylov and Chao Qin and Abdelrahman M Shaker and Mike Zhang and Mahardika Krisna Ihsani and Amiel Esplana and Monil Gokani and Shachar Mirkin and Harsh Singh and Ashay Srivastava and Endre Hamerlik and Fathinah Asma Izzati and Fadillah Adamsyah Maani and Sebastian Cavada and Jenny Chim and Rohit Gupta and Sanjay Manjunath and Kamila Zhumakhanova and Feno Heriniaina Rabevohitra and Azril Amirudin and Muhammad Ridzuan and Daniya Kareem and Ketan More and Kunyang Li and Pramesh Shakya and Muhammad Saad and Amirpouya Ghasemaghaei and Amirbek Djanibekov and Dilshod Azizov and Branislava Jankovic and Naman Bhatia and Alvaro Cabrera and Johan Obando-Ceron and Olympiah Otieno and Fabian Farestam and Muztoba Rabbani and Sanoojan Baliah and Santosh Sanjeev and Abduragim Shtanchaev and Maheen Fatima and Thao Nguyen and Amrin Kareem and Toluwani Aremu and Nathan Xavier and Amit Bhatkal and Hawau Toyin and Aman Chadha and Hisham Cholakkal and Rao Muhammad Anwer and Michael Felsberg and Jorma Laaksonen and Thamar Solorio and Monojit Choudhury and Ivan Laptev and Mubarak Shah and Salman Khan and Fahad Khan},

url = {https://arxiv.org/abs/2411.16508

https://mbzuai-oryx.github.io/ALM-Bench/

https://huggingface.co/datasets/MBZUAI/ALM-Bench

https://github.com/mbzuai-oryx/ALM-Bench

https://drive.google.com/file/d/1B2yynH-o1z1UvMCl32XoY3pVIkAxgN0P/view?usp=sharing

https://docs.google.com/presentation/d/10lUrfoC-R1lgl0Dx0EYpx1ID3AqLGnRO/edit?usp=sharing&ouid=103839181464334047856&rtpof=true&sd=true

https://www.youtube.com/watch?v=fvcX4yi5AiY&list=PLtIMlmzwbtty7ie1quj1fq2QoTRK9eWHX&ab_channel=AshmalVayani},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Existing Large Multimodal Models (LMMs) generally focus on only a few regions and languages. As LMMs continue to improve, it is increasingly important to ensure they understand cultural contexts, respect local sensitivities, and support low-resource languages, all while effectively integrating corresponding visual cues. In pursuit of culturally diverse global multimodal models, our proposed All Languages Matter Benchmark (ALM-bench) represents the largest and most comprehensive effort to date for evaluating LMMs across 100 languages. ALM-bench challenges existing models by testing their ability to understand and reason about culturally diverse images paired with text in various languages, including many low-resource languages traditionally underrepresented in LMM research. The benchmark offers a robust and nuanced evaluation framework featuring various question formats, including true/false, multiple choice, and open-ended questions, which are further divided into short and long-answer categories. ALM-bench design ensures a comprehensive assessment of a model's ability to handle varied levels of difficulty in visual and linguistic reasoning. To capture the rich tapestry of global cultures, ALM-bench carefully curates content from 13 distinct cultural aspects, ranging from traditions and rituals to famous personalities and celebrations. Through this, ALM-bench not only provides a rigorous testing ground for state-of-the-art open and closed-source LMMs but also highlights the importance of cultural and linguistic inclusivity, encouraging the development of models that can serve diverse global populations effectively. Our benchmark is publicly available.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Huynh2025b,

title = {CoLLM:ALargeLanguage Model for Composed Image Retrieva},

author = {Chuong Huynh and Jinyu Yang and Ashish Tawari and Mubarak Shah and Son Tran and Raffay Hamid and Trishul Chilimbi and Abhinav Shrivastava},

url = {https://www.amazon.science/publications/collm-a-large-language-model-for-composed-image-retrieval

https://assets.amazon.science/55/e0/7ead8a494dc49fce25cb9947654c/collm-a-large-language-model-for-composed-image-retrieval.pdf

https://collm-cvpr25.github.io/},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Composed Image Retrieval (CIR) is a complex task that aims to retrieve images based on a multimodal query. Typical training data consists of triplets containing a reference image, a textual description of desired modifications, and the target image, which are expensive and time-consuming to acquire. The scarcity of CIR datasets has led to zero-shot approaches utilizing synthetic triplets or leveraging vision-language models (VLMs) with ubiquitous web-crawled image-caption pairs. However, these methods have significant limitations: synthetic triplets suffer from limited scale, lack of diversity, and unnatural modification text, while image-caption pairs hinder joint embedding learning of the multimodal query due to the absence of triplet data. Moreover, existing approaches struggle with complex and nuanced modification texts that demand sophisticated fusion and understanding of vision and language modalities. We present CoLLM, a one-stop framework that effectively addresses these limitations. Our approach generates triplets on-the-fly from image-caption pairs, enabling supervised training without manual annotation. We leverage Large Language Models (LLMs) to generate joint embeddings of reference images and modification texts, facilitating deeper multimodal fusion. Additionally, we introduce Multi-Text CIR (MTCIR), a large-scale dataset comprising 3.4M samples, and refine existing CIR benchmarks (CIRR and Fashion-IQ) to enhance evaluation reliability. Experimental results demonstrate that CoLLM achieves state-of0the-art performance across multiple CIR benchmarks and settings. MTCIR yields competitive results, with up to 15% performance improvement. Our refined benchmarks provide more reliable evaluation metrics for CIR models, contributing to the advancement of this important field. Project page is at collm-cvpr25.github.io.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hu2025,

title = {M-LLMBasedVideo Frame Selection for Efficient Video Understanding},

author = {Kai Hu and Feng Gao and Xiaohan Nie and Peng Zhou and Son Tran and Tal Neiman and Lingyun Wang and Mubarak Shah and Raffay Hamid and Bing Yin and Trishul Chilimbi},

url = {https://www.amazon.science/publications/m-llm-based-video-frame-selection-for-efficient-video-understanding

https://assets.amazon.science/1a/4e/6794e1dd4fb2acb30eda428a5119/m-llm-based-video-frame-selection-for-efficient-video-understanding.pdf

},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Recent advances in Multi-Modal Large Language Models (M-LLMs) show promising results in video reasoning. Popular Multi-Modal Large Language Model (M-LLM) frameworks usually apply naive uniform sampling to reduce the number of video frames that are fed into an M-LLM, particularly for long context videos. However, it could lose crucial context in certain periods of a video, so that the downstream M-LLM may not have sufficient visual information to answer a question. To attack this pain point, we propose a light-weight M-LLM-based frame selection method that adaptively select frames that are more relevant to users’ queries. In order to train the proposed frame selector, we introduce two supervision signals (i) Spatial signal, where single frame importance score by prompting a M-LLM; (ii) Temporal signal, in which multiple frames selection by prompting Large Language Model (LLM) using the captions of all frame candidates. The selected frames are then digested by a frozen downstream video M-LLM for visual reasoning and question answering. Empirical results show that the proposed M-LLM video frame selector improves the performances various downstream video Large Language Model (video-LLM) across medium (ActivityNet, NExT-QA) and long (EgoSchema, LongVideoBench) context video question answering benchmarks.

},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Croitoru2025,

title = {Curriculum Direct Preference Optimization for Diffusion and Consistency Models},

author = {Florinel-Alin Croitoru and Vlad Hondru and Radu Tudor Ionescu and Nicu Sebe and Mubarak Shah},

url = {https://arxiv.org/abs/2405.13637

https://croitorualin.github.io/cl-dpo/

https://github.com/CroitoruAlin/Curriculum-DPO







},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Direct Preference Optimization (DPO) has been proposed as an effective and efficient alternative to reinforcement learning from human feedback (RLHF). In this paper, we propose a novel and enhanced version of DPO based on curriculum learning for text-to-image generation. Our method is divided into two training stages. First, a ranking of the examples generated for each prompt is obtained by employing a reward model. Then, increasingly difficult pairs of examples are sampled and provided to a text-to-image generative (diffusion or consistency) model. Generated samples that are far apart in the ranking are considered to form easy pairs, while those that are close in the ranking form hard pairs. In other words, we use the rank difference between samples as a measure of difficulty. The sampled pairs are split into batches according to their difficulty levels, which are gradually used to train the generative model. Our approach, Curriculum DPO, is compared against state-of-the-art fine-tuning approaches on nine benchmarks, outperforming the competing methods in terms of text alignment, aesthetics and human preference. Our code is available at this https URL.

},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2025b,

title = {Enhancing Privacy-Utility Trade-offs to Mitigate Memorization in Diffusion Models},

author = {Chen Chen and Daochang Liu and Mubarak Shah and Chang Xu},

url = {https://cvpr.thecvf.com/virtual/2025/poster/34842

https://chenchen-usyd.github.io/PRSS-Project-Page/},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Text-to-image diffusion models have demonstrated remarkable capabilities in creating images highly aligned with user prompts, yet their proclivity for memorizing training set images has sparked concerns about the originality of the generated images and privacy issues, potentially leading to legal complications for both model owners and users, particularly when the memorized images contain proprietary content. Although methods to mitigate these issues have been suggested, enhancing privacy often results in a significant decrease in the utility of the outputs, as indicated by text-alignment scores. To bridge the research gap, we introduce a novel method, PRSS, which refines the classifier-free guidance approach in diffusion models by integrating prompt re-anchoring (PR) to improve privacy and incorporating semantic prompt search (SS) to enhance utility. Extensive experiments across various privacy levels demonstrate that our approach consistently improves the privacy-utility trade-off, establishing a new state-of-the-art.



},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Enhancing Privacy-Utility Trade-offs to Mitigate Memorization in Diffusion Models},

author = {Chen Chen and Daochang Liu and Mubarak Shah and Chang Xu},

url = {https://chenchen-usyd.github.io/PRSS-Project-Page/},

year  = {2025},

date = {2025-04-25},

urldate = {2025-04-25},

abstract = {Text-to-image diffusion models have demonstrated remark-

able capabilities in creating images highly aligned with

user prompts, yet their proclivity for memorizing training

set images has sparked concerns about the originality of the

generated images and privacy issues, potentially leading to

legal complications for both model owners and users, par-

ticularly when the memorized images contain proprietary

content. Although methods to mitigate these issues have

been suggested, enhancing privacy often results in a signif-

icant decrease in the utility of the outputs, as indicated by

text-alignment scores. To bridge the research gap, we in-

troduce a novel method, PRSS, which refines the classifier-

free guidance approach in diffusion models by integrating

prompt re-anchoring (PR) to improve privacy and incorpo-

rating semantic prompt search (SS) to enhance utility. Ex-

tensive experiments across various privacy levels demon-

strate that our approach consistently improves the privacy-

utility trade-off, establishing a new state-of-the-art.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Intent3D: 3D Object Detection in RGB-D Scans Based on Human Intention},

author = {Weitai Kang and Mengxue Qu and Jyoti Kini and Yunchao Wei and Mubarak Shah and Yan Yan},

url = {https://openreview.net/forum?id=5GgjiRzYp3&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/3450_Intent3D_3D_Object_Detect.pdf.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {In real-life scenarios, humans seek out objects in the 3D world to fulfill their daily needs or intentions. This inspires us to introduce 3D intention grounding, a new task in 3D object detection employing RGB-D, based on human intention, such as “I want something to support my back.” Closely related, 3D visual grounding focuses on understanding human reference. To achieve detection based on human intention, it relies on humans to observe the scene, reason out the target that aligns with their intention (“pillow” in this case), and finally provide a reference to the AI system, such as “A pillow on the couch”. Instead, 3D intention grounding challenges AI agents to automatically observe, reason and detect the desired target solely based on human intention. To tackle this challenge, we introduce the new Intent3D dataset, consisting of 44,990 intention texts associated with 209 f ine-grained classes from 1,042 scenes of the ScanNet [Dai et al., 2017] dataset. We also establish several baselines based on different language-based 3D object detection models on our benchmark. Finally, we propose IntentNet, our unique approach, designed to tackle this intention-based detection problem. It focuses on three key aspects: intention understanding, reasoning to identify object candidates, and cascaded adaptive learning that leverages the intrinsic priority logic of different losses for multiple objective optimization.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chhipa2025,

title = {ASTrA: Adversarial Self-supervised Training with Adaptive-Attacks},

author = {Prakash Chandra Chhipa and Gautam Vashishtha and Settur Jithamanyu and Rajkumar Saini and Mubarak Shah and Marcus Liwicki},

url = {https://prakashchhipa.github.io/projects/ASTrA

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7338_ASTrA_Adversarial_Self_su.pdf.pdf},

year  = {2025},

date = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Existing self-supervised adversarial training (self-AT) methods rely on handcrafted adversarial attack strategies for PGD attacks, which fail to adapt to the evolving learning dynamics of the model and do not account for instancespecific characteristics of images. This results in sub-optimal adversarial robustness and limits the alignment between clean and adversarial data distributions. To address this, we propose ASTrA (Adversarial Self-supervised Training with Adaptive-Attacks), a novel framework introducing a learnable, self-supervised attack strategy network that autonomously discovers optimal attack parameters through exploration-exploitation in a single training episode. ASTrA leverages a reward mechanism based on contrastive loss, optimized with REINFORCE, enabling adaptive attack strategies without labeled data or additional hyperparameters. We further introduce a mixed contrastive objective to align the distribution of clean and adversarial examples in representation space. ASTrA achieves state-of-the-art results on CIFAR10, CIFAR100, and STL10 while integrating seamlessly as a plug-and-play module for other self-AT methods. ASTrAshows scalability to larger datasets, demonstrates strong semi-supervised performance, and is resilient to robust overfitting, backed by explainability analysis on optimal attack strategies. Project page for source code and other details at https://prakashchhipa.github.io/projects/ASTrA.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fioresi2025,

title = {ALBAR: Adversarial Learning approach to mitigate Biases in Action Recognition},

author = {Joseph Fioresi and Ishan Rajendrakumar Dave and Mubarak Shah},

url = {https://openreview.net/forum?id=9KiE3t6CsL&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/7609_ALBAR_Adversarial_Learnin.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {Bias in machine learning models can lead to unfair decision making, and while it has been well-studied in the image and text domains, it remains underexplored in action recognition. Action recognition models often suffer from background bias (i.e., inferring actions based on background cues) and foreground bias (i.e., relying on subject appearance), which can be detrimental to real-life applications such as autonomous vehicles or assisted living monitoring. While prior approaches have mainly focused on mitigating background bias using specialized augmentations, we thoroughly study both biases. We propose approachname, a novel adversarial training method that mitigates foreground and background biases without requiring specialized knowledge of the bias attributes. Our framework applies an adversarial cross-entropy loss to the sampled static clip (where all the frames are the same) and aims to make its class probabilities uniform using a proposed textit{entropy maximization} loss. Additionally, we introduce a textit{gradient penalty} loss for regularization against the debiasing process. We evaluate our method on established background and foreground bias protocols, setting a new state-of-the-art and strongly improving combined debiasing performance by over textbf{12%} on HMDB51. Furthermore, we identify an issue of background leakage in the existing UCF101 protocol for bias evaluation which provides a shortcut to predict actions and does not provide an accurate measure of the debiasing capability of a model. We address this issue by proposing more fine-grained segmentation boundaries for the actor, where our method also outperforms existing approaches.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Cui2025,

title = {AdaIR: Adaptive All-in-One Image Restoration via Frequency Mining and Modulation},

author = {Yuning Cui and Syed Waqas Zamir and Salman Khan and Alois Knoll and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://openreview.net/forum?id=M5t0WvjfCg&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/8311_AdaIR_Adaptive_All_in_One.pdf.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {In the image acquisition process, various forms of degradation, including noise, blur, haze, and rain, are frequently introduced. These degradations typically arise from the inherent limitations of cameras or unfavorable ambient conditions. To recover clean images from their degraded versions, numerous specialized restoration methods have been developed, each targeting a specific type of degradation. Recently, all-in-one algorithms have garnered significant attention by addressing different types of degradations within a single model without requiring the prior information of the input degradation type. However, most methods purely operate in the spatial domain and do not delve into the distinct frequency variations inherent to different degradation types. To address this gap, we propose an adaptive all-in-one image restoration network based on frequency mining and modulation. Our approach is motivated by the observation that different degradation types impact the image content on different frequency subbands, thereby requiring different treatments for each restoration task. Specifically, we first mine low- and high-frequency information from the input features, guided by the adaptively decoupled spectra of the degraded image. The extracted features are then modulated by a bidirectional operator to facilitate interactions between different frequency components. Finally, the modulated features are merged into the original input for a progressively guided restoration. With this approach, the model achieves adaptive reconstruction by accentuating the informative frequency subbands according to different input degradations. Extensive experiments demonstrate that the proposed method, named AdaIR, achieves state-of-the-art performance on different image restoration tasks, including image denoising, dehazing, deraining, motion deblurring, and low-light image enhancement. Our code and models will be made publicly available.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2025,

title = {Exploring Local Memorization in Diffusion Models via Bright Ending Attention},

author = {Chen Chen and Daochang Liu and Mubarak Shah and Chang Xu},

url = {https://openreview.net/forum?id=p4cLtzk4oe&referrer=%5BAuthor%20Console%5D(%2Fgroup%3Fid%3DICLR.cc%2F2025%2FConference%2FAuthors%23your-submissions)

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/9283_Exploring_Local_Memorizat.pdf},

year  = {2025},

date = {2025-04-24},

urldate = {2025-04-24},

booktitle = {Thirteenth International Conference on Learning Representations (ICLR)},

abstract = {In this paper, we identify and leverage a novel `bright ending' (BE) anomaly in diffusion models prone to memorizing training images to address a new task: locating localized memorization regions within these models. BE refers to a distinct cross-attention pattern observed in text-to-image generations using diffusion models. Specifically, memorized image patches exhibit significantly greater attention to the end token during the final inference step compared to non-memorized patches. This attention map effectively highlights regions where the generated image replicates training data. Furthermore, driven by our observation that local memorization significantly underperforms in existing tasks of measuring, detecting, and mitigating memorization in diffusion models compared to global memorization, we propose a simple yet effective method to integrate BE and the results of the new localization task into these existing frameworks. This integration effectively improves their performances by narrowing the performance gap caused by local memorization. Our results not only demonstrate the successful execution of the new localization task but also establish new state-of-the-art performance across all existing tasks, underscoring the significance of the BE phenomenon.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wu2024,

title = {PTQ4DiT: Post-training Quantization for Diffusion Transformers},

author = {Junyi Wu and Haoxuan Wang and Yuzhang Shang and Mubarak Shah and Yan Yan},

url = {https://nips.cc/virtual/2024/poster/95445

https://arxiv.org/pdf/2405.16005

https://github.com/adreamwu/PTQ4DiT},

year  = {2024},

date = {2024-12-13},

urldate = {2024-12-13},

publisher = {Thirty-Eighth Annual Conference on Neural Information Processing Systems (NeurIPS)},

abstract = {The recent introduction of Diffusion Transformers (DiTs) has demonstrated exceptional capabilities in image generation by using a different backbone architecture, departing from traditional U-Nets and embracing the scalable nature of transformers. Despite their advanced capabilities, the wide deployment of DiTs, particularly for real-time applications, is currently hampered by considerable computational demands at the inference stage. Post-training Quantization (PTQ) has emerged as a fast and data-efficient solution that can significantly reduce computation and memory footprint by using low-bit weights and activations. However, its applicability to DiTs has not yet been explored and faces non-trivial difficulties due to the unique design of DiTs. In this paper, we propose PTQ4DiT, a specifically designed PTQ method for DiTs. We discover two primary quantization challenges inherent in DiTs, notably the presence of salient channels with extreme magnitudes and the temporal variability in distributions of salient activation over multiple timesteps. To tackle these challenges, we propose Channel-wise Salience Balancing (CSB) and Spearmen's 

ρ

-guided Salience Calibration (SSC). CSB leverages the complementarity property of channel magnitudes to redistribute the extremes, alleviating quantization errors for both activations and weights. SSC extends this approach by dynamically adjusting the balanced salience to capture the temporal variations in activation. Additionally, to eliminate extra computational costs caused by PTQ4DiT during inference, we design an offline re-parameterization strategy for DiTs. Experiments demonstrate that our PTQ4DiT successfully quantizes DiTs to 8-bit precision (W8A8) while preserving comparable generation ability and further enables effective quantization to 4-bit weight precision (W4A8) for the first time.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kumar2024,

title = {Sparse Points to Dense Clouds: Enhancing 3D Detection with Limited LiDAR Data},

author = {Aakash Kumar and Chen Chen and Ajmal Mian and Neils Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2404.06715v1.pdf

https://aakashjuseja-aj.github.io/Sparse_to_Dense/},

year  = {2024},

date = {2024-10-14},

publisher = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

abstract = {3D detection is a critical task that enables machines to identify and locate objects in three-dimensional space. It has a broad range of applications in several fields, including autonomous driving, robotics and augmented reality. Monocular 3D detection is attractive as it requires only a single camera, however, it lacks the accuracy and robustness required for real world applications. High resolution LiDAR on the other hand, can be expensive and lead to interference problems in heavy traffic given their active transmissions. We propose a balanced approach that combines the advantages of monocular and point cloud-based 3D detection. Our method requires only a small number of 3D points, that can be obtained from a low-cost, low-resolution sensor. Specifically, we use only 512 points, which is just 1% of a full LiDAR frame in the KITTI dataset. Our method reconstructs a complete 3D point cloud from this limited 3D information combined with a single image. The reconstructed 3D point cloud and corresponding image can be used by any multi-modal off-the-shelf detector for 3D object detection. By using the proposed network architecture with an off-the-shelf multi-modal 3D detector, the accuracy of 3D detection improves by 20% compared to the state-of-the-art monocular detection methods and 6% to 9% compare to the baseline multi-modal methods on KITTI and JackRabbot datasets.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Möbius Transform for Mitigating Perspective Distortions in Representation Learning},

author = {Prakash Chandra Chhipa and Meenakshi Subhash Chippa and Kanjar De and Rajkumar

Saini and Marcus Liwicki and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/MPD_ECCV2024_CameraReady.pdf

https://prakashchhipa.github.io/projects/mpd

https://youtu.be/MKh9NE_XEMY

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/MPD_presentation_ECCV2024_final.pdf},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Perspective distortion (PD) causes unprecedented changes in shape, size, orientation, angles, and other spatial relationships of visual concepts in images. Precisely estimating camera intrinsic and extrinsic parameters is a challenging task that prevents synthesizing perspective distortion. Non-availability of dedicated training data poses a critical barrier to developing robust computer vision methods. Additionally, distortion correction methods make other computer vision tasks a multi-step approach and lack performance. In this work, we propose

mitigating perspective distortion (MPD) by employing a fine-grained parameter control on a specific family of Möbius transform to model real-world distortion without estimating camera intrinsic and extrinsic parameters and without the need for actual distorted data. Also, we present a dedicated perspectively distorted benchmark dataset, ImageNet-PD, to benchmark the robustness of deep learning models against this new dataset. The proposed method  outperforms existing benchmarks, ImageNet-E and ImageNet-X. Additionally, it significantly

improves performance on ImageNet-PD while consistently performing on standard data distribution. Notably, our method shows improved performance on three PD-affected real-world applications—crowd counting, fisheye image recognition, and person re-identification—and one PD-affected challenging CV task: object detection. The source code, dataset, and models are available on the project webpage at https://prakashchhipa.github.io/projects/mpd.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kang2024,

title = {SegVG: Transferring Object Bounding Box to Segmentation for Visual Grounding},

author = {Weitai Kang and Gaowen Liu and Mubarak Shah and Yan Yan},

url = {https://arxiv.org/pdf/2407.03200

},

doi = {https://doi.org/10.48550/arXiv.2407.03200},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Different from Object Detection, Visual Grounding deals with detecting a bounding box for each text-image pair. This one box for each text-image data provides sparse supervision signals. Although previous works achieve impressive results, their passive utilization of annotation, i.e. the sole use of the box annotation as regression ground truth, results in a suboptimal performance. In this paper, we present SegVG, a novel method transfers the box-level annotation as Segmentation signals to provide an additional pixel-level supervision for Visual Grounding. Specifically, we propose the Multi-layer Multi-task Encoder-Decoder as the target grounding stage, where we learn a regression query and multiple segmentation queries to ground the target by regression and segmentation of the box in each decoding layer, respectively. This approach allows us to iteratively exploit the annotation as signals for both box-level regression and pixel-level segmentation. Moreover, as the backbones are typically initialized by pretrained parameters learned from unimodal tasks and the queries for both regression and segmentation are static learnable embeddings, a domain discrepancy remains among these three types of features, which impairs subsequent target grounding. To mitigate this discrepancy, we introduce the Triple Alignment module, where the query, text, and vision tokens are triangularly updated to share the same space by triple attention mechanism. Extensive experiments on five widely used datasets validate our state-of-the-art (SOTA) performance.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024,

title = {FinePseudo: Improving Pseudo-Labelling through Temporal-Alignablity for Semi-Supervised Fine-Grained Action Recognition},

author = {Ishan Rajendrakumar Dave and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/finepsuedo_eccv24_dave.pdf

https://daveishan.github.io/finepsuedo-webpage/

https://youtu.be/bWOd8_JpjQs?si=WWRDhdg5ADWL0uwB},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Real-life applications of action recognition often require a

fine-grained understanding of subtle movements, e.g., in sports analytics, user interactions in AR/VR, and surgical videos. Although fine-grained actions are more costly to annotate, existing semi-supervised action recognition has mainly focused on coarse-grained action recognition. Since fine-grained actions are more challenging due to the absence of scene bias,  classifying these actions requires an understanding of action-phases. Hence, existing coarse-grained semi-supervised methods

do not work effectively. In this work, we for the first time thoroughly investigate semi-supervised fine-grained action recognition (FGAR). We observe that alignment distances like dynamic time warping (DTW) provide a suitable action-phase-aware measure for comparing fine-grained actions, a concept previously unexploited in FGAR. However, since regular DTW distance is pairwise and assumes strict alignment between pairs, it is not directly suitable for classifying fine-grained actions. To

utilize such alignment distances in a limited-label setting, we propose an Alignability-Verification-based Metric learning technique to effectively discriminate between fine-grained action pairs. Our learnable alignability score provides a better phase-aware measure, which we use to refine the pseudo-labels of the primary video encoder. Our collaborative pseudolabeling-based framework ‘FinePseudo’ significantly outperforms prior methods on four fine-grained action recognition datasets: Diving48,  FineGym99, FineGym288, and FineDiving, and shows improvement on existing coarse-grained datasets: Kinetics400 and Something-SomethingV2. We also demonstrate the  robustness of our collaborative pseudo-labeling in handling novel unlabeled classes in open-world semi-supervised setups.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Swetha2024,

title = {X-Former: Unifying Contrastive and Reconstruction Learning for MLLMs},

author = {Sirnam Swetha and Jinyu Yang and Tal Neiman and Mamshad Nayeem Rizve and Son Tran and Benjamin Yao and Trishul Chilimbi and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2407.13851v1.pdf

https://arxiv.org/abs/2407.13851

https://swetha5.github.io/XFormer/},

doi = {https://doi.org/10.48550/arXiv.2407.13851},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Recent advancements in Multimodal Large Language Models (MLLMs) have revolutionized the field of vision-language understanding by integrating visual perception capabilities into Large Language Models (LLMs). The prevailing trend in this field involves the utilization of a vision encoder derived from vision-language contrastive learning (CL), showing expertise in capturing overall representations while facing difficulties in capturing detailed local patterns. In this work, we focus on enhancing the visual representations for MLLMs by combining high-frequency and detailed visual representations, obtained through masked image modeling (MIM), with semantically-enriched low-frequency representations captured by CL. To achieve this goal, we introduce X-Former which is a lightweight transformer module designed to exploit the complementary strengths of CL and MIM through an innovative interaction mechanism. Specifically, X-Former first bootstraps vision-language representation learning and multimodal-to-multimodal generative learning from two frozen vision encoders, i.e., CLIP-ViT (CL-based) and MAE-ViT (MIM-based). It further bootstraps vision-to-language generative learning from a frozen LLM to ensure visual features from X-Former can be interpreted by the LLM. To demonstrate the effectiveness of our approach, we assess its performance on tasks demanding detailed visual understanding. Extensive evaluations indicate that X-Former excels in visual reasoning tasks involving both structural and semantic categories in the GQA dataset. Assessment on fine-grained visual perception benchmark further confirms its superior capabilities in visual understanding.

},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2024,

title = {Open Vocabulary Multi-Label Video Classification},

author = {Rohit Gupta and Mamshad Nayeem Rizve and Ashish Tawari and Jayakrishnan Unnikrishnan and Son Tran and Mubarak Shah and benjamin Yao and Trishul Chilimbi},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_CameraReady-2.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/OVMLVidCLS_ECCV_2024_Supplementary.pdf

https://arxiv.org/html/2407.09073v1#S1},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

abstract = {Pre-trained vision-language models (VLMs) have enabled significant progress in open vocabulary computer vision tasks such as image classification, object detection and image segmentation. Some recent works have focused on extending VLMs to open vocabulary single label action classification in videos. However, previous methods fall short in holistic video understanding which requires the ability to simultaneously recognize multiple actions and entities e.g., objects in the video in an open vocabulary setting. We formulate this problem as open vocabulary multi-label video classification and propose a method to adapt a pre-trained VLM such as CLIP to solve this task. We leverage large language models (LLMs) to provide semantic guidance to the VLM about class labels to improve its open vocabulary performance with two key contributions. First, we propose an end-to-end trainable architecture that learns to prompt an LLM to generate soft attributes for the CLIP text-encoder to enable it to recognize novel classes. Second, we integrate a temporal modeling module into CLIP’s vision encoder to effectively model the spatio-temporal dynamics of video concepts as well as propose a novel regularized finetuning technique to ensure strong open vocabulary classification performance in the video domain. Our extensive experimentation showcases the efficacy of our approach on multiple benchmark datasets.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024b,

title = {Sync from the Sea: Retrieving Alignable Videos from Large-Scale Datasets},

author = {Ishan Rajendrakumar Dave and Fabian Caba Heilbron and Mubarak Shah and Simon Jenni},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/avr_eccv24_dave.pdf

https://daveishan.github.io/avr-webpage/

https://youtu.be/6euQwz7XdQk?si=v12dQH4e7UrTUrIU},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024, Oral (Top 3%)},

abstract = {Temporal video alignment aims to synchronize the key events

like object interactions or action phase transitions in two videos. Such methods could benefit various video editing, processing, and understanding tasks. However, existing approaches operate under the restrictive assumption that a suitable video pair for alignment is given, significantly limiting their broader applicability. To address this, we re-pose temporal alignment as a search problem and introduce the task of Alignable Video Retrieval (AVR). Given a query video, our approach can identify well-alignable videos from a large collection of clips and temporally synchronize them to the query. To achieve this, we make three key contributions: 1) we introduce DRAQ, a video alignability indicator to identify and re-rank the best alignable video from a set of candidates; 2) we propose an effective and generalizable frame-level video feature design to improve the alignment performance of several off-the-shelf feature representations, and 3) we propose a novel benchmark and evaluation protocol for AVR using cycle-consistency metrics. Our experiments on 3 datasets, including large-scale Kinetics700, demonstrate the effectiveness of our approach in identifying alignable video pairs from diverse datasets. },

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yang2024,

title = {Regulating Model Reliance on Non-Robust Features by Smoothing Input Marginal Density},

author = {Paiyu Yang and Naveed Akhtar and Mubarak Shah and Ajmal Mian},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/main_paper-1.pdf

https://arxiv.org/pdf/2407.04370

https://github.com/ypeiyu/input_density_reg},

year  = {2024},

date = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pillai2024,

title = {GAReT: Cross-view Video Geolocalization with Adapters and Auto-Regressive Transformers},

author = {Manu S Pillai and Mamshad Nayeem Rizve and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07875-supp.pdf

https://arxiv.org/abs/2408.02840

https://github.com/manupillai308/GAReT},

doi = {https://doi.org/10.48550/arXiv.2408.02840},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Cross-view video geo-localization (CVGL) aims to derive GPS trajectories from street-view videos by aligning them with aerial-view images. Despite their promising performance, current CVGL methods face significant challenges. These methods use camera and odometry data, typically absent in real-world scenarios. They utilize multiple adjacent frames and various encoders for feature extraction, resulting in high computational costs. Moreover, these approaches independently predict each street-view frame's location, resulting in temporally inconsistent GPS trajectories. To address these challenges, in this work, we propose GAReT, a fully transformer-based method for CVGL that does not require camera and odometry data. We introduce GeoAdapter, a transformer-adapter module designed to efficiently aggregate image-level representations and adapt them for video inputs. Specifically, we train a transformer encoder on video frames and aerial images, then freeze the encoder to optimize the GeoAdapter module to obtain video-level representation. To address temporally inconsistent trajectories, we introduce TransRetriever, an encoder-decoder transformer model that predicts GPS locations of street-view frames by encoding top-k nearest neighbor predictions per frame and auto-regressively decoding the best neighbor based on the previous frame's predictions. Our method's effectiveness is validated through extensive experiments, demonstrating state-of-the-art performance on benchmark datasets.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kulkarni2024,

title = {CityGuessr: City-Level Video Geo-Localization on a Global Scale},

author = {Parth Parag Kulkarni and Guarav Kumar Nayak and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CityGuessr.pdf

https://parthpk.github.io/cityguessr-webpage/

},

year  = {2024},

date = {2024-09-29},

urldate = {2024-09-29},

publisher = {The 18th European Conference on Computer Vision ECCV 2024},

abstract = {Video geolocalization is a crucial problem in current times. Given just a video, ascertaining where it was captured from can have a plethora of advantages. The problem of worldwide geolocalization has been tackled before, but only using the image modality. Its video counterpart remains relatively unexplored. Meanwhile, video geolocalization has also garnered some attention in the recent past, but the existing methods are all restricted to specific regions. This motivates us to explore the problem of video geolocalization at a global scale. Hence, we propose a novel problem of worldwide video geolocalization with the objective of hierarchically predicting the correct city, state/province, country, and continent, given a video. However, no large scale video datasets that have extensive worldwide coverage exist, to train models for solving this problem. To this end, we introduce a new dataset, “CityGuessr68k” comprising of 68,269 videos from 166 cities all over the world. We also propose a novel baseline approach to this problem, by designing a transformerbased architecture comprising of an elegant “Self-Cross Attention” module for incorporating scenes as well as a “TextLabel Alignment” strategy for distilling knowledge from textlabels in feature space. To further enhance our location prediction, we also utilize soft-scene labels. Finally we demonstrate the performance of our method on our new dataset as well as Mapillary(MSLS) [38]. },

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2024,

title = {VidLA: Video-Language Alignment at Scale},

author = {Mamshad Nayeem Rizve and Fan Fei and Jayakrishnan Unnikrishnan and Son Tran and Benjamin Z. Yao and Belinda Zeng and Mubarak Shah and Trishul Chilimbi},

url = {https://arxiv.org/abs/2403.14870},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {In this paper, we propose VidLA, an approach for video-language alignment at scale. There are two major limitations of previous video-language alignment approaches. First, they do not capture both short-range and long-range temporal dependencies and typically employ complex hierarchical deep network architectures that are hard to integrate with existing pretrained image-text foundation models. To effectively address this limitation, we instead keep the network architecture simple and use a set of data tokens that operate at different temporal resolutions in a hierarchical manner, accounting for the temporally hierarchical nature of videos. By employing a simple two-tower architecture, we are able to initialize our video-language model with pretrained image-text foundation models, thereby boosting the final performance. Second, existing video-language alignment works struggle due to the lack of semantically aligned large-scale training data. To overcome it, we leverage recent LLMs to curate the largest video-language dataset to date with better visual grounding. Furthermore, unlike existing video-text datasets which only contain short clips, our dataset is enriched with video clips of varying durations to aid our temporally hierarchical data tokens in extracting better representations at varying temporal scales. Overall, empirical results show that our proposed approach surpasses state-of-the-art methods on multiple retrieval benchmarks, especially on longer videos, and performs competitively on classification benchmarks.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ristea2024,

title = {Self-Distilled Masked Auto-Encoders are Efficient Video Anomaly Detectors},

author = {Nicolae Catalin Ristea and Florinel Alin Croitoru and Radu Tudor Ionescu and Marius Popescu and Fahad Khan and Mubarak Shah},

url = {https://arxiv.org/abs/2306.12041

https://github.com/ristea/aed-mae},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {We propose an efficient abnormal event detection model based on a lightweight masked auto-encoder (AE) applied at the video frame level. The novelty of the proposed model is threefold. First, we introduce an approach to weight tokens based on motion gradients, thus shifting the focus from the static background scene to the foreground objects. Second, we integrate a teacher decoder and a student decoder into our architecture, leveraging the discrepancy between the outputs given by the two decoders to improve anomaly detection. Third, we generate synthetic abnormal events to augment the training videos, and task the masked AE model to jointly reconstruct the original frames (without anomalies) and the corresponding pixel-level anomaly maps. Our design leads to an efficient and effective model, as demonstrated by the extensive experiments carried out on four benchmarks: Avenue, ShanghaiTech, UBnormal and UCSD Ped2. The empirical results show that our model achieves an excellent trade-off between speed and accuracy, obtaining competitive AUC scores, while processing 1655 FPS. Hence, our model is between 8 and 70 times faster than competing methods. We also conduct an ablation study to justify our design. },

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Thawakar2024,

title = {Composed Video Retrieval via Enriched Context and Discriminative Embeddings},

author = {Omkar Chakradhar Thawakar and Muzammal Naseer and Rao Muhammad Anwer and Salman Khan and Michael Felsberg and Mubarak Shah and Fahad Khan },

url = {https://arxiv.org/abs/2403.16997

https://github.com/OmkarThawakar/composed-video-retrieval},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

abstract = {Composed video retrieval (CoVR) is a challenging problem in computer vision which has recently highlighted the integration of modification text with visual queries for more sophisticated video search in large databases. Existing works predominantly rely on visual queries combined with modification text to distinguish relevant videos. However, such a strategy struggles to fully preserve the rich query-specific context in retrieved target videos and only represents the target video using visual embedding. We introduce a novel CoVR framework that leverages detailed language descriptions to explicitly encode query-specific contextual information and learns discriminative embeddings of vision only, text only and vision-text for better alignment to accurately retrieve matched target videos. Our proposed framework can be flexibly employed for both composed video (CoVR) and image (CoIR) retrieval tasks. Experiments on three datasets show that our approach obtains state-of-the-art performance for both CovR and zero-shot CoIR tasks, achieving gains as high as around 7% in terms of recall@K=1 score.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dutta2024,

title = {Multiview Aerial Visual RECognition (MAVREC) Dataset: Can Multi-view Improve Aerial Visual Perception?},

author = {Aritra Dutta and Srijan Das and Jacob Nielsen and Rajatsubhra Chakraborty and Mubarak Shah},

url = {https://arxiv.org/abs/2312.04548

https://mavrec.github.io/},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Despite the commercial abundance of UAVs, aerial data acquisition remains challenging, and the existing Asia and North America-centric open-source UAV datasets are small-scale or low-resolution and lack diversity in scene contextuality. Additionally, the color content of the scenes, solar-zenith angle, and population density of different geographies influence the data diversity. These two factors conjointly render suboptimal aerial-visual perception of the deep neural network (DNN) models trained primarily on the ground-view data, including the open-world foundational models.



To pave the way for a transformative era of aerial detection, we present Multiview Aerial Visual RECognition or MAVREC, a video dataset where we record synchronized scenes from different perspectives -- ground camera and drone-mounted camera. MAVREC consists of around 2.5 hours of industry-standard 2.7K resolution video sequences, more than 0.5 million frames, and 1.1 million annotated bounding boxes. This makes MAVREC the largest ground and aerial-view dataset, and the fourth largest among all drone-based datasets across all modalities and tasks. Through our extensive benchmarking on MAVREC, we recognize that augmenting object detectors with ground-view images from the corresponding geographical location is a superior pre-training strategy for aerial detection. Building on this strategy, we benchmark MAVREC with a curriculum-based semi-supervised object detection approach that leverages labeled (ground and aerial) and unlabeled (only aerial) images to enhance the aerial detection.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2024c,

title = {CodaMal: Contrastive Domain Adaptation for Malaria Detection in Low-Cost Microscopes },

author = {Ishan Rajendrakumar Dave and Tristan de Blegiers and Chen Chen and Mubarak Shah},

url = {https://arxiv.org/pdf/2402.10478

https://daveishan.github.io/codamal-webpage/},

year  = {2024},

date = {2024-02-16},

publisher = {31st IEEE International Conference on Image Processing (ICIP) (Oral)},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {No More Shortcuts: Realizing the Potential of Temporal Self-Supervision},

author = {Ishan Rajendrakumar Dave and Simon Jenni and Mubarak Shah},

url = {https://arxiv.org/pdf/2312.13008

https://daveishan.github.io/nms-webpage/

https://youtu.be/5MBnxmMBQh0},

year  = {2023},

date = {2023-12-20},

publisher = {AAAI Conference on Artificial Intelligence (AAAI), Main Technical Track},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}