Publications

@conference{Luo2026,

title = {Dictionary-Aligned Concept Control for Safeguarding Multimodal LLMs},

author = {Jinqi Luo and Jinyu Yang and Tal Neiman and Lei Fan and Bing Yina and Son Tran and Mubarak Shah and Rene Vidal},

url = {https://arxiv.org/abs/2604.08846},

year  = {2026},

date = {2026-04-10},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {Multimodal Large Language Models (MLLMs) have been shown to be vulnerable to malicious queries that can elicit unsafe responses. Recent work uses prompt engineering, response classification, or finetuning to improve MLLM safety. Nevertheless, such approaches are often ineffective against evolving malicious patterns, may require rerunning the query, or demand heavy computational resources. Steering the activations of a frozen model at inference time has recently emerged as a flexible and effective solution. However, existing steering methods for MLLMs typically handle only a narrow set of safety-related concepts or struggle to adjust specific concepts without affecting others. To address these challenges, we introduce Dictionary-Aligned Concept Control (DACO), a framework that utilizes a curated concept dictionary and a Sparse Autoencoder (SAE) to provide granular control over MLLM activations. First, we curate a dictionary of 15,000 multimodal concepts by retrieving over 400,000 caption-image stimuli and summarizing their activations into concept directions. We name the dataset DACO-400K. Second, we show that the curated dictionary can be used to intervene activations via sparse coding. Third, we propose a new steering approach that uses our dictionary to initialize the training of an SAE and automatically annotate the semantics of the SAE atoms for safeguarding MLLMs. Experiments on multiple MLLMs (e.g., QwenVL, LLaVA, InternVL) across safety benchmarks (e.g., MM-SafetyBench, JailBreakV) show that DACO significantly improves MLLM safety while maintaining general-purpose capabilities.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Kulkarni2026,

title = {VidTAG: Temporally Aligned Video to GPS Geolocalization with Denoising Sequence Prediction at a Global Scale},

author = {Parth Parag Kulkarni and Rohit Gupta and Prakash Chandra Chhipa and Mubarak Shah},

url = {https://arxiv.org/abs/2604.12159},

year  = {2026},

date = {2026-04-10},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {The task of video geolocalization aims to determine the precise GPS coordinates of a video’s origin and map its trajectory; with applications in forensics, social media, and exploration. Existing classification-based approaches operate at a coarse city-level granularity and fail to capture fine-grained details, while image retrieval methods are impractical on a global scale due to the need for extensive image galleries which are infeasible to compile. Comparatively, constructing a gallery of GPS coordinates is straightforward and inexpensive. We propose VidTAG, a dual-encoder framework that performs frameto-GPS retrieval using both self-supervised and languagealigned features. To address temporal inconsistencies in video predictions, we introduce the TempGeo module, which aligns frame embeddings, and the GeoRefiner module, an encoder-decoder architecture that refines GPS features using the aligned frame embeddings. Evaluations on Mapillary (MSLS) and GAMa datasets demonstrate our model’s ability to generate temporally consistent trajectories and outperform baselines, achieving a 20% improvement at the 1 km threshold over GeoCLIP. We also beat current Stateof-the-Art by 25% on global coarse grained video geolocalization (CityGuessr68k). Our approach enables finegrained video geolocalization and lays a strong foundation for future research. More details on the project webpage: parthpk.github.io/vidtag webpage.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chan-Santiago2026,

title = {Learnability-Guided Diffusion for Dataset Distillation},

author = {Jeffrey A. Chan-Santiago and Mubarak Shah},

url = {https://arxiv.org/abs/2604.00519},

year  = {2026},

date = {2026-04-01},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {Training machine learning models on massive datasets is expensive and time-consuming. Dataset distillation addresses this by creating a small synthetic dataset that achieves the same performance as the full dataset. Recent methods use diffusion models to generate distilled data, either by promoting diversity or matching training gradients. However, existing approaches produce redundant training signals, where samples convey overlapping information. Empirically, disjoint subsets of distilled datasets capture 80-90% overlapping signals. This redundancy stems from optimizing visual diversity or average training dynamics without accounting for similarity across samples, leading to datasets where multiple samples share similar information rather than complementary knowledge. We propose learnability-driven dataset distillation, which constructs synthetic datasets incrementally through successive stages. Starting from a small set, we train a model and generate new samples guided by learnability scores that identify what the current model can learn from, creating an adaptive curriculum. We introduce Learnability-Guided Diffusion (LGD), which balances training utility for the current model with validity under a reference model to generate curriculum-aligned samples. Our approach reduces redundancy by 39.1%, promotes specialization across training stages, and achieves state-of-the-art results on ImageNet-1K (60.1%), ImageNette (87.2%), and ImageWoof (72.9%). Our code is available on our project page this https URL.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Swetha2026,

title = {VRR-QA: Visual Relational Reasoning in Videos Beyond Explicit Cues},

author = {Sirnam Swetha and Rohit Gupta and Parth Parag Kulkarni and David G Shatwell and Jeffrey A Chan Santiago and Nyle Siddiqui and Joseph Fioresi and Mubarak Shah},

url = {https://arxiv.org/abs/2506.21742},

year  = {2026},

date = {2026-03-29},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {Video Question Answering (VideoQA) has made significant strides by leveraging multimodal learning to align visual and textual modalities. However, current benchmarks overwhelmingly focus on questions answerable through explicit visual content - actions, objects, and events - directly observable within individual frames or short clips. To truly understand videos as humans do, models must go beyond what is directly shown, inferring hidden relationships and contextual cues that are only implied across frames. Current benchmarks fail to capture this essential aspect of video understanding. To address this gap, we introduce VRR-QA, a benchmark for Visual Relational Reasoning Beyond Explicit Cues. We curate our benchmark from creative and cinematic videos such as movies, that deliberately employ storytelling techniques which omit direct depictions of certain events or relations, requiring viewers to infer them. VRR-QA comprises 1K meticulously expert-annotated QA pairs drawn from 1K creative video clips covering 15 genres across 7 decades of content, from both live-action and animated titles. Our extensive evaluations on 14 leading VideoQA models reveals consistent and significant performance degradation, underscoring their reliance on surface-level visual cues and highlighting the difficulty of implicit reasoning. Even the best model substantially underperforms human baselines with only 64% accuracy. Performance variations across models further illustrate the complexity and diversity of the challenges presented by VRR-QA. By releasing both dataset and data collection framework, VRR-QA establishes a rigorous, diverse, and reproducible testbed for advancing VideoQA: this https URL.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Shatwell2026,

title = {TIGeR: A Unified Framework for Time, Images and Geo-location Retrieval},

author = {David G. Shatwell and Sirnam Swetha and Mubarak Shah},

url = {https://arxiv.org/abs/2603.24749},

year  = {2026},

date = {2026-03-28},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {Many real-world applications in digital forensics, urban monitoring, and environmental analysis require jointly reasoning about visual appearance, location, and time. Beyond standard geo-localization and time-of-capture prediction, these applications increasingly demand more complex capabilities, such as retrieving an image captured at the same location as a query image but at a specified target time. We formalize this problem as Geo-Time Aware Image Retrieval and propose TIGeR, a unified framework for Time, Images and Geo-location Retrieval. TIGeR supports flexible input configurations (single-modality and multi-modality queries) and uses the same representation to perform (i) geo-localization, (ii) time-of-capture prediction, and (iii) geo-time-aware retrieval. By preserving the underlying location identity despite large appearance changes, TIGeR enables retrieval based on where and when a scene was captured, rather than purely on visual similarity. To support this task, we design a multistage data curation pipeline and propose a new diverse dataset of 4.5M paired image-location-time triplets for training and 86k high-quality triplets for evaluation. Extensive experiments show that TIGeR consistently outperforms strong baselines and state-of-the-art methods by up to 16% on time-of-year, 8% time-of-day prediction, and 14% in geo-time aware retrieval recall, highlighting the benefits of unified geo-temporal modeling.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2026,

title = {Sky2Ground: A Benchmark for Site Modeling under Varying Altitude},

author = {Zengyan Wang and Sirshapan Mitra and Rajat Modi and Hui Xian Grace Lim and Yogesh Singh Rawat},

url = {https://arxiv.org/abs/2603.13740},

year  = {2026},

date = {2026-03-14},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {We introduce Sky2Ground, a three-view dataset designed for varying altitude camera localization, correspondence learning, and reconstruction. The dataset combines structured synthetic imagery with real, in-the-wild images, providing both controlled multi-view geometry and realistic scene noise. Each of the 51 sites contains thousands of satellite, aerial, and ground images spanning wide altitude ranges and nearly orthogonal viewing angles, enabling rigorous evaluation across global-to-local contexts. We benchmark state of the art pose estimation models, including MASt3R, DUSt3R, Map Anything, and VGGT, and observe that the use of satellite imagery often degrades performance, highlighting the challenges under large altitude variations. We also examine reconstruction methods, highlighting the challenges introduced by sparse geometric overlap, varying perspectives, and the use of real imagery, which often introduces noise and reduces rendering quality. To address some of these challenges, we propose SkyNet, a model which enhances cross-view consistency when incorporating satellite imagery with a curriculum-based training strategy to progressively incorporate more satellite views. SkyNet significantly strengthens multi-view alignment and outperforms existing methods by 9.6% on RRA@5 and 18.1% on RTA@5 in terms of absolute performance. Sky2Ground and SkyNet together establish a comprehensive testbed and baseline for advancing large-scale, multi-altitude 3D perception and generalizable camera localization. Code and models will be released publicly for future this http URL page: this https URL},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Azad2026,

title = {StreamReady: Learning *What* to Answer and *When* in Long Streaming Videos},

author = {Shehreen Azad and Vibhav Vineet and Yogesh Singh Rawat},

url = {https://arxiv.org/abs/2603.08620

https://www.youtube.com/watch?v=WstC36siN-0},

year  = {2026},

date = {2026-03-09},

publisher = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2026)},

abstract = {Streaming video understanding often involves time-sensitive scenarios where models need to answer exactly when the supporting visual evidence appears: answering before the evidence reflects speculation, answering after it has passed reduces real-time utility. To capture this behavior, we introduce a readiness-aware formulation of streaming video understanding with the Answer Readiness Score (ARS), a timing-aware objective with asymmetric early and late penalties. When combined with correctness, ARS defines an effective accuracy that measures not just whether a model is right, but whether it answers at the appropriate moment. Building on this formulation, we introduce StreamReady, a framework to unify temporal reasoning with on-time answering through a lightweight readiness mechanism that decides if sufficient evidence has been observed before responding. To evaluate this capability, we further introduce ProReady-QA, a benchmark with annotated answer evidence windows and proactive multi-turn questions across local and global contexts. StreamReady achieves superior performance on ProReady-QA, and consistently outperforms prior methods across eight additional streaming and offline long-video benchmarks, demonstrating robust and broadly generalizable video understanding capability.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liang2025,

title = {DIFFER: Disentangling Identity Features via Semantic Cues for Clothes-Changing Person Re-ID},

author = {Xin Liang and Yogesh Singh Rawat},

url = {https://cvpr.thecvf.com/virtual/2025/poster/33555

https://openreview.net/pdf?id=tI3ZlbEhOM

https://openreview.net/attachment?id=tI3ZlbEhOM&name=supplementary_material},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {In this work, we focus on clothes-changing person re-identification (CC-ReID), which aims to recognize individuals under different clothing scenarios. Current CC-ReID approaches either concentrate on modeling body shape using additional modalities including silhouette, pose, and body mesh, potentially causing the model to overlook other critical biometric traits such as gender, age, and style, or they incorporate supervision through additional labels that the model tries to disregard or emphasize, such as clothing or personal attributes. However, these annotations are discrete in nature and do not capture comprehensive descriptions.In this work, we propose DIFFER: Disentangle Identity Features From Entangled Representations, a novel adversarial learning method that leverages textual descriptions to disentangle identity features. Recognizing that image features inherently mix inseparable information, DIFFER introduces NBDetach, a mechanism that utilizes the separable nature of text descriptions as disentanglement supervision to partition the feature space into distinct subspaces, enabling the effective separation of identity-related features from non-biometric features through gradient reversal. We evaluate DIFFER on 4 different benchmark datasets (LTCC, PRCC, CelebreID-Light, and CCVID) to demonstrate its effectiveness and provide state-of-the-art performance across all the benchmarks. DIFFER consistently outperforms the baseline method, with improvements in top-1 accuracy of 3.6% on LTCC, 3.4% on PRCC, 2.5% on CelebReID-Light, and 1% on CCVID. The code will be made publicly available.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Azad2025,

title = {HierarQ: Task-Aware Hierarchical Q-Former for Enhanced Video Understanding},

author = {Shehreen Azad and Vibhav Vineet and Yogesh Singh Rawat},

url = {https://arxiv.org/abs/2503.08585

},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Despite advancements in multimodal large language models (MLLMs), current approaches struggle in medium-to-long video understanding due to frame and context length limitations. As a result, these models often depend on frame sampling, which risks missing key information over time and lacks task-specific relevance. To address these challenges, we introduce HierarQ, a task-aware hierarchical Q-Former based framework that sequentially processes frames to bypass the need for frame sampling, while avoiding LLM's context length limitations. We introduce a lightweight two-stream language-guided feature modulator to incorporate task awareness in video understanding, with the entity stream capturing frame-level object information within a short context and the scene stream identifying their broader interactions over longer period of time. Each stream is supported by dedicated memory banks which enables our proposed Hierachical Querying transformer (HierarQ) to effectively capture short and long-term context. Extensive evaluations on 10 video benchmarks across video understanding, question answering, and captioning tasks demonstrate HierarQ's state-of-the-art performance across most datasets, proving its robustness and efficiency for comprehensive video analysis.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Garg2025,

title = {STPro: Spatial and Temporal Progressive Learning for Weakly Supervised Spatio-Temporal Grounding},

author = {Aaryan Garg and Akash Kumar and Yogesh Singh Rawat},

url = {https://aaryangrg.github.io/research/stpro},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Vayani2025,

title = {All Languages Matter: Evaluating LMMs on Culturally Diverse 100 Languages},

author = {Ashmal Vayani and Dinura Dissanayake and Hasindri Watawana and Noor Ahsan and Nevasini Sasikumar and Omkar Thawakar and Henok Biadglign Ademtew and Yahya Hmaiti and Amandeep Kumar and Kartik Kuckreja and Mykola Maslych and Wafa Al Ghallabi and Mihail Mihaylov and Chao Qin and Abdelrahman M Shaker and Mike Zhang and Mahardika Krisna Ihsani and Amiel Esplana and Monil Gokani and Shachar Mirkin and Harsh Singh and Ashay Srivastava and Endre Hamerlik and Fathinah Asma Izzati and Fadillah Adamsyah Maani and Sebastian Cavada and Jenny Chim and Rohit Gupta and Sanjay Manjunath and Kamila Zhumakhanova and Feno Heriniaina Rabevohitra and Azril Amirudin and Muhammad Ridzuan and Daniya Kareem and Ketan More and Kunyang Li and Pramesh Shakya and Muhammad Saad and Amirpouya Ghasemaghaei and Amirbek Djanibekov and Dilshod Azizov and Branislava Jankovic and Naman Bhatia and Alvaro Cabrera and Johan Obando-Ceron and Olympiah Otieno and Fabian Farestam and Muztoba Rabbani and Sanoojan Baliah and Santosh Sanjeev and Abduragim Shtanchaev and Maheen Fatima and Thao Nguyen and Amrin Kareem and Toluwani Aremu and Nathan Xavier and Amit Bhatkal and Hawau Toyin and Aman Chadha and Hisham Cholakkal and Rao Muhammad Anwer and Michael Felsberg and Jorma Laaksonen and Thamar Solorio and Monojit Choudhury and Ivan Laptev and Mubarak Shah and Salman Khan and Fahad Khan},

url = {https://arxiv.org/abs/2411.16508

https://mbzuai-oryx.github.io/ALM-Bench/

https://huggingface.co/datasets/MBZUAI/ALM-Bench

https://github.com/mbzuai-oryx/ALM-Bench

https://drive.google.com/file/d/1B2yynH-o1z1UvMCl32XoY3pVIkAxgN0P/view?usp=sharing

https://docs.google.com/presentation/d/10lUrfoC-R1lgl0Dx0EYpx1ID3AqLGnRO/edit?usp=sharing&ouid=103839181464334047856&rtpof=true&sd=true

https://www.youtube.com/watch?v=fvcX4yi5AiY&list=PLtIMlmzwbtty7ie1quj1fq2QoTRK9eWHX&ab_channel=AshmalVayani},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Existing Large Multimodal Models (LMMs) generally focus on only a few regions and languages. As LMMs continue to improve, it is increasingly important to ensure they understand cultural contexts, respect local sensitivities, and support low-resource languages, all while effectively integrating corresponding visual cues. In pursuit of culturally diverse global multimodal models, our proposed All Languages Matter Benchmark (ALM-bench) represents the largest and most comprehensive effort to date for evaluating LMMs across 100 languages. ALM-bench challenges existing models by testing their ability to understand and reason about culturally diverse images paired with text in various languages, including many low-resource languages traditionally underrepresented in LMM research. The benchmark offers a robust and nuanced evaluation framework featuring various question formats, including true/false, multiple choice, and open-ended questions, which are further divided into short and long-answer categories. ALM-bench design ensures a comprehensive assessment of a model's ability to handle varied levels of difficulty in visual and linguistic reasoning. To capture the rich tapestry of global cultures, ALM-bench carefully curates content from 13 distinct cultural aspects, ranging from traditions and rituals to famous personalities and celebrations. Through this, ALM-bench not only provides a rigorous testing ground for state-of-the-art open and closed-source LMMs but also highlights the importance of cultural and linguistic inclusivity, encouraging the development of models that can serve diverse global populations effectively. Our benchmark is publicly available.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Huynh2025b,

title = {CoLLM:ALargeLanguage Model for Composed Image Retrieva},

author = {Chuong Huynh and Jinyu Yang and Ashish Tawari and Mubarak Shah and Son Tran and Raffay Hamid and Trishul Chilimbi and Abhinav Shrivastava},

url = {https://www.amazon.science/publications/collm-a-large-language-model-for-composed-image-retrieval

https://assets.amazon.science/55/e0/7ead8a494dc49fce25cb9947654c/collm-a-large-language-model-for-composed-image-retrieval.pdf

https://collm-cvpr25.github.io/},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Composed Image Retrieval (CIR) is a complex task that aims to retrieve images based on a multimodal query. Typical training data consists of triplets containing a reference image, a textual description of desired modifications, and the target image, which are expensive and time-consuming to acquire. The scarcity of CIR datasets has led to zero-shot approaches utilizing synthetic triplets or leveraging vision-language models (VLMs) with ubiquitous web-crawled image-caption pairs. However, these methods have significant limitations: synthetic triplets suffer from limited scale, lack of diversity, and unnatural modification text, while image-caption pairs hinder joint embedding learning of the multimodal query due to the absence of triplet data. Moreover, existing approaches struggle with complex and nuanced modification texts that demand sophisticated fusion and understanding of vision and language modalities. We present CoLLM, a one-stop framework that effectively addresses these limitations. Our approach generates triplets on-the-fly from image-caption pairs, enabling supervised training without manual annotation. We leverage Large Language Models (LLMs) to generate joint embeddings of reference images and modification texts, facilitating deeper multimodal fusion. Additionally, we introduce Multi-Text CIR (MTCIR), a large-scale dataset comprising 3.4M samples, and refine existing CIR benchmarks (CIRR and Fashion-IQ) to enhance evaluation reliability. Experimental results demonstrate that CoLLM achieves state-of0the-art performance across multiple CIR benchmarks and settings. MTCIR yields competitive results, with up to 15% performance improvement. Our refined benchmarks provide more reliable evaluation metrics for CIR models, contributing to the advancement of this important field. Project page is at collm-cvpr25.github.io.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Hu2025,

title = {M-LLMBasedVideo Frame Selection for Efficient Video Understanding},

author = {Kai Hu and Feng Gao and Xiaohan Nie and Peng Zhou and Son Tran and Tal Neiman and Lingyun Wang and Mubarak Shah and Raffay Hamid and Bing Yin and Trishul Chilimbi},

url = {https://www.amazon.science/publications/m-llm-based-video-frame-selection-for-efficient-video-understanding

https://assets.amazon.science/1a/4e/6794e1dd4fb2acb30eda428a5119/m-llm-based-video-frame-selection-for-efficient-video-understanding.pdf

},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Recent advances in Multi-Modal Large Language Models (M-LLMs) show promising results in video reasoning. Popular Multi-Modal Large Language Model (M-LLM) frameworks usually apply naive uniform sampling to reduce the number of video frames that are fed into an M-LLM, particularly for long context videos. However, it could lose crucial context in certain periods of a video, so that the downstream M-LLM may not have sufficient visual information to answer a question. To attack this pain point, we propose a light-weight M-LLM-based frame selection method that adaptively select frames that are more relevant to users’ queries. In order to train the proposed frame selector, we introduce two supervision signals (i) Spatial signal, where single frame importance score by prompting a M-LLM; (ii) Temporal signal, in which multiple frames selection by prompting Large Language Model (LLM) using the captions of all frame candidates. The selected frames are then digested by a frozen downstream video M-LLM for visual reasoning and question answering. Empirical results show that the proposed M-LLM video frame selector improves the performances various downstream video Large Language Model (video-LLM) across medium (ActivityNet, NExT-QA) and long (EgoSchema, LongVideoBench) context video question answering benchmarks.

},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Croitoru2025,

title = {Curriculum Direct Preference Optimization for Diffusion and Consistency Models},

author = {Florinel-Alin Croitoru and Vlad Hondru and Radu Tudor Ionescu and Nicu Sebe and Mubarak Shah},

url = {https://arxiv.org/abs/2405.13637

https://croitorualin.github.io/cl-dpo/

https://github.com/CroitoruAlin/Curriculum-DPO







},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Direct Preference Optimization (DPO) has been proposed as an effective and efficient alternative to reinforcement learning from human feedback (RLHF). In this paper, we propose a novel and enhanced version of DPO based on curriculum learning for text-to-image generation. Our method is divided into two training stages. First, a ranking of the examples generated for each prompt is obtained by employing a reward model. Then, increasingly difficult pairs of examples are sampled and provided to a text-to-image generative (diffusion or consistency) model. Generated samples that are far apart in the ranking are considered to form easy pairs, while those that are close in the ranking form hard pairs. In other words, we use the rank difference between samples as a measure of difficulty. The sampled pairs are split into batches according to their difficulty levels, which are gradually used to train the generative model. Our approach, Curriculum DPO, is compared against state-of-the-art fine-tuning approaches on nine benchmarks, outperforming the competing methods in terms of text alignment, aesthetics and human preference. Our code is available at this https URL.

},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2025,

title = {Generative Zero-Shot Composed Image Retrieval},

author = {Lan Wang and Wei Ao and Vishnu Boddeti and Sernam Lim},

url = {https://hal.cse.msu.edu/papers/cig-generative-zero-shot-composed-image-retrieval/

https://hal.cse.msu.edu/assets/pdfs/papers/2025-cvpr-cig-generative-zero-shot-composed-image-retrieval.pdf

https://hal.cse.msu.edu/assets/pdfs/papers/2025-cvpr-cig-generative-zero-shot-composed-image-retrieval-supp.pdf

https://lan-lw.github.io/CIG/},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Composed Image Retrieval (CIR) is a vision-language task utilizing queries comprising images and textual descriptions to achieve precise image retrieval. This task seeks to find images that are visually similar to a reference image while incorporating specific changes or features described textually (visual delta). CIR enables a more flexible and user-specific retrieval by bridging visual data with verbal instructions. This paper introduces a novel generative method that augments Composed Image Retrieval by Composed Image Generation (CIG) to provide pseudotarget images. CIG utilizes a textual inversion network to map reference images into semantic word space, which generates pseudo-target images in combination with textual descriptions. These images serve as additional visual information, significantly improving the accuracy and relevance of retrieved images when integrated into existing retrieval frameworks. Experiments conducted across multiple CIR datasets and several baseline methods demonstrate improvements in retrieval performance, which shows the potential of our approach as an effective add-on for existing composed image retrieval.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{xinLiu2025,

title = {Unveiling the Ignorance of MLLMs: Seeing Clearly, Answering Incorrectly},

author = {Ye xin Liu and Zhengyang Liang and Yueze Wang and Xianfeng Wu and Feilong Tang and Muyang He and Jian Li and Zheng Liu and Harry Yang and Ser-Nam Lim and Bo Zhao},

url = {https://arxiv.org/html/2406.10638v2

https://arxiv.org/pdf/2406.10638v2},

year  = {2025},

date = {2025-06-11},

abstract = {Multimodal Large Language Models (MLLMs) have displayed remarkable performance in multi-modal tasks, particularly in visual comprehension. However, we reveal that MLLMs often generate incorrect answers even when they understand the visual content. To this end, we manually construct a benchmark with 12 categories and design evaluation metrics that assess the degree of error in MLLM responses even when the visual content is seemingly understood. Based on this benchmark, we test 15 leading MLLMs and analyze the distribution of attention maps and logits of some MLLMs. Our investigation identifies two primary issues: 1) most instruction tuning datasets predominantly feature questions that “directly” relate to the visual content, leading to a bias in MLLMs’ responses to other indirect questions, and 2) MLLMs’ attention to visual tokens is notably lower than to system and question tokens. We further observe that attention scores between questions and visual tokens as well as the model’s confidence in the answers are lower in response to misleading questions than to straightforward ones. To address the first challenge, we introduce a paired positive and negative data construction pipeline to diversify the dataset. For the second challenge, we propose to enhance the model’s focus on visual content during decoding by refining the text and visual prompt. For the text prompt, we propose a content guided refinement strategy that performs preliminary visual content analysis to generate structured information before answering the question. Additionally, we employ a visual attention refinement strategy that highlights question-relevant visual tokens to increase the model’s attention to visual content that aligns with the question. Extensive experiments demonstrate that these challenges can be significantly mitigated with our proposed dataset and techniques. The benchmark, training set, and code will be available.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ghosal2025,

title = {IMMUNELogo : Improving Safety Against Jailbreaks in Multi-modal LLMs via Inference-Time Alignment},

author = {Soumya Suvra Ghosal, Souradip Chakraborty, Vaibhav Singh, Tianrui Guan, Mengdi Wang, Ahmad Beirami, Furong Huang, Alvaro Velasquez, Dinesh Manocha, Amrit Singh Bedi},

url = {https://itsvaibhav01.github.io/immune-web/

https://arxiv.org/pdf/2411.18688

https://arxiv.org/abs/2411.18688

https://github.com/itsvaibhav01/Immune},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {With the widespread deployment of Multimodal Large Language Models (MLLMs) for visual-reasoning tasks, improving their safety has become crucial. Recent research indicates that despite training-time safety alignment, these models remain vulnerable to jailbreak attacks. In this work, we first highlight an important safety gap to describe that alignment achieved solely through safety training may be insufficient against jailbreak attacks. To address this vulnerability, we propose Immune, an inference-time defense framework that leverages a safe reward model through controlled decoding to defend against jailbreak attacks. Additionally, we provide a mathematical characterization of Immune, offering insights on why it improves safety against jailbreaks. Extensive evaluations on diverse jailbreak benchmarks using recent MLLMs reveal that Immune effectively enhances model safety while preserving the model's original capabilities. For instance, against text-based jailbreak attacks on LLaVA-1.6, Immune reduces the attack success rate by 57.82% and 16.78% compared to the base MLLM and state-of-the-art defense strategy, respectively.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2025b,

title = {Enhancing Privacy-Utility Trade-offs to Mitigate Memorization in Diffusion Models},

author = {Chen Chen and Daochang Liu and Mubarak Shah and Chang Xu},

url = {https://cvpr.thecvf.com/virtual/2025/poster/34842

https://chenchen-usyd.github.io/PRSS-Project-Page/},

year  = {2025},

date = {2025-06-11},

urldate = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Text-to-image diffusion models have demonstrated remarkable capabilities in creating images highly aligned with user prompts, yet their proclivity for memorizing training set images has sparked concerns about the originality of the generated images and privacy issues, potentially leading to legal complications for both model owners and users, particularly when the memorized images contain proprietary content. Although methods to mitigate these issues have been suggested, enhancing privacy often results in a significant decrease in the utility of the outputs, as indicated by text-alignment scores. To bridge the research gap, we introduce a novel method, PRSS, which refines the classifier-free guidance approach in diffusion models by integrating prompt re-anchoring (PR) to improve privacy and incorporating semantic prompt search (SS) to enhance utility. Extensive experiments across various privacy levels demonstrate that our approach consistently improves the privacy-utility trade-off, establishing a new state-of-the-art.



},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Deng2025,

title = {Motion-Grounded Video Reasoning: Understanding and Perceiving Motion at Pixel Level},

author = {Andong Deng and Tongjia Chen and Shoubin Yu and Taojiannan Yang and Lincoln Spencer and Yapeng Tian and Ajmal Mian and Mohit Bansal and Chen Chen},

url = {https://groundmore.github.io/

https://arxiv.org/abs/2411.09921

https://github.com/dengandong/GroundMoRe

https://huggingface.co/datasets/groundmore/GroundMoRe},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {In this paper, we introduce Motion-Grounded Video Reasoning, a new motion understanding task that requires generating spatiotemporal segmentation masks according to the input question, and hence needs implicit spatiotemporal reasoning and grounding. This task extends existing spatiotemporal grounding work that focuses on explicit action/motion recognition, to a more general format by enabling implicit motion reasoning via questions. To facilitate the development of advanced motion-grounding models on such a task, we collect a large-scale dataset called GroundMoRe, which comprises 1,715 video clips, 249K object masks that are deliberately designed with 4 types (Causal, Sequential, Counterfactual, and Descriptive) for benchmarking deep and comprehensive motion understanding abilities. Our GroundMoRe uniquely requires models to generate visual answers (spatiotemporal masks), providing a more concrete and visually interpretable response than plain text. It evaluates models on spatiotemporal grounding and reasoning, helping address complex challenges in video reasoning, temporal perception, and pixel-level understanding. To further facilitate the proposed task, we propose a baseline model, Motion-Grounded Video Reasoning Assistant (MoRA). MoRA incorporates the multimodal reasoning ability from Multimodal LLM and the pixel-level perception capability from the grounding model (SAM) as well as an additional temporal localization head. MoRA achieves respectable performance on GroundMoRe outperforming the best existing visual grounding baseline model by an average of 28.8% relatively, but there still remains substantial room for interesting future improvements by the community. We hope this novel and challenging task will pave the way for future advancements in robust and general motion understanding via video reasoning segmentation.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ren2025,

title = {DA-VPT: Semantic-Guided Visual Prompt Tuning for Vision Transformers},

author = {Li Ren and Chen Chen and Liqiang Wang and Kien A. Hua},

url = {https://cvpr.thecvf.com/virtual/2025/poster/33078},

year  = {2025},

date = {2025-06-11},

abstract = {Visual Prompt Tuning (VPT) has become a promising solution for Parameter-Efficient Fine-Tuning (PEFT) approach for Vision Transformer (ViT) models by partially fine-tuning learnable tokens while keeping most model parameters frozen. Recent research has explored modifying the connection structures of the prompts. However, the fundamental correlation and distribution between the prompts and image tokens remain unexplored. In this paper, we leverage textit{metric learning} techniques to investigate how the distribution of prompts affects fine-tuning performance. Specifically, we propose a novel framework, textbf{D}istribution textbf{A}ware textbf{V}isual textbf{P}rompt Tuning (DA-VPT), to guide the distributions of the prompts by learning the distance metric from their class-related semantic data. Our method demonstrates that the prompts can serve as an effective bridge to share semantic information between image patches and the class token. We extensively evaluated our approach on popular benchmarks in both recognition and segmentation tasks. The results demonstrate that our approach enables more effective and efficient fine-tuning of ViT models by leveraging semantic information to guide the learning of the prompts, leading to improved performance on various downstream vision tasks. The code will be released.

},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Deng2025b,

title = {Seq2Time: Sequential Knowledge Transfer for Video LLM Temporal Grounding},

author = {Andong Deng and Zhongpai Gao and Anwesa Choudhuri and Benjamin Planche and Meng Zheng and Bin Wang and Terrence Chen and Chen Chen and Ziyan Wu},

url = {https://arxiv.org/abs/2411.16932},

year  = {2025},

date = {2025-06-11},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Temporal awareness is essential for video large language models (LLMs) to understand and reason about events within long videos, enabling applications like dense video captioning and temporal video grounding in a unified system. However, the scarcity of long videos with detailed captions and precise temporal annotations limits their temporal awareness. In this paper, we propose Seq2Time, a data-oriented training paradigm that leverages sequences of images and short video clips to enhance temporal awareness in long videos. By converting sequence positions into temporal annotations, we transform large-scale image and clip captioning datasets into sequences that mimic the temporal structure of long videos, enabling self-supervised training with abundant time-sensitive data. To enable sequence-to-time knowledge transfer, we introduce a novel time representation that unifies positional information across image sequences, clip sequences, and long videos. Experiments demonstrate the effectiveness of our method, achieving a 27.6% improvement in F1 score and 44.8% in CIDEr on the YouCook2 benchmark and a 14.7% increase in recall on the Charades-STA benchmark compared to the baseline.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Liu2024,

title = {Multi-View Attentive Contextualization for Multi-View 3D Object Detection},

author = {Xianpeng Liu and Ce Zheng and Ming Qian and Nan Xue and Chen Chen and Zhebin Zhang and Chen Li and Tianfu Wu},

year  = {2024},

date = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Peng2024,

title = {A Dual-Augmentor Framework for Domain Generalization in 3D Human Pose Estimation},

author = {Qucheng Peng and Ce Zheng and Chen Chen},

url = {https://www.computer.org/csdl/proceedings-article/cvpr/2024/530000c240/20hM2VBrKik

https://github.com/davidpengucf/DAF-DG

},

doi = {10.1109/CVPR52733.2024.00218},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {3D human pose data collected in controlled laboratory settings present challenges for pose estimators that generalize across diverse scenarios. To address this, domain generalization is employed. Current methodologies in domain generalization for 3D human pose estimation typically utilize adversarial training to generate synthetic poses for training. Nonetheless, these approaches exhibit several limitations. First, the lack of prior information about the target domain complicates the application of suitable augmentation through a single pose augmentor, affecting generalization on target domains. Moreover, adversarial training’s discriminator tends to enforce similarity between source and synthesized poses, impeding the exploration of out-ofsource distributions. Furthermore, the pose estimator’s optimization is not exposed to domain shifts, limiting its overall generalization ability. 

 

To address these limitations, we propose a novel framework featuring two pose augmentors: the weak and the strong augmentors. Our framework employs differential strategies for generation and discrimination processes, facilitating the preservation of knowledge related to source poses and the exploration of out-of-source distributions without prior information about target poses. Besides, we leverage meta-optimization to simulate domain shifts in the optimization process of the pose estimator, thereby improving its generalization ability. Our proposed approach significantly outperforms existing methods, as demonstrated through comprehensive experiments on various benchmark datasets.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Yuan2024,

title = {Towards Surveillance Video-and-Language Understanding: New Dataset, Baselines, and Challenges},

author = {Tongtong Yuan and Xuange Zhang and Kun Liu and Bo Liu and Chen Chen and Jian Jin and Zhenzhen Jiao},

url = {https://cvpr.thecvf.com/virtual/2024/poster/30807

https://xuange923.github.io/Surveillance-Video-Understanding

https://github.com/Xuange923/Surveillance-Video-Understanding},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Surveillance videos are an essential component of daily life with various critical applications, particularly in public security. However, current surveillance video tasks mainly focus on classifying and localizing anomalous events. Existing methods are limited to detecting and classifying the predefined events with unsatisfactory semantic understanding, although they have obtained considerable performance. To address this issue, we propose a new research direction of surveillance video-and-language understanding, and construct the first multimodal surveillance video dataset. We manually annotate the real-world surveillance dataset UCF-Crime with fine-grained event content and timing. Our newly annotated dataset, UCA UCF-Crime Annotation), contains 23,542 sentences, with an average length of 20 words, and its annotated videos are as long as 110.7 hours. Furthermore, we benchmark SOTA models for four multimodal tasks on this newly created dataset, which serve as new baselines for surveillance video-and-language understanding. Through our experiments, we find that mainstream models used in previously publicly available datasets perform poorly on surveillance video, which demonstrates the new challenges in surveillance video-and-language understanding. To validate the effectiveness of our UCA, we conducted experiments on multimodal anomaly detection. The results demonstrate that our multimodal surveillance learning can improve the performance of conventional anomaly detection tasks. All the experiments highlight the necessity of constructing this dataset to advance surveillance AI.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Pinyoanuntapong2024,

title = {MMM: Generative Masked Motion Model},

author = {Ekkasit Pinyoanuntapong and Pu Wang and Minwoo Lee and Chen Chen},

url = {https://arxiv.org/pdf/2312.03596.pdf

https://exitudio.github.io/MMM-page/

https://github.com/exitudio/MMM/},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Recent advances in text-to-motion generation using diffusion and autoregressive models have shown promising results. However, these models often suffer from a trade-off between real-time performance, high fidelity, and motion editability. To address this gap, we introduce MMM, a novel yet simple motion generation paradigm based on Masked Motion Model. MMM consists of two key components: (1) a motion tokenizer that transforms 3D human motion into a sequence of discrete tokens in latent space, and (2) a conditional masked motion transformer that learns to predict randomly masked motion tokens, conditioned on the pre-computed text tokens. By attending to motion and text tokens in all directions, MMM explicitly captures inherent dependency among motion tokens and semantic mapping between motion and text tokens. During inference, this allows parallel and iterative decoding of multiple motion tokens that are highly consistent with fine-grained text descriptions, therefore simultaneously achieving high-fidelity and high-speed motion generation. In addition, MMM has innate motion editability. By simply placing mask tokens in the place that needs editing, MMM automatically fills the gaps while guaranteeing smooth transitions between editing and non-editing parts. Extensive experiments on the HumanML3D and KIT-ML datasets demonstrate that MMM surpasses current leading methods in generating high-quality motion (evidenced by superior FID scores of 0.08 and 0.429), while offering advanced editing features such as body-part modification, motion in-betweening, and the synthesis of long motion sequences. In addition, MMM is two orders of magnitude faster on a single mid-range GPU than editable motion diffusion models.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wang2024,

title = {Skeleton-in-Context: Unified Skeleton Sequence Modeling with In-Context Learning},

author = {Xinshun Wang and Zhongbin Fang and Xia Li, Xiangtai Li and Chen Chen and Mengyuan Liu},

url = {https://arxiv.org/pdf/2312.03703.pdf

https://bradleywang0416.github.io/skeletonincontext/

https://github.com/fanglaosi/Skeleton-in-Context},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {In-context learning provides a new perspective for multi-task modeling for vision and NLP. Under this setting, the model can perceive tasks from prompts and accomplish them without any extra task-specific head predictions or model finetuning. However, Skeleton sequence modeling via in-context learning remains unexplored. Directly applying existing in-context models from other areas onto skeleton sequences fails due to the inter-frame and cross-task pose similarity that makes it outstandingly hard to perceive the task correctly from a subtle context.



To address this challenge, we propose Skeleton-in-Context (SiC), an effective framework for in-context skeleton sequence modeling. Our SiC is able to handle multiple skeleton-based tasks simultaneously after a single training process and accomplish each task from context according to the given prompt. It can further generalize to new, unseen tasks according to customized prompts. To facilitate context perception, we additionally propose a task-unified prompt, which adaptively learns tasks of different natures, such as partial joint-level generation, sequence-level prediction, or 2D-to-3D motion prediction. We conduct extensive experiments to evaluate the effectiveness of our SiC on multiple tasks, including motion prediction, pose estimation, joint completion, and future pose estimation. We also evaluate its generalization capability on unseen tasks such as motion-in-between. These experiments show that our model achieves state-of-the-art multi-task performance and even outperforms single-task methods on certain tasks.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chen2024,

title = {OST: Refining Text Knowledge with Optimal Spatio-Temporal Descriptor for General Video Recognition},

author = {Tongjia Chen and Hongshan Yu and Zhengeng Yang and Zechuan Li and Wei Sun and Chen Chen},

url = {https://arxiv.org/pdf/2312.00096.pdf

https://tomchen-ctj.github.io/OST/

https://github.com/tomchen-ctj/OST},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Due to the resource-intensive nature of training vision-language models on expansive video data, a majority of studies have centered on adapting pre-trained image-language models to the video domain. Dominant pipelines propose to tackle the visual discrepancies with additional temporal learners while overlooking the substantial discrepancy for web-scaled descriptive narratives and concise action category names, leading to less distinct semantic space and potential performance limitations. In this work, we prioritize the refinement of text knowledge to facilitate generalizable video recognition. To address the limitations of the less distinct semantic space of category names, we prompt a large language model (LLM) to augment action class names into Spatio-Temporal Descriptors thus bridging the textual discrepancy and serving as a knowledge base for general recognition. Moreover, to assign the best descriptors with different video instances, we propose Optimal Descriptor Solver, forming the video recognition problem as solving the optimal matching flow across frame-level representations and descriptors. Comprehensive evaluations in zero-shot, few-shot, and fully supervised video recognition highlight the effectiveness of our approach. Our best model achieves a state-of-the-art zero-shot accuracy of 75.1% on Kinetics-600.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2024,

title = {VidLA: Video-Language Alignment at Scale},

author = {Mamshad Nayeem Rizve and Fan Fei and Jayakrishnan Unnikrishnan and Son Tran and Benjamin Z. Yao and Belinda Zeng and Mubarak Shah and Trishul Chilimbi},

url = {https://arxiv.org/abs/2403.14870},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {In this paper, we propose VidLA, an approach for video-language alignment at scale. There are two major limitations of previous video-language alignment approaches. First, they do not capture both short-range and long-range temporal dependencies and typically employ complex hierarchical deep network architectures that are hard to integrate with existing pretrained image-text foundation models. To effectively address this limitation, we instead keep the network architecture simple and use a set of data tokens that operate at different temporal resolutions in a hierarchical manner, accounting for the temporally hierarchical nature of videos. By employing a simple two-tower architecture, we are able to initialize our video-language model with pretrained image-text foundation models, thereby boosting the final performance. Second, existing video-language alignment works struggle due to the lack of semantically aligned large-scale training data. To overcome it, we leverage recent LLMs to curate the largest video-language dataset to date with better visual grounding. Furthermore, unlike existing video-text datasets which only contain short clips, our dataset is enriched with video clips of varying durations to aid our temporally hierarchical data tokens in extracting better representations at varying temporal scales. Overall, empirical results show that our proposed approach surpasses state-of-the-art methods on multiple retrieval benchmarks, especially on longer videos, and performs competitively on classification benchmarks.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Ristea2024,

title = {Self-Distilled Masked Auto-Encoders are Efficient Video Anomaly Detectors},

author = {Nicolae Catalin Ristea and Florinel Alin Croitoru and Radu Tudor Ionescu and Marius Popescu and Fahad Khan and Mubarak Shah},

url = {https://arxiv.org/abs/2306.12041

https://github.com/ristea/aed-mae},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {We propose an efficient abnormal event detection model based on a lightweight masked auto-encoder (AE) applied at the video frame level. The novelty of the proposed model is threefold. First, we introduce an approach to weight tokens based on motion gradients, thus shifting the focus from the static background scene to the foreground objects. Second, we integrate a teacher decoder and a student decoder into our architecture, leveraging the discrepancy between the outputs given by the two decoders to improve anomaly detection. Third, we generate synthetic abnormal events to augment the training videos, and task the masked AE model to jointly reconstruct the original frames (without anomalies) and the corresponding pixel-level anomaly maps. Our design leads to an efficient and effective model, as demonstrated by the extensive experiments carried out on four benchmarks: Avenue, ShanghaiTech, UBnormal and UCSD Ped2. The empirical results show that our model achieves an excellent trade-off between speed and accuracy, obtaining competitive AUC scores, while processing 1655 FPS. Hence, our model is between 8 and 70 times faster than competing methods. We also conduct an ablation study to justify our design. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Thawakar2024,

title = {Composed Video Retrieval via Enriched Context and Discriminative Embeddings},

author = {Omkar Chakradhar Thawakar and Muzammal Naseer and Rao Muhammad Anwer and Salman Khan and Michael Felsberg and Mubarak Shah and Fahad Khan },

url = {https://arxiv.org/abs/2403.16997

https://github.com/OmkarThawakar/composed-video-retrieval},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

abstract = {Composed video retrieval (CoVR) is a challenging problem in computer vision which has recently highlighted the integration of modification text with visual queries for more sophisticated video search in large databases. Existing works predominantly rely on visual queries combined with modification text to distinguish relevant videos. However, such a strategy struggles to fully preserve the rich query-specific context in retrieved target videos and only represents the target video using visual embedding. We introduce a novel CoVR framework that leverages detailed language descriptions to explicitly encode query-specific contextual information and learns discriminative embeddings of vision only, text only and vision-text for better alignment to accurately retrieve matched target videos. Our proposed framework can be flexibly employed for both composed video (CoVR) and image (CoIR) retrieval tasks. Experiments on three datasets show that our approach obtains state-of-the-art performance for both CovR and zero-shot CoIR tasks, achieving gains as high as around 7% in terms of recall@K=1 score.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dutta2024,

title = {Multiview Aerial Visual RECognition (MAVREC) Dataset: Can Multi-view Improve Aerial Visual Perception?},

author = {Aritra Dutta and Srijan Das and Jacob Nielsen and Rajatsubhra Chakraborty and Mubarak Shah},

url = {https://arxiv.org/abs/2312.04548

https://mavrec.github.io/},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference (CVPR)},

abstract = {Despite the commercial abundance of UAVs, aerial data acquisition remains challenging, and the existing Asia and North America-centric open-source UAV datasets are small-scale or low-resolution and lack diversity in scene contextuality. Additionally, the color content of the scenes, solar-zenith angle, and population density of different geographies influence the data diversity. These two factors conjointly render suboptimal aerial-visual perception of the deep neural network (DNN) models trained primarily on the ground-view data, including the open-world foundational models.



To pave the way for a transformative era of aerial detection, we present Multiview Aerial Visual RECognition or MAVREC, a video dataset where we record synchronized scenes from different perspectives -- ground camera and drone-mounted camera. MAVREC consists of around 2.5 hours of industry-standard 2.7K resolution video sequences, more than 0.5 million frames, and 1.1 million annotated bounding boxes. This makes MAVREC the largest ground and aerial-view dataset, and the fourth largest among all drone-based datasets across all modalities and tasks. Through our extensive benchmarking on MAVREC, we recognize that augmenting object detectors with ground-view images from the corresponding geographical location is a superior pre-training strategy for aerial detection. Building on this strategy, we benchmark MAVREC with a curriculum-based semi-supervised object detection approach that leverages labeled (ground and aerial) and unlabeled (only aerial) images to enhance the aerial detection.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Azad2024,

title = {Activity-Biometrics: Person Identification from Daily Activities},

author = {Shehreen Azad and Yogesh Singh Rawat},

url = {https://arxiv.org/abs/2403.17360

https://github.com/sacrcv/Activity-Biometrics/

},

year  = {2024},

date = {2024-06-17},

urldate = {2024-06-17},

abstract = {In this work, we study a novel problem which focuses on person identification while performing daily activities. Learning biometric features from RGB videos is challenging due to spatio-temporal complexity and presence of appearance biases such as clothing color and background. We propose ABNet, a novel framework which leverages disentanglement of biometric and non-biometric features to perform effective person identification from daily activities. ABNet relies on a bias-less teacher to learn biometric features from RGB videos and explicitly disentangle non-biometric features with the help of biometric distortion. In addition, ABNet also exploits activity prior for biometrics which is enabled by joint biometric and activity learning. We perform comprehensive evaluation of the proposed approach across five different datasets which are derived from existing activity recognition benchmarks. Furthermore, we extensively compare ABNet with existing works in person identification and demonstrate its effectiveness for activity-based biometrics across all five datasets.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2023,

title = {R2Former: Unified retrieval and ranking Transformer for Place Recognition},

author = {Sijie Zhu and Linjie Yang and Chen Chen and Mubarak Shah and Xiaohui Shen and Heng Wang},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR_2023_PlaceRecognitionFinal.pdf

https://arxiv.org/pdf/2304.03410.pdf

https://github.com/Jeff-Zilence/R2Former},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Visual Place Recognition (VPR) estimates the location of query images by matching them with images in a reference database. Conventional methods generally adopt aggregated CNN features for global retrieval and RANSAC-based geometric verification for reranking. However, RANSAC only considers geometric information but ignores other possible information that could be useful for reranking, e.g. local feature correlation, and attention values. In this paper, we propose a unified place recognition framework that handles both retrieval and reranking with a novel transformer model, named R2Former. The proposed reranking module takes feature correlation, attention value, and xy coordinates into account, and learns to determine whether the image pair is from the same location. The whole pipeline is end-to-end trainable and the reranking module alone can

also be adopted on other CNN or transformer backbones as a generic component. Remarkably, R2Former significantly

outperforms state-of-the-art methods on major VPR datasets with much less inference time and memory consumption.

It also achieves the state-of-the-art on the holdout MSLS challenge set and could serve as a simple yet strong solution for real-world large-scale applications. Experiments also show vision transformer tokens are comparable and sometimes better than CNN local features on local matching. The code will be publicly available. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Gupta2023b,

title = {Class Prototypes based Contrastive Learning for Classifying Multi-Label and Fine-Grained Educational Videos},

author = {Rohit Gupta and Anirban Roy and Sujeong Kim and Claire Christensen and Todd Grindal and Sarah Nixon Gerard and Madeline Cincebeaux and Ajay Divakaran and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Rohit_SRI_CVPR2023_Multi_Modal_Multi_Label_Contrastive_Learning_Camera_Ready-4.pdf

https://www.rohitg.xyz/MMContrast/

https://nusci.csl.sri.com/project/APPROVE},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {The recent growth in the consumption of online media by children during early childhood necessitates data-driven tools enabling educators to filter out appropriate educational content for young learners. This paper presents an approach for detecting educational content in online videos. We focus on two widely used educational content classes: literacy and math. For each class, we choose prominent codes (sub-classes) based on the Common Core Standards. For example, literacy codes include ‘letter names’, ‘letter sounds’, and math codes include ‘counting’, ‘sorting’. We pose this as a fine-grained multilabel classification problem as videos can contain multiple types of educational content and the content classes can get visually similar (e.g., ‘letter names’ vs ‘letter sounds’). We propose a novel class prototypes based supervised contrastive learning approach that can handle fine-grained samples associated with multiple labels. We learn a class prototype for each class and a loss function is employed to minimize the distances between a class prototype and the samples from the class. Similarly,

distances between a class prototype and the samples from other classes are maximized. As the alignment between visual

and audio cues are crucial for effective comprehension, we consider a multimodal transformer network to capture the interaction between visual and audio cues in videos while learning the embedding for videos. For evaluation, we present a dataset, APPROVE, employing educational videos from YouTube labeled with fine-grained education classes by education researchers. APPROVE consists of 193 hours of expert-annotated videos with 19 classes. The proposed approach outperforms strong baselines on APPROVE and other benchmarks such as Youtube-8M, and COIN. The dataset is available at https://nusci.csl.sri.com/project/APPROVE.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Dave2023,

title = {TimeBalance: Temporally-Invariant and Temporally-Distinctive Video Representations for Semi-Supervised Action Recognition},

author = {Ishan Rajendrakumar Dave and Mamshad Nayeem Rizve and Chen Chen and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/TimeBalance_CVPR23_arxiv.pdf

https://daveishan.github.io/timebalance_webpage/

https://github.com/DAVEISHAN/TimeBalance},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Semi-Supervised Learning can be more beneficial for the video domain compared to images because of its higher annotation

cost and dimensionality. Besides, any video understanding task requires reasoning over both spatial and temporal dimensions. In order to learn both the static and motion related features for the semi-supervised action recognition task, existing methods rely on hard input inductive biases like using two-modalities (RGB and Optical-flow) or two-stream of different playback rates.

Instead of utilizing unlabeled videos through diverse input streams, we rely on self-supervised video representations,

particularly, we utilize temporally-invariant and temporally-distinctive representations. We observe that these representations complement each other depending on the nature of the action. Based on this observation, we propose a student-teacher semi-supervised learning framework, TimeBalance, where we distill the knowledge from a temporally-invariant and a temporally-distinctive teacher. Depending on the nature of the unlabeled video, we dynamically combine the knowledge of these two  teachers based on a novel temporal similarity-based reweighting scheme. Our method achieves state-of-the-art performance

on three action recognition benchmarks: UCF101, HMDB51, and Kinetics400. Code: https://github.com/DAVEISHAN/TimeBalance.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2023,

title = {PivoTAL: Prior-Driven Supervision for Weakly-Supervised Temporal Action Localization},

author = {Mamshad Nayeem Rizve and Gaurav Mittal and Ye Yu and Matthew Hall and Sandra Sajeev and Mubarak Shah and Mei Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR_2023.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR_2023_Supplemental_Material.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PivoTAL_CVPR2023_Poster.pdf

https://www.youtube.com/watch?v=6kAoQjXfzio},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Urooj2023,

title = {Learning Situation Hyper-Graphs for Video Question Answering},

author = {Aisha Urooj and Hilde Kuehne and Bo Wu and Kim Chheu and Walid Bousselham and Chuang Gan and Niels Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/2023072364-4.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/SHG_VQA_CVPR2023_cam_ready_supp.pdf

},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Answering questions about complex situations in videos requires not only capturing the presence of actors, objects, and their relations but also the evolution of these relationships over time. A situation hyper-graph is a representation that describes situations as scene sub-graphs for video frames and hyper-edges for connected sub-graphs and has been proposed to capture all such information in a compact structured form. In this work, we propose an architecture for Video Question Answering (VQA) that enables answering questions related to video content by predicting situation hyper-graphs, coined Situation Hyper-Graph based Video Question Answering (SHG-VQA). To this end, we train a situation hyper-graph decoder to implicitly identify graph representations with actions and object/human-object relationships from the input video clip. and to use cross-attention

between the predicted situation hyper-graphs and the question embedding to predict the correct answer. The proposed

method is trained in an end-to-end manner and optimized by a VQA loss with the cross-entropy function and a Hungarian

matching loss for the situation graph prediction. The effectiveness of the proposed architecture is extensively evaluated

on two challenging benchmarks: AGQA and STAR. Our results show that learning the underlying situation hypergraphs

helps the system to significantly improve its performance for novel challenges of video question-answering tasks. },

keywords = {CVPR, REU},

pubstate = {published},

tppubtype = {conference}

}

@conference{Bhunia2023,

title = {Person Image Synthesis via Denoising Diffusion Model},

author = {Ankan Kumar Bhunia and Salman Khan and Hisham Cholakkal and Rao Muhammad Anwer and Jorma Tapio Laaksonen and Mubarak Shah and Fahad Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/person_image_synthesis_via_den-Camera-ready-PDF.pdf

https://lnkd.in/d-8v3r8B

https://lnkd.in/dGPTjvge

https://lnkd.in/dxcGQsUX

https://github.com/ankanbhunia/PIDM},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {The pose-guided person image generation task requires synthesizing photorealistic images of humans in arbitrary poses. The existing approaches use generative adversarial networks that do not necessarily maintain realistic textures or need dense correspondences that struggle to handle complex  deformations and severe occlusions. In this work, we show how denoising diffusion models can be applied for high-fidelity person image synthesis with

strong sample diversity and enhanced mode coverage of the learnt data distribution. Our proposed Person Image Diffusion Model (PIDM) disintegrates the complex transfer problem into a series of simpler forward-backward denoising steps. This helps in learning plausible source-to-target transformation trajectories that result in faithful textures and undistorted appearance details. We introduce a ‘texture diffusion module’ based on cross-attention to accurately model the correspondences between appearance and pose information available in source and target images. Further, we propose  disentangled classifier-free guidance’ to ensure close resemblance between the conditional inputs and the synthesized output in terms of both pose and appearance information. Our extensive results on two large-scale benchmarks and a user study demonstrate the photorealism of our proposed approach under challenging scenarios. We also show how our generated images can help in downstream tasks. Code is available at https://github.com/ankanbhunia/PIDM.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Wasim2023,

title = {Vita-CLIP: Video and text adaptive CLIP via Multimodal Prompting},

author = {Syed Talal Wasim and Muzammal Naseer and Salman Khan and Fahad Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/vita_clip_video_and_text_adapt-Camera-ready-PDF.pdf

},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Adopting contrastive image-text pretrained models like CLIP towards video classification has gained attention due to its cost-effectiveness and competitive performance. However, recent works in this area face a trade-off. Finetuning the pretrained model to achieve strong supervised performance results in low zero-shot generalization. Similarly, freezing the backbone to retain zero-shot capability causes significant drop in supervised accuracy. Because of this,

recent works in literature typically train separate models for supervised and zero-shot action recognition. In this work, we propose a multimodal prompt learning scheme that works to balance the supervised and zero-shot performance under a single unified training. Our prompting approach on the vision side caters for three aspects: 1) Global video-level prompts to model the data distribution; 2) Local frame-level prompts to provide per-frame discriminative

conditioning; and 3) a summary prompt to extract a condensed video representation. Additionally, we define a prompting scheme on the text side to augment the textual context. Through this prompting scheme, we can achieve state-of-the-art zero-shot performance on Kinetics-600, HMDB51 and UCF101 while remaining competitive in the supervised setting. By keeping the pretrained backbone frozen, we optimize a much lower number of parameters and retain the existing general representation which helps achieve the strong zero-shot performance. Our codes and models will be publicly released. },

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Clark2023,

title = {Where We Are and What We're Looking At: Query Based Worldwide Image Geo-localization Using Hierarchies and Scenes},

author = {Brandon Eric Clark and Alec Kerrigan and Parth Parag Kulkarni and Vicente Vivanco Cepeda and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Camera-Ready-Full-Paper.pdf

https://github.com/AHKerrigan/GeoGuessNet

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR23-Poster_THU-PM-246-1.pdf

https://www.youtube.com/watch?v=fp3hZGbwPqk},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Determining the exact latitude and longitude that a photo was taken is a useful and widely applicable task, yet it remains exceptionally difficult despite the accelerated progress of other computer vision tasks. Most previous approaches have opted to learn single representations of query images, which are then classified at different levels of geographic granularity. These approaches fail to exploit the different visual cues that give context to different hierarchies, such as the country, state, and city level. To this end, we introduce an end-to-end transformer-based architecture that exploits the relationship between different geographic levels (which we refer to as hierarchies) and the corresponding visual scene information in an image through hierarchical cross-attention. We achieve this by learning a query for each geographic hierarchy and scene type. Furthermore, we learn a separate representation for different environmental scenes, as different scenes in the same location are often defined by completely different visual features. We achieve state of the art accuracy on 4 standard geo-localization datasets : Im2GPS, Im2GPS3k, YFCC4k, and YFCC26k, as well as qualitatively demonstrate how our method learns different representations for different visual hierarchies and scenes, which has not been demonstrated in the previous methods. Above previous testing  datasets mostly consist of iconic landmarks or images taken from social media, which makes the dataset a simple memory task, or makes it biased towards certain places. To address this issue we introduce a much harder testing dataset, Google-World-Streets-15k, comprised of images taken from Google Streetview covering the whole planet and present state of the art results. Our code can be found at https://github.com/AHKerrigan/GeoGuessNet.

},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rana2023,

title = {Hybrid Active Learning via Deep Clustering for Video Action Detection},

author = {Aayush Rana and Yogesh Singh Rawat},

url = {https://www.crcv.ucf.edu/research/projects/hybrid-active-learning-via-deep-clustering-for-video-action-detection/},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Chantry2023,

title = {A Large-scale Robustness Analysis of Video Action Recognition Models},

author = {Madeline Chantry and Naman Biyani and Prudvi Kamtam and Shruti Vyas and Hamid Palangi and Vibhav Vineet and Yogesh Singh Rawat},

url = {https://sites.google.com/view/videorobustnessbenchmark/home

https://www.crcv.ucf.edu/research/projects/ucf101-ds-action-recognition-for-real-world-distribution-shifts/

https://github.com/Maddy12/ActionRecognitionRobustnessEval

https://youtu.be/pv2AJ_t-v90

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/PosterCVPR2023.png},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zhu2023b,

title = {TopNet: Transformer-based Object Placement Network for Image Compositing },

author = {Sijie Zhu and Zhe Lin and Scott Cohen and Jason Kuen and Zhifei Zhang and Chen Chen},

url = {https://arxiv.org/pdf/2304.03372.pdf},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zheng2023,

title = {FeatER: An Efficient Network for Human Reconstruction via Feature Map-Based TransformER},

author = {Ce Zheng and Matias Mendieta and Taojiannan Yang and Guo-Jun Qi and Chen Chen},

url = {https://arxiv.org/pdf/2205.15448.pdf

https://zczcwh.github.io/feater_page/

https://github.com/zczcwh/FeatER},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zheng2023b,

title = {POTTER: Pooling Attention Transformer for Efficient Human Mesh Recovery},

author = {Ce Zheng and Xianpeng Liu and Guo-Jun Qi and Chen Chen},

url = {https://arxiv.org/pdf/2303.13357.pdf

https://zczcwh.github.io/potter_page/

https://github.com/zczcwh/POTTER},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PoseFormerV2: Exploring Frequency Domain for Efficient and Robust 3D Human Pose Estimation},

author = {Qitao Zhao and Ce Zheng and Mengyuan Liu and Pichao Wang and Chen Chen},

url = {https://arxiv.org/pdf/2303.17472.pdf

https://qitaozhao.github.io/PoseFormerV2

https://github.com/QitaoZhao/PoseFormerV2},

year  = {2023},

date = {2023-06-18},

urldate = {2023-06-18},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Cui2023,

title = {On the Robustness of Large Multimodal Models Against Image Adversarial Attacks},

author = {Xuanming Cui and Alejandro Aparcedo and Young Kyun Jang and Ser-Nam Lim},

url = {https://arxiv.org/pdf/2312.03777

https://arxiv.org/abs/2312.03777},

doi = {https://doi.org/10.48550/arXiv.2312.03777},

year  = {2023},

date = {2023-06-18},

publisher = {IEEE/CVF Computer Vision and Pattern Recognition Conference},

abstract = {Recent advances in instruction tuning have led to the development of State-of-the-Art Large Multimodal Models (LMMs). Given the novelty of these models, the impact of visual adversarial attacks on LMMs has not been thoroughly examined. We conduct a comprehensive study of the robustness of various LMMs against different adversarial attacks, evaluated across tasks including image classification, image captioning, and Visual Question Answer (VQA). We find that in general LMMs are not robust to visual adversarial inputs. However, our findings suggest that context provided to the model via prompts, such as questions in a QA pair helps to mitigate the effects of visual adversarial inputs. Notably, the LMMs evaluated demonstrated remarkable resilience to such attacks on the ScienceQA task with only an 8.10% drop in performance compared to their visual counterparts which dropped 99.73%. We also propose a new approach to real-world image classification which we term query decomposition. By incorporating existence queries into our input prompt we observe diminished attack effectiveness and improvements in image classification accuracy. This research highlights a previously under-explored facet of LMM robustness and sets the stage for future work aimed at strengthening the resilience of multimodal systems in adversarial environments.},

keywords = {CVPR, REU},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Self-Supervised Predictive Convolutional Attentive Block for Anomaly Detection},

author = {Nicolae-Catalin Ristea and Neelu Madan and Radu Tudor Ionescu and Kamal Nasrollahi and Fahad Shahbaz Khan and Thomas B. Moeslund and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/SSPCAB_camera-arxiv.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Anomaly Detection, CVPR, Self-Supervised Learning, Self-Supervision},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {UNICON: Combating Label Noise Through Uniform Selection and Contrastive Learning},

author = {Nazmul Karim and Mamshad Nayeem Rizve and Nazanin Rahnavard and Ajmal Mian and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07363.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/07363-supp.pdf

https://github.com/nazmul-karim170/unicon-noisy-label},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Contrastive Learning, CVPR, Noisy Labels, Semi-supervised learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {UBnormal: New Benchmark for Supervised Open-Set Video Anomaly Detection},

author = {Andra Acsintoae and Andrei Florescu and Mariana-Iuliana Georgescu and Tudor Mare and Paul Sumedrea and Radu Tudor Ionescu and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/04315.pdf

https://github.com/lilygeorgescu/UBnormal},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Anomaly Detection, CVPR, Dataset},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {SPAct: Self-supervised Privacy Preservation for Action Recognition},

author = {Ishan Rajendrakumar Dave and Chen Chen and Mubarak Shah},

url = {https://arxiv.org/pdf/2203.15205.pdf

https://github.com/DAVEISHAN/SPAct

https://www.youtube.com/watch?v=_PAlMT7ozts},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Action Recognition, CVPR, Privacy Preservation},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {End-to-End Semi-Supervised Learning for Video Action Detection},

author = {Akash Kumar and Yogesh Singh Rawat},

url = {https://arxiv.org/pdf/2203.04251.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR, Semi-supervised learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {Local Learning Matters: Rethinking Data Heterogeneity in Federated Learning},

author = {Matias Mendieta and Taojiannan Yang and Pu Wang and Minwoo Lee and Zhengming Ding and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11405.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11405_supp.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {CVPR, Federated Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {TransGeo: Transformer Is All You Need for Cross-view Image Geo-localization},

author = {Sijie Zhu and Mubarak Shah and Chen Chen},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/11695-supp.pdf},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

keywords = {Cross-View, CVPR, Geo-Localization, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {PSTR: End-to-End One-Step Person Search With Transformers},

author = {Jiale Cao and Yenwai Pang and Rao Muhammad Anwer and Hisham Cholakkal and Jin Xie and Mubarak Shah and Fahad Shahbaz Khan},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/05237-2.pdf

https://github.com/JialeCao001/PSTR},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {We propose a novel one-step transformer-based person search framework, PSTR, that jointly performs person detection and re-identification (re-id) in a single architecture. PSTR comprises a person search-specialized (PSS) module that contains a detection encoder-decoder for person detection along with a discriminative re-id decoder for person re-id. The discriminative re-id decoder utilizes a multi-level supervision scheme with a shared decoder for

discriminative re-id feature learning and also comprises a part attention block to encode relationship between different parts of a person. We further introduce a simple multi-scale scheme to support re-id across person instances at different scales. PSTR jointly achieves the diverse objectives of object-level recognition (detection) and instance-level matching (re-id). To the best of our knowledge, we are the first to propose an end-to-end one-step

transformer-based person search framework. Experiments are performed on two popular benchmarks: CUHK-SYSU and PRW. Our extensive ablations reveal the merits of the proposed contributions. Further, the proposed PSTR sets a new state-of-the-art on both benchmarks. On the challenging

PRW benchmark, PSTR achieves a mean average precision (mAP) score of 56.5%. The source code is available at https://github.com/JialeCao001/PSTR.},

keywords = {CVPR, Re-Identification, Transformers, Visual Search},

pubstate = {published},

tppubtype = {conference}

}

@conference{nokey,

title = {OW-DETR: Open-world Detection Transformer},

author = {Akshita Gupta and Sanath Narayan and K J Joseph and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/03815-supp.pdf

https://github.com/akshitac8/OW-DETR.},

year  = {2022},

date = {2022-06-19},

urldate = {2022-06-19},

publisher = {IEEE Computer Vision and Pattern Recognition},

abstract = {Open-world object detection (OWOD) is a challenging computer vision problem, where the task is to detect a known set of object categories while  simultaneously identifying unknown objects. Additionally, the model must incrementally learn new classes that become known in the next training episodes. Distinct from standard object detection, the OWOD setting poses significant challenges for generating quality candidate proposals on potentially unknown objects, separating the unknown objects from the background and detecting diverse unknown objects. Here, we introduce a novel end-to-end transformer-based framework, OW-DETR, for open-world object detection. The proposed OW-DETR comprises three dedicated components

namely, attention-driven pseudo-labeling, novelty classification and objectness scoring to explicitly address the aforementioned OWOD challenges. Our OW-DETR explicitly encodes multi-scale contextual information, possesses less inductive bias, enables knowledge transfer from known classes to the unknown class and can better discriminate between unknown objects and background. Comprehensive experiments are performed on two benchmarks: MS-COCO and PASCAL VOC. The extensive ablations reveal the merits of our proposed contributions. Further, our model outperforms the recently introduced OWOD approach, ORE, with absolute gains ranging from 1.8% to 3.3% in terms of unknown recall on MS-COCO. In the case of incremental

object detection, OW-DETR outperforms the state-of-the art for all settings on PASCAL VOC. Our code is available at https://github.com/akshitac8/OW-DETR.},

keywords = {CVPR, Object Detection, Open World, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sultani2021,

title = {Dogfight: Detecting Drones from Drones Videos },

author = {Muhammad Waseem Ashraf and Waqas Sultani and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Dogfight.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Dogfight_Supp.mp4

https://github.com/mwaseema/Drone-Detection

http://im.itu.edu.pk/dogfight-detecting-drones-from-drones-videos/

https://docs.google.com/presentation/d/1huBSbYzyNUCs-gJHdSU2CZ0XfRlBWGOQ/edit#slide=id.p1},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {CVPR, Drone Video Analysis, Object Detection, UAV Video Analysis, UVA Video Analysis},

pubstate = {published},

tppubtype = {conference}

}

@conference{Tirupattur2021,

title = {Modeling Multi-Label Action Dependencies for Temporal Action Localization},

author = {Praveen Tirupattur and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization_Supp.zip

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Modeling-Multi-Label-Action-Dependencies-for-Temporal-Action-Localization-CVPR-2021-ORAL.mp4},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition (Oral)},

keywords = {CVPR, Transformers},

pubstate = {published},

tppubtype = {conference}

}

@conference{Zaeemzadeh2021,

title = {Out-of-Distribution Detection Using Union of 1-Dimensional Subspaces},

author = {Alireza Zaeemzadeh and Niccolò Bisagno and Zeno Sambugaro and Nicola Conci and Nazanin Rahnavard and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Out-of-Distribution-Detection-Using-Union-of-1-Dimensional-Subspaces.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Out-of-Distribution-Detection-Using-Union-of-1-Dimensional-Subspaces_Supp.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/CVPR21_presentation_video.mov},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {CVPR, Open World, Out of Distribution (OOD)},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2021b,

title = {Found a Reason for me? Weakly-supervised Grounded Visual Question Answering using Capsules},

author = {Aisha Urooj Khan and Hilde Kuehne and Kevin Duarte and Chuang Gan and Niels Da Vitoria Lobo and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Found-a-Reason-for-me.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Found-a-Reason-for-me_Supp.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/cvpr21_poster_v2.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/cvpr_2021_5min.mp4},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {Capsule Networks, CVPR, Grounding, VQA},

pubstate = {published},

tppubtype = {conference}

}

@conference{Rizve2021b,

title = {Exploring Complementary Strengths of Invariant and Equivariant Representations for Few-Shot Learning},

author = {Mamshad Nayeem Rizve and Salman Khan and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Exploring-Complementary-Strengths-of-Invariant-and-Equivariant.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/Exploring-Complementary-Strengths-of-Invariant-and-Equivariant_Supp.pdf

https://github.com/nayeemrizve/invariance-equivariance

https://www.crcv.ucf.edu/wp-content/uploads/2018/11/FSL_CVPR2021_Video_Final.mp4},

year  = {2021},

date = {2021-06-19},

urldate = {2021-06-19},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {CVPR, Equivariance, Few-Shot Learning, Invariance},

pubstate = {published},

tppubtype = {conference}

}

@conference{Jonediei2020,

title = {Select to Better Learn: Fast and Accurate Deep Learning using Data Selection from Nonlinear Manifolds},

author = {Mohsen Joneidi and Saeed Vahidian and Ashkan Esmaeili and Weijia Wang and Nazanin Rahnavard and Bill Lin and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/04/Select-to-Better-Learn.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2020/04/Select-to-Better-Learn_Supplementary.pdf},

year  = {2020},

date = {2020-06-14},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

abstract = {Finding a small subset of data whose linear combination spans other data points, also called column subset selection problem (CSSP), is an important open problem in computer science with many applications in computer vision and deep learning such as the ones shown in Fig. 1. There are some studies that solve CSSP in a polynomial time complexity w.r.t. the size of the original dataset. A simple and efficient selection algorithm with a linear complexity order, referred to as spectrum pursuit (SP), is proposed that pursuits spectral components of the dataset using available sample points. The proposed non-greedy algorithm aims to iteratively find K data samples whose span is close to that of the first K spectral components of entire data. SP has no parameter to be fine tuned and this desirable property makes it problem-independent. The simplicity of SP enables us to extend the underlying linear model to more complex models such as nonlinear manifolds and graph-based models. The nonlinear extension of SP is introduced as kernel-SP (KSP). The superiority of the proposed algorithms is demonstrated in a wide range of applications.},

keywords = {CVPR},

pubstate = {published},

tppubtype = {conference}

}

@conference{Duarte2020,

title = {Visual-textual Capsule Routing for Text-based Video Segmentation},

author = {Bruce McIntosh and Kevin Duarte and Yogesh Singh Rawat and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/06/Projects_Visual-textual-Capsule-Routing-for-Text-based-Video-Segmentation.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2020/06/Projects_Visual-textual-Capsule-Routing-for-Text-based-Video-Segmentation_Supplementary.zip

https://www.crcv.ucf.edu/research/projects/visual-textual-capsule-routing-for-text-based-video-segmentation/},

year  = {2020},

date = {2020-06-14},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition (Oral)},

edition = {(Oral)},

abstract = {Joint understanding of vision and natural language is a challenging problem with a wide range of applications in artificial intelligence. In this work, we focus on integration of video and text for the task of actor and action video segmentation from a sentence. We propose a capsule-based approach which performs pixel-level localization based on a natural language query describing the actor of interest. We encode both the video and textual input in the form of capsules, which provide a more effective representation in comparison with standard convolution based features. Our novel visual-textual routing mechanism allows for the fusion of video and text capsules to successfully localize the actor and action. The existing works on actor-action localization are mainly focused on localization in a single frame instead of the full video. Different from existing works, we propose to perform the localization on all frames of the video. To validate the potential of the proposed network for actor and action video localization, we extend an existing actor-action dataset (A2D) with annotations for all the frames. The experimental evaluation demonstrates the effectiveness of our capsule network for text selective actor and action localization in videos. The proposed method also improves upon the performance of the existing state-of-the art works on single frame-based localization. },

keywords = {Capsule Networks, CVPR, Video Object Segmentation},

pubstate = {published},

tppubtype = {conference}

}

@conference{Khan2020,

title = {iTAML : An Incremental Task-Agnostic Meta-learning Approach},

author = {Jathushan Rajasegaran and Salman Khan and Munawar Hayat and Fahad Shahbaz Khan and Mubarak Shah},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/04/iTAML.pdf

https://www.crcv.ucf.edu/wp-content/uploads/2020/04/iTAML-Supplementary.pdf},

year  = {2020},

date = {2020-06-14},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

abstract = {Humans can continuously learn new knowledge as their experience grows. In contrast, previous learning in deep neural networks can quickly fade out when they are trained on a new task. In this paper, we hypothesize this problem can be avoided by learning a set of generalized parameters, that are neither specific to old nor new tasks. In this pursuit, we introduce a novel meta-learning approach that seeks to maintain an equilibrium between all the encountered tasks. This is ensured by a new meta-update rule which avoids catastrophic forgetting. In comparison to previous metalearning techniques, our approach is task-agnostic. When presented with a continuum of data, our model automatically identifies the task and quickly adapts to it with just a single update. We perform extensive experiments on five datasets in a class-incremental setting, leading to significant improvements over the state of the art methods (e.g., a 21.3% boost on CIFAR100 with 10 incremental tasks). Specifically, on large-scale datasets that generally prove difficult cases for incremental learning, our approach delivers absolute gains as high as 19.1% and 7.4% on ImageNet and MS-Celeb datasets, respectively. Our codes are available at: https://github.com/brjathu/iTAML. },

keywords = {CVPR, Meta-Learning},

pubstate = {published},

tppubtype = {conference}

}

@conference{Spampinato2017,

title = {Deep Learning Human Mind for Automated Visual Classification},

author = {Concetto Spampinato and Simone Palazzo and Isaak Kavasidis and Daniela Giordano and Mubarak Shah and Nasim Souly},

url = {https://www.crcv.ucf.edu/wp-content/uploads/2020/08/Publications_Deep-Learning-Human-Mind-for-Automated-Visual-Classification.pdf

},

year  = {2017},

date = {2017-07-21},

urldate = {2017-07-21},

publisher = {IEEE Conference on Computer Vision and Pattern Recognition},

keywords = {Classification, CVPR, EEG/Brain},

pubstate = {published},

tppubtype = {conference}

}